Merge branch 'master' into add-test-43003

.github/workflows/pull_request.yml

@@ -172,7 +172,7 @@ jobs:
   ################################# Stage Final #################################
   #
   FinishCheck:
-    if: ${{ !cancelled() }}
+    if: ${{ !failure() }}
     needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:

README.md

@@ -34,14 +34,12 @@ curl https://clickhouse.com/ | sh
 
 Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
 
-* [v24.6 Community Call](https://clickhouse.com/company/events/v24-6-community-release-call) - Jul 2
+* [v24.7 Community Call](https://clickhouse.com/company/events/v24-7-community-release-call) - Jul 30
 
 ## Upcoming Events
 
 Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
 
-* [AWS Summit in DC](https://clickhouse.com/company/events/2024-06-aws-summit-dc) - Jun 26
-* [ClickHouse Meetup in Amsterdam](https://www.meetup.com/clickhouse-netherlands-user-group/events/300781068/) - Jun 27
 * [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/300783448/) - Jul 9
 * [ClickHouse Cloud - Live Update Call](https://clickhouse.com/company/events/202407-cloud-update-live) - Jul 9
 * [ClickHouse Meetup @ Ramp - New York City](https://www.meetup.com/clickhouse-new-york-user-group/events/300595845/) - Jul 9

base/base/demangle.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdlib>
 #include <memory>
 #include <string>
 

base/base/extended_types.h

@@ -108,6 +108,14 @@ struct make_unsigned // NOLINT(readability-identifier-naming)
     using type = std::make_unsigned_t<T>;
 };
 
+template <> struct make_unsigned<Int8> { using type = UInt8; };
+template <> struct make_unsigned<UInt8> { using type = UInt8; };
+template <> struct make_unsigned<Int16> { using type = UInt16; };
+template <> struct make_unsigned<UInt16> { using type = UInt16; };
+template <> struct make_unsigned<Int32> { using type = UInt32; };
+template <> struct make_unsigned<UInt32> { using type = UInt32; };
+template <> struct make_unsigned<Int64> { using type = UInt64; };
+template <> struct make_unsigned<UInt64> { using type = UInt64; };
 template <> struct make_unsigned<Int128> { using type = UInt128; };
 template <> struct make_unsigned<UInt128> { using type = UInt128; };
 template <> struct make_unsigned<Int256>  { using type = UInt256; };
@@ -121,6 +129,14 @@ struct make_signed // NOLINT(readability-identifier-naming)
     using type = std::make_signed_t<T>;
 };
 
+template <> struct make_signed<Int8> { using type = Int8; };
+template <> struct make_signed<UInt8> { using type = Int8; };
+template <> struct make_signed<Int16> { using type = Int16; };
+template <> struct make_signed<UInt16> { using type = Int16; };
+template <> struct make_signed<Int32> { using type = Int32; };
+template <> struct make_signed<UInt32> { using type = Int32; };
+template <> struct make_signed<Int64> { using type = Int64; };
+template <> struct make_signed<UInt64> { using type = Int64; };
 template <> struct make_signed<Int128>  { using type = Int128; };
 template <> struct make_signed<UInt128> { using type = Int128; };
 template <> struct make_signed<Int256>  { using type = Int256; };

base/base/isSharedPtrUnique.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <memory>
+
+template <typename T>
+bool isSharedPtrUnique(const std::shared_ptr<T> & ptr)
+{
+    return ptr.use_count() == 1;
+}

base/poco/Foundation/include/Poco/Format.h

@@ -232,7 +232,7 @@ void Foundation_API format(
     const Any & value10);
 
 
-void Foundation_API format(std::string & result, const std::string & fmt, const std::vector<Any> & values);
+void Foundation_API formatVector(std::string & result, const std::string & fmt, const std::vector<Any> & values);
 /// Supports a variable number of arguments and is used by
 /// all other variants of format().
 

base/poco/Foundation/include/Poco/RefCountedObject.h

@@ -21,6 +21,8 @@
 #include "Poco/AtomicCounter.h"
 #include "Poco/Foundation.h"
 
+#include <atomic>
+
 
 namespace Poco
 {

base/poco/Foundation/src/Format.cpp

@@ -51,8 +51,8 @@ namespace
 		}
 		if (width != 0) str.width(width);
 	}
-	
-	
+
+
 	void parsePrec(std::ostream& str, std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
 	{
 		if (itFmt != endFmt && *itFmt == '.')
@@ -67,7 +67,7 @@ namespace
 			if (prec >= 0) str.precision(prec);
 		}
 	}
-	
+
 	char parseMod(std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
 	{
 		char mod = 0;
@@ -77,13 +77,13 @@ namespace
 			{
 			case 'l':
 			case 'h':
-			case 'L': 
+			case 'L':
 			case '?': mod = *itFmt++; break;
 			}
 		}
 		return mod;
 	}
-	
+
 	std::size_t parseIndex(std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
 	{
 		int index = 0;
@@ -110,8 +110,8 @@ namespace
 		case 'f': str << std::fixed; break;
 		}
 	}
-	
-	
+
+
 	void writeAnyInt(std::ostream& str, const Any& any)
 	{
 		if (any.type() == typeid(char))
@@ -201,7 +201,7 @@ namespace
 					str << RefAnyCast<std::string>(*itVal++);
 					break;
 				case 'z':
-					str << AnyCast<std::size_t>(*itVal++); 
+					str << AnyCast<std::size_t>(*itVal++);
 					break;
 				case 'I':
 				case 'D':
@@ -303,7 +303,7 @@ void format(std::string& result, const std::string& fmt, const Any& value)
 {
 	std::vector<Any> args;
 	args.push_back(value);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -312,7 +312,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	std::vector<Any> args;
 	args.push_back(value1);
 	args.push_back(value2);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -322,7 +322,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value1);
 	args.push_back(value2);
 	args.push_back(value3);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -333,7 +333,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value2);
 	args.push_back(value3);
 	args.push_back(value4);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -345,7 +345,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value3);
 	args.push_back(value4);
 	args.push_back(value5);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -358,7 +358,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value4);
 	args.push_back(value5);
 	args.push_back(value6);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -372,7 +372,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value5);
 	args.push_back(value6);
 	args.push_back(value7);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -387,7 +387,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value6);
 	args.push_back(value7);
 	args.push_back(value8);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -403,7 +403,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value7);
 	args.push_back(value8);
 	args.push_back(value9);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -420,16 +420,16 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value8);
 	args.push_back(value9);
 	args.push_back(value10);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
-void format(std::string& result, const std::string& fmt, const std::vector<Any>& values)
+void formatVector(std::string& result, const std::string& fmt, const std::vector<Any>& values)
 {
 	std::string::const_iterator itFmt  = fmt.begin();
 	std::string::const_iterator endFmt = fmt.end();
 	std::vector<Any>::const_iterator itVal  = values.begin();
-	std::vector<Any>::const_iterator endVal = values.end(); 
+	std::vector<Any>::const_iterator endVal = values.end();
 	while (itFmt != endFmt)
 	{
 		switch (*itFmt)

base/poco/MongoDB/src/ObjectId.cpp

@@ -57,7 +57,7 @@ std::string ObjectId::toString(const std::string& fmt) const
 
 	for (int i = 0; i < 12; ++i)
 	{
-		s += format(fmt, (unsigned int) _id[i]);
+		s += Poco::format(fmt, (unsigned int) _id[i]);
 	}
 	return s;
 }

base/poco/MongoDB/src/OpMsgCursor.cpp

@@ -43,9 +43,9 @@ namespace Poco {
 namespace MongoDB {
 
 
-static const std::string keyCursor		{"cursor"};
-static const std::string keyFirstBatch	{"firstBatch"};
-static const std::string keyNextBatch	{"nextBatch"};
+[[ maybe_unused ]] static const std::string keyCursor		{"cursor"};
+[[ maybe_unused ]] static const std::string keyFirstBatch	{"firstBatch"};
+[[ maybe_unused ]] static const std::string keyNextBatch	{"nextBatch"};
 
 static Poco::Int64 cursorIdFromResponse(const MongoDB::Document& doc);
 
@@ -131,7 +131,7 @@ OpMsgMessage& OpMsgCursor::next(Connection& connection)
 			connection.readResponse(_response);
 		}
 		else
-#endif		
+#endif
 		{
 			_response.clear();
 			_query.setCursor(_cursorID, _batchSize);

base/poco/Net/include/Poco/Net/NameValueCollection.h

@@ -79,7 +79,7 @@ namespace Net
         /// Returns the value of the first name-value pair with the given name.
         /// If no value with the given name has been found, the defaultValue is returned.
 
-        const std::vector<std::reference_wrapper<const std::string>> getAll(const std::string & name) const;
+        std::vector<std::string> getAll(const std::string & name) const;
         /// Returns all values of all name-value pairs with the given name.
         ///
         /// Returns an empty vector if there are no name-value pairs with the given name.

base/poco/Net/src/HTTPMessage.cpp

@@ -17,9 +17,9 @@
 #include "Poco/NumberFormatter.h"
 #include "Poco/NumberParser.h"
 #include "Poco/String.h"
+#include <charconv>
 #include <format>
 
-
 using Poco::NumberFormatter;
 using Poco::NumberParser;
 using Poco::icompare;
@@ -75,7 +75,7 @@ void HTTPMessage::setContentLength(std::streamsize length)
 		erase(CONTENT_LENGTH);
 }
 
-	
+
 std::streamsize HTTPMessage::getContentLength() const
 {
 	const std::string& contentLength = get(CONTENT_LENGTH, EMPTY);
@@ -98,7 +98,7 @@ void HTTPMessage::setContentLength64(Poco::Int64 length)
 		erase(CONTENT_LENGTH);
 }
 
-	
+
 Poco::Int64 HTTPMessage::getContentLength64() const
 {
 	const std::string& contentLength = get(CONTENT_LENGTH, EMPTY);
@@ -133,13 +133,13 @@ void HTTPMessage::setChunkedTransferEncoding(bool flag)
 		setTransferEncoding(IDENTITY_TRANSFER_ENCODING);
 }
 
-	
+
 bool HTTPMessage::getChunkedTransferEncoding() const
 {
 	return icompare(getTransferEncoding(), CHUNKED_TRANSFER_ENCODING) == 0;
 }
 
-	
+
 void HTTPMessage::setContentType(const std::string& mediaType)
 {
 	if (mediaType.empty())
@@ -154,7 +154,7 @@ void HTTPMessage::setContentType(const MediaType& mediaType)
 	setContentType(mediaType.toString());
 }
 
-	
+
 const std::string& HTTPMessage::getContentType() const
 {
 	return get(CONTENT_TYPE, UNKNOWN_CONTENT_TYPE);

base/poco/Net/src/NameValueCollection.cpp

@@ -102,9 +102,9 @@ const std::string& NameValueCollection::get(const std::string& name, const std::
 		return defaultValue;
 }
 
-const std::vector<std::reference_wrapper<const std::string>> NameValueCollection::getAll(const std::string& name) const
+std::vector<std::string> NameValueCollection::getAll(const std::string& name) const
 {
-    std::vector<std::reference_wrapper<const std::string>> values;
+    std::vector<std::string> values;
     for (ConstIterator it = _map.find(name); it != _map.end(); it++)
         if (it->first == name)
             values.push_back(it->second);

contrib/avro

@@ -1 +1 @@
-Subproject commit d43acc84d3d455b016f847d6666fbc3cd27f16a9
+Subproject commit 545e7002683cbc2198164d93088ac8e4955b4628

contrib/aws-cmake/CMakeLists.txt

@@ -125,7 +125,7 @@ configure_file("${AWS_SDK_CORE_DIR}/include/aws/core/SDKConfig.h.in"
     "${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY)
 
 aws_get_version(AWS_CRT_CPP_VERSION_MAJOR AWS_CRT_CPP_VERSION_MINOR AWS_CRT_CPP_VERSION_PATCH FULL_VERSION GIT_HASH)
-configure_file("${AWS_CRT_DIR}/include/aws/crt/Config.h.in" "${AWS_CRT_DIR}/include/aws/crt/Config.h" @ONLY)
+configure_file("${AWS_CRT_DIR}/include/aws/crt/Config.h.in" "${CMAKE_CURRENT_BINARY_DIR}/include/aws/crt/Config.h" @ONLY)
 
 list(APPEND AWS_SOURCES ${AWS_SDK_CORE_SRC} ${AWS_SDK_CORE_NET_SRC} ${AWS_SDK_CORE_PLATFORM_SRC})
 

contrib/grpc

@@ -1 +1 @@
-Subproject commit 77b2737a709d43d8c6895e3f03ca62b00bd9201c
+Subproject commit f5b7fdc2dff09ada06dbf6c75df298fb40f898df

contrib/libpq-cmake/CMakeLists.txt

@@ -54,7 +54,6 @@ set(SRCS
         "${LIBPQ_SOURCE_DIR}/port/pgstrcasecmp.c"
         "${LIBPQ_SOURCE_DIR}/port/thread.c"
         "${LIBPQ_SOURCE_DIR}/port/path.c"
-        "${LIBPQ_SOURCE_DIR}/port/explicit_bzero.c"
         )
 
 add_library(_libpq ${SRCS})

contrib/orc

@@ -1 +1 @@
-Subproject commit 947cebaf9432d708253ac08dc3012daa6b4ede6f
+Subproject commit bcc025c09828c556f54cfbdf83a66b9acae7d17f

contrib/rocksdb

@@ -1 +1 @@
-Subproject commit 3a0b80ca9d6eebb38fad7ea3f41dfc9db4f6a984
+Subproject commit 078fa5638690004e1f744076d1bdcc4e93767304

contrib/rocksdb-cmake/CMakeLists.txt

@@ -1,7 +1,7 @@
-option (ENABLE_ROCKSDB "Enable rocksdb library" ${ENABLE_LIBRARIES})
+option (ENABLE_ROCKSDB "Enable RocksDB" ${ENABLE_LIBRARIES})
 
 if (NOT ENABLE_ROCKSDB)
-  message (STATUS "Not using rocksdb")
+  message (STATUS "Not using RocksDB")
   return()
 endif()
 

docker/test/fasttest/run.sh

@@ -84,6 +84,8 @@ function start_server
     echo "ClickHouse server pid '$server_pid' started and responded"
 }
 
+export -f start_server
+
 function clone_root
 {
     [ "$UID" -eq 0 ] && git config --global --add safe.directory "$FASTTEST_SOURCE"
@@ -254,6 +256,19 @@ function configure
     rm -f "$FASTTEST_DATA/config.d/secure_ports.xml"
 }
 
+function timeout_with_logging() {
+    local exit_code=0
+
+    timeout -s TERM --preserve-status "${@}" || exit_code="${?}"
+
+    if [[ "${exit_code}" -eq "124" ]]
+    then
+      echo "The command 'timeout ${*}' has been killed by timeout"
+    fi
+
+    return $exit_code
+}
+
 function run_tests
 {
     clickhouse-server --version
@@ -269,6 +284,11 @@ function run_tests
       NPROC=1
     fi
 
+    export CLICKHOUSE_CONFIG_DIR=$FASTTEST_DATA
+    export CLICKHOUSE_CONFIG="$FASTTEST_DATA/config.xml"
+    export CLICKHOUSE_USER_FILES="$FASTTEST_DATA/user_files"
+    export CLICKHOUSE_SCHEMA_FILES="$FASTTEST_DATA/format_schemas"
+
     local test_opts=(
         --hung-check
         --fast-tests-only
@@ -292,6 +312,8 @@ function run_tests
     clickhouse stop --pid-path "$FASTTEST_DATA"
 }
 
+export -f run_tests
+
 case "$stage" in
 "")
     ls -la
@@ -315,7 +337,7 @@ case "$stage" in
     configure 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/install_log.txt"
     ;&
 "run_tests")
-    run_tests
+    timeout_with_logging 35m bash -c run_tests ||:
     /process_functional_tests_result.py --in-results-dir "$FASTTEST_OUTPUT/" \
         --out-results-file "$FASTTEST_OUTPUT/test_results.tsv" \
         --out-status-file "$FASTTEST_OUTPUT/check_status.tsv" || echo -e "failure\tCannot parse results" > "$FASTTEST_OUTPUT/check_status.tsv"

docker/test/stateless/run.sh

@@ -6,19 +6,31 @@ source /setup_export_logs.sh
 # fail on errors, verbose and export all env variables
 set -e -x -a
 
+MAX_RUN_TIME=${MAX_RUN_TIME:-10800}
+MAX_RUN_TIME=$((MAX_RUN_TIME == 0 ? 10800 : MAX_RUN_TIME))
+
+USE_DATABASE_REPLICATED=${USE_DATABASE_REPLICATED:=0}
+USE_SHARED_CATALOG=${USE_SHARED_CATALOG:=0}
+
+# disable for now
+RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0
+
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] || [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+  RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0
+fi
+
 # Choose random timezone for this test run.
 #
 # NOTE: that clickhouse-test will randomize session_timezone by itself as well
 # (it will choose between default server timezone and something specific).
 TZ="$(rg -v '#' /usr/share/zoneinfo/zone.tab  | awk '{print $3}' | shuf | head -n1)"
-echo "Choosen random timezone $TZ"
+echo "Chosen random timezone $TZ"
 ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" > /etc/timezone
 
 dpkg -i package_folder/clickhouse-common-static_*.deb
 dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
-# Accept failure in the next two commands until 24.4 is released (for compatibility and Bugfix validation run)
-dpkg -i package_folder/clickhouse-odbc-bridge_*.deb || true
-dpkg -i package_folder/clickhouse-library-bridge_*.deb || true
+dpkg -i package_folder/clickhouse-odbc-bridge_*.deb
+dpkg -i package_folder/clickhouse-library-bridge_*.deb
 dpkg -i package_folder/clickhouse-server_*.deb
 dpkg -i package_folder/clickhouse-client_*.deb
 
@@ -55,12 +67,6 @@ if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; th
     rm /etc/clickhouse-server/users.d/s3_cache_new.xml
     rm /etc/clickhouse-server/config.d/zero_copy_destructive_operations.xml
 
-    #todo: remove these after 24.3 released.
-    sudo sed -i "s|<object_storage_type>azure<|<object_storage_type>azure_blob_storage<|" /etc/clickhouse-server/config.d/azure_storage_conf.xml
-
-    #todo: remove these after 24.3 released.
-    sudo sed -i "s|<object_storage_type>local<|<object_storage_type>local_blob_storage<|" /etc/clickhouse-server/config.d/storage_conf.xml
-
     function remove_keeper_config()
     {
         sudo sed -i "/<$1>$2<\/$1>/d" /etc/clickhouse-server/config.d/keeper_port.xml
@@ -93,10 +99,57 @@ if [ "$NUM_TRIES" -gt "1" ]; then
     mkdir -p /var/run/clickhouse-server
 fi
 
+# Run a CH instance to execute sequential tests on it in parallel with all other tests.
+if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+  mkdir -p /var/run/clickhouse-server3 /etc/clickhouse-server3 /var/lib/clickhouse3
+  cp -r -L /etc/clickhouse-server/* /etc/clickhouse-server3/
+
+  sudo chown clickhouse:clickhouse /var/run/clickhouse-server3 /var/lib/clickhouse3 /etc/clickhouse-server3/
+  sudo chown -R clickhouse:clickhouse /etc/clickhouse-server3/*
+
+  function replace(){
+      sudo find /etc/clickhouse-server3/ -type f -name '*.xml' -exec sed -i "$1" {} \;
+  }
+
+  replace "s|<port>9000</port>|<port>19000</port>|g"
+  replace "s|<port>9440</port>|<port>19440</port>|g"
+  replace "s|<port>9988</port>|<port>19988</port>|g"
+  replace "s|<port>9234</port>|<port>19234</port>|g"
+  replace "s|<port>9181</port>|<port>19181</port>|g"
+  replace "s|<https_port>8443</https_port>|<https_port>18443</https_port>|g"
+  replace "s|<tcp_port>9000</tcp_port>|<tcp_port>19000</tcp_port>|g"
+  replace "s|<tcp_port>9181</tcp_port>|<tcp_port>19181</tcp_port>|g"
+  replace "s|<tcp_port_secure>9440</tcp_port_secure>|<tcp_port_secure>19440</tcp_port_secure>|g"
+  replace "s|<tcp_with_proxy_port>9010</tcp_with_proxy_port>|<tcp_with_proxy_port>19010</tcp_with_proxy_port>|g"
+  replace "s|<mysql_port>9004</mysql_port>|<mysql_port>19004</mysql_port>|g"
+  replace "s|<postgresql_port>9005</postgresql_port>|<postgresql_port>19005</postgresql_port>|g"
+  replace "s|<interserver_http_port>9009</interserver_http_port>|<interserver_http_port>19009</interserver_http_port>|g"
+  replace "s|8123|18123|g"
+  replace "s|/var/lib/clickhouse/|/var/lib/clickhouse3/|g"
+  replace "s|/etc/clickhouse-server/|/etc/clickhouse-server3/|g"
+  # distributed cache
+  replace "s|<tcp_port>10001</tcp_port>|<tcp_port>10003</tcp_port>|g"
+  replace "s|<tcp_port>10002</tcp_port>|<tcp_port>10004</tcp_port>|g"
+
+  sudo -E -u clickhouse /usr/bin/clickhouse server --daemon --config /etc/clickhouse-server3/config.xml \
+  --pid-file /var/run/clickhouse-server3/clickhouse-server.pid \
+  -- --path /var/lib/clickhouse3/ --logger.stderr /var/log/clickhouse-server/stderr3.log \
+  --logger.log /var/log/clickhouse-server/clickhouse-server3.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server3.err.log \
+  --tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
+  --prometheus.port 19988 --keeper_server.raft_configuration.server.port 19234 --keeper_server.tcp_port 19181 \
+  --mysql_port 19004 --postgresql_port 19005
+
+  for _ in {1..100}
+  do
+      clickhouse-client --port 19000 --query "SELECT 1" && break
+      sleep 1
+  done
+fi
+
 # simplest way to forward env variables to server
 sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon --pid-file /var/run/clickhouse-server/clickhouse-server.pid
 
-if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
 
     sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_2/</filesystem_caches_path>|" /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
@@ -133,7 +186,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000))    # set to 2.5 hours if 0 (unlimited)
 fi
 
-if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
     | sed "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" \
     > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
@@ -213,15 +266,15 @@ function run_tests()
         ADDITIONAL_OPTIONS+=('--no-random-merge-tree-settings')
     fi
 
-    if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+    if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
         ADDITIONAL_OPTIONS+=('--shared-catalog')
     fi
 
-    if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+    if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
         ADDITIONAL_OPTIONS+=('--replicated-database')
         # Too many tests fail for DatabaseReplicated in parallel.
         ADDITIONAL_OPTIONS+=('--jobs')
-        ADDITIONAL_OPTIONS+=('2')
+        ADDITIONAL_OPTIONS+=('3')
     elif [[ 1 == $(clickhouse-client --query "SELECT value LIKE '%SANITIZE_COVERAGE%' FROM system.build_options WHERE name = 'CXX_FLAGS'") ]]; then
         # Coverage on a per-test basis could only be collected sequentially.
         # Do not set the --jobs parameter.
@@ -229,7 +282,11 @@ function run_tests()
     else
         # All other configurations are OK.
         ADDITIONAL_OPTIONS+=('--jobs')
-        ADDITIONAL_OPTIONS+=('8')
+        ADDITIONAL_OPTIONS+=('5')
+    fi
+
+    if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+        ADDITIONAL_OPTIONS+=('--run-sequential-tests-in-parallel')
     fi
 
     if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then
@@ -253,7 +310,7 @@ function run_tests()
     try_run_with_retry 10 clickhouse-client -q "insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')"
 
     set +e
-    timeout -s TERM --preserve-status 120m clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
+    clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
          --no-drop-if-fail --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
     | ts '%Y-%m-%d %H:%M:%S' \
     | tee -a test_output/test_result.txt
@@ -262,14 +319,17 @@ function run_tests()
 
 export -f run_tests
 
+
+# This should be enough to setup job and collect artifacts
+TIMEOUT=$((MAX_RUN_TIME - 600))
 if [ "$NUM_TRIES" -gt "1" ]; then
     # We don't run tests with Ordinary database in PRs, only in master.
     # So run new/changed tests with Ordinary at least once in flaky check.
-    timeout_with_logging "$MAX_RUN_TIME" bash -c 'NUM_TRIES=1; USE_DATABASE_ORDINARY=1; run_tests' \
+    timeout_with_logging "$TIMEOUT" bash -c 'NUM_TRIES=1; USE_DATABASE_ORDINARY=1; run_tests' \
       | sed 's/All tests have finished//' | sed 's/No tests were run//' ||:
 fi
 
-timeout_with_logging "$MAX_RUN_TIME" bash -c run_tests ||:
+timeout_with_logging "$TIMEOUT" bash -c run_tests ||:
 
 echo "Files in current directory"
 ls -la ./
@@ -290,7 +350,7 @@ do
     err=$(clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.gz' format TSVWithNamesAndTypes")
     echo "$err"
     [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
-    if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+    if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
         err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
         echo "$err"
         [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
@@ -299,7 +359,7 @@ do
         [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
     fi
 
-    if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+    if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
         err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
         echo "$err"
         [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
@@ -310,12 +370,17 @@ done
 # Why do we read data with clickhouse-local?
 # Because it's the simplest way to read it when server has crashed.
 sudo clickhouse stop ||:
-if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+
+if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+    sudo clickhouse stop --pid-path /var/run/clickhouse-server3 ||:
+fi
+
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
     sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
 fi
 
-if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
 fi
 
@@ -323,6 +388,12 @@ rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:
 rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||:
 zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &
 
+if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+  rg -Fa "<Fatal>" /var/log/clickhouse-server3/clickhouse-server.log ||:
+  rg -A50 -Fa "============" /var/log/clickhouse-server3/stderr.log ||:
+  zstd --threads=0 < /var/log/clickhouse-server3/clickhouse-server.log > /test_output/clickhouse-server3.log.zst &
+fi
+
 data_path_config="--path=/var/lib/clickhouse/"
 if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
     # We need s3 storage configuration (but it's more likely that clickhouse-local will fail for some reason)
@@ -342,12 +413,17 @@ if [ $failed_to_save_logs -ne 0 ]; then
     for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
     do
         clickhouse-local "$data_path_config" --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
-        if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+
+        if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+            clickhouse-local --path /var/lib/clickhouse3/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.3.tsv.zst ||:
+        fi
+
+        if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
             clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
             clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst ||:
         fi
 
-        if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+        if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
             clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
         fi
     done
@@ -383,7 +459,14 @@ rm -rf /var/lib/clickhouse/data/system/*/
 tar -chf /test_output/store.tar /var/lib/clickhouse/store ||:
 tar -chf /test_output/metadata.tar /var/lib/clickhouse/metadata/*.sql ||:
 
-if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+    rm -rf /var/lib/clickhouse3/data/system/*/
+    tar -chf /test_output/store.tar /var/lib/clickhouse3/store ||:
+    tar -chf /test_output/metadata.tar /var/lib/clickhouse3/metadata/*.sql ||:
+fi
+
+
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
     rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server2.log ||:
     zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
@@ -394,7 +477,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||:
 fi
 
-if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
     zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
     mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:

docker/test/stateless/utils.lib

@@ -38,7 +38,7 @@ function fn_exists() {
 function timeout_with_logging() {
     local exit_code=0
 
-    timeout "${@}" || exit_code="${?}"
+    timeout -s TERM --preserve-status "${@}" || exit_code="${?}"
 
     if [[ "${exit_code}" -eq "124" ]]
     then

docs/en/engines/table-engines/integrations/s3queue.md

@@ -75,7 +75,7 @@ SETTINGS
 Possible values:
 
 - unordered — With unordered mode, the set of all already processed files is tracked with persistent nodes in ZooKeeper.
-- ordered — With ordered mode, only the max name of the successfully consumed file, and the names of files that will be retried after unsuccessful loading attempt are being stored in ZooKeeper.
+- ordered — With ordered mode, the files are processed in lexicographic order. It means that if file named 'BBB' was processed at some point and later on a file named 'AA' is added to the bucket, it will be ignored. Only the max name (in lexicographic sense) of the successfully consumed file, and the names of files that will be retried after unsuccessful loading attempt are being stored in ZooKeeper.
 
 Default value: `ordered` in versions before 24.6. Starting with 24.6 there is no default value, the setting becomes required to be specified manually. For tables created on earlier versions the default value will remain `Ordered` for compatibility.
 

docs/en/interfaces/formats.md

@@ -1535,6 +1535,10 @@ the columns from input data will be mapped to the columns from the table by thei
 Otherwise, the first row will be skipped.
 If setting [input_format_with_types_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_types_use_header) is set to 1,
 the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
+If setting [output_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#output_format_binary_encode_types_in_binary_format) is set to 1,
+the types in header will be written using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes output format.
+If setting [input_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#input_format_binary_encode_types_in_binary_format) is set to 1,
+the types in header will be read using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes input format.
 :::
 
 ## RowBinaryWithDefaults {#rowbinarywithdefaults}

docs/en/operations/opentelemetry.md

@@ -2,15 +2,11 @@
 slug: /en/operations/opentelemetry
 sidebar_position: 62
 sidebar_label: Tracing ClickHouse with OpenTelemetry
-title: "[experimental] Tracing ClickHouse with OpenTelemetry"
+title: "Tracing ClickHouse with OpenTelemetry"
 ---
 
 [OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting traces and metrics from the distributed application. ClickHouse has some support for OpenTelemetry.
 
-:::note    
-This is an experimental feature that will change in backwards-incompatible ways in future releases.
-:::
-
 ## Supplying Trace Context to ClickHouse
 
 ClickHouse accepts trace context HTTP headers, as described by the [W3C recommendation](https://www.w3.org/TR/trace-context/). It also accepts trace context over a native protocol that is used for communication between ClickHouse servers or between the client and server. For manual testing, trace context headers conforming to the Trace Context recommendation can be supplied to `clickhouse-client` using `--opentelemetry-traceparent` and `--opentelemetry-tracestate` flags.

docs/en/operations/settings/settings-formats.md

@@ -1951,6 +1951,18 @@ The maximum allowed size for String in RowBinary format. It prevents allocating
 
 Default value: `1GiB`.
 
+### output_format_binary_encode_types_in_binary_format {#output_format_binary_encode_types_in_binary_format}
+
+Write data types in [binary format](../../sql-reference/data-types/data-types-binary-encoding.md) instead of type names in RowBinaryWithNamesAndTypes output format.
+
+Disabled by default.
+
+### input_format_binary_decode_types_in_binary_format {#input_format_binary_decode_types_in_binary_format}
+
+Read data types in [binary format](../../sql-reference/data-types/data-types-binary-encoding.md) instead of type names in RowBinaryWithNamesAndTypes input format.
+
+Disabled by default.
+
 ## Native format settings {#native-format-settings}
 
 ### input_format_native_allow_types_conversion {#input_format_native_allow_types_conversion}
@@ -1958,3 +1970,15 @@ Default value: `1GiB`.
 Allow types conversion in Native input format between columns from input data and requested columns.
 
 Enabled by default.
+
+### output_format_native_encode_types_in_binary_format {#output_format_native_encode_types_in_binary_format}
+
+Write data types in [binary format](../../sql-reference/data-types/data-types-binary-encoding.md) instead of type names in Native output format.
+
+Disabled by default.
+
+### input_format_native_decode_types_in_binary_format {#input_format_native_decode_types_in_binary_format}
+
+Read data types in [binary format](../../sql-reference/data-types/data-types-binary-encoding.md) instead of type names in Native input format.
+
+Disabled by default.

docs/en/operations/settings/settings.md

@@ -1170,6 +1170,10 @@ Data in the VALUES clause of INSERT queries is processed by a separate stream pa
 
 Default value: 262144 (= 256 KiB).
 
+:::note
+`max_query_size` cannot be set within an SQL query (e.g., `SELECT now() SETTINGS max_query_size=10000`) because ClickHouse needs to allocate a buffer to parse the query, and this buffer size is determined by the `max_query_size` setting, which must be configured before the query is executed.
+:::
+
 ## max_parser_depth {#max_parser_depth}
 
 Limits maximum recursion depth in the recursive descent parser. Allows controlling the stack size.
@@ -1354,12 +1358,25 @@ Connection pool size for PostgreSQL table engine and database engine.
 
 Default value: 16
 
+## postgresql_connection_attempt_timeout {#postgresql-connection-attempt-timeout}
+
+Connection timeout in seconds of a single attempt to connect PostgreSQL end-point.
+The value is passed as a `connect_timeout` parameter of the connection URL.
+
+Default value: `2`.
+
 ## postgresql_connection_pool_wait_timeout {#postgresql-connection-pool-wait-timeout}
 
 Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.
 
 Default value: 5000
 
+## postgresql_connection_pool_retries {#postgresql-connection-pool-retries}
+
+The maximum number of retries to establish a connection with the PostgreSQL end-point.
+
+Default value: `2`.
+
 ## postgresql_connection_pool_auto_close_connection {#postgresql-connection-pool-auto-close-connection}
 
 Close connection before returning connection to the pool.

docs/en/operations/startup-scripts.md

@@ -0,0 +1,30 @@
+---
+slug: /en/operations/startup-scripts
+sidebar_label: Startup Scripts
+---
+
+# Startup Scripts
+
+ClickHouse can run arbitrary SQL queries from the server configuration during startup. This can be useful for migrations or automatic schema creation.
+
+```xml
+<clickhouse>
+    <startup_scripts>
+        <scripts>
+            <query>CREATE ROLE OR REPLACE test_role</query>
+        </scripts>
+        <scripts>
+            <query>CREATE TABLE TestTable (id UInt64) ENGINE=TinyLog</query>
+            <condition>SELECT 1;</condition>
+        </scripts>
+    </startup_scripts>
+</clickhouse>
+```
+
+ClickHouse executes all queries from the `startup_scripts` sequentially in the specified order. If any of the queries fail, the execution of the following queries won't be interrupted.
+
+You can specify a conditional query in the config. In that case, the corresponding query executes only when the condition query returns the value `1` or `true`.
+
+:::note
+If the condition query returns any other value than `1` or `true`, the result will be interpreted as `false`, and the corresponding won't be executed.
+:::

docs/en/operations/system-tables/metrics.md

@@ -357,7 +357,7 @@ Number of currently running inserts to Kafka
 
 Number of alive connections
 
-### KeeperOutstandingRequets
+### KeeperOutstandingRequests
 
 Number of outstanding requests
 

docs/en/operations/utilities/clickhouse-disks.md

@@ -4,35 +4,56 @@ sidebar_position: 59
 sidebar_label: clickhouse-disks
 ---
 
-# clickhouse-disks
+# Clickhouse-disks
 
-A utility providing filesystem-like operations for ClickHouse disks.
+A utility providing filesystem-like operations for ClickHouse disks. It can work in both interactive and not interactive modes.
 
-Program-wide options:
+## Program-wide options
 
 * `--config-file, -C` -- path to ClickHouse config, defaults to `/etc/clickhouse-server/config.xml`.
 * `--save-logs` -- Log progress of invoked commands to `/var/log/clickhouse-server/clickhouse-disks.log`.
 * `--log-level` -- What [type](../server-configuration-parameters/settings#server_configuration_parameters-logger) of events to log, defaults to `none`.
 * `--disk` -- what disk to use for `mkdir, move, read, write, remove` commands. Defaults to `default`.
+* `--query, -q` -- single query that can be executed without launching interactive mode
+* `--help, -h` -- print all the options and commands with description
+
+## Default Disks
+After the launch two disks are initialized. The first one is a disk `local` that is supposed to imitate local file system from which clickhouse-disks utility was launched. The second one is a disk `default` that is mounted to the local filesystem in the directory that can be found in config as a parameter `clickhouse/path` (default value is `/var/lib/clickhouse`).
+
+## Clickhouse-disks state
+For each disk that was added the utility stores current directory (as in a usual filesystem). User can change current directory and switch between disks.
+
+State is reflected in a prompt "`disk_name`:`path_name`" 
 
 ## Commands
 
-* `copy [--disk-from d1] [--disk-to d2] <FROM_PATH> <TO_PATH>`.
-  Recursively copy data from `FROM_PATH` at disk `d1` (defaults to `disk` value if not provided)
-  to `TO_PATH` at disk `d2` (defaults to `disk` value if not provided).
-* `move <FROM_PATH> <TO_PATH>`.
-  Move file or directory from `FROM_PATH` to `TO_PATH`.
-* `remove <PATH>`.
-  Remove `PATH` recursively.
-* `link <FROM_PATH> <TO_PATH>`.
-  Create a hardlink from `FROM_PATH` to `TO_PATH`.
-* `list [--recursive] <PATH>...`
-  List files at `PATH`s. Non-recursive by default.
-* `list-disks`.
+In these documentation file all mandatory positional arguments are referred as `<parameter>`, named arguments are referred as `[--parameter value]`. All positional parameters could be mentioned as a named parameter with a corresponding name.
+
+* `cd (change-dir, change_dir) [--disk disk] <path>`
+  Change directory to path `path` on disk `disk` (default value is a current disk). No disk switching happens.
+* `copy (cp) [--disk-from disk_1] [--disk-to disk_2] <path-from> <path-to>`.
+  Recursively copy data from `path-from` at disk `disk_1` (default value is a current disk (parameter `disk` in a non-interactive mode))
+  to `path-to` at disk `disk_2` (default value is a current disk (parameter `disk` in a non-interactive mode)).
+* `current_disk_with_path (current, current_disk, current_path)`
+  Print current state in format: 
+    `Disk: "current_disk" Path: "current path on current disk"`
+* `help [<command>]`
+  Print help message about command `command`. If `command` is not specified print information about all commands.
+* `move (mv) <path-from> <path-to>`.
+  Move file or directory from `path-from` to `path-to` within current disk.
+* `remove (rm, delete) <path>`.
+  Remove `path` recursively on a current disk.
+* `link (ln) <path-from> <path-to>`.
+  Create a hardlink from `path-from` to `path-to` on a current disk.
+* `list (ls) [--recursive] <path>`
+  List files at `path`s on a current disk. Non-recursive by default.
+* `list-disks (list_disks, ls-disks, ls_disks)`.
   List disks names.
-* `mkdir [--recursive] <PATH>`.
+* `mkdir [--recursive] <path>` on a current disk.
   Create a directory. Non-recursive by default.
-* `read: <FROM_PATH> [<TO_PATH>]`
-  Read a file from `FROM_PATH` to `TO_PATH` (`stdout` if not supplied).
-* `write [FROM_PATH] <TO_PATH>`.
-  Write a file from `FROM_PATH` (`stdin` if not supplied) to `TO_PATH`.
+* `read (r) <path-from> [--path-to path]`
+  Read a file from `path-from` to `path` (`stdout` if not supplied).
+* `switch-disk [--path path] <disk>`
+  Switch to disk `disk` on path `path` (if `path` is not specified default value is a previous path on disk `disk`). 
+* `write (w) [--path-from path] <path-to>`.
+  Write a file from `path` (`stdin` if `path` is not supplied, input must finish by Ctrl+D) to `path-to`.

docs/en/sql-reference/aggregate-functions/reference/maxmap.md

@@ -5,23 +5,45 @@ sidebar_position: 165
 
 # maxMap
 
-Syntax: `maxMap(key, value)` or `maxMap(Tuple(key, value))`
-
 Calculates the maximum from `value` array according to the keys specified in the `key` array.
 
-Passing a tuple of keys and value arrays is identical to passing two arrays of keys and values.
+**Syntax**
 
-The number of elements in `key` and `value` must be the same for each row that is totaled.
+```sql
+maxMap(key, value)
+```
+or
+```sql
+maxMap(Tuple(key, value))
+```
 
-Returns a tuple of two arrays: keys and values calculated for the corresponding keys.
+Alias: `maxMappedArrays`
 
-Example:
+:::note
+- Passing a tuple of keys and value arrays is identical to passing two arrays of keys and values.
+- The number of elements in `key` and `value` must be the same for each row that is totaled.
+:::
+
+**Parameters**
+
+- `key` — Array of keys. [Array](../../data-types/array.md).
+- `value` — Array of values. [Array](../../data-types/array.md).
+
+**Returned value**
+
+- Returns a tuple of two arrays: keys in sorted order, and values calculated for the corresponding keys. [Tuple](../../data-types/tuple.md)([Array](../../data-types/array.md), [Array](../../data-types/array.md)).
+
+**Example**
+
+Query:
 
 ``` sql
 SELECT maxMap(a, b)
 FROM values('a Array(Char), b Array(Int64)', (['x', 'y'], [2, 2]), (['y', 'z'], [3, 1]))
 ```
 
+Result:
+
 ``` text
 ┌─maxMap(a, b)───────────┐
 │ [['x','y','z'],[2,3,1]]│

docs/en/sql-reference/aggregate-functions/reference/minmap.md

@@ -5,23 +5,45 @@ sidebar_position: 169
 
 # minMap
 
-Syntax: `minMap(key, value)` or `minMap(Tuple(key, value))`
-
 Calculates the minimum from `value` array according to the keys specified in the `key` array.
 
-Passing a tuple of keys and value ​​arrays is identical to passing two arrays of keys and values.
+**Syntax**
 
-The number of elements in `key` and `value` must be the same for each row that is totaled.
+```sql
+`minMap(key, value)`
+```
+or
+```sql
+minMap(Tuple(key, value))
+```
 
-Returns a tuple of two arrays: keys in sorted order, and values calculated for the corresponding keys.
+Alias: `minMappedArrays`
 
-Example:
+:::note
+- Passing a tuple of keys and value arrays is identical to passing an array of keys and an array of values.
+- The number of elements in `key` and `value` must be the same for each row that is totaled.
+:::
+
+**Parameters**
+
+- `key` — Array of keys. [Array](../../data-types/array.md).
+- `value` — Array of values. [Array](../../data-types/array.md).
+
+**Returned value**
+
+- Returns a tuple of two arrays: keys in sorted order, and values calculated for the corresponding keys. [Tuple](../../data-types/tuple.md)([Array](../../data-types/array.md), [Array](../../data-types/array.md)).
+
+**Example**
+
+Query:
 
 ``` sql
 SELECT minMap(a, b)
 FROM values('a Array(Int32), b Array(Int64)', ([1, 2], [2, 2]), ([2, 3], [1, 1]))
 ```
 
+Result:
+
 ``` text
 ┌─minMap(a, b)──────┐
 │ ([1,2,3],[2,1,1]) │

docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md

@@ -16,7 +16,7 @@ singleValueOrNull(x)
 
 **Parameters**
 
-- `x` — Column of any [data type](../../data-types/index.md).
+- `x` — Column of any [data type](../../data-types/index.md) (except [Map](../../data-types/map.md), [Array](../../data-types/array.md) or [Tuple](../../data-types/tuple) which cannot be of type [Nullable](../../data-types/nullable.md)).
 
 **Returned values**
 

docs/en/sql-reference/data-types/data-types-binary-encoding.md

@@ -0,0 +1,115 @@
+---
+slug: /en/sql-reference/data-types/data-types-binary-encoding
+sidebar_position: 56
+sidebar_label: Data types binary encoding specification.
+---
+
+
+# Data types binary encoding specification
+
+This specification describes the binary format that can be used for binary encoding and decoding of ClickHouse data types. This format is used in `Dynamic` column [binary serialization](dynamic.md#binary-output-format) and can be used in input/output formats [RowBinaryWithNamesAndTypes](../../interfaces/formats.md#rowbinarywithnamesandtypes) and [Native](../../interfaces/formats.md#native) under corresponding settings.
+
+The table below describes how each data type is represented in binary format. Each data type encoding consist of 1 byte that indicates the type and some optional additional information.
+`var_uint` in the binary encoding means that the size is encoded using Variable-Length Quantity compression.
+
+| ClickHouse data type                                                                 | Binary encoding                                                                                                                                                                                                                                                                                                        |
+|--------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `Nothing`                                                                            | `0x00`                                                                                                                                                                                                                                                                                                                 |
+| `UInt8`                                                                              | `0x01`                                                                                                                                                                                                                                                                                                                 |
+| `UInt16`                                                                             | `0x02`                                                                                                                                                                                                                                                                                                                 |
+| `UInt32`                                                                             | `0x03`                                                                                                                                                                                                                                                                                                                 |
+| `UInt64`                                                                             | `0x04`                                                                                                                                                                                                                                                                                                                 |
+| `UInt128`                                                                            | `0x05`                                                                                                                                                                                                                                                                                                                 |
+| `UInt256`                                                                            | `0x06`                                                                                                                                                                                                                                                                                                                 |
+| `Int8`                                                                               | `0x07`                                                                                                                                                                                                                                                                                                                 |
+| `Int16`                                                                              | `0x08`                                                                                                                                                                                                                                                                                                                 |
+| `Int32`                                                                              | `0x09`                                                                                                                                                                                                                                                                                                                 |
+| `Int64`                                                                              | `0x0A`                                                                                                                                                                                                                                                                                                                 |
+| `Int128`                                                                             | `0x0B`                                                                                                                                                                                                                                                                                                                 |
+| `Int256`                                                                             | `0x0C`                                                                                                                                                                                                                                                                                                                 |
+| `Float32`                                                                            | `0x0D`                                                                                                                                                                                                                                                                                                                 |
+| `Float64`                                                                            | `0x0E`                                                                                                                                                                                                                                                                                                                 |
+| `Date`                                                                               | `0x0F`                                                                                                                                                                                                                                                                                                                 |
+| `Date32`                                                                             | `0x10`                                                                                                                                                                                                                                                                                                                 |
+| `DateTime`                                                                           | `0x11`                                                                                                                                                                                                                                                                                                                 |
+| `DateTime(time_zone)`                                                                | `0x12<var_uint_time_zone_name_size><time_zone_name_data>`                                                                                                                                                                                                                                                              |
+| `DateTime64(P)`                                                                      | `0x13<uint8_precision>`                                                                                                                                                                                                                                                                                                |
+| `DateTime64(P, time_zone)`                                                           | `0x14<uint8_precision><var_uint_time_zone_name_size><time_zone_name_data>`                                                                                                                                                                                                                                             |
+| `String`                                                                             | `0x15`                                                                                                                                                                                                                                                                                                                 |
+| `FixedString(N)`                                                                     | `0x16<var_uint_size>`                                                                                                                                                                                                                                                                                                  |
+| `Enum8`                                                                              | `0x17<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int8_value_1>...<var_uint_name_size_N><name_data_N><int8_value_N>`                                                                                                                                                                               |
+| `Enum16`                                                                             | `0x18<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int16_little_endian_value_1>...><var_uint_name_size_N><name_data_N><int16_little_endian_value_N>`                                                                                                                                                |
+| `Decimal32(P, S)`                                                                    | `0x19<uint8_precision><uint8_scale>`                                                                                                                                                                                                                                                                                   |
+| `Decimal64(P, S)`                                                                    | `0x1A<uint8_precision><uint8_scale>`                                                                                                                                                                                                                                                                                   |
+| `Decimal128(P, S)`                                                                   | `0x1B<uint8_precision><uint8_scale>`                                                                                                                                                                                                                                                                                   |
+| `Decimal256(P, S)`                                                                   | `0x1C<uint8_precision><uint8_scale>`                                                                                                                                                                                                                                                                                   |
+| `UUID`                                                                               | `0x1D`                                                                                                                                                                                                                                                                                                                 |
+| `Array(T)`                                                                           | `0x1E<nested_type_encoding>`                                                                                                                                                                                                                                                                                           |
+| `Tuple(T1, ..., TN)`                                                                 | `0x1F<var_uint_number_of_elements><nested_type_encoding_1>...<nested_type_encoding_N>`                                                                                                                                                                                                                                 |
+| `Tuple(name1 T1, ..., nameN TN)`                                                     | `0x20<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N>`                                                                                                                                                           |
+| `Set`                                                                                | `0x21`                                                                                                                                                                                                                                                                                                                 |
+| `Interval`                                                                           | `0x22<interval_kind>` (see [interval kind binary encoding](#interval-kind-binary-encoding))                                                                                                                                                                                                                            |
+| `Nullable(T)`                                                                        | `0x23<nested_type_encoding>`                                                                                                                                                                                                                                                                                           |
+| `Function`                                                                           | `0x24<var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N><return_type_encoding>`                                                                                                                                                                                                      |
+| `AggregateFunction(function_name(param_1, ..., param_N), arg_T1, ..., arg_TN)`       | `0x25<var_uint_version><var_uint_function_name_size><function_name_data><var_uint_number_of_parameters><param_1>...<param_N><var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N>` (see [aggregate function parameter binary encoding](#aggregate-function-parameter-binary-encoding)) |
+| `LowCardinality(T)`                                                                  | `0x26<nested_type_encoding>`                                                                                                                                                                                                                                                                                           |
+| `Map(K, V)`                                                                          | `0x27<key_type_encoding><value_type_encoding>`                                                                                                                                                                                                                                                                         |
+| `IPv4`                                                                               | `0x28`                                                                                                                                                                                                                                                                                                                 |
+| `IPv6`                                                                               | `0x29`                                                                                                                                                                                                                                                                                                                 |
+| `Variant(T1, ..., TN)`                                                               | `0x2A<var_uint_number_of_variants><variant_type_encoding_1>...<variant_type_encoding_N>`                                                                                                                                                                                                                               |
+| `Dynamic(max_types=N)`                                                               | `0x2B<uint8_max_types>`                                                                                                                                                                                                                                                                                                |
+| `Custom type` (`Ring`, `Polygon`, etc)                                               | `0x2C<var_uint_type_name_size><type_name_data>`                                                                                                                                                                                                                                                                        |
+| `Bool`                                                                               | `0x2D`                                                                                                                                                                                                                                                                                                                 |
+| `SimpleAggregateFunction(function_name(param_1, ..., param_N), arg_T1, ..., arg_TN)` | `0x2E<var_uint_function_name_size><function_name_data><var_uint_number_of_parameters><param_1>...<param_N><var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N>` (see [aggregate function parameter binary encoding](#aggregate-function-parameter-binary-encoding))                   |
+| `Nested(name1 T1, ..., nameN TN)`                                                    | `0x2F<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N>`                                                                                                                                                           |
+
+
+### Interval kind binary encoding
+
+The table below describes how different interval kinds of `Interval` data type are encoded.
+
+| Interval kind | Binary encoding |
+|---------------|-----------------|
+| `Nanosecond`  | `0x00`          |
+| `Microsecond` | `0x01`          |
+| `Millisecond` | `0x02`          |
+| `Second`      | `0x03`          |
+| `Minute`      | `0x04`          |
+| `Hour`        | `0x05`          |
+| `Day`         | `0x06`          |
+| `Week`        | `0x07`          |
+| `Month`       | `0x08`          |
+| `Quarter`     | `0x09`          |
+| `Year`        | `0x1A`          |
+
+### Aggregate function parameter binary encoding
+
+The table below describes how parameters of `AggragateFunction` and `SimpleAggregateFunction` are encoded.
+The encoding of a parameter consists of 1 byte indicating the type of the parameter and the value itself.
+
+| Parameter type           | Binary encoding                                                                                                                |
+|--------------------------|--------------------------------------------------------------------------------------------------------------------------------|
+| `Null`                   | `0x00`                                                                                                                         |
+| `UInt64`                 | `0x01<var_uint_value>`                                                                                                         |
+| `Int64`                  | `0x02<var_int_value>`                                                                                                          |
+| `UInt128`                | `0x03<uint128_little_endian_value>`                                                                                            |
+| `Int128`                 | `0x04<int128_little_endian_value>`                                                                                             |
+| `UInt128`                | `0x05<uint128_little_endian_value>`                                                                                            |
+| `Int128`                 | `0x06<int128_little_endian_value>`                                                                                             |
+| `Float64`                | `0x07<float64_little_endian_value>`                                                                                            |
+| `Decimal32`              | `0x08<var_uint_scale><int32_little_endian_value>`                                                                              |
+| `Decimal64`              | `0x09<var_uint_scale><int64_little_endian_value>`                                                                              |
+| `Decimal128`             | `0x0A<var_uint_scale><int128_little_endian_value>`                                                                             |
+| `Decimal256`             | `0x0B<var_uint_scale><int256_little_endian_value>`                                                                             |
+| `String`                 | `0x0C<var_uint_size><data>`                                                                                                    |
+| `Array`                  | `0x0D<var_uint_size><value_encoding_1>...<value_encoding_N>`                                                                   |
+| `Tuple`                  | `0x0E<var_uint_size><value_encoding_1>...<value_encoding_N>`                                                                   |
+| `Map`                    | `0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_endoding_N><value_encoding_N>`                                   |
+| `IPv4`                   | `0x10<uint32_little_endian_value>`                                                                                             |
+| `IPv6`                   | `0x11<uint128_little_endian_value>`                                                                                            |
+| `UUID`                   | `0x12<uuid_value>`                                                                                                             |
+| `Bool`                   | `0x13<bool_value>`                                                                                                             |
+| `Object`                 | `0x14<var_uint_size><var_uint_key_size_1><key_data_1><value_encoding_1>...<var_uint_key_size_N><key_data_N><value_encoding_N>` |
+| `AggregateFunctionState` | `0x15<var_uint_name_size><name_data><var_uint_data_size><data>`                                                                |
+| `Negative infinity`      | `0xFE`                                                                                                                         |
+| `Positive infinity`      | `0xFF`                                                                                                                         |

docs/en/sql-reference/data-types/dynamic.md

@@ -493,3 +493,14 @@ SELECT count(), dynamicType(d), _part FROM test GROUP BY _part, dynamicType(d) O
 ```
 
 As we can see, ClickHouse kept the most frequent types `UInt64` and `Array(UInt64)` and casted all other types to `String`.
+
+
+### Binary output format
+
+In [RowBinary](../../interfaces/formats.md#rowbinary-rowbinary) format values of `Dynamic` type are serialized in the following format:
+
+```text
+<binary_encoded_data_type><value_in_binary_format_according_to_the_data_type>
+```
+
+See the [data types binary encoding specification](../../sql-reference/data-types/data-types-binary-encoding.md)

docs/en/sql-reference/data-types/json.md

@@ -5,11 +5,11 @@ sidebar_label: Object Data Type
 keywords: [object, data type]
 ---
 
-# Object Data Type
+# Object Data Type (deprecated)
 
-:::note
-This feature is not production-ready and is now deprecated. If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-ingestion/data-formats/json) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864)
-:::
+**This feature is not production-ready and is now deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-ingestion/data-formats/json) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864).
+
+<hr />
 
 Stores JavaScript Object Notation (JSON) documents in a single column.
 

docs/en/sql-reference/functions/date-time-functions.md

@@ -83,7 +83,57 @@ Result:
 ```
 ## makeDate32
 
-Like [makeDate](#makedate) but produces a [Date32](../data-types/date32.md).
+Creates a date of type [Date32](../../sql-reference/data-types/date32.md) from a year, month, day (or optionally a year and a day). 
+
+**Syntax**
+
+```sql
+makeDate32(year, [month,] day)
+```
+
+**Arguments**
+
+- `year` — Year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+- `month` — Month (optional). [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+- `day` — Day. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+
+:::note
+If `month` is omitted then `day` should take a value between `1` and `365`, otherwise it should take a value between `1` and `31`.
+:::
+
+**Returned values**
+
+- A date created from the arguments. [Date32](../../sql-reference/data-types/date32.md).
+
+**Examples**
+
+Create a date from a year, month, and day:
+
+Query:
+
+```sql
+SELECT makeDate32(2024, 1, 1);
+```
+
+Result:
+
+```response
+2024-01-01
+```
+
+Create a Date from a year and day of year:
+
+Query:
+
+``` sql
+SELECT makeDate32(2024, 100);
+```
+
+Result:
+
+```response
+2024-04-09
+```
 
 ## makeDateTime
 
@@ -125,12 +175,38 @@ Result:
 
 ## makeDateTime64
 
-Like [makeDateTime](#makedatetime) but produces a [DateTime64](../data-types/datetime64.md).
+Creates a [DateTime64](../../sql-reference/data-types/datetime64.md) data type value from its components: year, month, day, hour, minute, second. With optional sub-second precision.
 
 **Syntax**
 
+```sql
+makeDateTime64(year, month, day, hour, minute, second[, precision])
+```
+
+**Arguments**
+
+- `year` — Year (0-9999). [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+- `month` — Month (1-12). [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). 
+- `day` — Day (1-31). [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+- `hour` — Hour (0-23). [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+- `minute` — Minute (0-59). [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). 
+- `second` — Second (0-59). [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). 
+- `precision` — Optional precision of the sub-second component (0-9). [Integer](../../sql-reference/data-types/int-uint.md).
+
+**Returned value**
+
+- A date and time created from the supplied arguments. [DateTime64](../../sql-reference/data-types/datetime64.md).  
+
+**Example**
+
 ``` sql
-makeDateTime64(year, month, day, hour, minute, second[, fraction[, precision[, timezone]]])
+SELECT makeDateTime64(2023, 5, 15, 10, 30, 45, 779, 5);
+```
+
+```response
+┌─makeDateTime64(2023, 5, 15, 10, 30, 45, 779, 5)─┐
+│                       2023-05-15 10:30:45.00779 │
+└─────────────────────────────────────────────────┘
 ```
 
 ## timestamp

docs/en/sql-reference/functions/string-functions.md

@@ -12,9 +12,7 @@ Functions for [searching](string-search-functions.md) in strings and for [replac
 
 ## empty
 
-Checks whether the input string is empty.
-
-A string is considered non-empty if it contains at least one byte, even if this byte is a space or the null byte.
+Checks whether the input string is empty. A string is considered non-empty if it contains at least one byte, even if this byte is a space or the null byte.
 
 The function is also available for [arrays](array-functions.md#function-empty) and [UUIDs](uuid-functions.md#empty).
 
@@ -48,9 +46,7 @@ Result:
 
 ## notEmpty
 
-Checks whether the input string is non-empty.
-
-A string is considered non-empty if it contains at least one byte, even if this byte is a space or the null byte.
+Checks whether the input string is non-empty. A string is considered non-empty if it contains at least one byte, even if this byte is a space or the null byte.
 
 The function is also available for [arrays](array-functions.md#function-notempty) and [UUIDs](uuid-functions.md#notempty).
 
@@ -96,7 +92,7 @@ length(s)
 
 **Parameters**
 
-- `s`: An input string or array. [String](../data-types/string)/[Array](../data-types/array).
+- `s` — An input string or array. [String](../data-types/string)/[Array](../data-types/array).
 
 **Returned value**
 
@@ -149,7 +145,7 @@ lengthUTF8(s)
 
 **Parameters**
 
-- `s`: String containing valid UTF-8 encoded text. [String](../data-types/string).
+- `s` — String containing valid UTF-8 encoded text. [String](../data-types/string).
 
 **Returned value**
 
@@ -183,8 +179,8 @@ left(s, offset)
 
 **Parameters**
 
-- `s`: The string to calculate a substring from. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
-- `offset`: The number of bytes of the offset. [UInt*](../data-types/int-uint).
+- `s` — The string to calculate a substring from. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
+- `offset` — The number of bytes of the offset. [UInt*](../data-types/int-uint).
 
 **Returned value**
 
@@ -230,8 +226,8 @@ leftUTF8(s, offset)
 
 **Parameters**
 
-- `s`: The UTF-8 encoded string to calculate a substring from. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
-- `offset`: The number of bytes of the offset. [UInt*](../data-types/int-uint).
+- `s` — The UTF-8 encoded string to calculate a substring from. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
+- `offset` — The number of bytes of the offset. [UInt*](../data-types/int-uint).
 
 **Returned value**
 
@@ -347,8 +343,8 @@ right(s, offset)
 
 **Parameters**
 
-- `s`: The string to calculate a substring from. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
-- `offset`: The number of bytes of the offset. [UInt*](../data-types/int-uint).
+- `s` — The string to calculate a substring from. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
+- `offset` — The number of bytes of the offset. [UInt*](../data-types/int-uint).
 
 **Returned value**
 
@@ -394,8 +390,8 @@ rightUTF8(s, offset)
 
 **Parameters**
 
-- `s`: The UTF-8 encoded string to calculate a substring from. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
-- `offset`: The number of bytes of the offset. [UInt*](../data-types/int-uint).
+- `s` — The UTF-8 encoded string to calculate a substring from. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
+- `offset` — The number of bytes of the offset. [UInt*](../data-types/int-uint).
 
 **Returned value**
 
@@ -547,7 +543,7 @@ Alias: `ucase`
 
 **Parameters**
 
-- `input`: A string type [String](../data-types/string.md).
+- `input` — A string type [String](../data-types/string.md).
 
 **Returned value**
 
@@ -571,16 +567,47 @@ SELECT upper('clickhouse');
 
 Converts a string to lowercase, assuming that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
 
-Does not detect the language, e.g. for Turkish the result might not be exactly correct (i/İ vs. i/I).
+:::note
+Does not detect the language, e.g. for Turkish the result might not be exactly correct (i/İ vs. i/I). If the length of the UTF-8 byte sequence is different for upper and lower case of a code point (such as `ẞ` and `ß`), the result may be incorrect for this code point.
+:::
 
-If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point.
+**Syntax**
+
+```sql
+lowerUTF8(input)
+```
+
+**Parameters**
+
+- `input` — A string type [String](../data-types/string.md).
+
+**Returned value**
+
+- A [String](../data-types/string.md) data type value.
+
+**Example**
+
+Query:
+
+``` sql
+SELECT lowerUTF8('MÜNCHEN') as Lowerutf8;
+```
+
+Result:
+
+``` response
+┌─Lowerutf8─┐
+│ münchen   │
+└───────────┘
+```
 
 ## upperUTF8
 
 Converts a string to uppercase, assuming that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
 
-
-If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point.
+:::note
+Does not detect the language, e.g. for Turkish the result might not be exactly correct (i/İ vs. i/I). If the length of the UTF-8 byte sequence is different for upper and lower case of a code point (such as `ẞ` and `ß`), the result may be incorrect for this code point.
+:::
 
 **Syntax**
 
@@ -590,7 +617,7 @@ upperUTF8(input)
 
 **Parameters**
 
-- `input`: A string type [String](../data-types/string.md).
+- `input` — A string type [String](../data-types/string.md).
 
 **Returned value**
 
@@ -604,6 +631,8 @@ Query:
 SELECT upperUTF8('München') as Upperutf8;
 ```
 
+Result:
+
 ``` response
 ┌─Upperutf8─┐
 │ MÜNCHEN   │
@@ -614,6 +643,34 @@ SELECT upperUTF8('München') as Upperutf8;
 
 Returns 1, if the set of bytes constitutes valid UTF-8-encoded text, otherwise 0.
 
+**Syntax**
+
+``` sql
+isValidUTF8(input)
+```
+
+**Parameters**
+
+- `input` — A string type [String](../data-types/string.md).
+
+**Returned value**
+
+- Returns `1`, if the set of bytes constitutes valid UTF-8-encoded text, otherwise `0`.
+
+Query:
+
+``` sql
+SELECT isValidUTF8('\xc3\xb1') AS valid, isValidUTF8('\xc3\x28') AS invalid;
+```
+
+Result:
+
+``` response
+┌─valid─┬─invalid─┐
+│     1 │       0 │
+└───────┴─────────┘
+```
+
 ## toValidUTF8
 
 Replaces invalid UTF-8 characters by the `<60>` (U+FFFD) character. All running in a row invalid characters are collapsed into the one replacement character.
@@ -883,7 +940,7 @@ Returns the substring of a string `s` which starts at the specified byte index `
 substring(s, offset[, length])
 ```
 
-Alias:
+Aliases:
 - `substr`
 - `mid`
 - `byteSlice`
@@ -926,9 +983,9 @@ substringUTF8(s, offset[, length])
 
 **Arguments**
 
-- `s`: The string to calculate a substring from. [String](../data-types/string.md), [FixedString](../data-types/fixedstring.md) or [Enum](../data-types/enum.md)
-- `offset`: The starting position of the substring in `s` . [(U)Int*](../data-types/int-uint.md).
-- `length`: The maximum length of the substring. [(U)Int*](../data-types/int-uint.md). Optional.
+- `s` — The string to calculate a substring from. [String](../data-types/string.md), [FixedString](../data-types/fixedstring.md) or [Enum](../data-types/enum.md)
+- `offset` — The starting position of the substring in `s` . [(U)Int*](../data-types/int-uint.md).
+- `length` — The maximum length of the substring. [(U)Int*](../data-types/int-uint.md). Optional.
 
 **Returned value**
 
@@ -964,9 +1021,9 @@ Alias: `SUBSTRING_INDEX`
 
 **Arguments**
 
-- s: The string to extract substring from. [String](../data-types/string.md).
-- delim: The character to split. [String](../data-types/string.md).
-- count: The number of occurrences of the delimiter to count before extracting the substring. If count is positive, everything to the left of the final delimiter (counting from the left) is returned. If count is negative, everything to the right of the final delimiter (counting from the right) is returned. [UInt or Int](../data-types/int-uint.md)
+- s — The string to extract substring from. [String](../data-types/string.md).
+- delim — The character to split. [String](../data-types/string.md).
+- count — The number of occurrences of the delimiter to count before extracting the substring. If count is positive, everything to the left of the final delimiter (counting from the left) is returned. If count is negative, everything to the right of the final delimiter (counting from the right) is returned. [UInt or Int](../data-types/int-uint.md)
 
 **Example**
 
@@ -995,9 +1052,9 @@ substringIndexUTF8(s, delim, count)
 
 **Arguments**
 
-- `s`: The string to extract substring from. [String](../data-types/string.md).
-- `delim`: The character to split. [String](../data-types/string.md).
-- `count`: The number of occurrences of the delimiter to count before extracting the substring. If count is positive, everything to the left of the final delimiter (counting from the left) is returned. If count is negative, everything to the right of the final delimiter (counting from the right) is returned. [UInt or Int](../data-types/int-uint.md)
+- `s` — The string to extract substring from. [String](../data-types/string.md).
+- `delim` — The character to split. [String](../data-types/string.md).
+- `count` — The number of occurrences of the delimiter to count before extracting the substring. If count is positive, everything to the left of the final delimiter (counting from the left) is returned. If count is negative, everything to the right of the final delimiter (counting from the right) is returned. [UInt or Int](../data-types/int-uint.md)
 
 **Returned value**
 
@@ -1277,7 +1334,7 @@ tryBase64Decode(encoded)
 
 **Arguments**
 
-- `encoded`: [String](../data-types/string.md) column or constant. If the string is not a valid Base64-encoded value, returns an empty string.
+- `encoded` — [String](../data-types/string.md) column or constant. If the string is not a valid Base64-encoded value, returns an empty string.
 
 **Returned value**
 
@@ -1309,7 +1366,7 @@ tryBase64URLDecode(encodedUrl)
 
 **Parameters**
 
-- `encodedURL`: [String](../data-types/string.md) column or constant. If the string is not a valid Base64-encoded value with URL-specific modifications, returns an empty string.
+- `encodedURL` — [String](../data-types/string.md) column or constant. If the string is not a valid Base64-encoded value with URL-specific modifications, returns an empty string.
 
 **Returned value**
 
@@ -1555,7 +1612,7 @@ The result type is UInt64.
 
 ## normalizeQuery
 
-Replaces literals, sequences of literals and complex aliases with placeholders.
+Replaces literals, sequences of literals and complex aliases (containing whitespace, more than two digits or at least 36 bytes long such as UUIDs) with placeholder `?`.
 
 **Syntax**
 
@@ -1573,6 +1630,8 @@ normalizeQuery(x)
 
 **Example**
 
+Query:
+
 ``` sql
 SELECT normalizeQuery('[1, 2, 3, x]') AS query;
 ```
@@ -1585,9 +1644,44 @@ Result:
 └──────────┘
 ```
 
+## normalizeQueryKeepNames
+
+Replaces literals, sequences of literals with placeholder `?` but does not replace complex aliases (containing whitespace, more than two digits
+or at least 36 bytes long such as UUIDs). This helps better analyze complex query logs.
+
+**Syntax**
+
+``` sql
+normalizeQueryKeepNames(x)
+```
+
+**Arguments**
+
+- `x` — Sequence of characters. [String](../data-types/string.md).
+
+**Returned value**
+
+- Sequence of characters with placeholders. [String](../data-types/string.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT normalizeQuery('SELECT 1 AS aComplexName123'), normalizeQueryKeepNames('SELECT 1 AS aComplexName123');
+```
+
+Result:
+
+```result
+┌─normalizeQuery('SELECT 1 AS aComplexName123')─┬─normalizeQueryKeepNames('SELECT 1 AS aComplexName123')─┐
+│ SELECT ? AS `?`                               │ SELECT ? AS aComplexName123                            │
+└───────────────────────────────────────────────┴────────────────────────────────────────────────────────┘
+```
+
 ## normalizedQueryHash
 
-Returns identical 64bit hash values without the values of literals for similar queries. Can be helpful to analyze query log.
+Returns identical 64bit hash values without the values of literals for similar queries. Can be helpful to analyze query logs.
 
 **Syntax**
 
@@ -1605,6 +1699,8 @@ normalizedQueryHash(x)
 
 **Example**
 
+Query:
+
 ``` sql
 SELECT normalizedQueryHash('SELECT 1 AS `xyz`') != normalizedQueryHash('SELECT 1 AS `abc`') AS res;
 ```
@@ -1617,6 +1713,43 @@ Result:
 └─────┘
 ```
 
+## normalizedQueryHashKeepNames
+
+Like [normalizedQueryHash](#normalizedqueryhash) it returns identical 64bit hash values without the values of literals for similar queries but it does not replace complex aliases (containing whitespace, more than two digits
+or at least 36 bytes long such as UUIDs) with a placeholder before hashing. Can be helpful to analyze query logs.
+
+**Syntax**
+
+``` sql
+normalizedQueryHashKeepNames(x)
+```
+
+**Arguments**
+
+- `x` — Sequence of characters. [String](../data-types/string.md).
+
+**Returned value**
+
+- Hash value. [UInt64](../data-types/int-uint.md#uint-ranges).
+
+**Example**
+
+``` sql
+SELECT normalizedQueryHash('SELECT 1 AS `xyz123`') != normalizedQueryHash('SELECT 1 AS `abc123`') AS normalizedQueryHash;
+SELECT normalizedQueryHashKeepNames('SELECT 1 AS `xyz123`') != normalizedQueryHashKeepNames('SELECT 1 AS `abc123`') AS normalizedQueryHashKeepNames;
+```
+
+Result:
+
+```result
+┌─normalizedQueryHash─┐
+│                   0 │
+└─────────────────────┘
+┌─normalizedQueryHashKeepNames─┐
+│                            1 │
+└──────────────────────────────┘
+```
+
 ## normalizeUTF8NFC
 
 Converts a string to [NFC normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string is valid UTF8-encoded text.
@@ -1935,7 +2068,7 @@ soundex(val)
 
 **Arguments**
 
-- `val` - Input value. [String](../data-types/string.md)
+- `val` — Input value. [String](../data-types/string.md)
 
 **Returned value**
 
@@ -1968,7 +2101,7 @@ punycodeEncode(val)
 
 **Arguments**
 
-- `val` - Input value. [String](../data-types/string.md)
+- `val` — Input value. [String](../data-types/string.md)
 
 **Returned value**
 
@@ -2001,7 +2134,7 @@ punycodeEncode(val)
 
 **Arguments**
 
-- `val` - Punycode-encoded string. [String](../data-types/string.md)
+- `val` — Punycode-encoded string. [String](../data-types/string.md)
 
 **Returned value**
 
@@ -2027,7 +2160,7 @@ Like `punycodeDecode` but returns an empty string if no valid Punycode-encoded s
 
 ## idnaEncode
 
-Returns the the ASCII representation (ToASCII algorithm) of a domain name according to the [Internationalized Domain Names in Applications](https://en.wikipedia.org/wiki/Internationalized_domain_name#Internationalizing_Domain_Names_in_Applications) (IDNA) mechanism.
+Returns the ASCII representation (ToASCII algorithm) of a domain name according to the [Internationalized Domain Names in Applications](https://en.wikipedia.org/wiki/Internationalized_domain_name#Internationalizing_Domain_Names_in_Applications) (IDNA) mechanism.
 The input string must be UTF-encoded and translatable to an ASCII string, otherwise an exception is thrown.
 Note: No percent decoding or trimming of tabs, spaces or control characters is performed.
 
@@ -2039,7 +2172,7 @@ idnaEncode(val)
 
 **Arguments**
 
-- `val` - Input value. [String](../data-types/string.md)
+- `val` — Input value. [String](../data-types/string.md)
 
 **Returned value**
 
@@ -2065,7 +2198,7 @@ Like `idnaEncode` but returns an empty string in case of an error instead of thr
 
 ## idnaDecode
 
-Returns the the Unicode (UTF-8) representation (ToUnicode algorithm) of a domain name according to the [Internationalized Domain Names in Applications](https://en.wikipedia.org/wiki/Internationalized_domain_name#Internationalizing_Domain_Names_in_Applications) (IDNA) mechanism.
+Returns the Unicode (UTF-8) representation (ToUnicode algorithm) of a domain name according to the [Internationalized Domain Names in Applications](https://en.wikipedia.org/wiki/Internationalized_domain_name#Internationalizing_Domain_Names_in_Applications) (IDNA) mechanism.
 In case of an error (e.g. because the input is invalid), the input string is returned.
 Note that repeated application of `idnaEncode()` and `idnaDecode()` does not necessarily return the original string due to case normalization.
 
@@ -2077,7 +2210,7 @@ idnaDecode(val)
 
 **Arguments**
 
-- `val` - Input value. [String](../data-types/string.md)
+- `val` — Input value. [String](../data-types/string.md)
 
 **Returned value**
 
@@ -2121,7 +2254,7 @@ Result:
 └───────────────────────────────────────────┘
 ```
 
-Alias: mismatches
+Alias: `mismatches`
 
 ## stringJaccardIndex
 
@@ -2175,7 +2308,7 @@ Result:
 └─────────────────────────────────────┘
 ```
 
-Alias: levenshteinDistance
+Alias: `levenshteinDistance`
 
 ## editDistanceUTF8
 
@@ -2201,7 +2334,7 @@ Result:
 └─────────────────────────────────────┘
 ```
 
-Alias: levenshteinDistanceUTF8
+Alias: `levenshteinDistanceUTF8`
 
 ## damerauLevenshteinDistance
 
@@ -2279,13 +2412,93 @@ Result:
 
 Convert the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters.
 
+:::note
+Because `initCap` converts only the first letter of each word to upper case you may observe unexpected behaviour for words containing apostrophes or capital letters. For example:
+
+```sql
+SELECT initCap('mother''s daughter'), initCap('joe McAdam');
+```
+
+will return
+
+```response
+┌─initCap('mother\'s daughter')─┬─initCap('joe McAdam')─┐
+│ Mother'S Daughter             │ Joe Mcadam            │
+└───────────────────────────────┴───────────────────────┘
+```
+
+This is a known behaviour, with no plans currently to fix it.
+:::
+
+**Syntax**
+
+```sql
+initcap(val)
+```
+
+**Arguments**
+
+- `val` — Input value. [String](../data-types/string.md).
+
+**Returned value**
+
+- `val` with the first letter of each word converted to upper case. [String](../data-types/string.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT initcap('building for fast');
+```
+
+Result:
+
+```text
+┌─initcap('building for fast')─┐
+│ Building For Fast            │
+└──────────────────────────────┘
+```
+
 ## initcapUTF8
 
-Like [initcap](#initcap), assuming that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
-
-Does not detect the language, e.g. for Turkish the result might not be exactly correct (i/İ vs. i/I).
+Like [initcap](#initcap), `initcapUTF8` converts the first letter of each word to upper case and the rest to lower case. Assumes that the string contains valid UTF-8 encoded text. 
+If this assumption is violated, no exception is thrown and the result is undefined.
 
+:::note
+This function does not detect the language, e.g. for Turkish the result might not be exactly correct (i/İ vs. i/I).
 If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point.
+:::
+
+**Syntax**
+
+```sql
+initcapUTF8(val)
+```
+
+**Arguments**
+
+- `val` — Input value. [String](../data-types/string.md).
+
+**Returned value**
+
+- `val` with the first letter of each word converted to upper case. [String](../data-types/string.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT initcapUTF8('не тормозит');
+```
+
+Result:
+
+```text
+┌─initcapUTF8('не тормозит')─┐
+│ Не Тормозит                │
+└────────────────────────────┘
+```
 
 ## firstLine
 
@@ -2299,7 +2512,7 @@ firstLine(val)
 
 **Arguments**
 
-- `val` - Input value. [String](../data-types/string.md)
+- `val` — Input value. [String](../data-types/string.md)
 
 **Returned value**
 

docs/en/sql-reference/functions/string-replace-functions.md

@@ -34,7 +34,7 @@ Alias: `replace`.
 
 Replaces the first occurrence of the substring matching the regular expression `pattern` (in [re2 syntax](https://github.com/google/re2/wiki/Syntax)) in `haystack` by the `replacement` string.
 
-`replacement` can containing substitutions `\0-\9`.
+`replacement` can contain substitutions `\0-\9`.
 Substitutions `\1-\9` correspond to the 1st to 9th capturing group (submatch), substitution `\0` corresponds to the entire match.
 
 To use a verbatim `\` character in the `pattern` or `replacement` strings, escape it using `\`.

docs/en/sql-reference/functions/time-window-functions.md

@@ -6,44 +6,122 @@ sidebar_label: Time Window
 
 # Time Window Functions
 
-Time window functions return the inclusive lower and exclusive upper bound of the corresponding window. The functions for working with WindowView are listed below:
+Time window functions return the inclusive lower and exclusive upper bound of the corresponding window. The functions for working with [WindowView](../statements/create/view.md/#window-view-experimental) are listed below:
 
 ## tumble
 
 A tumbling time window assigns records to non-overlapping, continuous windows with a fixed duration (`interval`). 
 
+**Syntax**
+
 ``` sql
 tumble(time_attr, interval [, timezone])
 ```
 
 **Arguments**
-- `time_attr` - Date and time. [DateTime](../data-types/datetime.md) data type.
-- `interval` - Window interval in [Interval](../data-types/special-data-types/interval.md) data type.
+- `time_attr` — Date and time. [DateTime](../data-types/datetime.md).
+- `interval` — Window interval in [Interval](../data-types/special-data-types/interval.md).
 - `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional). 
 
 **Returned values**
 
-- The inclusive lower and exclusive upper bound of the corresponding tumbling window. [Tuple](../data-types/tuple.md)([DateTime](../data-types/datetime.md), [DateTime](../data-types/datetime.md))`.
+- The inclusive lower and exclusive upper bound of the corresponding tumbling window. [Tuple](../data-types/tuple.md)([DateTime](../data-types/datetime.md), [DateTime](../data-types/datetime.md)).
 
 **Example**
 
 Query:
 
 ``` sql
-SELECT tumble(now(), toIntervalDay('1'))
+SELECT tumble(now(), toIntervalDay('1'));
 ```
 
 Result:
 
 ``` text
 ┌─tumble(now(), toIntervalDay('1'))─────────────┐
-│ ['2020-01-01 00:00:00','2020-01-02 00:00:00'] │
+│ ('2024-07-04 00:00:00','2024-07-05 00:00:00') │
 └───────────────────────────────────────────────┘
 ```
 
+## tumbleStart
+
+Returns the inclusive lower bound of the corresponding [tumbling window](#tumble).
+
+**Syntax**
+
+``` sql
+tumbleStart(time_attr, interval [, timezone]);
+```
+
+**Arguments**
+
+- `time_attr` — Date and time. [DateTime](../data-types/datetime.md).
+- `interval` — Window interval in [Interval](../data-types/special-data-types/interval.md).
+- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional).
+
+The parameters above can also be passed to the function as a [tuple](../data-types/tuple.md).
+
+**Returned values**
+
+- The inclusive lower bound of the corresponding tumbling window. [DateTime](../data-types/datetime.md), [Tuple](../data-types/tuple.md) or [UInt32](../data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT tumbleStart(now(), toIntervalDay('1'));
+```
+
+Result:
+
+```response
+┌─tumbleStart(now(), toIntervalDay('1'))─┐
+│                    2024-07-04 00:00:00 │
+└────────────────────────────────────────┘
+```
+
+## tumbleEnd
+
+Returns the exclusive upper bound of the corresponding [tumbling window](#tumble).
+
+**Syntax**
+
+``` sql
+tumbleEnd(time_attr, interval [, timezone]);
+```
+
+**Arguments**
+
+- `time_attr` — Date and time. [DateTime](../data-types/datetime.md).
+- `interval` — Window interval in [Interval](../data-types/special-data-types/interval.md).
+- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional).
+
+The parameters above can also be passed to the function as a [tuple](../data-types/tuple.md).
+
+**Returned values**
+
+- The inclusive lower bound of the corresponding tumbling window. [DateTime](../data-types/datetime.md), [Tuple](../data-types/tuple.md) or [UInt32](../data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT tumbleEnd(now(), toIntervalDay('1'));
+```
+
+Result:
+
+```response
+┌─tumbleEnd(now(), toIntervalDay('1'))─┐
+│                  2024-07-05 00:00:00 │
+└──────────────────────────────────────┘
+```
+
 ## hop
 
-A hopping time window has a fixed duration (`window_interval`) and hops by a specified hop interval (`hop_interval`). If the `hop_interval` is smaller than the `window_interval`, hopping windows are overlapping. Thus, records can be assigned to multiple windows. 
+A hopping time window has a fixed duration (`window_interval`) and hops by a specified hop interval (`hop_interval`). If the `hop_interval` is smaller than the `window_interval`, hopping windows are overlapping. Thus, records can be assigned to multiple windows.
 
 ``` sql
 hop(time_attr, hop_interval, window_interval [, timezone])
@@ -51,65 +129,118 @@ hop(time_attr, hop_interval, window_interval [, timezone])
 
 **Arguments**
 
-- `time_attr` - Date and time. [DateTime](../data-types/datetime.md) data type.
-- `hop_interval` - Hop interval in [Interval](../data-types/special-data-types/interval.md) data type. Should be a positive number.
-- `window_interval` - Window interval in [Interval](../data-types/special-data-types/interval.md) data type. Should be a positive number.
-- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional). 
+- `time_attr` — Date and time. [DateTime](../data-types/datetime.md).
+- `hop_interval` — Positive Hop interval. [Interval](../data-types/special-data-types/interval.md).
+- `window_interval` — Positive Window interval. [Interval](../data-types/special-data-types/interval.md).
+- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional).
 
 **Returned values**
 
-- The inclusive lower and exclusive upper bound of the corresponding hopping window. Since one record can be assigned to multiple hop windows, the function only returns the bound of the **first** window when hop function is used **without** `WINDOW VIEW`. [Tuple](../data-types/tuple.md)([DateTime](../data-types/datetime.md), [DateTime](../data-types/datetime.md))`.
+- The inclusive lower and exclusive upper bound of the corresponding hopping window. [Tuple](../data-types/tuple.md)([DateTime](../data-types/datetime.md), [DateTime](../data-types/datetime.md))`.
+
+:::note
+Since one record can be assigned to multiple hop windows, the function only returns the bound of the **first** window when hop function is used **without** `WINDOW VIEW`.
+:::
 
 **Example**
 
 Query:
 
 ``` sql
-SELECT hop(now(), INTERVAL '1' SECOND, INTERVAL '2' SECOND)
+SELECT hop(now(), INTERVAL '1' DAY, INTERVAL '2' DAY);
 ```
 
 Result:
 
 ``` text
-┌─hop(now(), toIntervalSecond('1'), toIntervalSecond('2'))──┐
-│ ('2020-01-14 16:58:22','2020-01-14 16:58:24')             │
-└───────────────────────────────────────────────────────────┘
-```
-
-## tumbleStart
-
-Returns the inclusive lower bound of the corresponding tumbling window.
-
-``` sql
-tumbleStart(bounds_tuple);
-tumbleStart(time_attr, interval [, timezone]);
-```
-
-## tumbleEnd
-
-Returns the exclusive upper bound of the corresponding tumbling window.
-
-``` sql
-tumbleEnd(bounds_tuple);
-tumbleEnd(time_attr, interval [, timezone]);
+┌─hop(now(), toIntervalDay('1'), toIntervalDay('2'))─┐
+│ ('2024-07-03 00:00:00','2024-07-05 00:00:00')      │
+└────────────────────────────────────────────────────┘
 ```
 
 ## hopStart
 
-Returns the inclusive lower bound of the corresponding hopping window.
+Returns the inclusive lower bound of the corresponding [hopping window](#hop).
+
+**Syntax**
 
 ``` sql
-hopStart(bounds_tuple);
 hopStart(time_attr, hop_interval, window_interval [, timezone]);
 ```
+**Arguments**
+
+- `time_attr` — Date and time. [DateTime](../data-types/datetime.md).
+- `hop_interval` — Positive Hop interval. [Interval](../data-types/special-data-types/interval.md).
+- `window_interval` — Positive Window interval. [Interval](../data-types/special-data-types/interval.md).
+- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional).
+
+The parameters above can also be passed to the function as a [tuple](../data-types/tuple.md).
+
+**Returned values**
+
+- The inclusive lower bound of the corresponding hopping window. [DateTime](../data-types/datetime.md), [Tuple](../data-types/tuple.md) or [UInt32](../data-types/int-uint.md).
+
+:::note
+Since one record can be assigned to multiple hop windows, the function only returns the bound of the **first** window when hop function is used **without** `WINDOW VIEW`.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT hopStart(now(), INTERVAL '1' DAY, INTERVAL '2' DAY);
+```
+
+Result:
+
+``` text
+┌─hopStart(now(), toIntervalDay('1'), toIntervalDay('2'))─┐
+│                                     2024-07-03 00:00:00 │
+└─────────────────────────────────────────────────────────┘
+```
 
 ## hopEnd
 
-Returns the exclusive upper bound of the corresponding hopping window.
+Returns the exclusive upper bound of the corresponding [hopping window](#hop).
+
+**Syntax**
 
 ``` sql
-hopEnd(bounds_tuple);
 hopEnd(time_attr, hop_interval, window_interval [, timezone]);
+```
+**Arguments**
+
+- `time_attr` — Date and time. [DateTime](../data-types/datetime.md).
+- `hop_interval` — Positive Hop interval. [Interval](../data-types/special-data-types/interval.md). 
+- `window_interval` — Positive Window interval. [Interval](../data-types/special-data-types/interval.md).
+- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional).
+
+The parameters above can also be passed to the function as a [tuple](../data-types/tuple.md).
+
+**Returned values**
+
+- The exclusive upper bound of the corresponding hopping window. [DateTime](../data-types/datetime.md), [Tuple](../data-types/tuple.md) or [UInt32](../data-types/int-uint.md).
+
+:::note
+Since one record can be assigned to multiple hop windows, the function only returns the bound of the **first** window when hop function is used **without** `WINDOW VIEW`.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT hopEnd(now(), INTERVAL '1' DAY, INTERVAL '2' DAY);
+```
+
+Result:
+
+``` text
+┌─hopEnd(now(), toIntervalDay('1'), toIntervalDay('2'))─┐
+│                                   2024-07-05 00:00:00 │
+└───────────────────────────────────────────────────────┘
+
 ```
 
 ## Related content

docs/en/sql-reference/functions/tuple-functions.md

@@ -7,7 +7,7 @@ sidebar_label: Tuples
 ## tuple
 
 A function that allows grouping multiple columns.
-For columns with the types T1, T2, ..., it returns a Tuple(T1, T2, ...) type tuple containing these columns. There is no cost to execute the function.
+For columns C1, C2, ... with the types T1, T2, ..., it returns a named Tuple(C1 T1, C2 T2, ...) type tuple containing these columns if their names are unique and can be treated as unquoted identifiers, otherwise a Tuple(T1, T2, ...) is returned. There is no cost to execute the function.
 Tuples are normally used as intermediate values for an argument of IN operators, or for creating a list of formal parameters of lambda functions. Tuples can’t be written to a table.
 
 The function implements the operator `(x, y, ...)`.
@@ -259,6 +259,60 @@ Result:
 └───────────────────────────────────────┘
 ```
 
+## tupleNames
+
+Converts a tuple into an array of column names. For a tuple in the form `Tuple(a T, b T, ...)`, it returns an array of strings representing the named columns of the tuple. If the tuple elements do not have explicit names, their indices will be used as the column names instead.
+
+**Syntax**
+
+``` sql
+tupleNames(tuple)
+```
+
+**Arguments**
+
+- `tuple` — Named tuple. [Tuple](../../sql-reference/data-types/tuple.md) with any types of values.
+
+**Returned value**
+
+- An array with strings.
+
+Type: [Array](../../sql-reference/data-types/array.md)([Tuple](../../sql-reference/data-types/tuple.md)([String](../../sql-reference/data-types/string.md), ...)).
+
+**Example**
+
+Query:
+
+``` sql
+CREATE TABLE tupletest (col Tuple(user_ID UInt64, session_ID UInt64)) ENGINE = Memory;
+
+INSERT INTO tupletest VALUES (tuple(1, 2));
+
+SELECT tupleNames(col) FROM tupletest;
+```
+
+Result:
+
+``` text
+┌─tupleNames(col)──────────┐
+│ ['user_ID','session_ID'] │
+└──────────────────────────┘
+```
+
+If you pass a simple tuple to the function, ClickHouse uses the indexes of the columns as their names:
+
+``` sql
+SELECT tupleNames(tuple(3, 2, 1));
+```
+
+Result:
+
+``` text
+┌─tupleNames((3, 2, 1))─┐
+│ ['1','2','3']         │
+└───────────────────────┘
+```
+
 ## tuplePlus
 
 Calculates the sum of corresponding values of two tuples of the same size.

docs/en/sql-reference/functions/tuple-map-functions.md

@@ -600,7 +600,7 @@ mapApply(func, map)
 
 **Arguments**
 
-- `func`  - [Lambda function](../../sql-reference/functions/index.md#higher-order-functions---operator-and-lambdaparams-expr-function).
+- `func` — [Lambda function](../../sql-reference/functions/index.md#higher-order-functions---operator-and-lambdaparams-expr-function).
 - `map` — [Map](../data-types/map.md).
 
 **Returned value**
@@ -831,7 +831,39 @@ SELECT mapSort((k, v) -> v, map('key2', 2, 'key3', 1, 'key1', 3)) AS map;
 └──────────────────────────────┘
 ```
 
-For more details see the [reference](../../sql-reference/functions/array-functions.md#array_functions-sort) for `arraySort` function.
+For more details see the [reference](../../sql-reference/functions/array-functions.md#array_functions-sort) for `arraySort` function. 
+
+## mapPartialSort
+
+Sorts the elements of a map in ascending order with additional `limit` argument allowing partial sorting. 
+If the `func` function is specified, the sorting order is determined by the result of the `func` function applied to the keys and values of the map.
+
+**Syntax**
+
+```sql
+mapPartialSort([func,] limit, map)
+```
+**Arguments**
+
+- `func` – Optional function to apply to the keys and values of the map. [Lambda function](../../sql-reference/functions/index.md#higher-order-functions---operator-and-lambdaparams-expr-function).
+- `limit` – Elements in range [1..limit] are sorted. [(U)Int](../data-types/int-uint.md).
+- `map` – Map to sort. [Map](../data-types/map.md).
+
+**Returned value**
+
+- Partially sorted map. [Map](../data-types/map.md).
+
+**Example**
+
+``` sql
+SELECT mapPartialSort((k, v) -> v, 2, map('k1', 3, 'k2', 1, 'k3', 2));
+```
+
+``` text
+┌─mapPartialSort(lambda(tuple(k, v), v), 2, map('k1', 3, 'k2', 1, 'k3', 2))─┐
+│ {'k2':1,'k3':2,'k1':3}                                                    │
+└───────────────────────────────────────────────────────────────────────────┘
+```
 
 ## mapReverseSort(\[func,\], map)
 
@@ -861,3 +893,35 @@ SELECT mapReverseSort((k, v) -> v, map('key2', 2, 'key3', 1, 'key1', 3)) AS map;
 ```
 
 For more details see function [arrayReverseSort](../../sql-reference/functions/array-functions.md#array_functions-reverse-sort).
+
+## mapPartialReverseSort
+
+Sorts the elements of a map in descending order with additional `limit` argument allowing partial sorting.
+If the `func` function is specified, the sorting order is determined by the result of the `func` function applied to the keys and values of the map.
+
+**Syntax**
+
+```sql
+mapPartialReverseSort([func,] limit, map)
+```
+**Arguments**
+
+- `func` – Optional function to apply to the keys and values of the map. [Lambda function](../../sql-reference/functions/index.md#higher-order-functions---operator-and-lambdaparams-expr-function).
+- `limit` – Elements in range [1..limit] are sorted. [(U)Int](../data-types/int-uint.md).
+- `map` – Map to sort. [Map](../data-types/map.md).
+
+**Returned value**
+
+- Partially sorted map. [Map](../data-types/map.md).
+
+**Example**
+
+``` sql
+SELECT mapPartialReverseSort((k, v) -> v, 2, map('k1', 3, 'k2', 1, 'k3', 2));
+```
+
+``` text
+┌─mapPartialReverseSort(lambda(tuple(k, v), v), 2, map('k1', 3, 'k2', 1, 'k3', 2))─┐
+│ {'k1':3,'k3':2,'k2':1}                                                           │
+└──────────────────────────────────────────────────────────────────────────────────┘
+```

docs/en/sql-reference/statements/kill.md

@@ -58,6 +58,8 @@ KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'
 KILL QUERY WHERE user='username' SYNC
 ```
 
+:::tip If you are killing a query in ClickHouse Cloud or in a self-managed cluster, then be sure to use the ```ON CLUSTER [cluster-name]``` option, in order to ensure the query is killed on all replicas:::
+
 Read-only users can only stop their own queries.
 
 By default, the asynchronous version of queries is used (`ASYNC`), which does not wait for confirmation that queries have stopped.
@@ -131,6 +133,7 @@ KILL MUTATION WHERE database = 'default' AND table = 'table'
 -- Cancel the specific mutation:
 KILL MUTATION WHERE database = 'default' AND table = 'table' AND mutation_id = 'mutation_3.txt'
 ```
+:::tip If you are killing a mutation in ClickHouse Cloud or in a self-managed cluster, then be sure to use the ```ON CLUSTER [cluster-name]``` option, in order to ensure the mutation is killed on all replicas:::
 
 The query is useful when a mutation is stuck and cannot finish (e.g. if some function in the mutation query throws an exception when applied to the data contained in the table).
 

docs/en/sql-reference/table-functions/file.md

@@ -130,7 +130,9 @@ SELECT * FROM file('user_files/archives/archive{1..2}.zip :: table.csv');
 
 ## Globs in path 
 
-Paths may use globbing. Files must match the whole path pattern, not only the suffix or prefix.
+Paths may use globbing. Files must match the whole path pattern, not only the suffix or prefix. There is one exception that if the path refers to an existing
+directory and does not use globs, a `*` will be implicitly added to the path so
+all the files in the directory are selected.
 
 - `*` — Represents arbitrarily many characters except `/` but including the empty string.
 - `?` — Represents an arbitrary single character.
@@ -163,6 +165,12 @@ An alternative path expression which achieves the same:
 SELECT count(*) FROM file('{some,another}_dir/*', 'TSV', 'name String, value UInt32');
 ```
 
+Query the total number of rows in `some_dir` using the implicit `*`:
+
+```sql
+SELECT count(*) FROM file('some_dir', 'TSV', 'name String, value UInt32');
+```
+
 :::note
 If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
 :::

docs/ru/introduction/distinctive-features.md

@@ -12,7 +12,7 @@ sidebar_label: "Отличительные возможности ClickHouse"
 
 Этот пункт пришлось выделить, так как существуют системы, которые могут хранить значения отдельных столбцов по отдельности, но не могут эффективно выполнять аналитические запросы в силу оптимизации под другой сценарий работы. Примеры: HBase, BigTable, Cassandra, HyperTable. В этих системах вы получите пропускную способность в районе сотен тысяч строк в секунду, но не сотен миллионов строк в секунду.
 
-Также стоит заметить, что ClickHouse является системой управления базами данных, а не системой для одной базой данных. То есть, ClickHouse позволяет создавать таблицы и базы данных во время выполнения (runtime), загружать данные и выполнять запросы без переконфигурирования и перезапуска сервера.
+Также стоит заметить, что ClickHouse является системой управления базами данных, а не системой для одной базы данных. То есть, ClickHouse позволяет создавать таблицы и базы данных во время выполнения (runtime), загружать данные и выполнять запросы без переконфигурирования и перезапуска сервера.
 
 ## Сжатие данных {#szhatie-dannykh}
 

programs/client/Client.cpp

@@ -1117,6 +1117,7 @@ void Client::processOptions(const OptionsDescription & options_description,
         if (!options["user"].defaulted())
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "User and JWT flags can't be specified together");
         config().setString("jwt", options["jwt"].as<std::string>());
+        config().setString("user", "");
     }
     if (options.count("accept-invalid-certificate"))
     {

programs/disks/CMakeLists.txt

@@ -1,6 +1,8 @@
 set (CLICKHOUSE_DISKS_SOURCES
     DisksApp.cpp
+    DisksClient.cpp
     ICommand.cpp
+    CommandChangeDirectory.cpp
     CommandCopy.cpp
     CommandLink.cpp
     CommandList.cpp
@@ -9,10 +11,14 @@ set (CLICKHOUSE_DISKS_SOURCES
     CommandMove.cpp
     CommandRead.cpp
     CommandRemove.cpp
-    CommandWrite.cpp)
+    CommandSwitchDisk.cpp
+    CommandWrite.cpp
+    CommandHelp.cpp
+    CommandTouch.cpp
+    CommandGetCurrentDiskAndPath.cpp)
 
 if (CLICKHOUSE_CLOUD)
-    set (CLICKHOUSE_DISKS_SOURCES ${CLICKHOUSE_DISKS_SOURCES} CommandPackedIO.cpp)
+  set (CLICKHOUSE_DISKS_SOURCES ${CLICKHOUSE_DISKS_SOURCES} CommandPackedIO.cpp)
 endif ()
 
 set (CLICKHOUSE_DISKS_LINK

programs/disks/CommandChangeDirectory.cpp

@@ -0,0 +1,35 @@
+#include <Interpreters/Context.h>
+#include <Common/TerminalSize.h>
+#include "DisksApp.h"
+#include "DisksClient.h"
+#include "ICommand.h"
+
+namespace DB
+{
+
+class CommandChangeDirectory final : public ICommand
+{
+public:
+    explicit CommandChangeDirectory() : ICommand()
+    {
+        command_name = "cd";
+        description = "Change directory (makes sense only in interactive mode)";
+        options_description.add_options()("path", po::value<String>(), "the path to which we want to change (mandatory, positional)")(
+            "disk", po::value<String>(), "A disk where the path is changed (without disk switching)");
+        positional_options_description.add("path", 1);
+    }
+
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
+    {
+        DiskWithPath & disk = getDiskWithPath(client, options, "disk");
+        String path = getValueFromCommandLineOptionsThrow<String>(options, "path");
+        disk.setPath(path);
+    }
+};
+
+CommandPtr makeCommandChangeDirectory()
+{
+    return std::make_shared<DB::CommandChangeDirectory>();
+}
+
+}

programs/disks/CommandCopy.cpp

@@ -1,6 +1,8 @@
-#include "ICommand.h"
 #include <Interpreters/Context.h>
+#include "Common/Exception.h"
 #include <Common/TerminalSize.h>
+#include "DisksClient.h"
+#include "ICommand.h"
 
 namespace DB
 {
@@ -10,59 +12,89 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
+
 class CommandCopy final : public ICommand
 {
 public:
-    CommandCopy()
+    explicit CommandCopy() : ICommand()
     {
         command_name = "copy";
-        command_option_description.emplace(createOptionsDescription("Allowed options", getTerminalWidth()));
-        description = "Recursively copy data from `FROM_PATH` to `TO_PATH`";
-        usage = "copy [OPTION]... <FROM_PATH> <TO_PATH>";
-        command_option_description->add_options()
-            ("disk-from", po::value<String>(), "disk from which we copy")
-            ("disk-to", po::value<String>(), "disk to which we copy");
+        description = "Recursively copy data from `path-from` to `path-to`";
+        options_description.add_options()(
+            "disk-from", po::value<String>(), "disk from which we copy is executed (default value is a current disk)")(
+            "disk-to", po::value<String>(), "disk to which copy is executed (default value is a current disk)")(
+            "path-from", po::value<String>(), "path from which copy is executed (mandatory, positional)")(
+            "path-to", po::value<String>(), "path to which copy is executed (mandatory, positional)")(
+            "recursive,r", "recursively copy the directory (required to remove a directory)");
+        positional_options_description.add("path-from", 1);
+        positional_options_description.add("path-to", 1);
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration & config,
-        po::variables_map & options) const override
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
     {
-        if (options.count("disk-from"))
-            config.setString("disk-from", options["disk-from"].as<String>());
-        if (options.count("disk-to"))
-            config.setString("disk-to", options["disk-to"].as<String>());
-    }
+        auto disk_from = getDiskWithPath(client, options, "disk-from");
+        auto disk_to = getDiskWithPath(client, options, "disk-to");
+        String path_from = disk_from.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
+        String path_to = disk_to.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
+        bool recursive = options.count("recursive");
 
-    void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) override
-    {
-        if (command_arguments.size() != 2)
+        if (!disk_from.getDisk()->exists(path_from))
         {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
+            throw Exception(
+                ErrorCodes::BAD_ARGUMENTS,
+                "cannot stat '{}' on disk '{}': No such file or directory",
+                path_from,
+                disk_from.getDisk()->getName());
         }
+        else if (disk_from.getDisk()->isFile(path_from))
+        {
+            auto target_location = getTargetLocation(path_from, disk_to, path_to);
+            if (!disk_to.getDisk()->exists(target_location) || disk_to.getDisk()->isFile(target_location))
+            {
+                disk_from.getDisk()->copyFile(
+                    path_from,
+                    *disk_to.getDisk(),
+                    target_location,
+                    /* read_settings= */ {},
+                    /* write_settings= */ {},
+                    /* cancellation_hook= */ {});
+            }
+            else
+            {
+                throw Exception(
+                    ErrorCodes::BAD_ARGUMENTS, "cannot overwrite directory {} with non-directory {}", target_location, path_from);
+            }
+        }
+        else if (disk_from.getDisk()->isDirectory(path_from))
+        {
+            if (!recursive)
+            {
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "--recursive not specified; omitting directory {}", path_from);
+            }
+            auto target_location = getTargetLocation(path_from, disk_to, path_to);
 
-        String disk_name_from = config.getString("disk-from", config.getString("disk", "default"));
-        String disk_name_to = config.getString("disk-to", config.getString("disk", "default"));
-
-        const String & path_from = command_arguments[0];
-        const String & path_to =  command_arguments[1];
-
-        DiskPtr disk_from = disk_selector->get(disk_name_from);
-        DiskPtr disk_to = disk_selector->get(disk_name_to);
-
-        String relative_path_from = validatePathAndGetAsRelative(path_from);
-        String relative_path_to = validatePathAndGetAsRelative(path_to);
-
-        disk_from->copyDirectoryContent(relative_path_from, disk_to, relative_path_to, /* read_settings= */ {}, /* write_settings= */ {}, /* cancellation_hook= */ {});
+            if (disk_to.getDisk()->isFile(target_location))
+            {
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "cannot overwrite non-directory {} with directory {}", path_to, target_location);
+            }
+            else if (!disk_to.getDisk()->exists(target_location))
+            {
+                disk_to.getDisk()->createDirectory(target_location);
+            }
+            disk_from.getDisk()->copyDirectoryContent(
+                path_from,
+                disk_to.getDisk(),
+                target_location,
+                /* read_settings= */ {},
+                /* write_settings= */ {},
+                /* cancellation_hook= */ {});
+        }
     }
 };
+
+CommandPtr makeCommandCopy()
+{
+    return std::make_shared<DB::CommandCopy>();
 }
 
-std::unique_ptr <DB::ICommand> makeCommandCopy()
-{
-    return std::make_unique<DB::CommandCopy>();
 }

programs/disks/CommandGetCurrentDiskAndPath.cpp

@@ -0,0 +1,30 @@
+#include <Interpreters/Context.h>
+#include <Common/TerminalSize.h>
+#include "DisksApp.h"
+#include "DisksClient.h"
+#include "ICommand.h"
+
+namespace DB
+{
+
+class CommandGetCurrentDiskAndPath final : public ICommand
+{
+public:
+    explicit CommandGetCurrentDiskAndPath() : ICommand()
+    {
+        command_name = "current_disk_with_path";
+        description = "Prints current disk and path (which coincide with the prompt)";
+    }
+
+    void executeImpl(const CommandLineOptions &, DisksClient & client) override
+    {
+        auto disk = client.getCurrentDiskWithPath();
+        std::cout << "Disk: " << disk.getDisk()->getName() << "\nPath: " << disk.getCurrentPath() << std::endl;
+    }
+};
+
+CommandPtr makeCommandGetCurrentDiskAndPath()
+{
+    return std::make_shared<DB::CommandGetCurrentDiskAndPath>();
+}
+}

programs/disks/CommandHelp.cpp

@@ -0,0 +1,43 @@
+#include "DisksApp.h"
+#include "ICommand.h"
+
+#include <Interpreters/Context.h>
+#include <Common/TerminalSize.h>
+
+namespace DB
+{
+
+class CommandHelp final : public ICommand
+{
+public:
+    explicit CommandHelp(const DisksApp & disks_app_) : disks_app(disks_app_)
+    {
+        command_name = "help";
+        description = "Print help message about available commands";
+        options_description.add_options()(
+            "command", po::value<String>(), "A command to help with (optional, positional), if not specified, help lists all the commands");
+        positional_options_description.add("command", 1);
+    }
+
+    void executeImpl(const CommandLineOptions & options, DisksClient & /*client*/) override
+    {
+        std::optional<String> command = getValueFromCommandLineOptionsWithOptional<String>(options, "command");
+        if (command.has_value())
+        {
+            disks_app.printCommandHelpMessage(command.value());
+        }
+        else
+        {
+            disks_app.printAvailableCommandsHelpMessage();
+        }
+    }
+
+    const DisksApp & disks_app;
+};
+
+CommandPtr makeCommandHelp(const DisksApp & disks_app)
+{
+    return std::make_shared<DB::CommandHelp>(disks_app);
+}
+
+}

programs/disks/CommandLink.cpp

@@ -1,14 +1,9 @@
-#include "ICommand.h"
 #include <Interpreters/Context.h>
+#include "ICommand.h"
 
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
 class CommandLink final : public ICommand
 {
 public:
@@ -16,42 +11,27 @@ public:
     {
         command_name = "link";
         description = "Create hardlink from `from_path` to `to_path`";
-        usage = "link [OPTION]... <FROM_PATH> <TO_PATH>";
+        options_description.add_options()(
+            "path-from", po::value<String>(), "the path from which a hard link will be created (mandatory, positional)")(
+            "path-to", po::value<String>(), "the path where a hard link will be created (mandatory, positional)");
+        positional_options_description.add("path-from", 1);
+        positional_options_description.add("path-to", 1);
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration &,
-        po::variables_map &) const override
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
     {
-    }
+        auto disk = client.getCurrentDiskWithPath();
 
-    void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) override
-    {
-        if (command_arguments.size() != 2)
-        {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
-        }
+        const String & path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
+        const String & path_to = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
 
-        String disk_name = config.getString("disk", "default");
-
-        const String & path_from = command_arguments[0];
-        const String & path_to = command_arguments[1];
-
-        DiskPtr disk = disk_selector->get(disk_name);
-
-        String relative_path_from = validatePathAndGetAsRelative(path_from);
-        String relative_path_to = validatePathAndGetAsRelative(path_to);
-
-        disk->createHardLink(relative_path_from, relative_path_to);
+        disk.getDisk()->createHardLink(path_from, path_to);
     }
 };
+
+CommandPtr makeCommandLink()
+{
+    return std::make_shared<DB::CommandLink>();
 }
 
-std::unique_ptr <DB::ICommand> makeCommandLink()
-{
-    return std::make_unique<DB::CommandLink>();
 }

programs/disks/CommandList.cpp

@@ -1,98 +1,95 @@
-#include "ICommand.h"
 #include <Interpreters/Context.h>
 #include <Common/TerminalSize.h>
+#include "DisksApp.h"
+#include "DisksClient.h"
+#include "ICommand.h"
 
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
 class CommandList final : public ICommand
 {
 public:
-    CommandList()
+    explicit CommandList() : ICommand()
     {
         command_name = "list";
-        command_option_description.emplace(createOptionsDescription("Allowed options", getTerminalWidth()));
         description = "List files at path[s]";
-        usage = "list [OPTION]... <PATH>...";
-        command_option_description->add_options()
-            ("recursive", "recursively list all directories");
+        options_description.add_options()("recursive", "recursively list the directory")("all", "show hidden files")(
+            "path", po::value<String>(), "the path of listing (mandatory, positional)");
+        positional_options_description.add("path", 1);
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration & config,
-        po::variables_map & options) const override
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
     {
-        if (options.count("recursive"))
-            config.setBool("recursive", true);
-    }
-
-    void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) override
-    {
-        if (command_arguments.size() != 1)
-        {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
-        }
-
-        String disk_name = config.getString("disk", "default");
-
-        const String & path =  command_arguments[0];
-
-        DiskPtr disk = disk_selector->get(disk_name);
-
-        String relative_path = validatePathAndGetAsRelative(path);
-
-        bool recursive = config.getBool("recursive", false);
+        bool recursive = options.count("recursive");
+        bool show_hidden = options.count("all");
+        auto disk = client.getCurrentDiskWithPath();
+        String path = getValueFromCommandLineOptionsWithDefault<String>(options, "path", ".");
 
         if (recursive)
-            listRecursive(disk, relative_path);
+            listRecursive(disk, path, show_hidden);
         else
-            list(disk, relative_path);
+            list(disk, path, show_hidden);
     }
 
 private:
-    static void list(const DiskPtr & disk, const std::string & relative_path)
+    static void list(const DiskWithPath & disk, const std::string & path, bool show_hidden)
     {
-        std::vector<String> file_names;
-        disk->listFiles(relative_path, file_names);
+        std::vector<String> file_names = disk.listAllFilesByPath(path);
+        std::vector<String> selected_and_sorted_file_names{};
 
         for (const auto & file_name : file_names)
-            std::cout << file_name << '\n';
+            if (show_hidden || (!file_name.starts_with('.')))
+                selected_and_sorted_file_names.push_back(file_name);
+
+        std::sort(selected_and_sorted_file_names.begin(), selected_and_sorted_file_names.end());
+        for (const auto & file_name : selected_and_sorted_file_names)
+        {
+            std::cout << file_name << "\n";
+        }
     }
 
-    static void listRecursive(const DiskPtr & disk, const std::string & relative_path)
+    static void listRecursive(const DiskWithPath & disk, const std::string & relative_path, bool show_hidden)
     {
-        std::vector<String> file_names;
-        disk->listFiles(relative_path, file_names);
+        std::vector<String> file_names = disk.listAllFilesByPath(relative_path);
+        std::vector<String> selected_and_sorted_file_names{};
 
         std::cout << relative_path << ":\n";
 
-        if (!file_names.empty())
-        {
-            for (const auto & file_name : file_names)
-                std::cout << file_name << '\n';
-            std::cout << "\n";
-        }
-
         for (const auto & file_name : file_names)
+            if (show_hidden || (!file_name.starts_with('.')))
+                selected_and_sorted_file_names.push_back(file_name);
+
+        std::sort(selected_and_sorted_file_names.begin(), selected_and_sorted_file_names.end());
+        for (const auto & file_name : selected_and_sorted_file_names)
         {
-            auto path = relative_path.empty() ? file_name : (relative_path + "/" + file_name);
-            if (disk->isDirectory(path))
-                listRecursive(disk, path);
+            std::cout << file_name << "\n";
+        }
+        std::cout << "\n";
+
+        for (const auto & file_name : selected_and_sorted_file_names)
+        {
+            auto path = [&]() -> String
+            {
+                if (relative_path.ends_with("/"))
+                {
+                    return relative_path + file_name;
+                }
+                else
+                {
+                    return relative_path + "/" + file_name;
+                }
+            }();
+            if (disk.isDirectory(path))
+            {
+                listRecursive(disk, path, show_hidden);
+            }
         }
     }
 };
-}
 
-std::unique_ptr <DB::ICommand> makeCommandList()
+CommandPtr makeCommandList()
 {
-    return std::make_unique<DB::CommandList>();
+    return std::make_shared<DB::CommandList>();
+}
 }

programs/disks/CommandListDisks.cpp

@@ -1,68 +1,40 @@
-#include "ICommand.h"
+#include <algorithm>
 #include <Interpreters/Context.h>
+#include <Common/TerminalSize.h>
+#include "DisksClient.h"
+#include "ICommand.h"
 
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
 class CommandListDisks final : public ICommand
 {
 public:
-    CommandListDisks()
+    explicit CommandListDisks() : ICommand()
     {
         command_name = "list-disks";
-        description = "List disks names";
-        usage = "list-disks [OPTION]";
+        description = "Lists all available disks";
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration &,
-        po::variables_map &) const override
-    {}
-
-    void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> &,
-        Poco::Util::LayeredConfiguration & config) override
+    void executeImpl(const CommandLineOptions &, DisksClient & client) override
     {
-        if (!command_arguments.empty())
+        std::vector<String> sorted_and_selected{};
+        for (const auto & disk_name : client.getAllDiskNames())
         {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
+            sorted_and_selected.push_back(disk_name + ":" + client.getDiskWithPath(disk_name).getAbsolutePath(""));
         }
-
-        constexpr auto config_prefix = "storage_configuration.disks";
-        constexpr auto default_disk_name = "default";
-
-        Poco::Util::AbstractConfiguration::Keys keys;
-        config.keys(config_prefix, keys);
-
-        bool has_default_disk = false;
-
-        /// For the output to be ordered
-        std::set<String> disks;
-
-        for (const auto & disk_name : keys)
+        std::sort(sorted_and_selected.begin(), sorted_and_selected.end());
+        for (const auto & disk_name : sorted_and_selected)
         {
-            if (disk_name == default_disk_name)
-                has_default_disk = true;
-            disks.insert(disk_name);
+            std::cout << disk_name << "\n";
         }
-
-        if (!has_default_disk)
-            disks.insert(default_disk_name);
-
-        for (const auto & disk : disks)
-            std::cout << disk << '\n';
     }
-};
-}
 
-std::unique_ptr <DB::ICommand> makeCommandListDisks()
+private:
+};
+
+CommandPtr makeCommandListDisks()
 {
-    return std::make_unique<DB::CommandListDisks>();
+    return std::make_shared<DB::CommandListDisks>();
+}
 }

programs/disks/CommandMkDir.cpp

@@ -6,61 +6,35 @@
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
 class CommandMkDir final : public ICommand
 {
 public:
     CommandMkDir()
     {
         command_name = "mkdir";
-        command_option_description.emplace(createOptionsDescription("Allowed options", getTerminalWidth()));
-        description = "Create a directory";
-        usage = "mkdir [OPTION]... <PATH>";
-        command_option_description->add_options()
-            ("recursive", "recursively create directories");
+        description = "Creates a directory";
+        options_description.add_options()("parents", "recursively create directories")(
+            "path", po::value<String>(), "the path on which directory should be created (mandatory, positional)");
+        positional_options_description.add("path", 1);
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration & config,
-        po::variables_map & options) const override
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
     {
-        if (options.count("recursive"))
-            config.setBool("recursive", true);
-    }
+        bool recursive = options.count("parents");
+        auto disk = client.getCurrentDiskWithPath();
 
-    void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) override
-    {
-        if (command_arguments.size() != 1)
-        {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
-        }
-
-        String disk_name = config.getString("disk", "default");
-
-        const String & path = command_arguments[0];
-
-        DiskPtr disk = disk_selector->get(disk_name);
-
-        String relative_path = validatePathAndGetAsRelative(path);
-        bool recursive = config.getBool("recursive", false);
+        String path = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path"));
 
         if (recursive)
-            disk->createDirectories(relative_path);
+            disk.getDisk()->createDirectories(path);
         else
-            disk->createDirectory(relative_path);
+            disk.getDisk()->createDirectory(path);
     }
 };
+
+CommandPtr makeCommandMkDir()
+{
+    return std::make_shared<DB::CommandMkDir>();
 }
 
-std::unique_ptr <DB::ICommand> makeCommandMkDir()
-{
-    return std::make_unique<DB::CommandMkDir>();
 }

programs/disks/CommandMove.cpp

@@ -1,5 +1,5 @@
-#include "ICommand.h"
 #include <Interpreters/Context.h>
+#include "ICommand.h"
 
 namespace DB
 {
@@ -9,6 +9,7 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
+
 class CommandMove final : public ICommand
 {
 public:
@@ -16,44 +17,62 @@ public:
     {
         command_name = "move";
         description = "Move file or directory from `from_path` to `to_path`";
-        usage = "move [OPTION]... <FROM_PATH> <TO_PATH>";
+        options_description.add_options()("path-from", po::value<String>(), "path from which we copy (mandatory, positional)")(
+            "path-to", po::value<String>(), "path to which we copy (mandatory, positional)");
+        positional_options_description.add("path-from", 1);
+        positional_options_description.add("path-to", 1);
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration &,
-        po::variables_map &) const override
-    {}
-
-    void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) override
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
     {
-        if (command_arguments.size() != 2)
+        auto disk = client.getCurrentDiskWithPath();
+
+        String path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
+        String path_to = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
+
+        if (disk.getDisk()->isFile(path_from))
         {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
+            disk.getDisk()->moveFile(path_from, path_to);
+        }
+        else if (disk.getDisk()->isDirectory(path_from))
+        {
+            auto target_location = getTargetLocation(path_from, disk, path_to);
+            if (!disk.getDisk()->exists(target_location))
+            {
+                disk.getDisk()->createDirectory(target_location);
+                disk.getDisk()->moveDirectory(path_from, target_location);
+            }
+            else
+            {
+                if (disk.getDisk()->isFile(target_location))
+                {
+                    throw Exception(
+                        ErrorCodes::BAD_ARGUMENTS, "cannot overwrite non-directory '{}' with directory '{}'", target_location, path_from);
+                }
+                if (!disk.getDisk()->isDirectoryEmpty(target_location))
+                {
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "cannot move '{}' to '{}': Directory not empty", path_from, target_location);
+                }
+                else
+                {
+                    disk.getDisk()->moveDirectory(path_from, target_location);
+                }
+            }
+        }
+        else if (!disk.getDisk()->exists(path_from))
+        {
+            throw Exception(
+                ErrorCodes::BAD_ARGUMENTS,
+                "cannot stat '{}' on disk: '{}': No such file or directory",
+                path_from,
+                disk.getDisk()->getName());
         }
-
-        String disk_name = config.getString("disk", "default");
-
-        const String & path_from = command_arguments[0];
-        const String & path_to = command_arguments[1];
-
-        DiskPtr disk = disk_selector->get(disk_name);
-
-        String relative_path_from = validatePathAndGetAsRelative(path_from);
-        String relative_path_to = validatePathAndGetAsRelative(path_to);
-
-        if (disk->isFile(relative_path_from))
-            disk->moveFile(relative_path_from, relative_path_to);
-        else
-            disk->moveDirectory(relative_path_from, relative_path_to);
     }
 };
+
+CommandPtr makeCommandMove()
+{
+    return std::make_shared<DB::CommandMove>();
 }
 
-std::unique_ptr <DB::ICommand> makeCommandMove()
-{
-    return std::make_unique<DB::CommandMove>();
 }

programs/disks/CommandRead.cpp

@@ -1,78 +1,52 @@
-#include "ICommand.h"
-#include <Interpreters/Context.h>
 #include <IO/ReadBufferFromFile.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/copyData.h>
+#include <Interpreters/Context.h>
 #include <Common/TerminalSize.h>
+#include "ICommand.h"
 
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
 class CommandRead final : public ICommand
 {
 public:
     CommandRead()
     {
         command_name = "read";
-        command_option_description.emplace(createOptionsDescription("Allowed options", getTerminalWidth()));
-        description = "Read a file from `FROM_PATH` to `TO_PATH`";
-        usage = "read [OPTION]... <FROM_PATH> [<TO_PATH>]";
-        command_option_description->add_options()
-            ("output", po::value<String>(), "file to which we are reading, defaults to `stdout`");
+        description = "Read a file from `path-from` to `path-to`";
+        options_description.add_options()("path-from", po::value<String>(), "file from which we are reading (mandatory, positional)")(
+            "path-to", po::value<String>(), "file to which we are writing, defaults to `stdout`");
+        positional_options_description.add("path-from", 1);
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration & config,
-        po::variables_map & options) const override
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
     {
-        if (options.count("output"))
-            config.setString("output", options["output"].as<String>());
-    }
+        auto disk = client.getCurrentDiskWithPath();
+        String path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
+        std::optional<String> path_to = getValueFromCommandLineOptionsWithOptional<String>(options, "path-to");
 
-    void execute(
-        const std::vector<String> & command_arguments,
-       std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) override
-    {
-        if (command_arguments.size() != 1)
+        auto in = disk.getDisk()->readFile(path_from);
+        std::unique_ptr<WriteBufferFromFileBase> out = {};
+        if (path_to.has_value())
         {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
-        }
-
-        String disk_name = config.getString("disk", "default");
-
-        DiskPtr disk = disk_selector->get(disk_name);
-
-        String relative_path = validatePathAndGetAsRelative(command_arguments[0]);
-
-        String path_output = config.getString("output", "");
-
-        if (!path_output.empty())
-        {
-            String relative_path_output = validatePathAndGetAsRelative(path_output);
-
-            auto in = disk->readFile(relative_path);
-            auto out = disk->writeFile(relative_path_output);
+            String relative_path_to = disk.getRelativeFromRoot(path_to.value());
+            out = disk.getDisk()->writeFile(relative_path_to);
             copyData(*in, *out);
-            out->finalize();
         }
         else
         {
-            auto in = disk->readFile(relative_path);
-            std::unique_ptr<WriteBufferFromFileBase> out = std::make_unique<WriteBufferFromFileDescriptor>(STDOUT_FILENO);
+            out = std::make_unique<WriteBufferFromFileDescriptor>(STDOUT_FILENO);
             copyData(*in, *out);
+            out->write('\n');
         }
+        out->finalize();
     }
 };
+
+CommandPtr makeCommandRead()
+{
+    return std::make_shared<DB::CommandRead>();
 }
 
-std::unique_ptr <DB::ICommand> makeCommandRead()
-{
-    return std::make_unique<DB::CommandRead>();
 }

programs/disks/CommandRemove.cpp

@@ -1,5 +1,6 @@
-#include "ICommand.h"
 #include <Interpreters/Context.h>
+#include "Common/Exception.h"
+#include "ICommand.h"
 
 namespace DB
 {
@@ -9,46 +10,49 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
+
 class CommandRemove final : public ICommand
 {
 public:
     CommandRemove()
     {
         command_name = "remove";
-        description = "Remove file or directory with all children. Throws exception if file doesn't exists.\nPath should be in format './' or './path' or 'path'";
-        usage = "remove [OPTION]... <PATH>";
+        description = "Remove file or directory. Throws exception if file doesn't exists";
+        options_description.add_options()("path", po::value<String>(), "path that is going to be deleted (mandatory, positional)")(
+            "recursive,r", "recursively removes the directory (required to remove a directory)");
+        positional_options_description.add("path", 1);
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration &,
-        po::variables_map &) const override
-    {}
-
-    void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) override
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
     {
-        if (command_arguments.size() != 1)
+        auto disk = client.getCurrentDiskWithPath();
+        const String & path = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path"));
+        bool recursive = options.count("recursive");
+        if (!disk.getDisk()->exists(path))
         {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} on disk {} doesn't exist", path, disk.getDisk()->getName());
+        }
+        else if (disk.getDisk()->isDirectory(path))
+        {
+            if (!recursive)
+            {
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "cannot remove '{}': Is a directory", path);
+            }
+            else
+            {
+                disk.getDisk()->removeRecursive(path);
+            }
+        }
+        else
+        {
+            disk.getDisk()->removeFileIfExists(path);
         }
-
-        String disk_name = config.getString("disk", "default");
-
-        const String & path = command_arguments[0];
-
-        DiskPtr disk = disk_selector->get(disk_name);
-
-        String relative_path = validatePathAndGetAsRelative(path);
-
-        disk->removeRecursive(relative_path);
     }
 };
+
+CommandPtr makeCommandRemove()
+{
+    return std::make_shared<DB::CommandRemove>();
 }
 
-std::unique_ptr <DB::ICommand> makeCommandRemove()
-{
-    return std::make_unique<DB::CommandRemove>();
 }

programs/disks/CommandSwitchDisk.cpp

@@ -0,0 +1,35 @@
+#include <optional>
+#include <Interpreters/Context.h>
+#include <Common/TerminalSize.h>
+#include "DisksApp.h"
+#include "ICommand.h"
+
+namespace DB
+{
+
+class CommandSwitchDisk final : public ICommand
+{
+public:
+    explicit CommandSwitchDisk() : ICommand()
+    {
+        command_name = "switch-disk";
+        description = "Switch disk (makes sense only in interactive mode)";
+        options_description.add_options()("disk", po::value<String>(), "the disk to switch to (mandatory, positional)")(
+            "path", po::value<String>(), "the path to switch on the disk");
+        positional_options_description.add("disk", 1);
+    }
+
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
+    {
+        String disk = getValueFromCommandLineOptions<String>(options, "disk");
+        std::optional<String> path = getValueFromCommandLineOptionsWithOptional<String>(options, "path");
+
+        client.switchToDisk(disk, path);
+    }
+};
+
+CommandPtr makeCommandSwitchDisk()
+{
+    return std::make_shared<DB::CommandSwitchDisk>();
+}
+}

programs/disks/CommandTouch.cpp

@@ -0,0 +1,34 @@
+#include <Interpreters/Context.h>
+#include <Common/TerminalSize.h>
+#include "DisksApp.h"
+#include "DisksClient.h"
+#include "ICommand.h"
+
+namespace DB
+{
+
+class CommandTouch final : public ICommand
+{
+public:
+    explicit CommandTouch() : ICommand()
+    {
+        command_name = "touch";
+        description = "Create a file by path";
+        options_description.add_options()("path", po::value<String>(), "the path of listing (mandatory, positional)");
+        positional_options_description.add("path", 1);
+    }
+
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
+    {
+        auto disk = client.getCurrentDiskWithPath();
+        String path = getValueFromCommandLineOptionsThrow<String>(options, "path");
+
+        disk.getDisk()->createFile(disk.getRelativeFromRoot(path));
+    }
+};
+
+CommandPtr makeCommandTouch()
+{
+    return std::make_shared<DB::CommandTouch>();
+}
+}

programs/disks/CommandWrite.cpp

@@ -1,79 +1,57 @@
-#include "ICommand.h"
 #include <Interpreters/Context.h>
+#include "ICommand.h"
 
-#include <Common/TerminalSize.h>
 #include <IO/ReadBufferFromFile.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/copyData.h>
+#include <Common/TerminalSize.h>
 
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
 class CommandWrite final : public ICommand
 {
 public:
     CommandWrite()
     {
         command_name = "write";
-        command_option_description.emplace(createOptionsDescription("Allowed options", getTerminalWidth()));
-        description = "Write a file from `FROM_PATH` to `TO_PATH`";
-        usage = "write [OPTION]... [<FROM_PATH>] <TO_PATH>";
-        command_option_description->add_options()
-            ("input", po::value<String>(), "file from which we are reading, defaults to `stdin`");
+        description = "Write a file from `path-from` to `path-to`";
+        options_description.add_options()("path-from", po::value<String>(), "file from which we are reading, defaults to `stdin` (input from `stdin` is finished by Ctrl+D)")(
+            "path-to", po::value<String>(), "file to which we are writing (mandatory, positional)");
+        positional_options_description.add("path-to", 1);
     }
 
-    void processOptions(
-        Poco::Util::LayeredConfiguration & config,
-        po::variables_map & options) const override
+
+    void executeImpl(const CommandLineOptions & options, DisksClient & client) override
     {
-        if (options.count("input"))
-            config.setString("input", options["input"].as<String>());
-    }
+        auto disk = client.getCurrentDiskWithPath();
 
-    void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) override
-    {
-        if (command_arguments.size() != 1)
+        std::optional<String> path_from = getValueFromCommandLineOptionsWithOptional<String>(options, "path-from");
+
+        String path_to = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
+
+        auto in = [&]() -> std::unique_ptr<ReadBufferFromFileBase>
         {
-            printHelpMessage();
-            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
-        }
+            if (!path_from.has_value())
+            {
+                return std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
+            }
+            else
+            {
+                String relative_path_from = disk.getRelativeFromRoot(path_from.value());
+                return disk.getDisk()->readFile(relative_path_from);
+            }
+        }();
 
-        String disk_name = config.getString("disk", "default");
-
-        const String & path = command_arguments[0];
-
-        DiskPtr disk = disk_selector->get(disk_name);
-
-        String relative_path = validatePathAndGetAsRelative(path);
-
-        String path_input = config.getString("input", "");
-        std::unique_ptr<ReadBufferFromFileBase> in;
-        if (path_input.empty())
-        {
-            in = std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
-        }
-        else
-        {
-            String relative_path_input = validatePathAndGetAsRelative(path_input);
-            in = disk->readFile(relative_path_input);
-        }
-
-        auto out = disk->writeFile(relative_path);
+        auto out = disk.getDisk()->writeFile(path_to);
         copyData(*in, *out);
         out->finalize();
     }
 };
+
+CommandPtr makeCommandWrite()
+{
+    return std::make_shared<DB::CommandWrite>();
 }
 
-std::unique_ptr <DB::ICommand> makeCommandWrite()
-{
-    return std::make_unique<DB::CommandWrite>();
 }

programs/disks/DisksApp.cpp

@@ -1,11 +1,22 @@
 #include "DisksApp.h"
+#include <Client/ClientBase.h>
+#include <Client/ReplxxLineReader.h>
+#include "Common/Exception.h"
+#include "Common/filesystemHelpers.h"
+#include <Common/Config/ConfigProcessor.h>
+#include "DisksClient.h"
 #include "ICommand.h"
+#include "ICommand_fwd.h"
+
+#include <cstring>
+#include <filesystem>
+#include <memory>
+#include <optional>
 
 #include <Disks/registerDisks.h>
 
-#include <Common/TerminalSize.h>
 #include <Formats/registerFormats.h>
-
+#include <Common/TerminalSize.h>
 
 namespace DB
 {
@@ -13,74 +24,289 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
+    extern const int LOGICAL_ERROR;
+};
+
+LineReader::Patterns DisksApp::query_extenders = {"\\"};
+LineReader::Patterns DisksApp::query_delimiters = {""};
+String DisksApp::word_break_characters = " \t\v\f\a\b\r\n";
+
+CommandPtr DisksApp::getCommandByName(const String & command) const
+{
+    try
+    {
+        if (auto it = aliases.find(command); it != aliases.end())
+            return command_descriptions.at(it->second);
+
+        return command_descriptions.at(command);
+    }
+    catch (std::out_of_range &)
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "The command `{}` is unknown", command);
+    }
 }
 
-size_t DisksApp::findCommandPos(std::vector<String> & common_arguments)
+std::vector<String> DisksApp::getEmptyCompletion(String command_name) const
 {
-    for (size_t i = 0; i < common_arguments.size(); i++)
-        if (supported_commands.contains(common_arguments[i]))
-            return i + 1;
-    return common_arguments.size();
+    auto command_ptr = command_descriptions.at(command_name);
+    std::vector<String> answer{};
+    if (multidisk_commands.contains(command_ptr->command_name))
+    {
+        answer = client->getAllFilesByPatternFromAllDisks("");
+    }
+    else
+    {
+        answer = client->getCurrentDiskWithPath().getAllFilesByPattern("");
+    }
+    for (const auto & disk_name : client->getAllDiskNames())
+    {
+        answer.push_back(disk_name);
+    }
+    for (const auto & option : command_ptr->options_description.options())
+    {
+        answer.push_back("--" + option->long_name());
+    }
+    if (command_name == "help")
+    {
+        for (const auto & [current_command_name, description] : command_descriptions)
+        {
+            answer.push_back(current_command_name);
+        }
+    }
+    std::sort(answer.begin(), answer.end());
+    return answer;
 }
 
-void DisksApp::printHelpMessage(ProgramOptionsDescription & command_option_description)
+std::vector<String> DisksApp::getCommandsToComplete(const String & command_prefix) const
 {
-    std::optional<ProgramOptionsDescription> help_description =
-        createOptionsDescription("Help Message for clickhouse-disks", getTerminalWidth());
-
-    help_description->add(command_option_description);
-
-    std::cout << "ClickHouse disk management tool\n";
-    std::cout << "Usage: ./clickhouse-disks [OPTION]\n";
-    std::cout << "clickhouse-disks\n\n";
-
-    for (const auto & current_command : supported_commands)
-        std::cout << command_descriptions[current_command]->command_name
-                  << "\t"
-                  << command_descriptions[current_command]->description
-                  << "\n\n";
-
-    std::cout << command_option_description << '\n';
+    std::vector<String> answer{};
+    for (const auto & [word, _] : command_descriptions)
+    {
+        if (word.starts_with(command_prefix))
+        {
+            answer.push_back(word);
+        }
+    }
+    if (!answer.empty())
+    {
+        std::sort(answer.begin(), answer.end());
+        return answer;
+    }
+    for (const auto & [word, _] : aliases)
+    {
+        if (word.starts_with(command_prefix))
+        {
+            answer.push_back(word);
+        }
+    }
+    if (!answer.empty())
+    {
+        std::sort(answer.begin(), answer.end());
+        return answer;
+    }
+    return {command_prefix};
 }
 
-String DisksApp::getDefaultConfigFileName()
+std::vector<String> DisksApp::getCompletions(const String & prefix) const
 {
-    return "/etc/clickhouse-server/config.xml";
+    auto arguments = po::split_unix(prefix, word_break_characters);
+    if (arguments.empty())
+    {
+        return {};
+    }
+    if (word_break_characters.contains(prefix.back()))
+    {
+        CommandPtr command;
+        try
+        {
+            command = getCommandByName(arguments[0]);
+        }
+        catch (...)
+        {
+            return {arguments.back()};
+        }
+        return getEmptyCompletion(command->command_name);
+    }
+    else if (arguments.size() == 1)
+    {
+        String command_prefix = arguments[0];
+        return getCommandsToComplete(command_prefix);
+    }
+    else
+    {
+        String last_token = arguments.back();
+        CommandPtr command;
+        try
+        {
+            command = getCommandByName(arguments[0]);
+        }
+        catch (...)
+        {
+            return {last_token};
+        }
+        std::vector<String> answer = {};
+        if (command->command_name == "help")
+        {
+            return getCommandsToComplete(last_token);
+        }
+        else
+        {
+            answer = [&]() -> std::vector<String>
+            {
+                if (multidisk_commands.contains(command->command_name))
+                {
+                    return client->getAllFilesByPatternFromAllDisks(last_token);
+                }
+                else
+                {
+                    return client->getCurrentDiskWithPath().getAllFilesByPattern(last_token);
+                }
+            }();
+
+            for (const auto & disk_name : client->getAllDiskNames())
+            {
+                if (disk_name.starts_with(last_token))
+                {
+                    answer.push_back(disk_name);
+                }
+            }
+            for (const auto & option : command->options_description.options())
+            {
+                String option_sign = "--" + option->long_name();
+                if (option_sign.starts_with(last_token))
+                {
+                    answer.push_back(option_sign);
+                }
+            }
+        }
+        if (!answer.empty())
+        {
+            std::sort(answer.begin(), answer.end());
+            return answer;
+        }
+        else
+        {
+            return {last_token};
+        }
+    }
 }
 
-void DisksApp::addOptions(
-    ProgramOptionsDescription & options_description_,
-    boost::program_options::positional_options_description & positional_options_description
-)
+bool DisksApp::processQueryText(const String & text)
 {
-    options_description_.add_options()
-        ("help,h", "Print common help message")
-        ("config-file,C", po::value<String>(), "Set config file")
-        ("disk", po::value<String>(), "Set disk name")
-        ("command_name", po::value<String>(), "Name for command to do")
-        ("save-logs", "Save logs to a file")
-        ("log-level", po::value<String>(), "Logging level")
-        ;
+    if (text.find_first_not_of(word_break_characters) == std::string::npos)
+    {
+        return true;
+    }
+    if (exit_strings.find(text) != exit_strings.end())
+        return false;
+    CommandPtr command;
+    try
+    {
+        auto arguments = po::split_unix(text, word_break_characters);
+        command = getCommandByName(arguments[0]);
+        arguments.erase(arguments.begin());
+        command->execute(arguments, *client);
+    }
+    catch (DB::Exception & err)
+    {
+        int code = getCurrentExceptionCode();
+        if (code == ErrorCodes::LOGICAL_ERROR)
+        {
+            throw std::move(err);
+        }
+        else if (code == ErrorCodes::BAD_ARGUMENTS)
+        {
+            std::cerr << err.message() << "\n"
+                      << "\n";
+            if (command.get())
+            {
+                std::cerr << "COMMAND: " << command->command_name << "\n";
+                std::cerr << command->options_description << "\n";
+            }
+            else
+            {
+                printAvailableCommandsHelpMessage();
+            }
+        }
+        else
+        {
+            std::cerr << err.message() << "\n";
+        }
+    }
+    catch (std::exception & err)
+    {
+        std::cerr << err.what() << "\n";
+    }
 
-    positional_options_description.add("command_name", 1);
+    return true;
+}
 
-    supported_commands = {"list-disks", "list", "move", "remove", "link", "copy", "write", "read", "mkdir"};
-#ifdef CLICKHOUSE_CLOUD
-    supported_commands.insert("packed-io");
-#endif
+void DisksApp::runInteractiveReplxx()
+{
+    ReplxxLineReader lr(
+        suggest,
+        history_file,
+        /* multiline= */ false,
+        query_extenders,
+        query_delimiters,
+        word_break_characters.c_str(),
+        /* highlighter_= */ {});
+    lr.enableBracketedPaste();
+
+    while (true)
+    {
+        DiskWithPath disk_with_path = client->getCurrentDiskWithPath();
+        String prompt = "\x1b[1;34m" + disk_with_path.getDisk()->getName() + "\x1b[0m:" + "\x1b[1;31m" + disk_with_path.getCurrentPath()
+            + "\x1b[0m$ ";
+
+        auto input = lr.readLine(prompt, "\x1b[1;31m:-] \x1b[0m");
+        if (input.empty())
+            break;
+
+        if (!processQueryText(input))
+            break;
+    }
+}
+
+void DisksApp::parseAndCheckOptions(
+    const std::vector<String> & arguments, const ProgramOptionsDescription & options_description, CommandLineOptions & options)
+{
+    auto parser = po::command_line_parser(arguments).options(options_description).allow_unregistered();
+    po::parsed_options parsed = parser.run();
+    po::store(parsed, options);
+}
+
+void DisksApp::addOptions()
+{
+    options_description.add_options()("help,h", "Print common help message")("config-file,C", po::value<String>(), "Set config file")(
+        "disk", po::value<String>(), "Set disk name")("save-logs", "Save logs to a file")(
+        "log-level", po::value<String>(), "Logging level")("query,q", po::value<String>(), "Query for a non-interactive mode")(
+        "test-mode", "Interactive interface in test regyme");
 
     command_descriptions.emplace("list-disks", makeCommandListDisks());
+    command_descriptions.emplace("copy", makeCommandCopy());
     command_descriptions.emplace("list", makeCommandList());
+    command_descriptions.emplace("cd", makeCommandChangeDirectory());
     command_descriptions.emplace("move", makeCommandMove());
     command_descriptions.emplace("remove", makeCommandRemove());
     command_descriptions.emplace("link", makeCommandLink());
-    command_descriptions.emplace("copy", makeCommandCopy());
     command_descriptions.emplace("write", makeCommandWrite());
     command_descriptions.emplace("read", makeCommandRead());
     command_descriptions.emplace("mkdir", makeCommandMkDir());
+    command_descriptions.emplace("switch-disk", makeCommandSwitchDisk());
+    command_descriptions.emplace("current_disk_with_path", makeCommandGetCurrentDiskAndPath());
+    command_descriptions.emplace("touch", makeCommandTouch());
+    command_descriptions.emplace("help", makeCommandHelp(*this));
 #ifdef CLICKHOUSE_CLOUD
     command_descriptions.emplace("packed-io", makeCommandPackedIO());
 #endif
+    for (const auto & [command_name, command_ptr] : command_descriptions)
+    {
+        if (command_name != command_ptr->command_name)
+        {
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Command name inside map doesn't coincide with actual command name");
+        }
+    }
 }
 
 void DisksApp::processOptions()
@@ -93,76 +319,122 @@ void DisksApp::processOptions()
         config().setBool("save-logs", true);
     if (options.count("log-level"))
         config().setString("log-level", options["log-level"].as<String>());
+    if (options.count("test-mode"))
+        config().setBool("test-mode", true);
+    if (options.count("query"))
+        query = std::optional{options["query"].as<String>()};
 }
 
-DisksApp::~DisksApp()
+
+void DisksApp::printEntryHelpMessage() const
 {
-    if (global_context)
-        global_context->shutdown();
+    std::cout << "\x1b[1;33m ClickHouse disk management tool \x1b[0m \n";
+    std::cout << options_description << '\n';
 }
 
-void DisksApp::init(std::vector<String> & common_arguments)
+
+void DisksApp::printAvailableCommandsHelpMessage() const
 {
-    stopOptionsProcessing();
+    std::cout << "\x1b[1;32mAvailable commands:\x1b[0m\n";
+    std::vector<std::pair<String, CommandPtr>> commands_with_aliases_and_descrtiptions{};
+    size_t maximal_command_length = 0;
+    for (const auto & [command_name, command_ptr] : command_descriptions)
+    {
+        std::string command_string = getCommandLineWithAliases(command_ptr);
+        maximal_command_length = std::max(maximal_command_length, command_string.size());
+        commands_with_aliases_and_descrtiptions.push_back({std::move(command_string), command_descriptions.at(command_name)});
+    }
+    for (const auto & [command_with_aliases, command_ptr] : commands_with_aliases_and_descrtiptions)
+    {
+        std::cout << "\x1b[1;33m" << command_with_aliases << "\x1b[0m" << std::string(5, ' ') << "\x1b[1;33m" << command_ptr->description
+                  << "\x1b[0m \n";
+        std::cout << command_ptr->options_description;
+        std::cout << std::endl;
+    }
+}
 
-    ProgramOptionsDescription options_description{createOptionsDescription("clickhouse-disks", getTerminalWidth())};
+void DisksApp::printCommandHelpMessage(CommandPtr command) const
+{
+    String command_name_with_aliases = getCommandLineWithAliases(command);
+    std::cout << "\x1b[1;32m" << command_name_with_aliases << "\x1b[0m" << std::string(2, ' ') << command->description << "\n";
+    std::cout << command->options_description;
+}
 
-    po::positional_options_description positional_options_description;
+void DisksApp::printCommandHelpMessage(String command_name) const
+{
+    printCommandHelpMessage(getCommandByName(command_name));
+}
 
-    addOptions(options_description, positional_options_description);
+String DisksApp::getCommandLineWithAliases(CommandPtr command) const
+{
+    String command_string = command->command_name;
+    bool need_comma = false;
+    for (const auto & [alias_name, alias_command_name] : aliases)
+    {
+        if (alias_command_name == command->command_name)
+        {
+            if (std::exchange(need_comma, true))
+                command_string += ",";
+            else
+                command_string += "(";
+            command_string += alias_name;
+        }
+    }
+    command_string += (need_comma ? ")" : "");
+    return command_string;
+}
 
-    size_t command_pos = findCommandPos(common_arguments);
-    std::vector<String> global_flags(command_pos);
-    command_arguments.resize(common_arguments.size() - command_pos);
-    copy(common_arguments.begin(), common_arguments.begin() + command_pos, global_flags.begin());
-    copy(common_arguments.begin() + command_pos, common_arguments.end(), command_arguments.begin());
+void DisksApp::initializeHistoryFile()
+{
+    String home_path;
+    const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
+    if (home_path_cstr)
+        home_path = home_path_cstr;
+    if (config().has("history-file"))
+        history_file = config().getString("history-file");
+    else
+        history_file = home_path + "/.disks-file-history";
 
-    parseAndCheckOptions(options_description, positional_options_description, global_flags);
+    if (!history_file.empty() && !fs::exists(history_file))
+    {
+        try
+        {
+            FS::createFile(history_file);
+        }
+        catch (const ErrnoException & e)
+        {
+            if (e.getErrno() != EEXIST)
+                throw;
+        }
+    }
+}
+
+void DisksApp::init(const std::vector<String> & common_arguments)
+{
+    addOptions();
+    parseAndCheckOptions(common_arguments, options_description, options);
 
     po::notify(options);
 
     if (options.count("help"))
     {
-        printHelpMessage(options_description);
+        printEntryHelpMessage();
+        printAvailableCommandsHelpMessage();
         exit(0); // NOLINT(concurrency-mt-unsafe)
     }
 
-    if (!supported_commands.contains(command_name))
-    {
-        std::cerr << "Unknown command name:  " << command_name << "\n";
-        printHelpMessage(options_description);
-        throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
-    }
-
     processOptions();
 }
 
-void DisksApp::parseAndCheckOptions(
-    ProgramOptionsDescription & options_description_,
-    boost::program_options::positional_options_description & positional_options_description,
-    std::vector<String> & arguments)
+String DisksApp::getDefaultConfigFileName()
 {
-    auto parser = po::command_line_parser(arguments)
-        .options(options_description_)
-        .positional(positional_options_description)
-        .allow_unregistered();
-
-    po::parsed_options parsed = parser.run();
-    po::store(parsed, options);
-
-    auto positional_arguments = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::include_positional);
-    for (const auto & arg : positional_arguments)
-    {
-        if (command_descriptions.contains(arg))
-        {
-            command_name = arg;
-            break;
-        }
-    }
+    return "/etc/clickhouse-server/config.xml";
 }
 
 int DisksApp::main(const std::vector<String> & /*args*/)
 {
+    std::vector<std::string> keys;
+    config().keys(keys);
     if (config().has("config-file") || fs::exists(getDefaultConfigFileName()))
     {
         String config_path = config().getString("config-file", getDefaultConfigFileName());
@@ -173,9 +445,13 @@ int DisksApp::main(const std::vector<String> & /*args*/)
     }
     else
     {
+        printEntryHelpMessage();
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "No config-file specified");
     }
 
+    config().keys(keys);
+    initializeHistoryFile();
+
     if (config().has("save-logs"))
     {
         auto log_level = config().getString("log-level", "trace");
@@ -200,61 +476,68 @@ int DisksApp::main(const std::vector<String> & /*args*/)
     global_context->setApplicationType(Context::ApplicationType::DISKS);
 
     String path = config().getString("path", DBMS_DEFAULT_PATH);
+
     global_context->setPath(path);
 
-    auto & command = command_descriptions[command_name];
+    String main_disk = config().getString("disk", "default");
 
-    auto command_options = command->getCommandOptions();
-    std::vector<String> args;
-    if (command_options)
+    auto validator = [](const Poco::Util::AbstractConfiguration &, const std::string &, const std::string &) { return true; };
+
+    constexpr auto config_prefix = "storage_configuration.disks";
+    auto disk_selector = std::make_shared<DiskSelector>(std::unordered_set<String>{"cache", "encrypted"});
+    disk_selector->initialize(config(), config_prefix, global_context, validator);
+
+    std::vector<std::pair<DiskPtr, std::optional<String>>> disks_with_path;
+
+    for (const auto & [_, disk_ptr] : disk_selector->getDisksMap())
     {
-        auto parser = po::command_line_parser(command_arguments).options(*command_options).allow_unregistered();
-        po::parsed_options parsed = parser.run();
-        po::store(parsed, options);
-        po::notify(options);
+        disks_with_path.emplace_back(
+            disk_ptr, (disk_ptr->getName() == "local") ? std::optional{fs::current_path().string()} : std::nullopt);
+    }
 
-        args = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::include_positional);
-        command->processOptions(config(), options);
+
+    client = std::make_unique<DisksClient>(std::move(disks_with_path), main_disk);
+
+    suggest.setCompletionsCallback([&](const String & prefix, size_t /* prefix_length */) { return getCompletions(prefix); });
+
+    if (!query.has_value())
+    {
+        runInteractive();
     }
     else
     {
-        auto parser = po::command_line_parser(command_arguments).options({}).allow_unregistered();
-        po::parsed_options parsed = parser.run();
-        args = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::include_positional);
+        processQueryText(query.value());
     }
 
-    std::unordered_set<std::string> disks
-    {
-        config().getString("disk", "default"),
-        config().getString("disk-from", config().getString("disk", "default")),
-        config().getString("disk-to", config().getString("disk", "default")),
-    };
-
-    auto validator = [&disks](
-        const Poco::Util::AbstractConfiguration & config,
-        const std::string & disk_config_prefix,
-        const std::string & disk_name)
-    {
-        if (!disks.contains(disk_name))
-            return false;
-
-        const auto disk_type = config.getString(disk_config_prefix + ".type", "local");
-
-        if (disk_type == "cache")
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk type 'cache' of disk {} is not supported by clickhouse-disks", disk_name);
-
-        return true;
-    };
-
-    constexpr auto config_prefix = "storage_configuration.disks";
-    auto disk_selector = std::make_shared<DiskSelector>();
-    disk_selector->initialize(config(), config_prefix, global_context, validator);
-
-    command->execute(args, disk_selector, config());
-
     return Application::EXIT_OK;
 }
 
+DisksApp::~DisksApp()
+{
+    client.reset(nullptr);
+    if (global_context)
+        global_context->shutdown();
+}
+
+void DisksApp::runInteractiveTestMode()
+{
+    for (String input; std::getline(std::cin, input);)
+    {
+        if (!processQueryText(input))
+            break;
+
+        std::cout << "\a\a\a\a" << std::endl;
+        std::cerr << std::flush;
+    }
+}
+
+void DisksApp::runInteractive()
+{
+    if (config().hasOption("test-mode"))
+        runInteractiveTestMode();
+    else
+        runInteractiveReplxx();
+}
 }
 
 int mainEntryClickHouseDisks(int argc, char ** argv)
@@ -269,16 +552,16 @@ int mainEntryClickHouseDisks(int argc, char ** argv)
     catch (const DB::Exception & e)
     {
         std::cerr << DB::getExceptionMessage(e, false) << std::endl;
-        return 1;
+        return 0;
     }
     catch (const boost::program_options::error & e)
     {
         std::cerr << "Bad arguments: " << e.what() << std::endl;
-        return DB::ErrorCodes::BAD_ARGUMENTS;
+        return 0;
     }
     catch (...)
     {
         std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
-        return 1;
+        return 0;
     }
 }

programs/disks/DisksApp.h

@@ -1,61 +1,107 @@
 #pragma once
 
+#include <unordered_map>
+#include <vector>
+#include <Client/ReplxxLineReader.h>
 #include <Loggers/Loggers.h>
+#include "DisksClient.h"
+#include "ICommand_fwd.h"
 
 #include <Interpreters/Context.h>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
 #include <Poco/Util/Application.h>
 
-#include <boost/program_options.hpp>
-
 namespace DB
 {
 
-class ICommand;
-using CommandPtr = std::unique_ptr<ICommand>;
-
-namespace po = boost::program_options;
 using ProgramOptionsDescription = boost::program_options::options_description;
 using CommandLineOptions = boost::program_options::variables_map;
 
-class DisksApp : public Poco::Util::Application, public Loggers
+class DisksApp : public Poco::Util::Application
 {
 public:
-    DisksApp() = default;
-    ~DisksApp() override;
+    void addOptions();
 
-    void init(std::vector<String> & common_arguments);
-
-    int main(const std::vector<String> & args) override;
-
-protected:
-    static String getDefaultConfigFileName();
-
-    void addOptions(
-        ProgramOptionsDescription & options_description,
-        boost::program_options::positional_options_description & positional_options_description);
     void processOptions();
 
-    void printHelpMessage(ProgramOptionsDescription & command_option_description);
+    bool processQueryText(const String & text);
 
-    size_t findCommandPos(std::vector<String> & common_arguments);
+    void init(const std::vector<String> & common_arguments);
+
+    int main(const std::vector<String> & /*args*/) override;
+
+    CommandPtr getCommandByName(const String & command) const;
+
+    void initializeHistoryFile();
+
+    static void parseAndCheckOptions(
+        const std::vector<String> & arguments, const ProgramOptionsDescription & options_description, CommandLineOptions & options);
+
+    void printEntryHelpMessage() const;
+    void printAvailableCommandsHelpMessage() const;
+    void printCommandHelpMessage(String command_name) const;
+    void printCommandHelpMessage(CommandPtr command) const;
+    String getCommandLineWithAliases(CommandPtr command) const;
+
+
+    std::vector<String> getCompletions(const String & prefix) const;
+
+    std::vector<String> getEmptyCompletion(String command_name) const;
+
+    ~DisksApp() override;
 
 private:
-    void parseAndCheckOptions(
-        ProgramOptionsDescription & options_description,
-        boost::program_options::positional_options_description & positional_options_description,
-        std::vector<String> & arguments);
+    void runInteractive();
+    void runInteractiveReplxx();
+    void runInteractiveTestMode();
+
+    String getDefaultConfigFileName();
+
+    std::vector<String> getCommandsToComplete(const String & command_prefix) const;
+
+    // Fields responsible for the REPL work
+    String history_file;
+    LineReader::Suggest suggest;
+    static LineReader::Patterns query_extenders;
+    static LineReader::Patterns query_delimiters;
+    static String word_break_characters;
+
+    // General command line arguments parsing fields
 
-protected:
-    ContextMutablePtr global_context;
     SharedContextHolder shared_context;
-
-    String command_name;
-    std::vector<String> command_arguments;
-
-    std::unordered_set<String> supported_commands;
+    ContextMutablePtr global_context;
+    ProgramOptionsDescription options_description;
+    CommandLineOptions options;
     std::unordered_map<String, CommandPtr> command_descriptions;
 
-    po::variables_map options;
-};
+    std::optional<String> query;
 
+    const std::unordered_map<String, String> aliases
+        = {{"cp", "copy"},
+           {"mv", "move"},
+           {"ls", "list"},
+           {"list_disks", "list-disks"},
+           {"ln", "link"},
+           {"rm", "remove"},
+           {"cat", "read"},
+           {"r", "read"},
+           {"w", "write"},
+           {"create", "touch"},
+           {"delete", "remove"},
+           {"ls-disks", "list-disks"},
+           {"ls_disks", "list-disks"},
+           {"packed_io", "packed-io"},
+           {"change-dir", "cd"},
+           {"change_dir", "cd"},
+           {"switch_disk", "switch-disk"},
+           {"current", "current_disk_with_path"},
+           {"current_disk", "current_disk_with_path"},
+           {"current_path", "current_disk_with_path"},
+           {"cur", "current_disk_with_path"}};
+
+    std::set<String> multidisk_commands = {"copy", "packed-io", "switch-disk", "cd"};
+
+    std::unique_ptr<DisksClient> client{};
+};
 }

programs/disks/DisksClient.cpp

@@ -0,0 +1,263 @@
+#include "DisksClient.h"
+#include <Client/ClientBase.h>
+#include <Client/ReplxxLineReader.h>
+#include <Disks/registerDisks.h>
+#include <Common/Config/ConfigProcessor.h>
+
+#include <Formats/registerFormats.h>
+
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+    extern const int LOGICAL_ERROR;
+};
+
+namespace DB
+{
+DiskWithPath::DiskWithPath(DiskPtr disk_, std::optional<String> path_) : disk(disk_)
+{
+    if (path_.has_value())
+    {
+        if (!fs::path{path_.value()}.is_absolute())
+        {
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Initializing path {} is not absolute", path_.value());
+        }
+        path = path_.value();
+    }
+    else
+    {
+        path = String{"/"};
+    }
+
+    String relative_path = normalizePathAndGetAsRelative(path);
+    if (disk->isDirectory(relative_path) || (relative_path.empty() && (disk->isDirectory("/"))))
+    {
+        return;
+    }
+    throw Exception(
+        ErrorCodes::BAD_ARGUMENTS,
+        "Initializing path {} (normalized path: {}) at disk {} is not a directory",
+        path,
+        relative_path,
+        disk->getName());
+}
+
+std::vector<String> DiskWithPath::listAllFilesByPath(const String & any_path) const
+{
+    if (isDirectory(any_path))
+    {
+        std::vector<String> file_names;
+        disk->listFiles(getRelativeFromRoot(any_path), file_names);
+        return file_names;
+    }
+    else
+    {
+        return {};
+    }
+}
+
+std::vector<String> DiskWithPath::getAllFilesByPattern(const String & pattern) const
+{
+    auto [path_before, path_after] = [&]() -> std::pair<String, String>
+    {
+        auto slash_pos = pattern.find_last_of('/');
+        if (slash_pos >= pattern.size())
+        {
+            return {"", pattern};
+        }
+        else
+        {
+            return {pattern.substr(0, slash_pos + 1), pattern.substr(slash_pos + 1, pattern.size() - slash_pos - 1)};
+        }
+    }();
+
+    if (!isDirectory(path_before))
+    {
+        return {};
+    }
+    else
+    {
+        std::vector<String> file_names = listAllFilesByPath(path_before);
+
+        std::vector<String> answer;
+
+        for (const auto & file_name : file_names)
+        {
+            if (file_name.starts_with(path_after))
+            {
+                String file_pattern = path_before + file_name;
+                if (isDirectory(file_pattern))
+                {
+                    file_pattern = file_pattern + "/";
+                }
+                answer.push_back(file_pattern);
+            }
+        }
+        return answer;
+    }
+};
+
+void DiskWithPath::setPath(const String & any_path)
+{
+    if (isDirectory(any_path))
+    {
+        path = getAbsolutePath(any_path);
+    }
+    else
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} at disk {} is not a directory", any_path, disk->getName());
+    }
+}
+
+String DiskWithPath::validatePathAndGetAsRelative(const String & path)
+{
+    String lexically_normal_path = fs::path(path).lexically_normal();
+    if (lexically_normal_path.find("..") != std::string::npos)
+        throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Path {} is not normalized", path);
+
+    /// If path is absolute we should keep it as relative inside disk, so disk will look like
+    /// an ordinary filesystem with root.
+    if (fs::path(lexically_normal_path).is_absolute())
+        return lexically_normal_path.substr(1);
+
+    return lexically_normal_path;
+}
+
+String DiskWithPath::normalizePathAndGetAsRelative(const String & messyPath)
+{
+    std::filesystem::path path(messyPath);
+    std::filesystem::path canonical_path = std::filesystem::weakly_canonical(path);
+    String npath = canonical_path.make_preferred().string();
+    return validatePathAndGetAsRelative(npath);
+}
+
+String DiskWithPath::normalizePath(const String & path)
+{
+    std::filesystem::path canonical_path = std::filesystem::weakly_canonical(path);
+    return canonical_path.make_preferred().string();
+}
+
+DisksClient::DisksClient(std::vector<std::pair<DiskPtr, std::optional<String>>> && disks_with_paths, std::optional<String> begin_disk)
+{
+    if (disks_with_paths.empty())
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Initializing array of disks is empty");
+    }
+    if (!begin_disk.has_value())
+    {
+        begin_disk = disks_with_paths[0].first->getName();
+    }
+    bool has_begin_disk = false;
+    for (auto & [disk, path] : disks_with_paths)
+    {
+        addDisk(disk, path);
+        if (disk->getName() == begin_disk.value())
+        {
+            has_begin_disk = true;
+        }
+    }
+    if (!has_begin_disk)
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no begin_disk '{}' in initializing array", begin_disk.value());
+    }
+    current_disk = std::move(begin_disk.value());
+}
+
+const DiskWithPath & DisksClient::getDiskWithPath(const String & disk) const
+{
+    try
+    {
+        return disks.at(disk);
+    }
+    catch (...)
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "The disk '{}' is unknown", disk);
+    }
+}
+
+DiskWithPath & DisksClient::getDiskWithPath(const String & disk)
+{
+    try
+    {
+        return disks.at(disk);
+    }
+    catch (...)
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "The disk '{}' is unknown", disk);
+    }
+}
+
+const DiskWithPath & DisksClient::getCurrentDiskWithPath() const
+{
+    try
+    {
+        return disks.at(current_disk);
+    }
+    catch (...)
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no current disk in client");
+    }
+}
+
+DiskWithPath & DisksClient::getCurrentDiskWithPath()
+{
+    try
+    {
+        return disks.at(current_disk);
+    }
+    catch (...)
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no current disk in client");
+    }
+}
+
+void DisksClient::switchToDisk(const String & disk_, const std::optional<String> & path_)
+{
+    if (disks.contains(disk_))
+    {
+        if (path_.has_value())
+        {
+            disks.at(disk_).setPath(path_.value());
+        }
+        current_disk = disk_;
+    }
+    else
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "The disk '{}' is unknown", disk_);
+    }
+}
+
+std::vector<String> DisksClient::getAllDiskNames() const
+{
+    std::vector<String> answer{};
+    answer.reserve(disks.size());
+    for (const auto & [disk_name, _] : disks)
+    {
+        answer.push_back(disk_name);
+    }
+    return answer;
+}
+
+std::vector<String> DisksClient::getAllFilesByPatternFromAllDisks(const String & pattern) const
+{
+    std::vector<String> answer{};
+    for (const auto & [_, disk] : disks)
+    {
+        for (auto & word : disk.getAllFilesByPattern(pattern))
+        {
+            answer.push_back(word);
+        }
+    }
+    return answer;
+}
+
+void DisksClient::addDisk(DiskPtr disk_, const std::optional<String> & path_)
+{
+    String disk_name = disk_->getName();
+    if (disks.contains(disk_->getName()))
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "The disk '{}' already exists", disk_name);
+    }
+    disks.emplace(disk_name, DiskWithPath{disk_, path_});
+}
+}

programs/disks/DisksClient.h

@@ -0,0 +1,89 @@
+#pragma once
+
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <Client/ReplxxLineReader.h>
+#include <Loggers/Loggers.h>
+#include "Disks/IDisk.h"
+
+#include <Interpreters/Context.h>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+
+namespace fs = std::filesystem;
+
+namespace DB
+{
+
+std::vector<String> split(const String & text, const String & delimiters);
+
+using ProgramOptionsDescription = boost::program_options::options_description;
+using CommandLineOptions = boost::program_options::variables_map;
+
+class DiskWithPath
+{
+public:
+    explicit DiskWithPath(DiskPtr disk_, std::optional<String> path_ = std::nullopt);
+
+    String getAbsolutePath(const String & any_path) const { return normalizePath(fs::path(path) / any_path); }
+
+    String getCurrentPath() const { return path; }
+
+    bool isDirectory(const String & any_path) const
+    {
+        return disk->isDirectory(getRelativeFromRoot(any_path)) || (getRelativeFromRoot(any_path).empty() && (disk->isDirectory("/")));
+    }
+
+    std::vector<String> listAllFilesByPath(const String & any_path) const;
+
+    std::vector<String> getAllFilesByPattern(const String & pattern) const;
+
+    DiskPtr getDisk() const { return disk; }
+
+    void setPath(const String & any_path);
+
+    String getRelativeFromRoot(const String & any_path) const { return normalizePathAndGetAsRelative(getAbsolutePath(any_path)); }
+
+private:
+    static String validatePathAndGetAsRelative(const String & path);
+    static std::string normalizePathAndGetAsRelative(const std::string & messyPath);
+    static std::string normalizePath(const std::string & messyPath);
+
+    const DiskPtr disk;
+    String path;
+};
+
+class DisksClient
+{
+public:
+    explicit DisksClient(std::vector<std::pair<DiskPtr, std::optional<String>>> && disks_with_paths, std::optional<String> begin_disk);
+
+    const DiskWithPath & getDiskWithPath(const String & disk) const;
+
+    DiskWithPath & getDiskWithPath(const String & disk);
+
+    const DiskWithPath & getCurrentDiskWithPath() const;
+
+    DiskWithPath & getCurrentDiskWithPath();
+
+    DiskPtr getCurrentDisk() const { return getCurrentDiskWithPath().getDisk(); }
+
+    DiskPtr getDisk(const String & disk) const { return getDiskWithPath(disk).getDisk(); }
+
+    void switchToDisk(const String & disk_, const std::optional<String> & path_);
+
+    std::vector<String> getAllDiskNames() const;
+
+    std::vector<String> getAllFilesByPatternFromAllDisks(const String & pattern) const;
+
+
+private:
+    void addDisk(DiskPtr disk_, const std::optional<String> & path_);
+
+    String current_disk;
+    std::unordered_map<String, DiskWithPath> disks;
+};
+}

programs/disks/ICommand.cpp

@@ -1,5 +1,5 @@
 #include "ICommand.h"
-#include <iostream>
+#include "DisksClient.h"
 
 
 namespace DB
@@ -10,43 +10,42 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
-void ICommand::printHelpMessage() const
+CommandLineOptions ICommand::processCommandLineArguments(const Strings & commands)
 {
-    std::cout << "Command: " << command_name << '\n';
-    std::cout << "Description: " << description << '\n';
-    std::cout << "Usage: " << usage << '\n';
+    CommandLineOptions options;
+    auto parser = po::command_line_parser(commands);
+    parser.options(options_description).positional(positional_options_description);
 
-    if (command_option_description)
+    po::parsed_options parsed = parser.run();
+    po::store(parsed, options);
+
+    return options;
+}
+
+void ICommand::execute(const Strings & commands, DisksClient & client)
+{
+    try
     {
-        auto options = *command_option_description;
-        if (!options.options().empty())
-            std::cout << options << '\n';
+        processCommandLineArguments(commands);
+    }
+    catch (std::exception & exc)
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}", exc.what());
+    }
+    executeImpl(processCommandLineArguments(commands), client);
+}
+
+DiskWithPath & ICommand::getDiskWithPath(DisksClient & client, const CommandLineOptions & options, const String & name)
+{
+    auto disk_name = getValueFromCommandLineOptionsWithOptional<String>(options, name);
+    if (disk_name.has_value())
+    {
+        return client.getDiskWithPath(disk_name.value());
+    }
+    else
+    {
+        return client.getCurrentDiskWithPath();
     }
 }
 
-void ICommand::addOptions(ProgramOptionsDescription & options_description)
-{
-    if (!command_option_description || command_option_description->options().empty())
-        return;
-
-    options_description.add(*command_option_description);
-}
-
-String ICommand::validatePathAndGetAsRelative(const String & path)
-{
-    /// If path contain non-normalized symbols like . we will normalized them. If the resulting normalized path
-    /// still contain '..' it can be dangerous, disallow such paths. Also since clickhouse-disks
-    /// is not an interactive program (don't track you current path) it's OK to disallow .. paths.
-    String lexically_normal_path = fs::path(path).lexically_normal();
-    if (lexically_normal_path.find("..") != std::string::npos)
-        throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Path {} is not normalized", path);
-
-    /// If path is absolute we should keep it as relative inside disk, so disk will look like
-    /// an ordinary filesystem with root.
-    if (fs::path(lexically_normal_path).is_absolute())
-        return lexically_normal_path.substr(1);
-
-    return lexically_normal_path;
-}
-
 }

programs/disks/ICommand.h

@@ -1,66 +1,146 @@
 #pragma once
 
-#include <Disks/IDisk.h>
+#include <optional>
 #include <Disks/DiskSelector.h>
+#include <Disks/IDisk.h>
 
+#include <boost/any/bad_any_cast.hpp>
 #include <boost/program_options.hpp>
 
-#include <Common/Config/ConfigProcessor.h>
 #include <Poco/Util/Application.h>
+#include "Common/Exception.h"
+#include <Common/Config/ConfigProcessor.h>
 
-#include <memory>
+#include <boost/program_options/positional_options.hpp>
+
+#include "DisksApp.h"
+
+#include "DisksClient.h"
+
+#include "ICommand_fwd.h"
 
 namespace DB
 {
 
 namespace po = boost::program_options;
-using ProgramOptionsDescription = boost::program_options::options_description;
-using CommandLineOptions = boost::program_options::variables_map;
+using ProgramOptionsDescription = po::options_description;
+using PositionalProgramOptionsDescription = po::positional_options_description;
+using CommandLineOptions = po::variables_map;
+
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
 
 class ICommand
 {
 public:
-    ICommand() = default;
+    explicit ICommand() = default;
 
     virtual ~ICommand() = default;
 
-    virtual void execute(
-        const std::vector<String> & command_arguments,
-        std::shared_ptr<DiskSelector> & disk_selector,
-        Poco::Util::LayeredConfiguration & config) = 0;
+    void execute(const Strings & commands, DisksClient & client);
 
-    const std::optional<ProgramOptionsDescription> & getCommandOptions() const { return command_option_description; }
+    virtual void executeImpl(const CommandLineOptions & options, DisksClient & client) = 0;
 
-    void addOptions(ProgramOptionsDescription & options_description);
-
-    virtual void processOptions(Poco::Util::LayeredConfiguration & config, po::variables_map & options) const = 0;
+    CommandLineOptions processCommandLineArguments(const Strings & commands);
 
 protected:
-    void printHelpMessage() const;
+    template <typename T>
+    static T getValueFromCommandLineOptions(const CommandLineOptions & options, const String & name)
+    {
+        try
+        {
+            return options[name].as<T>();
+        }
+        catch (boost::bad_any_cast &)
+        {
+            throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Argument '{}' has wrong type and can't be parsed", name);
+        }
+    }
+
+    template <typename T>
+    static T getValueFromCommandLineOptionsThrow(const CommandLineOptions & options, const String & name)
+    {
+        if (options.count(name))
+        {
+            return getValueFromCommandLineOptions<T>(options, name);
+        }
+        else
+        {
+            throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Mandatory argument '{}' is missing", name);
+        }
+    }
+
+    template <typename T>
+    static T getValueFromCommandLineOptionsWithDefault(const CommandLineOptions & options, const String & name, const T & default_value)
+    {
+        if (options.count(name))
+        {
+            return getValueFromCommandLineOptions<T>(options, name);
+        }
+        else
+        {
+            return default_value;
+        }
+    }
+
+    template <typename T>
+    static std::optional<T> getValueFromCommandLineOptionsWithOptional(const CommandLineOptions & options, const String & name)
+    {
+        if (options.count(name))
+        {
+            return std::optional{getValueFromCommandLineOptions<T>(options, name)};
+        }
+        else
+        {
+            return std::nullopt;
+        }
+    }
+
+    DiskWithPath & getDiskWithPath(DisksClient & client, const CommandLineOptions & options, const String & name);
+
+    String getTargetLocation(const String & path_from, DiskWithPath & disk_to, const String & path_to)
+    {
+        if (!disk_to.getDisk()->isDirectory(path_to))
+        {
+            return path_to;
+        }
+        String copied_path_from = path_from;
+        if (copied_path_from.ends_with('/'))
+        {
+            copied_path_from.pop_back();
+        }
+        String plain_filename = fs::path(copied_path_from).filename();
+
+        return fs::path{path_to} / plain_filename;
+    }
 
-    static String validatePathAndGetAsRelative(const String & path);
 
 public:
     String command_name;
     String description;
+    ProgramOptionsDescription options_description;
 
 protected:
-    std::optional<ProgramOptionsDescription> command_option_description;
-    String usage;
-    po::positional_options_description positional_options_description;
+    PositionalProgramOptionsDescription positional_options_description;
 };
 
-using CommandPtr = std::unique_ptr<ICommand>;
-
-}
-
 DB::CommandPtr makeCommandCopy();
-DB::CommandPtr makeCommandLink();
-DB::CommandPtr makeCommandList();
 DB::CommandPtr makeCommandListDisks();
+DB::CommandPtr makeCommandList();
+DB::CommandPtr makeCommandChangeDirectory();
+DB::CommandPtr makeCommandLink();
 DB::CommandPtr makeCommandMove();
 DB::CommandPtr makeCommandRead();
 DB::CommandPtr makeCommandRemove();
 DB::CommandPtr makeCommandWrite();
 DB::CommandPtr makeCommandMkDir();
+DB::CommandPtr makeCommandSwitchDisk();
+DB::CommandPtr makeCommandGetCurrentDiskAndPath();
+DB::CommandPtr makeCommandHelp(const DisksApp & disks_app);
+DB::CommandPtr makeCommandTouch();
+#ifdef CLICKHOUSE_CLOUD
 DB::CommandPtr makeCommandPackedIO();
+#endif
+}

programs/disks/ICommand_fwd.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <memory>
+
+namespace DB
+{
+class ICommand;
+
+using CommandPtr = std::shared_ptr<ICommand>;
+}

programs/keeper/Keeper.cpp

@@ -264,6 +264,35 @@ HTTPContextPtr httpContext()
     return std::make_shared<KeeperHTTPContext>(Context::getGlobalContextInstance());
 }
 
+String getKeeperPath(Poco::Util::LayeredConfiguration & config)
+{
+    String path;
+    if (config.has("keeper_server.storage_path"))
+    {
+        path = config.getString("keeper_server.storage_path");
+    }
+    else if (config.has("keeper_server.log_storage_path"))
+    {
+        path = std::filesystem::path(config.getString("keeper_server.log_storage_path")).parent_path();
+    }
+    else if (config.has("keeper_server.snapshot_storage_path"))
+    {
+        path = std::filesystem::path(config.getString("keeper_server.snapshot_storage_path")).parent_path();
+    }
+    else if (std::filesystem::is_directory(std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination"))
+    {
+        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG,
+                        "By default 'keeper_server.storage_path' could be assigned to {}, but the directory {} already exists. Please specify 'keeper_server.storage_path' in the keeper configuration explicitly",
+                        KEEPER_DEFAULT_PATH, String{std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination"});
+    }
+    else
+    {
+        path = KEEPER_DEFAULT_PATH;
+    }
+    return path;
+}
+
+
 }
 
 int Keeper::main(const std::vector<std::string> & /*args*/)
@@ -316,31 +345,7 @@ try
 
     updateMemorySoftLimitInConfig(config());
 
-    std::string path;
-
-    if (config().has("keeper_server.storage_path"))
-    {
-        path = config().getString("keeper_server.storage_path");
-    }
-    else if (config().has("keeper_server.log_storage_path"))
-    {
-        path = std::filesystem::path(config().getString("keeper_server.log_storage_path")).parent_path();
-    }
-    else if (config().has("keeper_server.snapshot_storage_path"))
-    {
-        path = std::filesystem::path(config().getString("keeper_server.snapshot_storage_path")).parent_path();
-    }
-    else if (std::filesystem::is_directory(std::filesystem::path{config().getString("path", DBMS_DEFAULT_PATH)} / "coordination"))
-    {
-        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG,
-                        "By default 'keeper_server.storage_path' could be assigned to {}, but the directory {} already exists. Please specify 'keeper_server.storage_path' in the keeper configuration explicitly",
-                        KEEPER_DEFAULT_PATH, String{std::filesystem::path{config().getString("path", DBMS_DEFAULT_PATH)} / "coordination"});
-    }
-    else
-    {
-        path = KEEPER_DEFAULT_PATH;
-    }
-
+    std::string path = getKeeperPath(config());
     std::filesystem::create_directories(path);
 
     /// Check that the process user id matches the owner of the data.
@@ -554,7 +559,7 @@ try
     auto main_config_reloader = std::make_unique<ConfigReloader>(
         config_path,
         extra_paths,
-        config().getString("path", KEEPER_DEFAULT_PATH),
+        getKeeperPath(config()),
         std::move(unused_cache),
         unused_event,
         [&](ConfigurationPtr config, bool /* initial_loading */)

programs/keeper/conf.d/local.yaml

@@ -0,0 +1,9 @@
+logger:
+    log:
+        "@remove": remove
+    errorlog:
+        "@remove": remove
+    console: 1
+keeper_server:
+    log_storage_path: ./logs
+    snapshot_storage_path: ./snapshots

programs/server/Server.cpp

@@ -587,6 +587,54 @@ static void sanityChecks(Server & server)
     }
 }
 
+void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, ContextMutablePtr context, Poco::Logger * log)
+{
+    try
+    {
+        Poco::Util::AbstractConfiguration::Keys keys;
+        config.keys("startup_scripts", keys);
+
+        SetResultDetailsFunc callback;
+        for (const auto & key : keys)
+        {
+            std::string full_prefix = "startup_scripts." + key;
+
+            if (config.has(full_prefix + ".condition"))
+            {
+                auto condition = config.getString(full_prefix + ".condition");
+                auto condition_read_buffer = ReadBufferFromString(condition);
+                auto condition_write_buffer = WriteBufferFromOwnString();
+
+                LOG_DEBUG(log, "Checking startup query condition `{}`", condition);
+                executeQuery(condition_read_buffer, condition_write_buffer, true, context, callback, QueryFlags{ .internal = true }, std::nullopt, {});
+
+                auto result = condition_write_buffer.str();
+
+                if (result != "1\n" && result != "true\n")
+                {
+                    if (result != "0\n" && result != "false\n")
+                        context->addWarningMessage(fmt::format("The condition query returned `{}`, which can't be interpreted as a boolean (`0`, `false`, `1`, `true`). Will skip this query.", result));
+
+                    continue;
+                }
+
+                LOG_DEBUG(log, "Condition is true, will execute the query next");
+            }
+
+            auto query = config.getString(full_prefix + ".query");
+            auto read_buffer = ReadBufferFromString(query);
+            auto write_buffer = WriteBufferFromOwnString();
+
+            LOG_DEBUG(log, "Executing query `{}`", query);
+            executeQuery(read_buffer, write_buffer, true, context, callback, QueryFlags{ .internal = true }, std::nullopt, {});
+        }
+    }
+    catch (...)
+    {
+        tryLogCurrentException(log, "Failed to parse startup scripts file");
+    }
+}
+
 static void initializeAzureSDKLogger(
     [[ maybe_unused ]] const ServerSettings & server_settings,
     [[ maybe_unused ]] int server_logs_level)
@@ -1990,6 +2038,11 @@ try
         /// otherwise there is a race condition between the system database initialization
         /// and creation of new tables in the database.
         waitLoad(TablesLoaderForegroundPoolId, system_startup_tasks);
+
+        /// Startup scripts can depend on the system log tables.
+        if (config().has("startup_scripts") && !server_settings.prepare_system_log_tables_on_startup.changed)
+            global_context->setServerSetting("prepare_system_log_tables_on_startup", true);
+
         /// After attaching system databases we can initialize system log.
         global_context->initializeSystemLogs();
         global_context->setSystemZooKeeperLogAfterInitializationIfNeeded();
@@ -2138,6 +2191,9 @@ try
         load_metadata_tasks.clear();
         load_metadata_tasks.shrink_to_fit();
 
+        if (config().has("startup_scripts"))
+            loadStartupScripts(config(), global_context, log);
+
         {
             std::lock_guard lock(servers_lock);
             for (auto & server : servers)

src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp

@@ -10,6 +10,7 @@
 #include <Analyzer/Utils.h>
 
 #include <DataTypes/DataTypeLowCardinality.h>
+#include <DataTypes/DataTypesNumber.h>
 
 namespace DB
 {
@@ -26,12 +27,103 @@ static constexpr std::array boolean_functions{
     "like"sv,     "notLike"sv,     "ilike"sv,  "notILike"sv,        "empty"sv,        "notEmpty"sv,        "not"sv,    "and"sv,
     "or"sv};
 
-static bool isBooleanFunction(const String & func_name)
+
+bool isBooleanFunction(const String & func_name)
 {
     return std::any_of(
         boolean_functions.begin(), boolean_functions.end(), [&](const auto boolean_func) { return func_name == boolean_func; });
 }
 
+bool isNodeFunction(const QueryTreeNodePtr & node, const String & func_name)
+{
+    if (const auto * function_node = node->as<FunctionNode>())
+        return function_node->getFunctionName() == func_name;
+    return false;
+}
+
+QueryTreeNodePtr getFunctionArgument(const QueryTreeNodePtr & node, size_t idx)
+{
+    if (const auto * function_node = node->as<FunctionNode>())
+    {
+        const auto & args = function_node->getArguments().getNodes();
+        if (idx < args.size())
+            return args[idx];
+    }
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected '{}' to be a function with at least {} arguments", node->formatASTForErrorMessage(), idx + 1);
+}
+
+QueryTreeNodePtr findEqualsFunction(const QueryTreeNodes & nodes)
+{
+    for (const auto & node : nodes)
+    {
+        const auto * function_node = node->as<FunctionNode>();
+        if (function_node && function_node->getFunctionName() == "equals" &&
+            function_node->getArguments().getNodes().size() == 2)
+        {
+            return node;
+        }
+    }
+    return nullptr;
+}
+
+bool isBooleanConstant(const QueryTreeNodePtr & node, bool expected_value)
+{
+    const auto * constant_node = node->as<ConstantNode>();
+    if (!constant_node || !constant_node->getResultType()->equals(DataTypeUInt8()))
+        return false;
+
+    UInt64 constant_value;
+    return (constant_node->getValue().tryGet<UInt64>(constant_value) && constant_value == expected_value);
+}
+
+/// Returns true if expression consists of only conjunctions of functions with the specified name or true constants
+bool isOnlyConjunctionOfFunctions(
+    const QueryTreeNodePtr & node,
+    const String & func_name,
+    const QueryTreeNodePtrWithHashSet & allowed_arguments)
+{
+    if (isBooleanConstant(node, true))
+        return true;
+
+    const auto * node_function = node->as<FunctionNode>();
+    if (!node_function)
+        return false;
+
+    if (node_function->getFunctionName() == func_name
+        && allowed_arguments.contains(node_function->getArgumentsNode()))
+        return true;
+
+    if (node_function->getFunctionName() == "and")
+    {
+        for (const auto & and_argument : node_function->getArguments().getNodes())
+        {
+            if (!isOnlyConjunctionOfFunctions(and_argument, func_name, allowed_arguments))
+                return false;
+        }
+        return true;
+    }
+    return false;
+}
+
+/// We can rewrite to a <=> b only if we are joining on a and b,
+/// because the function is not yet implemented for other cases.
+bool isTwoArgumentsFromDifferentSides(const FunctionNode & node_function, const JoinNode & join_node)
+{
+    const auto & argument_nodes = node_function.getArguments().getNodes();
+    if (argument_nodes.size() != 2)
+        return false;
+
+    auto first_src = getExpressionSource(argument_nodes[0]);
+    auto second_src = getExpressionSource(argument_nodes[1]);
+    if (!first_src || !second_src)
+        return false;
+
+    const auto & lhs_join = *join_node.getLeftTableExpression();
+    const auto & rhs_join = *join_node.getRightTableExpression();
+    return (first_src->isEqual(lhs_join) && second_src->isEqual(rhs_join)) ||
+           (first_src->isEqual(rhs_join) && second_src->isEqual(lhs_join));
+}
+
 /// Visitor that optimizes logical expressions _only_ in JOIN ON section
 class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext<JoinOnLogicalExpressionOptimizerVisitor>
 {
@@ -47,15 +139,16 @@ public:
     {
         auto * function_node = node->as<FunctionNode>();
 
-        if (!function_node)
-            return;
+        QueryTreeNodePtr new_node = nullptr;
+        if (function_node && function_node->getFunctionName() == "or")
+            new_node = tryOptimizeJoinOnNulls(function_node->getArguments().getNodes(), getContext());
+        else
+            new_node = tryOptimizeJoinOnNulls({node}, getContext());
 
-        if (function_node->getFunctionName() == "or")
+        if (new_node)
         {
-            bool is_argument_type_changed = tryOptimizeIsNotDistinctOrIsNull(node, getContext());
-            if (is_argument_type_changed)
-                need_rerun_resolve = true;
-            return;
+            need_rerun_resolve |= !new_node->getResultType()->equals(*node->getResultType());
+            node = new_node;
         }
     }
 
@@ -72,15 +165,11 @@ private:
     const JoinNode * join_node;
     bool need_rerun_resolve = false;
 
-    /// Returns true if type of some operand is changed and parent function needs to be re-resolved
-    bool tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node, const ContextPtr & context)
+    /// Returns optimized node or nullptr if nothing have been changed
+    QueryTreeNodePtr tryOptimizeJoinOnNulls(const QueryTreeNodes & nodes, const ContextPtr & context)
     {
-        auto & function_node = node->as<FunctionNode &>();
-        chassert(function_node.getFunctionName() == "or");
-
-
         QueryTreeNodes or_operands;
-        or_operands.reserve(function_node.getArguments().getNodes().size());
+        or_operands.reserve(nodes.size());
 
         /// Indices of `equals` or `isNotDistinctFrom` functions in the vector above
         std::vector<size_t> equals_functions_indices;
@@ -93,47 +182,73 @@ private:
           *     b => [(a IS NULL AND b IS NULL)]
           *     c => [(a IS NULL AND c IS NULL)]
           * }
-          * Then for each a <=> b we can find all operands that contains both a IS NULL and b IS NULL
+          * Then for each equality a = b we can check if we have operand (a IS NULL AND b IS NULL)
           */
         QueryTreeNodePtrWithHashMap<std::vector<size_t>> is_null_argument_to_indices;
 
-        for (const auto & argument : function_node.getArguments())
-        {
-            or_operands.push_back(argument);
+        bool is_anything_changed = false;
 
-            auto * argument_function = argument->as<FunctionNode>();
+        for (const auto & node : nodes)
+        {
+            if (isBooleanConstant(node, false))
+            {
+                /// Remove false constants from OR
+                is_anything_changed = true;
+                continue;
+            }
+
+            or_operands.push_back(node);
+            auto * argument_function = node->as<FunctionNode>();
             if (!argument_function)
                 continue;
 
             const auto & func_name = argument_function->getFunctionName();
             if (func_name == "equals" || func_name == "isNotDistinctFrom")
             {
-                const auto & argument_nodes = argument_function->getArguments().getNodes();
-                if (argument_nodes.size() != 2)
-                    continue;
-                /// We can rewrite to a <=> b only if we are joining on a and b,
-                /// because the function is not yet implemented for other cases.
-                auto first_src = getExpressionSource(argument_nodes[0]);
-                auto second_src = getExpressionSource(argument_nodes[1]);
-                if (!first_src || !second_src)
-                    continue;
-                const auto & lhs_join = *join_node->getLeftTableExpression();
-                const auto & rhs_join = *join_node->getRightTableExpression();
-                bool arguments_from_both_sides = (first_src->isEqual(lhs_join) && second_src->isEqual(rhs_join)) ||
-                                                 (first_src->isEqual(rhs_join) && second_src->isEqual(lhs_join));
-                if (!arguments_from_both_sides)
-                    continue;
-                equals_functions_indices.push_back(or_operands.size() - 1);
+                if (isTwoArgumentsFromDifferentSides(*argument_function, *join_node))
+                    equals_functions_indices.push_back(or_operands.size() - 1);
             }
             else if (func_name == "and")
             {
-                for (const auto & and_argument : argument_function->getArguments().getNodes())
+                const auto & and_arguments = argument_function->getArguments().getNodes();
+                bool all_are_is_null = and_arguments.size() == 2 && isNodeFunction(and_arguments[0], "isNull") && isNodeFunction(and_arguments[1], "isNull");
+                if (all_are_is_null)
                 {
-                    auto * and_argument_function = and_argument->as<FunctionNode>();
-                    if (and_argument_function && and_argument_function->getFunctionName() == "isNull")
+                    is_null_argument_to_indices[getFunctionArgument(and_arguments.front(), 0)].push_back(or_operands.size() - 1);
+                    is_null_argument_to_indices[getFunctionArgument(and_arguments.back(), 0)].push_back(or_operands.size() - 1);
+                }
+
+                /// Expression `a = b AND (a IS NOT NULL) AND true AND (b IS NOT NULL)` we can be replaced with `a = b`
+                /// Even though this expression are not equivalent (first is NULL on NULLs, while second is FALSE),
+                /// it is still correct since for JOIN ON condition NULL is treated as FALSE
+                if (const auto & equals_function = findEqualsFunction(and_arguments))
+                {
+                    const auto & equals_arguments = equals_function->as<FunctionNode>()->getArguments().getNodes();
+                    /// Expected isNotNull arguments
+                    QueryTreeNodePtrWithHashSet allowed_arguments;
+                    allowed_arguments.insert(QueryTreeNodePtrWithHash(std::make_shared<ListNode>(QueryTreeNodes{equals_arguments[0]})));
+                    allowed_arguments.insert(QueryTreeNodePtrWithHash(std::make_shared<ListNode>(QueryTreeNodes{equals_arguments[1]})));
+
+                    bool can_be_optimized = true;
+                    for (const auto & and_argument : and_arguments)
                     {
-                        const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0];
-                        is_null_argument_to_indices[is_null_argument].push_back(or_operands.size() - 1);
+                        if (and_argument.get() == equals_function.get())
+                            continue;
+
+                        if (isOnlyConjunctionOfFunctions(and_argument, "isNotNull", allowed_arguments))
+                            continue;
+
+                        can_be_optimized = false;
+                        break;
+                    }
+
+                    if (can_be_optimized)
+                    {
+                        is_anything_changed = true;
+                        or_operands.pop_back();
+                        or_operands.push_back(equals_function);
+                        if (isTwoArgumentsFromDifferentSides(equals_function->as<FunctionNode &>(), *join_node))
+                            equals_functions_indices.push_back(or_operands.size() - 1);
                     }
                 }
             }
@@ -144,9 +259,9 @@ private:
 
         for (size_t equals_function_idx : equals_functions_indices)
         {
-            auto * equals_function = or_operands[equals_function_idx]->as<FunctionNode>();
+            const auto * equals_function = or_operands[equals_function_idx]->as<FunctionNode>();
 
-            /// For a <=> b we are looking for expressions containing both `a IS NULL` and `b IS NULL` combined with AND
+            /// For a = b we are looking for all expressions `a IS NULL AND b IS NULL`
             const auto & argument_nodes = equals_function->getArguments().getNodes();
             const auto & lhs_is_null_parents = is_null_argument_to_indices[argument_nodes[0]];
             const auto & rhs_is_null_parents = is_null_argument_to_indices[argument_nodes[1]];
@@ -161,60 +276,40 @@ private:
 
             for (size_t to_optimize_idx : operands_to_optimize)
             {
-                /// We are looking for operand `a IS NULL AND b IS NULL AND ...`
-                auto * operand_to_optimize = or_operands[to_optimize_idx]->as<FunctionNode>();
-
-                /// Remove `a IS NULL` and `b IS NULL` arguments from AND
-                QueryTreeNodes new_arguments;
-                for (const auto & and_argument : operand_to_optimize->getArguments().getNodes())
-                {
-                    bool to_eliminate = false;
-
-                    const auto * and_argument_function = and_argument->as<FunctionNode>();
-                    if (and_argument_function && and_argument_function->getFunctionName() == "isNull")
-                    {
-                        const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0];
-                        to_eliminate = (is_null_argument->isEqual(*argument_nodes[0]) || is_null_argument->isEqual(*argument_nodes[1]));
-                    }
-
-                    if (to_eliminate)
-                        arguments_to_reresolve.insert(to_optimize_idx);
-                    else
-                        new_arguments.emplace_back(and_argument);
-                }
-                /// If less than two arguments left, we will remove or replace the whole AND below
-                operand_to_optimize->getArguments().getNodes() = std::move(new_arguments);
+                /// Remove `a IS NULL AND b IS NULL`
+                or_operands[to_optimize_idx] = nullptr;
+                is_anything_changed = true;
             }
         }
 
-        if (arguments_to_reresolve.empty())
+        if (arguments_to_reresolve.empty() && !is_anything_changed)
             /// Nothing have been changed
-            return false;
+            return nullptr;
 
         auto and_function_resolver = FunctionFactory::instance().get("and", context);
         auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", context);
 
-        bool need_reresolve = false;
         QueryTreeNodes new_or_operands;
         for (size_t i = 0; i < or_operands.size(); ++i)
         {
             if (arguments_to_reresolve.contains(i))
             {
-                auto * function = or_operands[i]->as<FunctionNode>();
+                const auto * function = or_operands[i]->as<FunctionNode>();
                 if (function->getFunctionName() == "equals")
                 {
                     /// We should replace `a = b` with `a <=> b` because we removed checks for IS NULL
-                    need_reresolve |= function->getResultType()->isNullable();
-                    function->resolveAsFunction(strict_equals_function_resolver);
-                    new_or_operands.emplace_back(std::move(or_operands[i]));
+                    auto new_function = or_operands[i]->clone();
+                    new_function->as<FunctionNode>()->resolveAsFunction(strict_equals_function_resolver);
+                    new_or_operands.emplace_back(std::move(new_function));
                 }
                 else if (function->getFunctionName() == "and")
                 {
                     const auto & and_arguments = function->getArguments().getNodes();
                     if (and_arguments.size() > 1)
                     {
-                        function->resolveAsFunction(and_function_resolver);
-                        new_or_operands.emplace_back(std::move(or_operands[i]));
+                        auto new_function = or_operands[i]->clone();
+                        new_function->as<FunctionNode>()->resolveAsFunction(and_function_resolver);
+                        new_or_operands.emplace_back(std::move(new_function));
                     }
                     else if (and_arguments.size() == 1)
                     {
@@ -223,25 +318,26 @@ private:
                     }
                 }
                 else
-                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function name: '{}'", function->getFunctionName());
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function '{}'", function->getFunctionName());
             }
-            else
+            else if (or_operands[i])
             {
                 new_or_operands.emplace_back(std::move(or_operands[i]));
             }
         }
 
+        if (new_or_operands.empty())
+            return nullptr;
+
         if (new_or_operands.size() == 1)
-        {
-            node = std::move(new_or_operands[0]);
-            return need_reresolve;
-        }
+            return new_or_operands[0];
 
         /// Rebuild OR function
         auto or_function_resolver = FunctionFactory::instance().get("or", context);
-        function_node.getArguments().getNodes() = std::move(new_or_operands);
-        function_node.resolveAsFunction(or_function_resolver);
-        return need_reresolve;
+        auto function_node = std::make_shared<FunctionNode>("or");
+        function_node->getArguments().getNodes() = std::move(new_or_operands);
+        function_node->resolveAsFunction(or_function_resolver);
+        return function_node;
     }
 };
 

src/Analyzer/Passes/SumIfToCountIfPass.cpp

@@ -33,7 +33,7 @@ public:
             return;
 
         auto * function_node = node->as<FunctionNode>();
-        if (!function_node || !function_node->isAggregateFunction())
+        if (!function_node || !function_node->isAggregateFunction() || !function_node->getResultType()->equals(DataTypeUInt64()))
             return;
 
         auto function_name = function_node->getFunctionName();

src/Analyzer/QueryTreeBuilder.cpp

@@ -940,6 +940,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
                 table_join.locality,
                 result_join_strictness,
                 result_join_kind);
+            join_node->setOriginalAST(table_element.table_join);
 
             /** Original AST is not set because it will contain only join part and does
               * not include left table expression.

src/Backups/BackupOperationInfo.h

@@ -3,6 +3,8 @@
 #include <Backups/BackupStatus.h>
 #include <Common/ProfileEvents.h>
 
+#include <exception>
+
 namespace DB
 {
 

src/Client/ConnectionEstablisher.cpp

@@ -8,6 +8,7 @@ namespace ProfileEvents
     extern const Event DistributedConnectionUsable;
     extern const Event DistributedConnectionMissingTable;
     extern const Event DistributedConnectionStaleReplica;
+    extern const Event DistributedConnectionFailTry;
 }
 
 namespace DB
@@ -97,6 +98,8 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::
     }
     catch (const Exception & e)
     {
+        ProfileEvents::increment(ProfileEvents::DistributedConnectionFailTry);
+
         if (e.code() != ErrorCodes::NETWORK_ERROR && e.code() != ErrorCodes::SOCKET_TIMEOUT
             && e.code() != ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF && e.code() != ErrorCodes::DNS_ERROR)
             throw;

src/Client/HedgedConnections.cpp

@@ -187,9 +187,9 @@ void HedgedConnections::sendQuery(
             modified_settings.group_by_two_level_threshold_bytes = 0;
         }
 
-        const bool enable_sample_offset_parallel_processing = settings.max_parallel_replicas > 1 && settings.allow_experimental_parallel_reading_from_replicas == 0;
+        const bool enable_offset_parallel_processing = context->canUseOffsetParallelReplicas();
 
-        if (offset_states.size() > 1 && enable_sample_offset_parallel_processing)
+        if (offset_states.size() > 1 && enable_offset_parallel_processing)
         {
             modified_settings.parallel_replicas_count = offset_states.size();
             modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset;
@@ -201,7 +201,8 @@ void HedgedConnections::sendQuery(
         /// all servers involved in the distributed query processing.
         modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
 
-        replica.connection->sendQuery(timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {});
+        replica.connection->sendQuery(
+            timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {});
         replica.change_replica_timeout.setRelative(timeouts.receive_data_timeout);
         replica.packet_receiver->setTimeout(hedged_connections_factory.getConnectionTimeouts().receive_timeout);
     };

src/Client/HedgedConnectionsFactory.cpp

@@ -7,7 +7,6 @@
 namespace ProfileEvents
 {
     extern const Event HedgedRequestsChangeReplica;
-    extern const Event DistributedConnectionFailTry;
     extern const Event DistributedConnectionFailAtAll;
 }
 
@@ -327,7 +326,6 @@ HedgedConnectionsFactory::State HedgedConnectionsFactory::processFinishedConnect
     {
         ShuffledPool & shuffled_pool = shuffled_pools[index];
         LOG_INFO(log, "Connection failed at try №{}, reason: {}", (shuffled_pool.error_count + 1), fail_message);
-        ProfileEvents::increment(ProfileEvents::DistributedConnectionFailTry);
 
         shuffled_pool.error_count = std::min(pool->getMaxErrorCup(), shuffled_pool.error_count + 1);
         shuffled_pool.slowdown_count = 0;

src/Client/MultiplexedConnections.cpp

@@ -2,6 +2,7 @@
 
 #include <Common/thread_local_rng.h>
 #include <Core/Protocol.h>
+#include <Interpreters/Context.h>
 #include <IO/ConnectionTimeouts.h>
 #include <IO/Operators.h>
 #include <Interpreters/ClientInfo.h>
@@ -23,8 +24,8 @@ namespace ErrorCodes
 }
 
 
-MultiplexedConnections::MultiplexedConnections(Connection & connection, const Settings & settings_, const ThrottlerPtr & throttler)
-    : settings(settings_)
+MultiplexedConnections::MultiplexedConnections(Connection & connection, ContextPtr context_, const ThrottlerPtr & throttler)
+    : context(std::move(context_)), settings(context->getSettingsRef())
 {
     connection.setThrottler(throttler);
 
@@ -36,9 +37,9 @@ MultiplexedConnections::MultiplexedConnections(Connection & connection, const Se
 }
 
 
-MultiplexedConnections::MultiplexedConnections(std::shared_ptr<Connection> connection_ptr_, const Settings & settings_, const ThrottlerPtr & throttler)
-    : settings(settings_)
-    , connection_ptr(connection_ptr_)
+MultiplexedConnections::MultiplexedConnections(
+    std::shared_ptr<Connection> connection_ptr_, ContextPtr context_, const ThrottlerPtr & throttler)
+    : context(std::move(context_)), settings(context->getSettingsRef()), connection_ptr(connection_ptr_)
 {
     connection_ptr->setThrottler(throttler);
 
@@ -50,9 +51,8 @@ MultiplexedConnections::MultiplexedConnections(std::shared_ptr<Connection> conne
 }
 
 MultiplexedConnections::MultiplexedConnections(
-        std::vector<IConnectionPool::Entry> && connections,
-        const Settings & settings_, const ThrottlerPtr & throttler)
-    : settings(settings_)
+    std::vector<IConnectionPool::Entry> && connections, ContextPtr context_, const ThrottlerPtr & throttler)
+    : context(std::move(context_)), settings(context->getSettingsRef())
 {
     /// If we didn't get any connections from pool and getMany() did not throw exceptions, this means that
     /// `skip_unavailable_shards` was set. Then just return.
@@ -156,18 +156,18 @@ void MultiplexedConnections::sendQuery(
     /// all servers involved in the distributed query processing.
     modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
 
-    const bool enable_sample_offset_parallel_processing = settings.max_parallel_replicas > 1 && settings.allow_experimental_parallel_reading_from_replicas == 0;
+    const bool enable_offset_parallel_processing = context->canUseOffsetParallelReplicas();
 
     size_t num_replicas = replica_states.size();
     if (num_replicas > 1)
     {
-        if (enable_sample_offset_parallel_processing)
+        if (enable_offset_parallel_processing)
             /// Use multiple replicas for parallel query processing.
             modified_settings.parallel_replicas_count = num_replicas;
 
         for (size_t i = 0; i < num_replicas; ++i)
         {
-            if (enable_sample_offset_parallel_processing)
+            if (enable_offset_parallel_processing)
                 modified_settings.parallel_replica_offset = i;
 
             replica_states[i].connection->sendQuery(

src/Client/MultiplexedConnections.h

@@ -10,7 +10,6 @@
 namespace DB
 {
 
-
 /** To retrieve data directly from multiple replicas (connections) from one shard
   * within a single thread. As a degenerate case, it can also work with one connection.
   * It is assumed that all functions except sendCancel are always executed in one thread.
@@ -21,14 +20,12 @@ class MultiplexedConnections final : public IConnections
 {
 public:
     /// Accepts ready connection.
-    MultiplexedConnections(Connection & connection, const Settings & settings_, const ThrottlerPtr & throttler_);
+    MultiplexedConnections(Connection & connection, ContextPtr context_, const ThrottlerPtr & throttler_);
     /// Accepts ready connection and keep it alive before drain
-    MultiplexedConnections(std::shared_ptr<Connection> connection_, const Settings & settings_, const ThrottlerPtr & throttler_);
+    MultiplexedConnections(std::shared_ptr<Connection> connection_, ContextPtr context_, const ThrottlerPtr & throttler_);
 
     /// Accepts a vector of connections to replicas of one shard already taken from pool.
-    MultiplexedConnections(
-        std::vector<IConnectionPool::Entry> && connections,
-        const Settings & settings_, const ThrottlerPtr & throttler_);
+    MultiplexedConnections(std::vector<IConnectionPool::Entry> && connections, ContextPtr context_, const ThrottlerPtr & throttler_);
 
     void sendScalarsData(Scalars & data) override;
     void sendExternalTablesData(std::vector<ExternalTablesData> & data) override;
@@ -86,6 +83,7 @@ private:
     /// Mark the replica as invalid.
     void invalidateReplica(ReplicaState & replica_state);
 
+    ContextPtr context;
     const Settings & settings;
 
     /// The current number of valid connections to the replicas of this shard.

src/Columns/ColumnAggregateFunction.cpp

@@ -267,7 +267,11 @@ bool ColumnAggregateFunction::structureEquals(const IColumn & to) const
 }
 
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start, size_t length)
+#else
+void ColumnAggregateFunction::doInsertRangeFrom(const IColumn & from, size_t start, size_t length)
+#endif
 {
     const ColumnAggregateFunction & from_concrete = assert_cast<const ColumnAggregateFunction &>(from);
 
@@ -462,7 +466,11 @@ void ColumnAggregateFunction::insertFromWithOwnership(const IColumn & from, size
     insertMergeFrom(from, n);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnAggregateFunction::insertFrom(const IColumn & from, size_t n)
+#else
+void ColumnAggregateFunction::doInsertFrom(const IColumn & from, size_t n)
+#endif
 {
     insertRangeFrom(from, n, 1);
 }

src/Columns/ColumnAggregateFunction.h

@@ -145,7 +145,14 @@ public:
 
     void insertData(const char * pos, size_t length) override;
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn & from, size_t n) override;
+#else
+    using IColumn::insertFrom;
+
+    void doInsertFrom(const IColumn & from, size_t n) override;
+#endif
+
 
     void insertFrom(ConstAggregateDataPtr place);
 
@@ -182,7 +189,11 @@ public:
 
     void protect() override;
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn & from, size_t start, size_t length) override;
+#else
+    void doInsertRangeFrom(const IColumn & from, size_t start, size_t length) override;
+#endif
 
     void popBack(size_t n) override;
 
@@ -201,7 +212,11 @@ public:
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t, size_t, const IColumn &, int) const override
+#else
+    int doCompareAt(size_t, size_t, const IColumn &, int) const override
+#endif
     {
         return 0;
     }

src/Columns/ColumnArray.cpp

@@ -337,7 +337,11 @@ bool ColumnArray::tryInsert(const Field & x)
     return true;
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnArray::insertFrom(const IColumn & src_, size_t n)
+#else
+void ColumnArray::doInsertFrom(const IColumn & src_, size_t n)
+#endif
 {
     const ColumnArray & src = assert_cast<const ColumnArray &>(src_);
     size_t size = src.sizeAt(n);
@@ -392,7 +396,11 @@ int ColumnArray::compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int nan
             : 1);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 int ColumnArray::compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const
+#else
+int ColumnArray::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const
+#endif
 {
     return compareAtImpl(n, m, rhs_, nan_direction_hint);
 }
@@ -535,7 +543,11 @@ void ColumnArray::getExtremes(Field & min, Field & max) const
 }
 
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t length)
+#else
+void ColumnArray::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
+#endif
 {
     if (length == 0)
         return;

src/Columns/ColumnArray.h

@@ -84,10 +84,18 @@ public:
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void updateWeakHash32(WeakHash32 & hash) const override;
     void updateHashFast(SipHash & hash) const override;
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#else
+    void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#endif
     void insert(const Field & x) override;
     bool tryInsert(const Field & x) override;
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn & src_, size_t n) override;
+#else
+    void doInsertFrom(const IColumn & src_, size_t n) override;
+#endif
     void insertDefault() override;
     void popBack(size_t n) override;
     ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
@@ -95,7 +103,11 @@ public:
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     ColumnPtr index(const IColumn & indexes, size_t limit) const override;
     template <typename Type> ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
+#else
+    int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
+#endif
     int compareAtWithCollation(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint, const Collator & collator) const override;
     void getPermutation(PermutationSortDirection direction, PermutationSortStability stability,
                             size_t limit, int nan_direction_hint, Permutation & res) const override;

src/Columns/ColumnCompressed.h

@@ -85,7 +85,11 @@ public:
     bool isDefaultAt(size_t) const override { throwMustBeDecompressed(); }
     void insert(const Field &) override { throwMustBeDecompressed(); }
     bool tryInsert(const Field &) override { throwMustBeDecompressed(); }
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn &, size_t, size_t) override { throwMustBeDecompressed(); }
+#else
+    void doInsertRangeFrom(const IColumn &, size_t, size_t) override { throwMustBeDecompressed(); }
+#endif
     void insertData(const char *, size_t) override { throwMustBeDecompressed(); }
     void insertDefault() override { throwMustBeDecompressed(); }
     void popBack(size_t) override { throwMustBeDecompressed(); }
@@ -100,7 +104,11 @@ public:
     void expand(const Filter &, bool) override { throwMustBeDecompressed(); }
     ColumnPtr permute(const Permutation &, size_t) const override { throwMustBeDecompressed(); }
     ColumnPtr index(const IColumn &, size_t) const override { throwMustBeDecompressed(); }
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t, size_t, const IColumn &, int) const override { throwMustBeDecompressed(); }
+#else
+    int doCompareAt(size_t, size_t, const IColumn &, int) const override { throwMustBeDecompressed(); }
+#endif
     void compareColumn(const IColumn &, size_t, PaddedPODArray<UInt64> *, PaddedPODArray<Int8> &, int, int) const override
     {
         throwMustBeDecompressed();

src/Columns/ColumnConst.h

@@ -32,6 +32,8 @@ private:
     ColumnConst(const ColumnConst & src) = default;
 
 public:
+    bool isConst() const override { return true; }
+
     ColumnPtr convertToFullColumn() const;
 
     ColumnPtr convertToFullColumnIfConst() const override
@@ -121,7 +123,11 @@ public:
         return data->isNullAt(0);
     }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn &, size_t /*start*/, size_t length) override
+#else
+    void doInsertRangeFrom(const IColumn &, size_t /*start*/, size_t length) override
+#endif
     {
         s += length;
     }
@@ -145,12 +151,20 @@ public:
         ++s;
     }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn &, size_t) override
+#else
+    void doInsertFrom(const IColumn &, size_t) override
+#endif
     {
         ++s;
     }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertManyFrom(const IColumn & /*src*/, size_t /* position */, size_t length) override { s += length; }
+#else
+    void doInsertManyFrom(const IColumn & /*src*/, size_t /* position */, size_t length) override { s += length; }
+#endif
 
     void insertDefault() override
     {
@@ -223,7 +237,11 @@ public:
         return data->allocatedBytes() + sizeof(s);
     }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t, size_t, const IColumn & rhs, int nan_direction_hint) const override
+#else
+    int doCompareAt(size_t, size_t, const IColumn & rhs, int nan_direction_hint) const override
+#endif
     {
         return data->compareAt(0, 0, *assert_cast<const ColumnConst &>(rhs).data, nan_direction_hint);
     }

src/Columns/ColumnDecimal.cpp

@@ -32,7 +32,11 @@ namespace ErrorCodes
 }
 
 template <is_decimal T>
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const
+#else
+int ColumnDecimal<T>::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int) const
+#endif
 {
     auto & other = static_cast<const Self &>(rhs_);
     const T & a = data[n];
@@ -331,7 +335,11 @@ void ColumnDecimal<T>::insertData(const char * src, size_t /*length*/)
 }
 
 template <is_decimal T>
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnDecimal<T>::insertRangeFrom(const IColumn & src, size_t start, size_t length)
+#else
+void ColumnDecimal<T>::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
+#endif
 {
     const ColumnDecimal & src_vec = assert_cast<const ColumnDecimal &>(src);
 

src/Columns/ColumnDecimal.h

@@ -55,9 +55,17 @@ public:
     void reserve(size_t n) override { data.reserve_exact(n); }
     void shrinkToFit() override { data.shrink_to_fit(); }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
+#else
+    void doInsertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
+#endif
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override
+#else
+    void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override
+#endif
     {
         ValueType v = assert_cast<const Self &>(src).getData()[position];
         data.resize_fill(data.size() + length, v);
@@ -68,7 +76,11 @@ public:
     void insertManyDefaults(size_t length) override { data.resize_fill(data.size() + length); }
     void insert(const Field & x) override { data.push_back(x.get<T>()); }
     bool tryInsert(const Field & x) override;
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#else
+    void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#endif
 
     void popBack(size_t n) override
     {
@@ -92,7 +104,11 @@ public:
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void updateWeakHash32(WeakHash32 & hash) const override;
     void updateHashFast(SipHash & hash) const override;
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
+#else
+    int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
+#endif
     void getPermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability,
                         size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override;
     void updatePermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability,

src/Columns/ColumnDynamic.cpp

@@ -4,7 +4,9 @@
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeVariant.h>
 #include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeNothing.h>
 #include <DataTypes/FieldToDataType.h>
+#include <DataTypes/DataTypesBinaryEncoding.h>
 #include <Common/Arena.h>
 #include <Common/SipHash.h>
 #include <Processors/Transforms/ColumnGathererTransform.h>
@@ -213,7 +215,11 @@ bool ColumnDynamic::tryInsert(const DB::Field & x)
 }
 
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnDynamic::insertFrom(const DB::IColumn & src_, size_t n)
+#else
+void ColumnDynamic::doInsertFrom(const DB::IColumn & src_, size_t n)
+#endif
 {
     const auto & dynamic_src = assert_cast<const ColumnDynamic &>(src_);
 
@@ -263,7 +269,11 @@ void ColumnDynamic::insertFrom(const DB::IColumn & src_, size_t n)
     variant_col.insertIntoVariantFrom(string_variant_discr, *tmp_string_column, 0);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnDynamic::insertRangeFrom(const DB::IColumn & src_, size_t start, size_t length)
+#else
+void ColumnDynamic::doInsertRangeFrom(const DB::IColumn & src_, size_t start, size_t length)
+#endif
 {
     if (start + length > src_.size())
         throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Parameter out of bound in ColumnDynamic::insertRangeFrom method. "
@@ -429,7 +439,11 @@ void ColumnDynamic::insertRangeFrom(const DB::IColumn & src_, size_t start, size
     }
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnDynamic::insertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
+#else
+void ColumnDynamic::doInsertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
+#endif
 {
     const auto & dynamic_src = assert_cast<const ColumnDynamic &>(src_);
 
@@ -481,7 +495,7 @@ StringRef ColumnDynamic::serializeValueIntoArena(size_t n, DB::Arena & arena, co
     /// We cannot use Variant serialization here as it serializes discriminator + value,
     /// but Dynamic doesn't have fixed mapping discriminator <-> variant type
     /// as different Dynamic column can have different Variants.
-    /// Instead, we serialize null bit + variant type name (size + bytes) + value.
+    /// Instead, we serialize null bit + variant type in binary format (size + bytes) + value.
     const auto & variant_col = assert_cast<const ColumnVariant &>(*variant_column);
     auto discr = variant_col.globalDiscriminatorAt(n);
     StringRef res;
@@ -495,14 +509,15 @@ StringRef ColumnDynamic::serializeValueIntoArena(size_t n, DB::Arena & arena, co
         return res;
     }
 
-    const auto & variant_name = variant_info.variant_names[discr];
-    size_t variant_name_size = variant_name.size();
-    char * pos = arena.allocContinue(sizeof(UInt8) + sizeof(size_t) + variant_name.size(), begin);
+    const auto & variant_type = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariant(discr);
+    String variant_type_binary_data = encodeDataType(variant_type);
+    size_t variant_type_binary_data_size = variant_type_binary_data.size();
+    char * pos = arena.allocContinue(sizeof(UInt8) + sizeof(size_t) + variant_type_binary_data.size(), begin);
     memcpy(pos, &null_bit, sizeof(UInt8));
-    memcpy(pos + sizeof(UInt8), &variant_name_size, sizeof(size_t));
-    memcpy(pos + sizeof(UInt8) + sizeof(size_t), variant_name.data(), variant_name.size());
+    memcpy(pos + sizeof(UInt8), &variant_type_binary_data_size, sizeof(size_t));
+    memcpy(pos + sizeof(UInt8) + sizeof(size_t), variant_type_binary_data.data(), variant_type_binary_data.size());
     res.data = pos;
-    res.size = sizeof(UInt8) + sizeof(size_t) + variant_name.size();
+    res.size = sizeof(UInt8) + sizeof(size_t) + variant_type_binary_data.size();
 
     auto value_ref = variant_col.getVariantByGlobalDiscriminator(discr).serializeValueIntoArena(variant_col.offsetAt(n), arena, begin);
     res.data = value_ref.data - res.size;
@@ -521,13 +536,15 @@ const char * ColumnDynamic::deserializeAndInsertFromArena(const char * pos)
         return pos;
     }
 
-    /// Read variant type name.
-    const size_t variant_name_size = unalignedLoad<size_t>(pos);
-    pos += sizeof(variant_name_size);
-    String variant_name;
-    variant_name.resize(variant_name_size);
-    memcpy(variant_name.data(), pos, variant_name_size);
-    pos += variant_name_size;
+    /// Read variant type in binary format.
+    const size_t variant_type_binary_data_size = unalignedLoad<size_t>(pos);
+    pos += sizeof(variant_type_binary_data_size);
+    String variant_type_binary_data;
+    variant_type_binary_data.resize(variant_type_binary_data_size);
+    memcpy(variant_type_binary_data.data(), pos, variant_type_binary_data_size);
+    pos += variant_type_binary_data_size;
+    auto variant_type = decodeDataType(variant_type_binary_data);
+    auto variant_name = variant_type->getName();
     /// If we already have such variant, just deserialize it into corresponding variant column.
     auto it = variant_info.variant_name_to_discriminator.find(variant_name);
     if (it != variant_info.variant_name_to_discriminator.end())
@@ -537,7 +554,6 @@ const char * ColumnDynamic::deserializeAndInsertFromArena(const char * pos)
     }
 
     /// If we don't have such variant, add it.
-    auto variant_type = DataTypeFactory::instance().get(variant_name);
     if (likely(addNewVariant(variant_type)))
     {
         auto discr = variant_info.variant_name_to_discriminator[variant_name];
@@ -563,13 +579,13 @@ const char * ColumnDynamic::skipSerializedInArena(const char * pos) const
     if (null_bit)
         return pos;
 
-    const size_t variant_name_size = unalignedLoad<size_t>(pos);
-    pos += sizeof(variant_name_size);
-    String variant_name;
-    variant_name.resize(variant_name_size);
-    memcpy(variant_name.data(), pos, variant_name_size);
-    pos += variant_name_size;
-    auto tmp_variant_column = DataTypeFactory::instance().get(variant_name)->createColumn();
+    const size_t variant_type_binary_data_size = unalignedLoad<size_t>(pos);
+    pos += sizeof(variant_type_binary_data_size);
+    String variant_type_binary_data;
+    variant_type_binary_data.resize(variant_type_binary_data_size);
+    memcpy(variant_type_binary_data.data(), pos, variant_type_binary_data_size);
+    pos += variant_type_binary_data_size;
+    auto tmp_variant_column = decodeDataType(variant_type_binary_data)->createColumn();
     return tmp_variant_column->skipSerializedInArena(pos);
 }
 
@@ -587,7 +603,11 @@ void ColumnDynamic::updateHashWithValue(size_t n, SipHash & hash) const
     variant_col.getVariantByGlobalDiscriminator(discr).updateHashWithValue(variant_col.offsetAt(n), hash);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 int ColumnDynamic::compareAt(size_t n, size_t m, const DB::IColumn & rhs, int nan_direction_hint) const
+#else
+int ColumnDynamic::doCompareAt(size_t n, size_t m, const DB::IColumn & rhs, int nan_direction_hint) const
+#endif
 {
     const auto & left_variant = assert_cast<const ColumnVariant &>(*variant_column);
     const auto & right_dynamic = assert_cast<const ColumnDynamic &>(rhs);

src/Columns/ColumnDynamic.h

@@ -142,9 +142,16 @@ public:
 
     void insert(const Field & x) override;
     bool tryInsert(const Field & x) override;
+
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
+#else
+    void doInsertFrom(const IColumn & src_, size_t n) override;
+    void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+    void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override;
+#endif
 
     void insertDefault() override
     {
@@ -213,7 +220,11 @@ public:
         return scattered_columns;
     }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
+#else
+    int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
+#endif
 
     bool hasEqualValues() const override
     {

src/Columns/ColumnFixedString.cpp

@@ -74,7 +74,11 @@ bool ColumnFixedString::tryInsert(const Field & x)
     return true;
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnFixedString::insertFrom(const IColumn & src_, size_t index)
+#else
+void ColumnFixedString::doInsertFrom(const IColumn & src_, size_t index)
+#endif
 {
     const ColumnFixedString & src = assert_cast<const ColumnFixedString &>(src_);
 
@@ -86,7 +90,11 @@ void ColumnFixedString::insertFrom(const IColumn & src_, size_t index)
     memcpySmallAllowReadWriteOverflow15(chars.data() + old_size, &src.chars[n * index], n);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnFixedString::insertManyFrom(const IColumn & src, size_t position, size_t length)
+#else
+void ColumnFixedString::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
+#endif
 {
     const ColumnFixedString & src_concrete = assert_cast<const ColumnFixedString &>(src);
     if (n != src_concrete.getN())
@@ -219,7 +227,11 @@ size_t ColumnFixedString::estimateCardinalityInPermutedRange(const Permutation &
     return elements.size();
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_t length)
+#else
+void ColumnFixedString::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
+#endif
 {
     const ColumnFixedString & src_concrete = assert_cast<const ColumnFixedString &>(src);
     chassert(this->n == src_concrete.n);

src/Columns/ColumnFixedString.h

@@ -98,9 +98,17 @@ public:
 
     bool tryInsert(const Field & x) override;
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn & src_, size_t index) override;
+#else
+    void doInsertFrom(const IColumn & src_, size_t index) override;
+#endif
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
+#else
+    void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override;
+#endif
 
     void insertData(const char * pos, size_t length) override;
 
@@ -129,7 +137,11 @@ public:
 
     void updateHashFast(SipHash & hash) const override;
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
+#else
+    int doCompareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
+#endif
     {
         const ColumnFixedString & rhs = assert_cast<const ColumnFixedString &>(rhs_);
         chassert(this->n == rhs.n);
@@ -144,7 +156,11 @@ public:
 
     size_t estimateCardinalityInPermutedRange(const Permutation & permutation, const EqualRange & equal_range) const override;
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#else
+    void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#endif
 
     ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
 

src/Columns/ColumnFunction.cpp

@@ -72,7 +72,11 @@ ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
     return ColumnFunction::create(length, function, capture, is_short_circuit_argument, is_function_compiled);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnFunction::insertFrom(const IColumn & src, size_t n)
+#else
+void ColumnFunction::doInsertFrom(const IColumn & src, size_t n)
+#endif
 {
     const ColumnFunction & src_func = assert_cast<const ColumnFunction &>(src);
 
@@ -89,7 +93,11 @@ void ColumnFunction::insertFrom(const IColumn & src, size_t n)
     ++elements_size;
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnFunction::insertRangeFrom(const IColumn & src, size_t start, size_t length)
+#else
+void ColumnFunction::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
+#endif
 {
     const ColumnFunction & src_func = assert_cast<const ColumnFunction &>(src);
 

src/Columns/ColumnFunction.h

@@ -94,8 +94,16 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot insert into {}", getName());
     }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn & src, size_t n) override;
+#else
+    void doInsertFrom(const IColumn & src, size_t n) override;
+#endif
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn &, size_t start, size_t length) override;
+#else
+    void doInsertRangeFrom(const IColumn &, size_t start, size_t length) override;
+#endif
 
     void insertData(const char *, size_t) override
     {
@@ -137,7 +145,11 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "popBack is not implemented for {}", getName());
     }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t, size_t, const IColumn &, int) const override
+#else
+    int doCompareAt(size_t, size_t, const IColumn &, int) const override
+#endif
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "compareAt is not implemented for {}", getName());
     }

src/Columns/ColumnLowCardinality.cpp

@@ -159,7 +159,11 @@ void ColumnLowCardinality::insertDefault()
     idx.insertPosition(getDictionary().getDefaultValueIndex());
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnLowCardinality::insertFrom(const IColumn & src, size_t n)
+#else
+void ColumnLowCardinality::doInsertFrom(const IColumn & src, size_t n)
+#endif
 {
     const auto * low_cardinality_src = typeid_cast<const ColumnLowCardinality *>(&src);
 
@@ -187,7 +191,11 @@ void ColumnLowCardinality::insertFromFullColumn(const IColumn & src, size_t n)
     idx.insertPosition(getDictionary().uniqueInsertFrom(src, n));
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, size_t length)
+#else
+void ColumnLowCardinality::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
+#endif
 {
     const auto * low_cardinality_src = typeid_cast<const ColumnLowCardinality *>(&src);
 
@@ -364,7 +372,11 @@ int ColumnLowCardinality::compareAtImpl(size_t n, size_t m, const IColumn & rhs,
     return getDictionary().compareAt(n_index, m_index, low_cardinality_column.getDictionary(), nan_direction_hint);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 int ColumnLowCardinality::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
+#else
+int ColumnLowCardinality::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
+#endif
 {
     return compareAtImpl(n, m, rhs, nan_direction_hint);
 }

src/Columns/ColumnLowCardinality.h

@@ -78,10 +78,18 @@ public:
     bool tryInsert(const Field & x) override;
     void insertDefault() override;
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn & src, size_t n) override;
+#else
+    void doInsertFrom(const IColumn & src, size_t n) override;
+#endif
     void insertFromFullColumn(const IColumn & src, size_t n);
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#else
+    void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#endif
     void insertRangeFromFullColumn(const IColumn & src, size_t start, size_t length);
     void insertRangeFromDictionaryEncodedColumn(const IColumn & keys, const IColumn & positions);
 
@@ -127,7 +135,11 @@ public:
         return ColumnLowCardinality::create(dictionary.getColumnUniquePtr(), getIndexes().index(indexes_, limit));
     }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
+#else
+    int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
+#endif
 
     int compareAtWithCollation(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint, const Collator &) const override;
 

src/Columns/ColumnMap.cpp

@@ -153,17 +153,29 @@ void ColumnMap::updateHashFast(SipHash & hash) const
     nested->updateHashFast(hash);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnMap::insertFrom(const IColumn & src, size_t n)
+#else
+void ColumnMap::doInsertFrom(const IColumn & src, size_t n)
+#endif
 {
     nested->insertFrom(assert_cast<const ColumnMap &>(src).getNestedColumn(), n);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnMap::insertManyFrom(const IColumn & src, size_t position, size_t length)
+#else
+void ColumnMap::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
+#endif
 {
     assert_cast<ColumnArray &>(*nested).insertManyFrom(assert_cast<const ColumnMap &>(src).getNestedColumn(), position, length);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnMap::insertRangeFrom(const IColumn & src, size_t start, size_t length)
+#else
+void ColumnMap::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
+#endif
 {
     nested->insertRangeFrom(
         assert_cast<const ColumnMap &>(src).getNestedColumn(),
@@ -210,7 +222,11 @@ MutableColumns ColumnMap::scatter(ColumnIndex num_columns, const Selector & sele
     return res;
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 int ColumnMap::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
+#else
+int ColumnMap::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
+#endif
 {
     const auto & rhs_map = assert_cast<const ColumnMap &>(rhs);
     return nested->compareAt(n, m, rhs_map.getNestedColumn(), nan_direction_hint);

src/Columns/ColumnMap.h

@@ -66,16 +66,28 @@ public:
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void updateWeakHash32(WeakHash32 & hash) const override;
     void updateHashFast(SipHash & hash) const override;
+
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#else
+    void doInsertFrom(const IColumn & src_, size_t n) override;
+    void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override;
+    void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+#endif
+
     ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
     void expand(const Filter & mask, bool inverted) override;
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     ColumnPtr index(const IColumn & indexes, size_t limit) const override;
     ColumnPtr replicate(const Offsets & offsets) const override;
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
+#if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
+#else
+    int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
+#endif
     void getExtremes(Field & min, Field & max) const override;
     void getPermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability,
                         size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override;

src/Columns/ColumnNullable.cpp

@@ -221,7 +221,11 @@ const char * ColumnNullable::skipSerializedInArena(const char * pos) const
     return pos;
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnNullable::insertRangeFrom(const IColumn & src, size_t start, size_t length)
+#else
+void ColumnNullable::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
+#endif
 {
     const ColumnNullable & nullable_col = assert_cast<const ColumnNullable &>(src);
     getNullMapColumn().insertRangeFrom(*nullable_col.null_map, start, length);
@@ -258,7 +262,11 @@ bool ColumnNullable::tryInsert(const Field & x)
     return true;
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnNullable::insertFrom(const IColumn & src, size_t n)
+#else
+void ColumnNullable::doInsertFrom(const IColumn & src, size_t n)
+#endif
 {
     const ColumnNullable & src_concrete = assert_cast<const ColumnNullable &>(src);
     getNestedColumn().insertFrom(src_concrete.getNestedColumn(), n);
@@ -266,7 +274,11 @@ void ColumnNullable::insertFrom(const IColumn & src, size_t n)
 }
 
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 void ColumnNullable::insertManyFrom(const IColumn & src, size_t position, size_t length)
+#else
+void ColumnNullable::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
+#endif
 {
     const ColumnNullable & src_concrete = assert_cast<const ColumnNullable &>(src);
     getNestedColumn().insertManyFrom(src_concrete.getNestedColumn(), position, length);
@@ -402,7 +414,11 @@ int ColumnNullable::compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int
     return getNestedColumn().compareAt(n, m, nested_rhs, null_direction_hint);
 }
 
+#if !defined(ABORT_ON_LOGICAL_ERROR)
 int ColumnNullable::compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const
+#else
+int ColumnNullable::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const
+#endif
 {
     return compareAtImpl(n, m, rhs_, null_direction_hint);
 }