Merge remote-tracking branch 'upstream/master' into replicated-database-forbid-create-as-select

.github/workflows/backport_branches.yml

@@ -157,7 +157,8 @@ jobs:
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:
-    if: ${{ !failure() && !cancelled() }}
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
       - RunConfig
       - BuilderDebAarch64
@@ -177,7 +178,8 @@ jobs:
       run_command: |
         python3 build_report_check.py "$CHECK_NAME"
   BuilderSpecialReport:
-    if: ${{ !failure() && !cancelled() }}
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
       - RunConfig
       - BuilderBinDarwin

.github/workflows/master.yml

@@ -262,6 +262,8 @@ jobs:
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
       - RunConfig
       - BuilderBinRelease
@@ -272,7 +274,6 @@ jobs:
       - BuilderDebRelease
       - BuilderDebTsan
       - BuilderDebUBsan
-    if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse build check
@@ -285,7 +286,8 @@ jobs:
       run_command: |
         python3 build_report_check.py "$CHECK_NAME"
   BuilderSpecialReport:
-    if: ${{ !failure() && !cancelled() }}
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
       - RunConfig
       - BuilderBinAarch64

.github/workflows/pull_request.yml

@@ -291,6 +291,8 @@ jobs:
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
       - RunConfig
       - BuilderBinRelease
@@ -301,7 +303,6 @@ jobs:
       - BuilderDebRelease
       - BuilderDebTsan
       - BuilderDebUBsan
-    if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse build check
@@ -314,7 +315,8 @@ jobs:
       run_command: |
         python3 build_report_check.py "$CHECK_NAME"
   BuilderSpecialReport:
-    if: ${{ !failure() && !cancelled() }}
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
       - RunConfig
       - BuilderBinAarch64

.github/workflows/release_branches.yml

@@ -172,6 +172,8 @@ jobs:
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
       - RunConfig
       - BuilderDebRelease
@@ -181,7 +183,6 @@ jobs:
       - BuilderDebUBsan
       - BuilderDebMsan
       - BuilderDebDebug
-    if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse build check
@@ -194,7 +195,8 @@ jobs:
       run_command: |
         python3 build_report_check.py "$CHECK_NAME"
   BuilderSpecialReport:
-    if: ${{ !failure() && !cancelled() }}
+    # run report check for failed builds to indicate the CI error
+    if: ${{ !cancelled() }}
     needs:
       - RunConfig
       - BuilderDebRelease

.github/workflows/reusable_build.yml

@@ -76,6 +76,8 @@ jobs:
         run: |
           python3 "$GITHUB_WORKSPACE/tests/ci/build_check.py" "$BUILD_NAME"
       - name: Post
+        # it still be build report to upload for failed build job
+        if: always()
         run: |
           python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.build_name}}'
       - name: Mark as done

base/poco/Foundation/include/Poco/StreamUtil.h

@@ -69,6 +69,9 @@
 // init() is called in the MyIOS constructor.
 // Therefore we replace each call to init() with
 // the poco_ios_init macro defined below.
+//
+// Also this macro will adjust exceptions() flags, since by default std::ios
+// will hide exceptions, while in ClickHouse it is better to pass them through.
 
 
 #if !defined(POCO_IOS_INIT_HACK)
@@ -79,7 +82,10 @@
 #if defined(POCO_IOS_INIT_HACK)
 #    define poco_ios_init(buf)
 #else
-#    define poco_ios_init(buf) init(buf)
+#    define poco_ios_init(buf) do {                         \
+    init(buf);                                              \
+    this->exceptions(std::ios::failbit | std::ios::badbit); \
+} while (0)
 #endif
 
 

base/poco/Foundation/include/Poco/UTF32Encoding.h

@@ -70,6 +70,15 @@ public:
     int queryConvert(const unsigned char * bytes, int length) const;
     int sequenceLength(const unsigned char * bytes, int length) const;
 
+protected:
+	static int safeToInt(Poco::UInt32 value)
+	{
+		if (value <= 0x10FFFF)
+			return static_cast<int>(value);
+		else
+			return -1;
+	}
+
 private:
     bool _flipBytes;
     static const char * _names[];

base/poco/Foundation/src/UTF32Encoding.cpp

@@ -30,22 +30,22 @@ const char* UTF32Encoding::_names[] =
 
 const TextEncoding::CharacterMap UTF32Encoding::_charMap = 
 {
-	/* 00 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 10 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 20 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 30 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 40 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 50 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 60 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 70 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 80 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* 90 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* a0 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* b0 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* c0 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* d0 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* e0 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
-	/* f0 */	-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 
+	/* 00 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 10 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 20 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 30 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 40 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 50 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 60 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 70 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 80 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* 90 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* a0 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* b0 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* c0 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* d0 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* e0 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
+	/* f0 */	-4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
 };
 
 
@@ -118,7 +118,7 @@ const TextEncoding::CharacterMap& UTF32Encoding::characterMap() const
 int UTF32Encoding::convert(const unsigned char* bytes) const
 {
 	UInt32 uc;
-	unsigned char* p = (unsigned char*) &uc;
+	unsigned char* p = reinterpret_cast<unsigned char*>(&uc);
 	*p++ = *bytes++;
 	*p++ = *bytes++;
 	*p++ = *bytes++;
@@ -129,7 +129,7 @@ int UTF32Encoding::convert(const unsigned char* bytes) const
 		ByteOrder::flipBytes(uc);
 	}
 
-	return uc;
+	return safeToInt(uc);
 }
 
 
@@ -138,7 +138,7 @@ int UTF32Encoding::convert(int ch, unsigned char* bytes, int length) const
 	if (bytes && length >= 4)
 	{
 		UInt32 ch1 = _flipBytes ? ByteOrder::flipBytes((UInt32) ch) : (UInt32) ch;
-		unsigned char* p = (unsigned char*) &ch1;
+		unsigned char* p = reinterpret_cast<unsigned char*>(&ch1);
 		*bytes++ = *p++;
 		*bytes++ = *p++;
 		*bytes++ = *p++;
@@ -155,14 +155,14 @@ int UTF32Encoding::queryConvert(const unsigned char* bytes, int length) const
 	if (length >= 4)
 	{
 		UInt32 uc;
-		unsigned char* p = (unsigned char*) &uc;
+		unsigned char* p = reinterpret_cast<unsigned char*>(&uc);
 		*p++ = *bytes++;
 		*p++ = *bytes++;
 		*p++ = *bytes++;
 		*p++ = *bytes++;
 		if (_flipBytes) 
 			ByteOrder::flipBytes(uc);
-		return uc;
+		ret = safeToInt(uc);
 	}
 
 	return ret;

contrib/azure

@@ -1 +1 @@
-Subproject commit a852d81f92f153e109de165ee08546741e3f2a68
+Subproject commit 060c54dfb0abe869c065143303a9d3e9c54c29e3

contrib/azure-cmake/CMakeLists.txt

@@ -8,31 +8,21 @@ endif()
 set(AZURE_DIR "${ClickHouse_SOURCE_DIR}/contrib/azure")
 set(AZURE_SDK_LIBRARY_DIR "${AZURE_DIR}/sdk")
 
-file(GLOB AZURE_SDK_CORE_SRC
+file(GLOB AZURE_SDK_SRC
     "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/*.cpp"
     "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/cryptography/*.cpp"
     "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/*.cpp"
     "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/http/curl/*.cpp"
     "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/io/*.cpp"
-)
-
-file(GLOB AZURE_SDK_IDENTITY_SRC
+    "${AZURE_SDK_LIBRARY_DIR}/core/azure-core/src/tracing/*.cpp"
     "${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/src/*.cpp"
-)
-
-file(GLOB AZURE_SDK_STORAGE_COMMON_SRC
+    "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/*.cpp"
+    "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/private/*.cpp"
     "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/src/*.cpp"
 )
 
-file(GLOB AZURE_SDK_STORAGE_BLOBS_SRC
-    "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/*.cpp"
-)
-
 file(GLOB AZURE_SDK_UNIFIED_SRC
-    ${AZURE_SDK_CORE_SRC}
-    ${AZURE_SDK_IDENTITY_SRC}
-    ${AZURE_SDK_STORAGE_COMMON_SRC}
-    ${AZURE_SDK_STORAGE_BLOBS_SRC}
+    ${AZURE_SDK_SRC}
 )
 
 set(AZURE_SDK_INCLUDES

docker/test/integration/runner/compose/docker_compose_minio.yml

@@ -34,7 +34,7 @@ services:
 
   # Empty container to run proxy resolver.
   resolver:
-    image: clickhouse/python-bottle
+    image: clickhouse/python-bottle:${DOCKER_PYTHON_BOTTLE_TAG:-latest}
     expose:
       - "8080"
     tty: true

docker/test/stateful/run.sh

@@ -40,6 +40,12 @@ if [ "$cache_policy" = "SLRU" ]; then
     mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
 fi
 
+if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
+    # It is not needed, we will explicitly create tables on s3.
+    # We do not have statefull tests with s3 storage run in public repository, but this is needed for another repository.
+    rm /etc/clickhouse-server/config.d/s3_storage_policy_for_merge_tree_by_default.xml
+fi
+
 function start()
 {
     if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
@@ -123,8 +129,76 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
 else
     clickhouse-client --query "CREATE DATABASE test"
     clickhouse-client --query "SHOW TABLES FROM test"
-    clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
-    clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
+    if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
+        clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64,  JavaEnable UInt8,  Title String,  GoodEvent Int16,
+            EventTime DateTime,  EventDate Date,  CounterID UInt32,  ClientIP UInt32,  ClientIP6 FixedString(16),  RegionID UInt32,
+            UserID UInt64,  CounterClass Int8,  OS UInt8,  UserAgent UInt8,  URL String,  Referer String,  URLDomain String,
+            RefererDomain String,  Refresh UInt8,  IsRobot UInt8,  RefererCategories Array(UInt16),  URLCategories Array(UInt16),
+            URLRegions Array(UInt32),  RefererRegions Array(UInt32),  ResolutionWidth UInt16,  ResolutionHeight UInt16,  ResolutionDepth UInt8,
+            FlashMajor UInt8, FlashMinor UInt8,  FlashMinor2 String,  NetMajor UInt8,  NetMinor UInt8, UserAgentMajor UInt16,
+            UserAgentMinor FixedString(2),  CookieEnable UInt8, JavascriptEnable UInt8,  IsMobile UInt8,  MobilePhone UInt8,
+            MobilePhoneModel String,  Params String,  IPNetworkID UInt32,  TraficSourceID Int8, SearchEngineID UInt16,
+            SearchPhrase String,  AdvEngineID UInt8,  IsArtifical UInt8,  WindowClientWidth UInt16,  WindowClientHeight UInt16,
+            ClientTimeZone Int16,  ClientEventTime DateTime,  SilverlightVersion1 UInt8, SilverlightVersion2 UInt8,  SilverlightVersion3 UInt32,
+            SilverlightVersion4 UInt16,  PageCharset String,  CodeVersion UInt32,  IsLink UInt8,  IsDownload UInt8,  IsNotBounce UInt8,
+            FUniqID UInt64,  HID UInt32,  IsOldCounter UInt8, IsEvent UInt8,  IsParameter UInt8,  DontCountHits UInt8,  WithHash UInt8,
+            HitColor FixedString(1),  UTCEventTime DateTime,  Age UInt8,  Sex UInt8,  Income UInt8,  Interests UInt16,  Robotness UInt8,
+            GeneralInterests Array(UInt16), RemoteIP UInt32,  RemoteIP6 FixedString(16),  WindowName Int32,  OpenerName Int32,
+            HistoryLength Int16,  BrowserLanguage FixedString(2),  BrowserCountry FixedString(2),  SocialNetwork String,  SocialAction String,
+            HTTPError UInt16, SendTiming Int32,  DNSTiming Int32,  ConnectTiming Int32,  ResponseStartTiming Int32,  ResponseEndTiming Int32,
+            FetchTiming Int32,  RedirectTiming Int32, DOMInteractiveTiming Int32,  DOMContentLoadedTiming Int32,  DOMCompleteTiming Int32,
+            LoadEventStartTiming Int32,  LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32,  FirstPaintTiming Int32,
+            RedirectCount Int8, SocialSourceNetworkID UInt8,  SocialSourcePage String,  ParamPrice Int64, ParamOrderID String,
+            ParamCurrency FixedString(3),  ParamCurrencyID UInt16, GoalsReached Array(UInt32),  OpenstatServiceName String,
+            OpenstatCampaignID String,  OpenstatAdID String,  OpenstatSourceID String,  UTMSource String, UTMMedium String,
+            UTMCampaign String,  UTMContent String,  UTMTerm String, FromTag String,  HasGCLID UInt8,  RefererHash UInt64,
+            URLHash UInt64,  CLID UInt32,  YCLID UInt64,  ShareService String,  ShareURL String,  ShareTitle String,
+            ParsedParams Nested(Key1 String,  Key2 String, Key3 String, Key4 String, Key5 String,  ValueDouble Float64),
+            IslandID FixedString(16),  RequestNum UInt32,  RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
+            ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
+        clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32,  StartDate Date,  Sign Int8,  IsNew UInt8,
+            VisitID UInt64,  UserID UInt64,  StartTime DateTime,  Duration UInt32,  UTCStartTime DateTime,  PageViews Int32,
+            Hits Int32,  IsBounce UInt8,  Referer String,  StartURL String,  RefererDomain String,  StartURLDomain String,
+            EndURL String,  LinkURL String,  IsDownload UInt8,  TraficSourceID Int8,  SearchEngineID UInt16,  SearchPhrase String,
+            AdvEngineID UInt8,  PlaceID Int32,  RefererCategories Array(UInt16),  URLCategories Array(UInt16),  URLRegions Array(UInt32),
+            RefererRegions Array(UInt32),  IsYandex UInt8,  GoalReachesDepth Int32,  GoalReachesURL Int32,  GoalReachesAny Int32,
+            SocialSourceNetworkID UInt8,  SocialSourcePage String,  MobilePhoneModel String,  ClientEventTime DateTime,  RegionID UInt32,
+            ClientIP UInt32,  ClientIP6 FixedString(16),  RemoteIP UInt32,  RemoteIP6 FixedString(16),  IPNetworkID UInt32,
+            SilverlightVersion3 UInt32,  CodeVersion UInt32,  ResolutionWidth UInt16,  ResolutionHeight UInt16,  UserAgentMajor UInt16,
+            UserAgentMinor UInt16,  WindowClientWidth UInt16,  WindowClientHeight UInt16,  SilverlightVersion2 UInt8,  SilverlightVersion4 UInt16,
+            FlashVersion3 UInt16,  FlashVersion4 UInt16,  ClientTimeZone Int16,  OS UInt8,  UserAgent UInt8,  ResolutionDepth UInt8,
+            FlashMajor UInt8,  FlashMinor UInt8,  NetMajor UInt8,  NetMinor UInt8,  MobilePhone UInt8,  SilverlightVersion1 UInt8,
+            Age UInt8,  Sex UInt8,  Income UInt8,  JavaEnable UInt8,  CookieEnable UInt8,  JavascriptEnable UInt8,  IsMobile UInt8,
+            BrowserLanguage UInt16,  BrowserCountry UInt16,  Interests UInt16,  Robotness UInt8,  GeneralInterests Array(UInt16),
+            Params Array(String),  Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime,  Price Int64,  OrderID String, CurrencyID UInt32),
+            WatchIDs Array(UInt64),  ParamSumPrice Int64,  ParamCurrency FixedString(3),  ParamCurrencyID UInt16,  ClickLogID UInt64,
+            ClickEventID Int32,  ClickGoodEvent Int32,  ClickEventTime DateTime,  ClickPriorityID Int32,  ClickPhraseID Int32,  ClickPageID Int32,
+            ClickPlaceID Int32,  ClickTypeID Int32,  ClickResourceID Int32,  ClickCost UInt32,  ClickClientIP UInt32,  ClickDomainID UInt32,
+            ClickURL String,  ClickAttempt UInt8,  ClickOrderID UInt32,  ClickBannerID UInt32,  ClickMarketCategoryID UInt32,  ClickMarketPP UInt32,
+            ClickMarketCategoryName String,  ClickMarketPPName String,  ClickAWAPSCampaignName String,  ClickPageName String,  ClickTargetType UInt16,
+            ClickTargetPhraseID UInt64,  ClickContextType UInt8,  ClickSelectType Int8,  ClickOptions String,  ClickGroupBannerID Int32,
+            OpenstatServiceName String,  OpenstatCampaignID String,  OpenstatAdID String,  OpenstatSourceID String,  UTMSource String,
+            UTMMedium String,  UTMCampaign String,  UTMContent String,  UTMTerm String,  FromTag String,  HasGCLID UInt8,  FirstVisit DateTime,
+            PredLastVisit Date,  LastVisit Date,  TotalVisits UInt32,  TraficSource    Nested(ID Int8,  SearchEngineID UInt16, AdvEngineID UInt8,
+            PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String),  Attendance FixedString(16),
+            CLID UInt32,  YCLID UInt64,  NormalizedRefererHash UInt64,  SearchPhraseHash UInt64,  RefererDomainHash UInt64,  NormalizedStartURLHash UInt64,
+            StartURLDomainHash UInt64,  NormalizedEndURLHash UInt64,  TopLevelDomain UInt64,  URLScheme UInt64,  OpenstatServiceNameHash UInt64,
+            OpenstatCampaignIDHash UInt64,  OpenstatAdIDHash UInt64,  OpenstatSourceIDHash UInt64,  UTMSourceHash UInt64,  UTMMediumHash UInt64,
+            UTMCampaignHash UInt64,  UTMContentHash UInt64,  UTMTermHash UInt64,  FromHash UInt64,  WebVisorEnabled UInt8,  WebVisorActivity UInt32,
+            ParsedParams    Nested(Key1 String,  Key2 String,  Key3 String,  Key4 String, Key5 String, ValueDouble    Float64),
+            Market Nested(Type UInt8, GoalID UInt32, OrderID String,  OrderPrice Int64,  PP UInt32,  DirectPlaceID UInt32,  DirectOrderID  UInt32,
+            DirectBannerID UInt32,  GoodID String, GoodName String, GoodQuantity Int32,  GoodPrice Int64),  IslandID FixedString(16))
+            ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
+            SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
+
+        clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
+        clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
+        clickhouse-client --query "DROP TABLE datasets.visits_v1 SYNC"
+        clickhouse-client --query "DROP TABLE datasets.hits_v1 SYNC"
+    else
+        clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
+        clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
+    fi
     clickhouse-client --query "CREATE TABLE test.hits_s3  (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
     clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0"
 fi
@@ -144,6 +218,10 @@ function run_tests()
         ADDITIONAL_OPTIONS+=('--replicated-database')
     fi
 
+    if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
+        ADDITIONAL_OPTIONS+=('--s3-storage')
+    fi
+
     if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then
         ADDITIONAL_OPTIONS+=('--db-engine=Ordinary')
     fi

docker/test/stateless/run.sh

@@ -58,6 +58,7 @@ if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; th
 
     # it contains some new settings, but we can safely remove it
     rm /etc/clickhouse-server/users.d/s3_cache_new.xml
+    rm /etc/clickhouse-server/config.d/zero_copy_destructive_operations.xml
 fi
 
 # For flaky check we also enable thread fuzzer

docs/en/operations/system-tables/asynchronous_metrics.md

@@ -239,6 +239,10 @@ The amount of virtual memory mapped for the pages of machine code of the server
 
 The amount of virtual memory mapped for the use of stack and for the allocated memory, in bytes. It is unspecified whether it includes the per-thread stacks and most of the allocated memory, that is allocated with the 'mmap' system call. This metric exists only for completeness reasons. I recommend to use the `MemoryResident` metric for monitoring.
 
+### MemoryResidentMax
+
+Maximum amount of physical memory used by the server process, in bytes.
+
 ### MemoryResident
 
 The amount of physical memory used by the server process, in bytes.

docs/en/sql-reference/statements/insert-into.md

@@ -11,7 +11,7 @@ Inserts data into a table.
 **Syntax**
 
 ``` sql
-INSERT INTO [TABLE] [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ...
+INSERT INTO [TABLE] [db.]table [(c1, c2, c3)] [SETTINGS ...] VALUES (v11, v12, v13), (v21, v22, v23), ...
 ```
 
 You can specify a list of columns to insert using  the `(c1, c2, c3)`. You can also use an expression with column [matcher](../../sql-reference/statements/select/index.md#asterisk) such as `*` and/or [modifiers](../../sql-reference/statements/select/index.md#select-modifiers) such as [APPLY](../../sql-reference/statements/select/index.md#apply-modifier), [EXCEPT](../../sql-reference/statements/select/index.md#except-modifier), [REPLACE](../../sql-reference/statements/select/index.md#replace-modifier).
@@ -126,7 +126,7 @@ To insert a default value instead of `NULL` into a column with not nullable data
 **Syntax**
 
 ``` sql
-INSERT INTO [TABLE] [db.]table [(c1, c2, c3)] FROM INFILE file_name [COMPRESSION type] FORMAT format_name
+INSERT INTO [TABLE] [db.]table [(c1, c2, c3)] FROM INFILE file_name [COMPRESSION type] [SETTINGS ...] [FORMAT format_name]
 ```
 
 Use the syntax above to insert data from a file, or files, stored on the **client** side. `file_name` and `type` are string literals. Input file [format](../../interfaces/formats.md) must be set in the `FORMAT` clause.

docs/en/sql-reference/statements/select/from.md

@@ -34,7 +34,7 @@ Queries that use `FINAL` are executed slightly slower than similar queries that
 - Data is merged during query execution.
 - Queries with `FINAL` read primary key columns in addition to the columns specified in the query.
 
-**In most cases, avoid using `FINAL`.** The common approach is to use different queries that assume the background processes of the `MergeTree` engine haven’t happened yet and deal with it by applying aggregation (for example, to discard duplicates).
+`FINAL` requires additional compute and memory resources, as the processing that normally would occur at merge time must occur in memory at the time of the query. However, using FINAL is sometimes necessary in order to produce accurate results, and is less expensive than running `OPTIMIZE` to force a merge. It is also sometimes possible to use different queries that assume the background processes of the `MergeTree` engine haven’t happened yet and deal with it by applying aggregation (for example, to discard duplicates). If you need to use FINAL in your queries in order to get the required results, then it is okay to do so but be aware of the additional processing required.
 
 `FINAL` can be applied automatically using [FINAL](../../../operations/settings/settings.md#final) setting to all tables in a query using a session or a user profile.
 

docs/en/sql-reference/statements/select/join.md

@@ -43,6 +43,7 @@ Additional join types available in ClickHouse:
 - `LEFT ANTI JOIN` and `RIGHT ANTI JOIN`, a blacklist on “join keys”, without producing a cartesian product.
 - `LEFT ANY JOIN`, `RIGHT ANY JOIN` and `INNER ANY JOIN`, partially (for opposite side of `LEFT` and `RIGHT`) or completely (for `INNER` and `FULL`) disables the cartesian product for standard `JOIN` types.
 - `ASOF JOIN` and `LEFT ASOF JOIN`, joining sequences with a non-exact match. `ASOF JOIN` usage is described below.
+- `PASTE JOIN`, performs a horizontal concatenation of two tables.
 
 :::note
 When [join_algorithm](../../../operations/settings/settings.md#join_algorithm) is set to `partial_merge`, `RIGHT JOIN` and `FULL JOIN` are supported only with `ALL` strictness (`SEMI`, `ANTI`, `ANY`, and `ASOF` are not supported).
@@ -269,6 +270,33 @@ For example, consider the following tables:
 `ASOF` join is **not** supported in the [Join](../../../engines/table-engines/special/join.md) table engine.
 :::
 
+## PASTE JOIN Usage
+
+The result of `PASTE JOIN` is a table that contains all columns from left subquery followed by all columns from the right subquery.
+The rows are matched based on their positions in the original tables (the order of rows should be defined). 
+If the subqueries return a different number of rows, extra rows will be cut.
+
+Example:
+```SQL
+SELECT *
+FROM
+(
+    SELECT number AS a
+    FROM numbers(2)
+) AS t1
+PASTE JOIN
+(
+    SELECT number AS a
+    FROM numbers(2)
+    ORDER BY a DESC
+) AS t2
+
+┌─a─┬─t2.a─┐
+│ 0 │    1 │
+│ 1 │    0 │
+└───┴──────┘
+```
+
 ## Distributed JOIN
 
 There are two ways to execute join involving distributed tables:

programs/benchmark/Benchmark.cpp

@@ -35,7 +35,6 @@
 #include <Common/StudentTTest.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/ErrorCodes.h>
-#include <filesystem>
 
 
 /** A tool for evaluating ClickHouse performance.

programs/format/Format.cpp

@@ -24,6 +24,7 @@
 #include <TableFunctions/registerTableFunctions.h>
 #include <Storages/StorageFactory.h>
 #include <Storages/registerStorages.h>
+#include <Storages/MergeTree/MergeTreeSettings.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <Formats/FormatFactory.h>
 #include <Formats/registerFormats.h>
@@ -32,6 +33,9 @@
 #pragma GCC diagnostic ignored "-Wunused-function"
 #pragma GCC diagnostic ignored "-Wmissing-declarations"
 
+extern const char * auto_time_zones[];
+
+
 namespace DB
 {
 namespace ErrorCodes
@@ -133,9 +137,25 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
 
             auto all_known_storage_names = StorageFactory::instance().getAllRegisteredNames();
             auto all_known_data_type_names = DataTypeFactory::instance().getAllRegisteredNames();
+            auto all_known_settings = Settings().getAllRegisteredNames();
+            auto all_known_merge_tree_settings = MergeTreeSettings().getAllRegisteredNames();
 
             additional_names.insert(all_known_storage_names.begin(), all_known_storage_names.end());
             additional_names.insert(all_known_data_type_names.begin(), all_known_data_type_names.end());
+            additional_names.insert(all_known_settings.begin(), all_known_settings.end());
+            additional_names.insert(all_known_merge_tree_settings.begin(), all_known_merge_tree_settings.end());
+
+            for (auto * it = auto_time_zones; *it; ++it)
+            {
+                String time_zone_name = *it;
+
+                /// Example: Europe/Amsterdam
+                Strings split;
+                boost::split(split, time_zone_name, [](char c){ return c == '/'; });
+                for (const auto & word : split)
+                    if (!word.empty())
+                        additional_names.insert(word);
+            }
 
             KnownIdentifierFunc is_known_identifier = [&](std::string_view name)
             {

src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp

@@ -4,6 +4,7 @@
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/IQueryTreeNode.h>
 #include <Analyzer/InDepthQueryTreeVisitor.h>
+#include <Analyzer/Utils.h>
 
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeEnum.h>
@@ -41,22 +42,6 @@ DataTypePtr getEnumType(const std::set<std::string> & string_values)
         return getDataEnumType<DataTypeEnum8>(string_values);
 }
 
-QueryTreeNodePtr createCastFunction(QueryTreeNodePtr from, DataTypePtr result_type, ContextPtr context)
-{
-    auto enum_literal = std::make_shared<ConstantValue>(result_type->getName(), std::make_shared<DataTypeString>());
-    auto enum_literal_node = std::make_shared<ConstantNode>(std::move(enum_literal));
-
-    auto cast_function = FunctionFactory::instance().get("_CAST", std::move(context));
-    QueryTreeNodes arguments{ std::move(from), std::move(enum_literal_node) };
-
-    auto function_node = std::make_shared<FunctionNode>("_CAST");
-    function_node->getArguments().getNodes() = std::move(arguments);
-
-    function_node->resolveAsFunction(cast_function->build(function_node->getArgumentColumns()));
-
-    return function_node;
-}
-
 /// if(arg1, arg2, arg3) will be transformed to if(arg1, _CAST(arg2, Enum...), _CAST(arg3, Enum...))
 /// where Enum is generated based on the possible values stored in string_values
 void changeIfArguments(

src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp

@@ -9,6 +9,8 @@
 #include <Analyzer/HashUtils.h>
 #include <Analyzer/Utils.h>
 
+#include <DataTypes/DataTypeLowCardinality.h>
+
 namespace DB
 {
 
@@ -323,8 +325,21 @@ private:
             /// Because we reduce the number of operands here by eliminating the same equality checks,
             /// the only situation we can end up here is we had AND check where all the equality checks are the same so we know the type is UInt8.
             /// Otherwise, we will have > 1 operands and we don't have to do anything.
-            assert(!function_node.getResultType()->isNullable() && and_operands[0]->getResultType()->equals(*function_node.getResultType()));
-            node = std::move(and_operands[0]);
+
+            auto operand_type = and_operands[0]->getResultType();
+            auto function_type = function_node.getResultType();
+            assert(!function_type->isNullable());
+            if (!function_type->equals(*operand_type))
+            {
+                /// Result of equality operator can be low cardinality, while AND always returns UInt8.
+                /// In that case we replace `(lc = 1) AND (lc = 1)` with `(lc = 1) AS UInt8`
+                assert(function_type->equals(*removeLowCardinality(operand_type)));
+                node = createCastFunction(std::move(and_operands[0]), function_type, getContext());
+            }
+            else
+            {
+                node = std::move(and_operands[0]);
+            }
             return;
         }
 
@@ -389,11 +404,14 @@ private:
                 continue;
             }
 
+            bool is_any_nullable = false;
             Tuple args;
             args.reserve(equals_functions.size());
             /// first we create tuple from RHS of equals functions
             for (const auto & equals : equals_functions)
             {
+                is_any_nullable |= equals->getResultType()->isNullable();
+
                 const auto * equals_function = equals->as<FunctionNode>();
                 assert(equals_function && equals_function->getFunctionName() == "equals");
 
@@ -421,8 +439,20 @@ private:
 
             in_function->getArguments().getNodes() = std::move(in_arguments);
             in_function->resolveAsFunction(in_function_resolver);
-
-            or_operands.push_back(std::move(in_function));
+            /** For `k :: UInt8`, expression `k = 1 OR k = NULL` with result type Nullable(UInt8)
+              * is replaced with `k IN (1, NULL)` with result type UInt8.
+              * Convert it back to Nullable(UInt8).
+              */
+            if (is_any_nullable && !in_function->getResultType()->isNullable())
+            {
+                auto nullable_result_type = std::make_shared<DataTypeNullable>(in_function->getResultType());
+                auto in_function_nullable = createCastFunction(std::move(in_function), std::move(nullable_result_type), getContext());
+                or_operands.push_back(std::move(in_function_nullable));
+            }
+            else
+            {
+                or_operands.push_back(std::move(in_function));
+            }
         }
 
         if (or_operands.size() == function_node.getArguments().getNodes().size())

src/Analyzer/Utils.cpp

@@ -667,4 +667,20 @@ NameSet collectIdentifiersFullNames(const QueryTreeNodePtr & node)
     return out;
 }
 
+QueryTreeNodePtr createCastFunction(QueryTreeNodePtr node, DataTypePtr result_type, ContextPtr context)
+{
+    auto enum_literal = std::make_shared<ConstantValue>(result_type->getName(), std::make_shared<DataTypeString>());
+    auto enum_literal_node = std::make_shared<ConstantNode>(std::move(enum_literal));
+
+    auto cast_function = FunctionFactory::instance().get("_CAST", std::move(context));
+    QueryTreeNodes arguments{ std::move(node), std::move(enum_literal_node) };
+
+    auto function_node = std::make_shared<FunctionNode>("_CAST");
+    function_node->getArguments().getNodes() = std::move(arguments);
+
+    function_node->resolveAsFunction(cast_function->build(function_node->getArgumentColumns()));
+
+    return function_node;
+}
+
 }

src/Analyzer/Utils.h

@@ -99,4 +99,7 @@ void rerunFunctionResolve(FunctionNode * function_node, ContextPtr context);
 /// Just collect all identifiers from query tree
 NameSet collectIdentifiersFullNames(const QueryTreeNodePtr & node);
 
+/// Wrap node into `_CAST` function
+QueryTreeNodePtr createCastFunction(QueryTreeNodePtr node, DataTypePtr result_type, ContextPtr context);
+
 }

src/Backups/BackupEntriesCollector.cpp

@@ -88,18 +88,19 @@ BackupEntriesCollector::BackupEntriesCollector(
     , read_settings(read_settings_)
     , context(context_)
     , on_cluster_first_sync_timeout(context->getConfigRef().getUInt64("backups.on_cluster_first_sync_timeout", 180000))
-    , collect_metadata_timeout(context->getConfigRef().getUInt64("backups.collect_metadata_timeout", context->getConfigRef().getUInt64("backups.consistent_metadata_snapshot_timeout", 600000)))
+    , collect_metadata_timeout(context->getConfigRef().getUInt64(
+          "backups.collect_metadata_timeout", context->getConfigRef().getUInt64("backups.consistent_metadata_snapshot_timeout", 600000)))
     , attempts_to_collect_metadata_before_sleep(context->getConfigRef().getUInt("backups.attempts_to_collect_metadata_before_sleep", 2))
-    , min_sleep_before_next_attempt_to_collect_metadata(context->getConfigRef().getUInt64("backups.min_sleep_before_next_attempt_to_collect_metadata", 100))
-    , max_sleep_before_next_attempt_to_collect_metadata(context->getConfigRef().getUInt64("backups.max_sleep_before_next_attempt_to_collect_metadata", 5000))
+    , min_sleep_before_next_attempt_to_collect_metadata(
+          context->getConfigRef().getUInt64("backups.min_sleep_before_next_attempt_to_collect_metadata", 100))
+    , max_sleep_before_next_attempt_to_collect_metadata(
+          context->getConfigRef().getUInt64("backups.max_sleep_before_next_attempt_to_collect_metadata", 5000))
     , compare_collected_metadata(context->getConfigRef().getBool("backups.compare_collected_metadata", true))
     , log(&Poco::Logger::get("BackupEntriesCollector"))
     , global_zookeeper_retries_info(
-        "BackupEntriesCollector",
-        log,
-        context->getSettingsRef().backup_restore_keeper_max_retries,
-        context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms,
-        context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms)
+          context->getSettingsRef().backup_restore_keeper_max_retries,
+          context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms,
+          context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms)
     , threadpool(threadpool_)
 {
 }
@@ -572,7 +573,7 @@ std::vector<std::pair<ASTPtr, StoragePtr>> BackupEntriesCollector::findTablesInD
     {
         /// Database or table could be replicated - so may use ZooKeeper. We need to retry.
         auto zookeeper_retries_info = global_zookeeper_retries_info;
-        ZooKeeperRetriesControl retries_ctl("getTablesForBackup", zookeeper_retries_info, nullptr);
+        ZooKeeperRetriesControl retries_ctl("getTablesForBackup", log, zookeeper_retries_info, nullptr);
         retries_ctl.retryLoop([&](){ db_tables = database->getTablesForBackup(filter_by_table_name, context); });
     }
     catch (Exception & e)

src/Backups/BackupImpl.cpp

@@ -157,11 +157,16 @@ BackupImpl::~BackupImpl()
 void BackupImpl::open()
 {
     std::lock_guard lock{mutex};
-    LOG_INFO(log, "{} backup: {}", ((open_mode == OpenMode::WRITE) ? "Writing" : "Reading"), backup_name_for_logging);
-    ProfileEvents::increment((open_mode == OpenMode::WRITE) ? ProfileEvents::BackupsOpenedForWrite : ProfileEvents::BackupsOpenedForRead);
 
-    if (open_mode == OpenMode::WRITE)
+    if (open_mode == OpenMode::READ)
     {
+        ProfileEvents::increment(ProfileEvents::BackupsOpenedForRead);
+        LOG_INFO(log, "Reading backup: {}", backup_name_for_logging);
+    }
+    else
+    {
+        ProfileEvents::increment(ProfileEvents::BackupsOpenedForWrite);
+        LOG_INFO(log, "Writing backup: {}", backup_name_for_logging);
         timestamp = std::time(nullptr);
         if (!uuid)
             uuid = UUIDHelpers::generateV4();
@@ -189,7 +194,7 @@ void BackupImpl::open()
 void BackupImpl::close()
 {
     std::lock_guard lock{mutex};
-    closeArchive();
+    closeArchive(/* finalize= */ false);
 
     if (!is_internal_backup && writer && !writing_finalized)
         removeAllFilesAfterFailure();
@@ -222,8 +227,11 @@ void BackupImpl::openArchive()
     }
 }
 
-void BackupImpl::closeArchive()
+void BackupImpl::closeArchive(bool finalize)
 {
+    if (finalize && archive_writer)
+        archive_writer->finalize();
+
     archive_reader.reset();
     archive_writer.reset();
 }
@@ -978,7 +986,7 @@ void BackupImpl::finalizeWriting()
     {
         LOG_TRACE(log, "Finalizing backup {}", backup_name_for_logging);
         writeBackupMetadata();
-        closeArchive();
+        closeArchive(/* finalize= */ true);
         setCompressedSize();
         removeLockFile();
         LOG_TRACE(log, "Finalized backup {}", backup_name_for_logging);

src/Backups/BackupImpl.h

@@ -89,7 +89,7 @@ private:
     void close();
 
     void openArchive();
-    void closeArchive();
+    void closeArchive(bool finalize);
 
     /// Writes the file ".backup" containing backup's metadata.
     void writeBackupMetadata() TSA_REQUIRES(mutex);

src/Backups/BackupOperationInfo.h

@@ -17,6 +17,9 @@ struct BackupOperationInfo
     /// Operation name, a string like "Disk('backups', 'my_backup')"
     String name;
 
+    /// Base Backup Operation name, a string like "Disk('backups', 'my_base_backup')"
+    String base_backup_name;
+
     /// This operation is internal and should not be shown in system.backups
     bool internal = false;
 

src/Backups/BackupsWorker.cpp

@@ -394,9 +394,13 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
 
     auto backup_info = BackupInfo::fromAST(*backup_query->backup_name);
     String backup_name_for_logging = backup_info.toStringForLogging();
+    String base_backup_name;
+    if (backup_settings.base_backup_info)
+        base_backup_name = backup_settings.base_backup_info->toString();
+
     try
     {
-        addInfo(backup_id, backup_name_for_logging, backup_settings.internal, BackupStatus::CREATING_BACKUP);
+        addInfo(backup_id, backup_name_for_logging, base_backup_name, backup_settings.internal, BackupStatus::CREATING_BACKUP);
 
         /// Prepare context to use.
         ContextPtr context_in_use = context;
@@ -745,8 +749,11 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
     {
         auto backup_info = BackupInfo::fromAST(*restore_query->backup_name);
         String backup_name_for_logging = backup_info.toStringForLogging();
+        String base_backup_name;
+        if (restore_settings.base_backup_info)
+            base_backup_name = restore_settings.base_backup_info->toString();
 
-        addInfo(restore_id, backup_name_for_logging, restore_settings.internal, BackupStatus::RESTORING);
+        addInfo(restore_id, backup_name_for_logging, base_backup_name, restore_settings.internal, BackupStatus::RESTORING);
 
         /// Prepare context to use.
         ContextMutablePtr context_in_use = context;
@@ -1005,11 +1012,12 @@ void BackupsWorker::restoreTablesData(const OperationID & restore_id, BackupPtr
 }
 
 
-void BackupsWorker::addInfo(const OperationID & id, const String & name, bool internal, BackupStatus status)
+void BackupsWorker::addInfo(const OperationID & id, const String & name, const String & base_backup_name, bool internal, BackupStatus status)
 {
     BackupOperationInfo info;
     info.id = id;
     info.name = name;
+    info.base_backup_name = base_backup_name;
     info.internal = internal;
     info.status = status;
     info.start_time = std::chrono::system_clock::now();

src/Backups/BackupsWorker.h

@@ -83,7 +83,7 @@ private:
     /// Run data restoring tasks which insert data to tables.
     void restoreTablesData(const BackupOperationID & restore_id, BackupPtr backup, DataRestoreTasks && tasks, ThreadPool & thread_pool);
 
-    void addInfo(const BackupOperationID & id, const String & name, bool internal, BackupStatus status);
+    void addInfo(const BackupOperationID & id, const String & name, const String & base_backup_name, bool internal, BackupStatus status);
     void setStatus(const BackupOperationID & id, BackupStatus status, bool throw_if_error = true);
     void setStatusSafe(const String & id, BackupStatus status) { setStatus(id, status, false); }
     void setNumFilesAndSize(const BackupOperationID & id, size_t num_files, UInt64 total_size, size_t num_entries,

src/Backups/RestorerFromBackup.cpp

@@ -43,14 +43,6 @@ namespace Stage = BackupCoordinationStage;
 
 namespace
 {
-    /// Uppercases the first character of a passed string.
-    String toUpperFirst(const String & str)
-    {
-        String res = str;
-        res[0] = std::toupper(res[0]);
-        return res;
-    }
-
     /// Outputs "table <name>" or "temporary table <name>"
     String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_upper)
     {
@@ -145,7 +137,7 @@ RestorerFromBackup::DataRestoreTasks RestorerFromBackup::run(Mode mode)
 
 void RestorerFromBackup::setStage(const String & new_stage, const String & message)
 {
-    LOG_TRACE(log, fmt::runtime(toUpperFirst(new_stage)));
+    LOG_TRACE(log, "Setting stage: {}", new_stage);
     current_stage = new_stage;
 
     if (restore_coordination)

src/Backups/WithRetries.cpp

@@ -20,22 +20,19 @@ WithRetries::KeeperSettings WithRetries::KeeperSettings::fromContext(ContextPtr
     };
 }
 
-WithRetries::WithRetries(Poco::Logger * log_, zkutil::GetZooKeeper get_zookeeper_, const KeeperSettings & settings_, RenewerCallback callback_)
+WithRetries::WithRetries(
+    Poco::Logger * log_, zkutil::GetZooKeeper get_zookeeper_, const KeeperSettings & settings_, RenewerCallback callback_)
     : log(log_)
     , get_zookeeper(get_zookeeper_)
     , settings(settings_)
     , callback(callback_)
     , global_zookeeper_retries_info(
-        log->name(),
-        log,
-        settings.keeper_max_retries,
-        settings.keeper_retry_initial_backoff_ms,
-        settings.keeper_retry_max_backoff_ms)
+          settings.keeper_max_retries, settings.keeper_retry_initial_backoff_ms, settings.keeper_retry_max_backoff_ms)
 {}
 
 WithRetries::RetriesControlHolder::RetriesControlHolder(const WithRetries * parent, const String & name)
     : info(parent->global_zookeeper_retries_info)
-    , retries_ctl(name, info, nullptr)
+    , retries_ctl(name, parent->log, info, nullptr)
     , faulty_zookeeper(parent->getFaultyZooKeeper())
 {}
 

src/Client/Suggest.cpp

@@ -48,7 +48,7 @@ Suggest::Suggest()
               "GRANT",        "REVOKE",        "OPTION",       "ADMIN",      "EXCEPT",   "REPLACE",     "IDENTIFIED", "HOST",
               "NAME",         "READONLY",      "WRITABLE",     "PERMISSIVE", "FOR",      "RESTRICTIVE", "RANDOMIZED", "INTERVAL",
               "LIMITS",       "ONLY",          "TRACKING",     "IP",         "REGEXP",   "ILIKE",       "CLEANUP",    "APPEND",
-              "IGNORE NULLS", "RESPECT NULLS", "OVER"});
+              "IGNORE NULLS", "RESPECT NULLS", "OVER",         "PASTE"});
 }
 
 static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggestion)

src/Common/AsynchronousMetrics.cpp

@@ -8,6 +8,7 @@
 #include <IO/MMappedFileCache.h>
 #include <IO/ReadHelpers.h>
 #include <base/errnoToString.h>
+#include <sys/resource.h>
 #include <chrono>
 
 #include "config.h"
@@ -655,6 +656,19 @@ void AsynchronousMetrics::update(TimePoint update_time)
             total_memory_tracker.setRSS(rss, free_memory_in_allocator_arenas);
         }
     }
+
+    {
+        struct rusage rusage{};
+        if (!getrusage(RUSAGE_SELF, &rusage))
+        {
+            new_values["MemoryResidentMax"] = { rusage.ru_maxrss * 1024 /* KiB -> bytes */,
+                "Maximum amount of physical memory used by the server process, in bytes." };
+        }
+        else
+        {
+            LOG_ERROR(log, "Cannot obtain resource usage: {}", errnoToString(errno));
+        }
+    }
 #endif
 
 #if defined(OS_LINUX)

src/Common/FieldVisitorToString.cpp

@@ -18,16 +18,37 @@ template <typename T>
 static inline String formatQuoted(T x)
 {
     WriteBufferFromOwnString wb;
-    writeQuoted(x, wb);
-    return wb.str();
-}
 
-template <typename T>
-static inline void writeQuoted(const DecimalField<T> & x, WriteBuffer & buf)
-{
-    writeChar('\'', buf);
-    writeText(x.getValue(), x.getScale(), buf, {});
-    writeChar('\'', buf);
+    if constexpr (is_decimal_field<T>)
+    {
+        writeChar('\'', wb);
+        writeText(x.getValue(), x.getScale(), wb, {});
+        writeChar('\'', wb);
+    }
+    else if constexpr (is_big_int_v<T>)
+    {
+        writeChar('\'', wb);
+        writeText(x, wb);
+        writeChar('\'', wb);
+    }
+    else
+    {
+        /// While `writeQuoted` sounds like it will always write the value in quotes,
+        /// in fact it means: write according to the rules of the quoted format, like VALUES,
+        /// where strings, dates, date-times, UUID are in quotes, and numbers are not.
+
+        /// That's why we take extra care to put Decimal and big integers inside quotes
+        /// when formatting literals in SQL language,
+        /// because it is different from the quoted formats like VALUES.
+
+        /// In fact, there are no Decimal and big integer literals in SQL,
+        /// but they can appear if we format the query from a modified AST.
+
+        /// We can fix this idiosyncrasy later.
+
+        writeQuoted(x, wb);
+    }
+    return wb.str();
 }
 
 /** In contrast to writeFloatText (and writeQuoted),

src/Common/ProfileEvents.cpp

@@ -250,9 +250,9 @@ Number of times data after merge is not byte-identical to the data on another re
 7. Manual modification of source data after server startup.
 8. Manual modification of checksums stored in ZooKeeper.
 9. Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas.
-The server successfully detected this situation and will download merged part from replica to force byte-identical result.
+The server successfully detected this situation and will download merged part from the replica to force the byte-identical result.
 )") \
-    M(DataAfterMutationDiffersFromReplica, "Number of times data after mutation is not byte-identical to the data on another replicas. In addition to the reasons described in 'DataAfterMergeDiffersFromReplica', it is also possible due to non-deterministic mutation.") \
+    M(DataAfterMutationDiffersFromReplica, "Number of times data after mutation is not byte-identical to the data on other replicas. In addition to the reasons described in 'DataAfterMergeDiffersFromReplica', it is also possible due to non-deterministic mutation.") \
     M(PolygonsAddedToPool, "A polygon has been added to the cache (pool) for the 'pointInPolygon' function.") \
     M(PolygonsInPoolAllocatedBytes, "The number of bytes for polygons added to the cache (pool) for the 'pointInPolygon' function.") \
     \
@@ -272,12 +272,12 @@ The server successfully detected this situation and will download merged part fr
     M(PartsLockWaitMicroseconds, "Total time spent waiting for data parts lock in MergeTree tables") \
     \
     M(RealTimeMicroseconds, "Total (wall clock) time spent in processing (queries and other tasks) threads (note that this is a sum).") \
-    M(UserTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in user mode. This include time CPU pipeline was stalled due to main memory access, cache misses, branch mispredictions, hyper-threading, etc.") \
+    M(UserTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in user mode. This includes time CPU pipeline was stalled due to main memory access, cache misses, branch mispredictions, hyper-threading, etc.") \
     M(SystemTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in OS kernel mode. This is time spent in syscalls, excluding waiting time during blocking syscalls.") \
     M(MemoryOvercommitWaitTimeMicroseconds, "Total time spent in waiting for memory to be freed in OvercommitTracker.") \
     M(MemoryAllocatorPurge, "Total number of times memory allocator purge was requested") \
     M(MemoryAllocatorPurgeTimeMicroseconds, "Total number of times memory allocator purge was requested") \
-    M(SoftPageFaults, "The number of soft page faults in query execution threads. Soft page fault usually means a miss in the memory allocator cache which required a new memory mapping from the OS and subsequent allocation of a page of physical memory.") \
+    M(SoftPageFaults, "The number of soft page faults in query execution threads. Soft page fault usually means a miss in the memory allocator cache, which requires a new memory mapping from the OS and subsequent allocation of a page of physical memory.") \
     M(HardPageFaults, "The number of hard page faults in query execution threads. High values indicate either that you forgot to turn off swap on your server, or eviction of memory pages of the ClickHouse binary during very high memory pressure, or successful usage of the 'mmap' read method for the tables data.") \
     \
     M(OSIOWaitMicroseconds, "Total time a thread spent waiting for a result of IO operation, from the OS point of view. This is real IO that doesn't include page cache.") \
@@ -290,8 +290,8 @@ The server successfully detected this situation and will download merged part fr
     \
     M(PerfCpuCycles, "Total cycles. Be wary of what happens during CPU frequency scaling.")  \
     M(PerfInstructions, "Retired instructions. Be careful, these can be affected by various issues, most notably hardware interrupt counts.") \
-    M(PerfCacheReferences, "Cache accesses. Usually this indicates Last Level Cache accesses but this may vary depending on your CPU. This may include prefetches and coherency messages; again this depends on the design of your CPU.") \
-    M(PerfCacheMisses, "Cache misses. Usually this indicates Last Level Cache misses; this is intended to be used in con‐junction with the PERFCOUNTHWCACHEREFERENCES event to calculate cache miss rates.") \
+    M(PerfCacheReferences, "Cache accesses. Usually, this indicates Last Level Cache accesses, but this may vary depending on your CPU. This may include prefetches and coherency messages; again this depends on the design of your CPU.") \
+    M(PerfCacheMisses, "Cache misses. Usually this indicates Last Level Cache misses; this is intended to be used in conjunction with the PERFCOUNTHWCACHEREFERENCES event to calculate cache miss rates.") \
     M(PerfBranchInstructions, "Retired branch instructions. Prior to Linux 2.6.35, this used the wrong event on AMD processors.") \
     M(PerfBranchMisses, "Mispredicted branch instructions.") \
     M(PerfBusCycles, "Bus cycles, which can be different from total cycles.") \
@@ -457,7 +457,7 @@ The server successfully detected this situation and will download merged part fr
     M(FileSegmentWaitReadBufferMicroseconds, "Metric per file segment. Time spend waiting for internal read buffer (includes cache waiting)") \
     M(FileSegmentReadMicroseconds, "Metric per file segment. Time spend reading from file") \
     M(FileSegmentCacheWriteMicroseconds, "Metric per file segment. Time spend writing data to cache") \
-    M(FileSegmentPredownloadMicroseconds, "Metric per file segment. Time spent predownloading data to cache (predownloading - finishing file segment download (after someone who failed to do that) up to the point current thread was requested to do)") \
+    M(FileSegmentPredownloadMicroseconds, "Metric per file segment. Time spent pre-downloading data to cache (pre-downloading - finishing file segment download (after someone who failed to do that) up to the point current thread was requested to do)") \
     M(FileSegmentUsedBytes, "Metric per file segment. How many bytes were actually used from current file segment") \
     \
     M(ReadBufferSeekCancelConnection, "Number of seeks which lead to new connection (s3, http)") \
@@ -466,12 +466,12 @@ The server successfully detected this situation and will download merged part fr
     M(SleepFunctionMicroseconds, "Time set to sleep in a sleep function (sleep, sleepEachRow).") \
     M(SleepFunctionElapsedMicroseconds, "Time spent sleeping in a sleep function (sleep, sleepEachRow).") \
     \
-    M(ThreadPoolReaderPageCacheHit, "Number of times the read inside ThreadPoolReader was done from page cache.") \
-    M(ThreadPoolReaderPageCacheHitBytes, "Number of bytes read inside ThreadPoolReader when it was done from page cache.") \
+    M(ThreadPoolReaderPageCacheHit, "Number of times the read inside ThreadPoolReader was done from the page cache.") \
+    M(ThreadPoolReaderPageCacheHitBytes, "Number of bytes read inside ThreadPoolReader when it was done from the page cache.") \
     M(ThreadPoolReaderPageCacheHitElapsedMicroseconds, "Time spent reading data from page cache in ThreadPoolReader.") \
     M(ThreadPoolReaderPageCacheMiss, "Number of times the read inside ThreadPoolReader was not done from page cache and was hand off to thread pool.") \
     M(ThreadPoolReaderPageCacheMissBytes, "Number of bytes read inside ThreadPoolReader when read was not done from page cache and was hand off to thread pool.") \
-    M(ThreadPoolReaderPageCacheMissElapsedMicroseconds, "Time spent reading data inside the asynchronous job in ThreadPoolReader - when read was not done from page cache.") \
+    M(ThreadPoolReaderPageCacheMissElapsedMicroseconds, "Time spent reading data inside the asynchronous job in ThreadPoolReader - when read was not done from the page cache.") \
     \
     M(AsynchronousReadWaitMicroseconds, "Time spent in waiting for asynchronous reads in asynchronous local read.") \
     M(SynchronousReadWaitMicroseconds, "Time spent in waiting for synchronous reads in asynchronous local read.") \
@@ -512,7 +512,7 @@ The server successfully detected this situation and will download merged part fr
     M(SchemaInferenceCacheSchemaHits, "Number of times the schema is found in schema cache during schema inference") \
     M(SchemaInferenceCacheNumRowsHits, "Number of times the number of rows is found in schema cache during count from files") \
     M(SchemaInferenceCacheMisses, "Number of times the requested source is not in schema cache") \
-    M(SchemaInferenceCacheSchemaMisses, "Number of times the requested source is in cache but the schema is not in cache while schema inference") \
+    M(SchemaInferenceCacheSchemaMisses, "Number of times the requested source is in cache but the schema is not in cache during schema inference") \
     M(SchemaInferenceCacheNumRowsMisses, "Number of times the requested source is in cache but the number of rows is not in cache while count from files") \
     M(SchemaInferenceCacheEvictions, "Number of times a schema from cache was evicted due to overflow") \
     M(SchemaInferenceCacheInvalidations, "Number of times a schema in cache became invalid due to changes in data") \
@@ -570,7 +570,7 @@ The server successfully detected this situation and will download merged part fr
     \
     M(ReadTaskRequestsSent, "The number of callbacks requested from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the remote server side.") \
     M(MergeTreeReadTaskRequestsSent, "The number of callbacks requested from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the remote server side.") \
-    M(MergeTreeAllRangesAnnouncementsSent, "The number of announcement sent from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.") \
+    M(MergeTreeAllRangesAnnouncementsSent, "The number of announcements sent from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.") \
     M(ReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for s3Cluster table function and similar). Measured on the remote server side.") \
     M(MergeTreeReadTaskRequestsSentElapsedMicroseconds, "Time spent in callbacks requested from the remote server back to the initiator server to choose the read task (for MergeTree tables). Measured on the remote server side.") \
     M(MergeTreeAllRangesAnnouncementsSentElapsedMicroseconds, "Time spent in sending the announcement from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.") \
@@ -586,6 +586,8 @@ The server successfully detected this situation and will download merged part fr
     M(LogWarning, "Number of log messages with level Warning") \
     M(LogError, "Number of log messages with level Error") \
     M(LogFatal, "Number of log messages with level Fatal") \
+    \
+    M(ParallelReplicasUsedCount, "Number of replicas used to execute a query with task-based parallel replicas") \
 
 #ifdef APPLY_FOR_EXTERNAL_EVENTS
     #define APPLY_FOR_EVENTS(M) APPLY_FOR_BUILTIN_EVENTS(M) APPLY_FOR_EXTERNAL_EVENTS(M)

src/Common/SystemLogBase.cpp

@@ -188,6 +188,9 @@ typename SystemLogQueue<LogElement>::Index SystemLogQueue<LogElement>::pop(std::
                                                                            bool & should_prepare_tables_anyway,
                                                                            bool & exit_this_thread)
 {
+    /// Call dtors and deallocate strings without holding the global lock
+    output.resize(0);
+
     std::unique_lock lock(mutex);
     flush_event.wait_for(lock,
         std::chrono::milliseconds(settings.flush_interval_milliseconds),
@@ -200,7 +203,6 @@ typename SystemLogQueue<LogElement>::Index SystemLogQueue<LogElement>::pop(std::
     queue_front_index += queue.size();
     // Swap with existing array from previous flush, to save memory
     // allocations.
-    output.resize(0);
     queue.swap(output);
 
     should_prepare_tables_anyway = is_force_prepare_tables;

src/Core/Joins.cpp

@@ -13,6 +13,7 @@ const char * toString(JoinKind kind)
         case JoinKind::Full: return "FULL";
         case JoinKind::Cross: return "CROSS";
         case JoinKind::Comma: return "COMMA";
+        case JoinKind::Paste: return "PASTE";
     }
 };
 

src/Core/Joins.h

@@ -13,7 +13,8 @@ enum class JoinKind
     Right,
     Full,
     Cross, /// Direct product. Strictness and condition doesn't matter.
-    Comma /// Same as direct product. Intended to be converted to INNER JOIN with conditions from WHERE.
+    Comma, /// Same as direct product. Intended to be converted to INNER JOIN with conditions from WHERE.
+    Paste, /// Used to join parts without `ON` clause.
 };
 
 const char * toString(JoinKind kind);
@@ -27,6 +28,7 @@ inline constexpr bool isRightOrFull(JoinKind kind)  { return kind == JoinKind::R
 inline constexpr bool isLeftOrFull(JoinKind kind)   { return kind == JoinKind::Left  || kind == JoinKind::Full; }
 inline constexpr bool isInnerOrRight(JoinKind kind) { return kind == JoinKind::Inner || kind == JoinKind::Right; }
 inline constexpr bool isInnerOrLeft(JoinKind kind)  { return kind == JoinKind::Inner || kind == JoinKind::Left; }
+inline constexpr bool isPaste(JoinKind kind)        { return kind == JoinKind::Paste; }
 
 /// Allows more optimal JOIN for typical cases.
 enum class JoinStrictness

src/Core/Settings.cpp

@@ -107,9 +107,7 @@ std::vector<String> Settings::getAllRegisteredNames() const
 {
     std::vector<String> all_settings;
     for (const auto & setting_field : all())
-    {
         all_settings.push_back(setting_field.getName());
-    }
     return all_settings;
 }
 

src/Disks/DiskLocal.cpp

@@ -373,7 +373,7 @@ void DiskLocal::removeDirectory(const String & path)
 {
     auto fs_path = fs::path(disk_path) / path;
     if (0 != rmdir(fs_path.c_str()))
-        ErrnoException::throwFromPath(ErrorCodes::CANNOT_RMDIR, fs_path, "Cannot rmdir {}", fs_path);
+        ErrnoException::throwFromPath(ErrorCodes::CANNOT_RMDIR, fs_path, "Cannot remove directory {}", fs_path);
 }
 
 void DiskLocal::removeRecursive(const String & path)

src/Formats/JSONUtils.cpp

@@ -24,7 +24,6 @@ namespace ErrorCodes
 
 namespace JSONUtils
 {
-
     template <const char opening_bracket, const char closing_bracket>
     static std::pair<bool, size_t>
     fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, DB::Memory<> & memory, size_t min_bytes, size_t min_rows, size_t max_rows)
@@ -72,7 +71,7 @@ namespace JSONUtils
             }
             else
             {
-                pos = find_first_symbols<opening_bracket, closing_bracket, '\\', '"'>(pos, in.buffer().end());
+                pos = find_first_symbols<opening_bracket, closing_bracket, '"'>(pos, in.buffer().end());
 
                 if (pos > in.buffer().end())
                     throw Exception(ErrorCodes::LOGICAL_ERROR, "Position in buffer is out of bounds. There must be a bug.");
@@ -89,19 +88,13 @@ namespace JSONUtils
                     --balance;
                     ++pos;
                 }
-                else if (*pos == '\\')
-                {
-                    ++pos;
-                    if (loadAtPosition(in, memory, pos))
-                        ++pos;
-                }
                 else if (*pos == '"')
                 {
                     quotes = true;
                     ++pos;
                 }
 
-                if (balance == 0)
+                if (!quotes && balance == 0)
                 {
                     ++number_of_rows;
                     if ((number_of_rows >= min_rows)
@@ -115,13 +108,14 @@ namespace JSONUtils
         return {loadAtPosition(in, memory, pos), number_of_rows};
     }
 
-    std::pair<bool, size_t> fileSegmentationEngineJSONEachRow(ReadBuffer & in, DB::Memory<> & memory, size_t min_bytes, size_t max_rows)
+    std::pair<bool, size_t> fileSegmentationEngineJSONEachRow(
+        ReadBuffer & in, DB::Memory<> & memory, size_t min_bytes, size_t max_rows)
     {
         return fileSegmentationEngineJSONEachRowImpl<'{', '}'>(in, memory, min_bytes, 1, max_rows);
     }
 
-    std::pair<bool, size_t>
-    fileSegmentationEngineJSONCompactEachRow(ReadBuffer & in, DB::Memory<> & memory, size_t min_bytes, size_t min_rows, size_t max_rows)
+    std::pair<bool, size_t> fileSegmentationEngineJSONCompactEachRow(
+        ReadBuffer & in, DB::Memory<> & memory, size_t min_bytes, size_t min_rows, size_t max_rows)
     {
         return fileSegmentationEngineJSONEachRowImpl<'[', ']'>(in, memory, min_bytes, min_rows, max_rows);
     }

src/Formats/registerFormats.cpp

@@ -14,7 +14,6 @@ void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory);
 void registerFileSegmentationEngineRegexp(FormatFactory & factory);
 void registerFileSegmentationEngineJSONAsString(FormatFactory & factory);
 void registerFileSegmentationEngineJSONAsObject(FormatFactory & factory);
-void registerFileSegmentationEngineJSONCompactEachRow(FormatFactory & factory);
 #if USE_HIVE
 void registerFileSegmentationEngineHiveText(FormatFactory & factory);
 #endif
@@ -161,7 +160,6 @@ void registerFormats()
     registerFileSegmentationEngineJSONEachRow(factory);
     registerFileSegmentationEngineJSONAsString(factory);
     registerFileSegmentationEngineJSONAsObject(factory);
-    registerFileSegmentationEngineJSONCompactEachRow(factory);
 #if USE_HIVE
     registerFileSegmentationEngineHiveText(factory);
 #endif
@@ -294,4 +292,3 @@ void registerFormats()
 }
 
 }
-

src/Functions/FunctionsConversion.h

@@ -1413,10 +1413,10 @@ inline bool tryParseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuff
 template <>
 inline bool tryParseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool)
 {
-    time_t tmp = 0;
-    if (!tryReadDateTimeText(tmp, rb, *time_zone))
+    time_t time = 0;
+    if (!tryReadDateTimeText(time, rb, *time_zone))
         return false;
-    x = static_cast<UInt32>(tmp);
+    convertFromTime<DataTypeDateTime>(x, time);
     return true;
 }
 
@@ -1697,7 +1697,6 @@ struct ConvertThroughParsing
                                     break;
                                 }
                             }
-
                             parseImpl<ToDataType>(vec_to[i], read_buffer, local_time_zone, precise_float_parsing);
                         } while (false);
                     }
@@ -3291,7 +3290,6 @@ private:
         {
             /// In case when converting to Nullable type, we apply different parsing rule,
             /// that will not throw an exception but return NULL in case of malformed input.
-
             FunctionPtr function = FunctionConvertFromString<ToDataType, FunctionName, ConvertFromStringExceptionMode::Null>::create();
             return createFunctionAdaptor(function, from_type);
         }

src/Functions/FunctionsStringHashFixedString.cpp

@@ -172,7 +172,7 @@ struct SHA512Impl256
         /// SSL library that we use, for S390X architecture only OpenSSL is supported. But the SHA512-256, SHA512_256_Init,
         /// SHA512_256_Update, SHA512_256_Final methods to calculate hash (similar to the other SHA functions) aren't available
         /// in the current version of OpenSSL that we use which necessitates the use of the EVP interface.
-        auto md_ctx = EVP_MD_CTX_create();
+        auto * md_ctx = EVP_MD_CTX_create();
         EVP_DigestInit_ex(md_ctx, EVP_sha512_256(), nullptr /*engine*/);
         EVP_DigestUpdate(md_ctx, begin, size);
         EVP_DigestFinal_ex(md_ctx, out_char_data, nullptr /*size*/);

src/Functions/concat.cpp

@@ -145,13 +145,13 @@ private:
                 }
                 write_helper.finalize();
 
-                /// Same as the normal `ColumnString` branch
-                has_column_string = true;
-                data[i] = &converted_col_str->getChars();
-                offsets[i] = &converted_col_str->getOffsets();
-
                 /// Keep the pointer alive
                 converted_col_ptrs[i] = std::move(converted_col_str);
+
+                /// Same as the normal `ColumnString` branch
+                has_column_string = true;
+                data[i] = &converted_col_ptrs[i]->getChars();
+                offsets[i] = &converted_col_ptrs[i]->getOffsets();
             }
         }
 

src/Functions/format.cpp

@@ -108,13 +108,13 @@ public:
                 }
                 write_helper.finalize();
 
-                /// Same as the normal `ColumnString` branch
-                has_column_string = true;
-                data[i - 1] = &converted_col_str->getChars();
-                offsets[i - 1] = &converted_col_str->getOffsets();
-
                 /// Keep the pointer alive
                 converted_col_ptrs[i - 1] = std::move(converted_col_str);
+
+                /// Same as the normal `ColumnString` branch
+                has_column_string = true;
+                data[i - 1] = &converted_col_ptrs[i - 1]->getChars();
+                offsets[i - 1] = &converted_col_ptrs[i - 1]->getOffsets();
             }
         }
 

src/IO/Archives/IArchiveWriter.h

@@ -13,7 +13,7 @@ class WriteBufferFromFileBase;
 class IArchiveWriter : public std::enable_shared_from_this<IArchiveWriter>, boost::noncopyable
 {
 public:
-    /// Destructors finalizes writing the archive.
+    /// Call finalize() before destructing IArchiveWriter.
     virtual ~IArchiveWriter() = default;
 
     /// Starts writing a file to the archive. The function returns a write buffer,
@@ -26,6 +26,10 @@ public:
     /// This function should be used mostly for debugging purposes.
     virtual bool isWritingFile() const = 0;
 
+    /// Finalizes writing of the archive. This function must be always called at the end of writing.
+    /// (Unless an error appeared and the archive is in fact no longer needed.)
+    virtual void finalize() = 0;
+
     static constexpr const int kDefaultCompressionLevel = -1;
 
     /// Sets compression method and level.

src/IO/Archives/ZipArchiveWriter.cpp

@@ -15,86 +15,56 @@ namespace ErrorCodes
     extern const int CANNOT_PACK_ARCHIVE;
     extern const int SUPPORT_IS_DISABLED;
     extern const int LOGICAL_ERROR;
+    extern const int NOT_IMPLEMENTED;
 }
 
-using RawHandle = zipFile;
 
-
-/// Holds a raw handle, calls acquireRawHandle() in the constructor and releaseRawHandle() in the destructor.
-class ZipArchiveWriter::HandleHolder
+namespace
 {
-public:
-    HandleHolder() = default;
-
-    explicit HandleHolder(const std::shared_ptr<ZipArchiveWriter> & writer_) : writer(writer_), raw_handle(writer->acquireRawHandle()) { }
-
-    ~HandleHolder()
+    void checkResultCodeImpl(int code, const String & file_name)
     {
-        if (raw_handle)
+        if (code >= ZIP_OK)
+            return;
+
+        String message = "Code = ";
+        switch (code)
         {
-            try
-            {
-                int err = zipCloseFileInZip(raw_handle);
-                /// If err == ZIP_PARAMERROR the file is already closed.
-                if (err != ZIP_PARAMERROR)
-                    checkResult(err);
-            }
-            catch (...)
-            {
-                tryLogCurrentException("ZipArchiveWriter");
-            }
-            writer->releaseRawHandle(raw_handle);
+            case ZIP_ERRNO: message += "ERRNO, errno = " + errnoToString(); break;
+            case ZIP_PARAMERROR: message += "PARAMERROR"; break;
+            case ZIP_BADZIPFILE: message += "BADZIPFILE"; break;
+            case ZIP_INTERNALERROR: message += "INTERNALERROR"; break;
+            default: message += std::to_string(code); break;
         }
+        throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Couldn't pack zip archive: {}, filename={}", message, quoteString(file_name));
     }
-
-    HandleHolder(HandleHolder && src) noexcept
-    {
-        *this = std::move(src);
-    }
-
-    HandleHolder & operator=(HandleHolder && src) noexcept
-    {
-        writer = std::exchange(src.writer, nullptr);
-        raw_handle = std::exchange(src.raw_handle, nullptr);
-        return *this;
-    }
-
-    RawHandle getRawHandle() const { return raw_handle; }
-    std::shared_ptr<ZipArchiveWriter> getWriter() const { return writer; }
-
-    void checkResult(int code) const { writer->checkResult(code); }
-
-private:
-    std::shared_ptr<ZipArchiveWriter> writer;
-    RawHandle raw_handle = nullptr;
-};
+}
 
 
 /// This class represents a WriteBuffer actually returned by writeFile().
 class ZipArchiveWriter::WriteBufferFromZipArchive : public WriteBufferFromFileBase
 {
 public:
-    WriteBufferFromZipArchive(HandleHolder && handle_, const String & filename_)
+    WriteBufferFromZipArchive(std::shared_ptr<ZipArchiveWriter> archive_writer_, const String & filename_)
         : WriteBufferFromFileBase(DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0)
-        , handle(std::move(handle_))
         , filename(filename_)
     {
-        auto compress_method = handle.getWriter()->compression_method;
-        auto compress_level = handle.getWriter()->compression_level;
+        zip_handle = archive_writer_->startWritingFile();
+        archive_writer = archive_writer_;
+
+        auto compress_method = archive_writer_->getCompressionMethod();
+        auto compress_level = archive_writer_->getCompressionLevel();
         checkCompressionMethodIsEnabled(compress_method);
 
         const char * password_cstr = nullptr;
-        const String & password_str = handle.getWriter()->password;
-        if (!password_str.empty())
+        String current_password = archive_writer_->getPassword();
+        if (!current_password.empty())
         {
             checkEncryptionIsEnabled();
-            password_cstr = password_str.c_str();
+            password_cstr = current_password.c_str();
         }
 
-        RawHandle raw_handle = handle.getRawHandle();
-
-        checkResult(zipOpenNewFileInZip3_64(
-            raw_handle,
+        int code = zipOpenNewFileInZip3_64(
+            zip_handle,
             filename_.c_str(),
             /* zipfi= */ nullptr,
             /* extrafield_local= */ nullptr,
@@ -110,21 +80,30 @@ public:
             /* strategy= */ 0,
             password_cstr,
             /* crc_for_crypting= */ 0,
-            /* zip64= */ true));
+            /* zip64= */ true);
+        checkResultCode(code);
     }
 
     ~WriteBufferFromZipArchive() override
     {
         try
         {
-            finalize();
+            closeFile(/* throw_if_error= */ false);
+            endWritingFile();
         }
         catch (...)
         {
-            tryLogCurrentException("ZipArchiveWriter");
+            tryLogCurrentException("WriteBufferFromZipArchive");
         }
     }
 
+    void finalizeImpl() override
+    {
+        next();
+        closeFile(/* throw_if_error= */ true);
+        endWritingFile();
+    }
+
     void sync() override { next(); }
     std::string getFileName() const override { return filename; }
 
@@ -133,110 +112,106 @@ private:
     {
         if (!offset())
             return;
-        RawHandle raw_handle = handle.getRawHandle();
-        int code = zipWriteInFileInZip(raw_handle, working_buffer.begin(), static_cast<uint32_t>(offset()));
-        checkResult(code);
+        chassert(zip_handle);
+        int code = zipWriteInFileInZip(zip_handle, working_buffer.begin(), static_cast<uint32_t>(offset()));
+        checkResultCode(code);
     }
 
-    void checkResult(int code) const { handle.checkResult(code); }
+    void closeFile(bool throw_if_error)
+    {
+        if (zip_handle)
+        {
+            int code = zipCloseFileInZip(zip_handle);
+            zip_handle = nullptr;
+            if (throw_if_error)
+                checkResultCode(code);
+        }
+    }
 
-    HandleHolder handle;
-    String filename;
+    void endWritingFile()
+    {
+        if (auto archive_writer_ptr = archive_writer.lock())
+        {
+            archive_writer_ptr->endWritingFile();
+            archive_writer.reset();
+        }
+    }
+
+    void checkResultCode(int code) const { checkResultCodeImpl(code, filename); }
+
+    std::weak_ptr<ZipArchiveWriter> archive_writer;
+    const String filename;
+    ZipHandle zip_handle;
 };
 
 
-namespace
+/// Provides a set of functions allowing the minizip library to write its output
+/// to a WriteBuffer instead of an ordinary file in the local filesystem.
+class ZipArchiveWriter::StreamInfo
 {
-    /// Provides a set of functions allowing the minizip library to write its output
-    /// to a WriteBuffer instead of an ordinary file in the local filesystem.
-    class StreamFromWriteBuffer
+public:
+    explicit StreamInfo(std::unique_ptr<WriteBuffer> write_buffer_)
+        : write_buffer(std::move(write_buffer_)), start_offset(write_buffer->count())
     {
-    public:
-        static RawHandle open(std::unique_ptr<WriteBuffer> archive_write_buffer)
-        {
-            Opaque opaque{std::move(archive_write_buffer)};
+    }
 
-            zlib_filefunc64_def func_def;
-            func_def.zopen64_file = &StreamFromWriteBuffer::openFileFunc;
-            func_def.zclose_file = &StreamFromWriteBuffer::closeFileFunc;
-            func_def.zread_file = &StreamFromWriteBuffer::readFileFunc;
-            func_def.zwrite_file = &StreamFromWriteBuffer::writeFileFunc;
-            func_def.zseek64_file = &StreamFromWriteBuffer::seekFunc;
-            func_def.ztell64_file = &StreamFromWriteBuffer::tellFunc;
-            func_def.zerror_file = &StreamFromWriteBuffer::testErrorFunc;
-            func_def.opaque = &opaque;
+    ~StreamInfo() = default;
 
-            return zipOpen2_64(
-                /* path= */ nullptr,
-                /* append= */ false,
-                /* globalcomment= */ nullptr,
-                &func_def);
-        }
+    ZipHandle makeZipHandle()
+    {
+        zlib_filefunc64_def func_def;
+        func_def.zopen64_file = &StreamInfo::openFileFunc;
+        func_def.zclose_file = &StreamInfo::closeFileFunc;
+        func_def.zread_file = &StreamInfo::readFileFunc;
+        func_def.zwrite_file = &StreamInfo::writeFileFunc;
+        func_def.zseek64_file = &StreamInfo::seekFunc;
+        func_def.ztell64_file = &StreamInfo::tellFunc;
+        func_def.zerror_file = &StreamInfo::testErrorFunc;
+        func_def.opaque = this;
 
-    private:
-        std::unique_ptr<WriteBuffer> write_buffer;
-        UInt64 start_offset = 0;
+        return zipOpen2_64(
+            /* path= */ nullptr,
+            /* append= */ false,
+            /* globalcomment= */ nullptr,
+            &func_def);
+    }
 
-        struct Opaque
-        {
-            std::unique_ptr<WriteBuffer> write_buffer;
-        };
+    WriteBuffer & getWriteBuffer() { return *write_buffer; }
 
-        static void * openFileFunc(void * opaque, const void *, int)
-        {
-            Opaque & opq = *reinterpret_cast<Opaque *>(opaque);
-            return new StreamFromWriteBuffer(std::move(opq.write_buffer));
-        }
+private:
+    /// We do nothing in openFileFunc() and in closeFileFunc() because we already have `write_buffer` (file is already opened).
+    static void * openFileFunc(void * opaque, const void *, int) { return opaque; }
+    static int closeFileFunc(void *, void *) { return ZIP_OK; }
 
-        explicit StreamFromWriteBuffer(std::unique_ptr<WriteBuffer> write_buffer_)
-            : write_buffer(std::move(write_buffer_)), start_offset(write_buffer->count()) {}
+    static unsigned long writeFileFunc(void * opaque, void *, const void * buf, unsigned long size) // NOLINT(google-runtime-int)
+    {
+        auto * stream_info = reinterpret_cast<StreamInfo *>(opaque);
+        stream_info->write_buffer->write(reinterpret_cast<const char *>(buf), size);
+        return size;
+    }
 
-        ~StreamFromWriteBuffer()
-        {
-            write_buffer->finalize();
-        }
+    static int testErrorFunc(void *, void *) { return ZIP_OK; }
 
-        static int closeFileFunc(void *, void * stream)
-        {
-            delete reinterpret_cast<StreamFromWriteBuffer *>(stream);
-            return ZIP_OK;
-        }
+    static ZPOS64_T tellFunc(void * opaque, void *)
+    {
+        auto * stream_info = reinterpret_cast<StreamInfo *>(opaque);
+        auto pos = stream_info->write_buffer->count() - stream_info->start_offset;
+        return pos;
+    }
 
-        static StreamFromWriteBuffer & get(void * ptr)
-        {
-            return *reinterpret_cast<StreamFromWriteBuffer *>(ptr);
-        }
+    static long seekFunc(void *, void *, ZPOS64_T, int) // NOLINT(google-runtime-int)
+    {
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "StreamInfo::seek() is not implemented");
+    }
 
-        static unsigned long writeFileFunc(void *, void * stream, const void * buf, unsigned long size) // NOLINT(google-runtime-int)
-        {
-            auto & strm = get(stream);
-            strm.write_buffer->write(reinterpret_cast<const char *>(buf), size);
-            return size;
-        }
+    static unsigned long readFileFunc(void *, void *, void *, unsigned long) // NOLINT(google-runtime-int)
+    {
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "StreamInfo::readFile() is not implemented");
+    }
 
-        static int testErrorFunc(void *, void *)
-        {
-            return ZIP_OK;
-        }
-
-        static ZPOS64_T tellFunc(void *, void * stream)
-        {
-            auto & strm = get(stream);
-            auto pos = strm.write_buffer->count() - strm.start_offset;
-            return pos;
-        }
-
-        static long seekFunc(void *, void *, ZPOS64_T, int) // NOLINT(google-runtime-int)
-        {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "StreamFromWriteBuffer::seek must not be called");
-        }
-
-        static unsigned long readFileFunc(void *, void *, void *, unsigned long) // NOLINT(google-runtime-int)
-        {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "StreamFromWriteBuffer::readFile must not be called");
-        }
-    };
-}
+    std::unique_ptr<WriteBuffer> write_buffer;
+    UInt64 start_offset;
+};
 
 
 ZipArchiveWriter::ZipArchiveWriter(const String & path_to_archive_)
@@ -248,21 +223,42 @@ ZipArchiveWriter::ZipArchiveWriter(const String & path_to_archive_, std::unique_
     : path_to_archive(path_to_archive_), compression_method(MZ_COMPRESS_METHOD_DEFLATE)
 {
     if (archive_write_buffer_)
-        handle = StreamFromWriteBuffer::open(std::move(archive_write_buffer_));
+    {
+        stream_info = std::make_unique<StreamInfo>(std::move(archive_write_buffer_));
+        zip_handle = stream_info->makeZipHandle();
+    }
     else
-        handle = zipOpen64(path_to_archive.c_str(), /* append= */ false);
-    if (!handle)
-        throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Couldn't create zip archive {}", quoteString(path_to_archive));
+    {
+        zip_handle = zipOpen64(path_to_archive.c_str(), /* append= */ false);
+    }
 
+    if (!zip_handle)
+        throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Couldn't create zip archive {}", quoteString(path_to_archive));
 }
 
 ZipArchiveWriter::~ZipArchiveWriter()
 {
-    if (handle)
+    if (!finalized)
+    {
+        /// It is totally OK to destroy instance without finalization when an exception occurs.
+        /// However it is suspicious to destroy instance without finalization at the green path.
+        if (!std::uncaught_exceptions() && std::current_exception() == nullptr)
+        {
+            Poco::Logger * log = &Poco::Logger::get("ZipArchiveWriter");
+            LOG_ERROR(log,
+                       "ZipArchiveWriter is not finalized when destructor is called. "
+                       "The zip archive might not be written at all or might be truncated. "
+                       "Stack trace: {}", StackTrace().toString());
+            chassert(false && "ZipArchiveWriter is not finalized in destructor.");
+        }
+    }
+
+    if (zip_handle)
     {
         try
         {
-            checkResult(zipClose(handle, /* global_comment= */ nullptr));
+            zipCloseFileInZip(zip_handle);
+            zipClose(zip_handle, /* global_comment= */ nullptr);
         }
         catch (...)
         {
@@ -273,13 +269,38 @@ ZipArchiveWriter::~ZipArchiveWriter()
 
 std::unique_ptr<WriteBufferFromFileBase> ZipArchiveWriter::writeFile(const String & filename)
 {
-    return std::make_unique<WriteBufferFromZipArchive>(acquireHandle(), filename);
+    return std::make_unique<WriteBufferFromZipArchive>(std::static_pointer_cast<ZipArchiveWriter>(shared_from_this()), filename);
 }
 
 bool ZipArchiveWriter::isWritingFile() const
 {
     std::lock_guard lock{mutex};
-    return !handle;
+    return is_writing_file;
+}
+
+void ZipArchiveWriter::finalize()
+{
+    std::lock_guard lock{mutex};
+    if (finalized)
+        return;
+
+    if (is_writing_file)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "ZipArchiveWriter::finalize() is called in the middle of writing a file into the zip archive. That's not allowed");
+
+    if (zip_handle)
+    {
+        int code = zipClose(zip_handle, /* global_comment= */ nullptr);
+        zip_handle = nullptr;
+        checkResultCode(code);
+    }
+
+    if (stream_info)
+    {
+        stream_info->getWriteBuffer().finalize();
+        stream_info.reset();
+    }
+
+    finalized = true;
 }
 
 void ZipArchiveWriter::setCompression(const String & compression_method_, int compression_level_)
@@ -289,12 +310,30 @@ void ZipArchiveWriter::setCompression(const String & compression_method_, int co
     compression_level = compression_level_;
 }
 
+int ZipArchiveWriter::getCompressionMethod() const
+{
+    std::lock_guard lock{mutex};
+    return compression_method;
+}
+
+int ZipArchiveWriter::getCompressionLevel() const
+{
+    std::lock_guard lock{mutex};
+    return compression_level;
+}
+
 void ZipArchiveWriter::setPassword(const String & password_)
 {
     std::lock_guard lock{mutex};
     password = password_;
 }
 
+String ZipArchiveWriter::getPassword() const
+{
+    std::lock_guard lock{mutex};
+    return password;
+}
+
 int ZipArchiveWriter::compressionMethodToInt(const String & compression_method_)
 {
     if (compression_method_.empty())
@@ -361,45 +400,24 @@ void ZipArchiveWriter::checkEncryptionIsEnabled()
 #endif
 }
 
-ZipArchiveWriter::HandleHolder ZipArchiveWriter::acquireHandle()
-{
-    return HandleHolder{std::static_pointer_cast<ZipArchiveWriter>(shared_from_this())};
-}
-
-RawHandle ZipArchiveWriter::acquireRawHandle()
+ZipArchiveWriter::ZipHandle ZipArchiveWriter::startWritingFile()
 {
     std::lock_guard lock{mutex};
-    if (!handle)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot have more than one write buffer while writing a zip archive");
-    return std::exchange(handle, nullptr);
+    if (is_writing_file)
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot write two files to a zip archive in parallel");
+    is_writing_file = true;
+    return zip_handle;
 }
 
-void ZipArchiveWriter::releaseRawHandle(RawHandle raw_handle_)
+void ZipArchiveWriter::endWritingFile()
 {
     std::lock_guard lock{mutex};
-    handle = raw_handle_;
+    is_writing_file = false;
 }
 
-void ZipArchiveWriter::checkResult(int code) const
+void ZipArchiveWriter::checkResultCode(int code) const
 {
-    if (code >= ZIP_OK)
-        return;
-
-    String message = "Code = ";
-    switch (code)
-    {
-        case ZIP_ERRNO: message += "ERRNO, errno = " + errnoToString(); break;
-        case ZIP_PARAMERROR: message += "PARAMERROR"; break;
-        case ZIP_BADZIPFILE: message += "BADZIPFILE"; break;
-        case ZIP_INTERNALERROR: message += "INTERNALERROR"; break;
-        default: message += std::to_string(code); break;
-    }
-    showError(message);
-}
-
-void ZipArchiveWriter::showError(const String & message) const
-{
-    throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Couldn't pack zip archive {}: {}", quoteString(path_to_archive), message);
+    checkResultCodeImpl(code, path_to_archive);
 }
 
 }

src/IO/Archives/ZipArchiveWriter.h

@@ -4,6 +4,7 @@
 
 #if USE_MINIZIP
 #include <IO/Archives/IArchiveWriter.h>
+#include <base/defines.h>
 #include <mutex>
 
 
@@ -22,7 +23,7 @@ public:
     /// Constructs an archive that will be written by using a specified `archive_write_buffer_`.
     ZipArchiveWriter(const String & path_to_archive_, std::unique_ptr<WriteBuffer> archive_write_buffer_);
 
-    /// Destructors finalizes writing the archive.
+    /// Call finalize() before destructing IArchiveWriter.
     ~ZipArchiveWriter() override;
 
     /// Starts writing a file to the archive. The function returns a write buffer,
@@ -35,6 +36,10 @@ public:
     /// This function should be used mostly for debugging purposes.
     bool isWritingFile() const override;
 
+    /// Finalizes writing of the archive. This function must be always called at the end of writing.
+    /// (Unless an error appeared and the archive is in fact no longer needed.)
+    void finalize() override;
+
     /// Supported compression methods.
     static constexpr const char kStore[] = "store";
     static constexpr const char kDeflate[] = "deflate";
@@ -68,22 +73,27 @@ public:
     static void checkEncryptionIsEnabled();
 
 private:
+    class StreamInfo;
+    using ZipHandle = void *;
     class WriteBufferFromZipArchive;
-    class HandleHolder;
-    using RawHandle = void *;
 
-    HandleHolder acquireHandle();
-    RawHandle acquireRawHandle();
-    void releaseRawHandle(RawHandle raw_handle_);
+    int getCompressionMethod() const;
+    int getCompressionLevel() const;
+    String getPassword() const;
 
-    void checkResult(int code) const;
-    [[noreturn]] void showError(const String & message) const;
+    ZipHandle startWritingFile();
+    void endWritingFile();
+
+    void checkResultCode(int code) const;
 
     const String path_to_archive;
-    int compression_method; /// By default the compression method is "deflate".
-    int compression_level = kDefaultCompressionLevel;
-    String password;
-    RawHandle handle = nullptr;
+    std::unique_ptr<StreamInfo> TSA_GUARDED_BY(mutex) stream_info;
+    int compression_method TSA_GUARDED_BY(mutex); /// By default the compression method is "deflate".
+    int compression_level TSA_GUARDED_BY(mutex) = kDefaultCompressionLevel;
+    String password TSA_GUARDED_BY(mutex);
+    ZipHandle zip_handle TSA_GUARDED_BY(mutex) = nullptr;
+    bool is_writing_file TSA_GUARDED_BY(mutex) = false;
+    bool finalized TSA_GUARDED_BY(mutex) = false;
     mutable std::mutex mutex;
 };
 

src/IO/ReadBufferFromIStream.cpp

@@ -34,6 +34,11 @@ bool ReadBufferFromIStream::nextImpl()
 ReadBufferFromIStream::ReadBufferFromIStream(std::istream & istr_, size_t size)
     : BufferWithOwnMemory<ReadBuffer>(size), istr(istr_)
 {
+    /// - badbit will be set if some exception will be throw from ios implementation
+    /// - failbit can be set when for instance read() reads less data, so we
+    ///   cannot set it, since we are requesting to read more data, then the
+    ///   buffer has now.
+    istr.exceptions(std::ios::badbit);
 }
 
 }

src/IO/ReadBufferFromS3.cpp

@@ -196,7 +196,7 @@ bool ReadBufferFromS3::nextImpl()
             next_result = impl->next();
             break;
         }
-        catch (Exception & e)
+        catch (Poco::Exception & e)
         {
             if (!processException(e, getPosition(), attempt) || last_attempt)
                 throw;

src/IO/SharedThreadPools.cpp

@@ -66,6 +66,8 @@ void StaticThreadPool::reloadConfiguration(size_t max_threads, size_t max_free_t
     if (!instance)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "The {} is not initialized", name);
 
+    std::lock_guard lock(mutex);
+
     instance->setMaxThreads(turbo_mode_enabled > 0 ? max_threads_turbo : max_threads);
     instance->setMaxFreeThreads(max_free_threads);
     instance->setQueueSize(queue_size);

src/IO/tests/gtest_archive_reader_and_writer.cpp

@@ -102,7 +102,8 @@ TEST_P(ArchiveReaderAndWriterTest, EmptyArchive)
 {
     /// Make an archive.
     {
-        createArchiveWriter(getPathToArchive());
+        auto writer = createArchiveWriter(getPathToArchive());
+        writer->finalize();
     }
 
     /// The created archive can be found in the local filesystem.
@@ -132,7 +133,9 @@ TEST_P(ArchiveReaderAndWriterTest, SingleFileInArchive)
         {
             auto out = writer->writeFile("a.txt");
             writeString(contents, *out);
+            out->finalize();
         }
+        writer->finalize();
     }
 
     /// Read the archive.
@@ -198,11 +201,14 @@ TEST_P(ArchiveReaderAndWriterTest, TwoFilesInArchive)
         {
             auto out = writer->writeFile("a.txt");
             writeString(a_contents, *out);
+            out->finalize();
         }
         {
             auto out = writer->writeFile("b/c.txt");
             writeString(c_contents, *out);
+            out->finalize();
         }
+        writer->finalize();
     }
 
     /// Read the archive.
@@ -281,11 +287,14 @@ TEST_P(ArchiveReaderAndWriterTest, InMemory)
         {
             auto out = writer->writeFile("a.txt");
             writeString(a_contents, *out);
+            out->finalize();
         }
         {
             auto out = writer->writeFile("b.txt");
             writeString(b_contents, *out);
+            out->finalize();
         }
+        writer->finalize();
     }
 
     /// The created archive is really in memory.
@@ -335,7 +344,9 @@ TEST_P(ArchiveReaderAndWriterTest, Password)
         {
             auto out = writer->writeFile("a.txt");
             writeString(contents, *out);
+            out->finalize();
         }
+        writer->finalize();
     }
 
     /// Read the archive.

src/Interpreters/ActionsVisitor.cpp

@@ -1414,7 +1414,10 @@ FutureSetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool
             set_key = right_in_operand->getTreeHash(/*ignore_aliases=*/ true);
 
         if (auto set = data.prepared_sets->findSubquery(set_key))
+        {
+            set->markAsINSubquery();
             return set;
+        }
 
         FutureSetPtr external_table_set;
 
@@ -1460,7 +1463,8 @@ FutureSetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool
             interpreter->buildQueryPlan(*source);
         }
 
-        return data.prepared_sets->addFromSubquery(set_key, std::move(source), nullptr, std::move(external_table_set), data.getContext()->getSettingsRef());
+        return data.prepared_sets->addFromSubquery(
+            set_key, std::move(source), nullptr, std::move(external_table_set), data.getContext()->getSettingsRef(), /*in_subquery=*/true);
     }
     else
     {

src/Interpreters/BackupLog.cpp

@@ -27,6 +27,7 @@ NamesAndTypesList BackupLogElement::getNamesAndTypes()
         {"event_time_microseconds", std::make_shared<DataTypeDateTime64>(6)},
         {"id", std::make_shared<DataTypeString>()},
         {"name", std::make_shared<DataTypeString>()},
+        {"base_backup_name", std::make_shared<DataTypeString>()},
         {"status", std::make_shared<DataTypeEnum8>(getBackupStatusEnumValues())},
         {"error", std::make_shared<DataTypeString>()},
         {"start_time", std::make_shared<DataTypeDateTime>()},
@@ -49,6 +50,7 @@ void BackupLogElement::appendToBlock(MutableColumns & columns) const
     columns[i++]->insert(event_time_usec);
     columns[i++]->insert(info.id);
     columns[i++]->insert(info.name);
+    columns[i++]->insert(info.base_backup_name);
     columns[i++]->insert(static_cast<Int8>(info.status));
     columns[i++]->insert(info.error_message);
     columns[i++]->insert(static_cast<UInt32>(std::chrono::system_clock::to_time_t(info.start_time)));

src/Interpreters/ExpressionAnalyzer.cpp

@@ -56,6 +56,7 @@
 #include <Core/Names.h>
 #include <Core/NamesAndTypes.h>
 #include <Common/logger_useful.h>
+#include <Interpreters/PasteJoin.h>
 #include <QueryPipeline/SizeLimits.h>
 
 
@@ -951,6 +952,9 @@ static std::shared_ptr<IJoin> tryCreateJoin(
     std::unique_ptr<QueryPlan> & joined_plan,
     ContextPtr context)
 {
+    if (analyzed_join->kind() == JoinKind::Paste)
+        return std::make_shared<PasteJoin>(analyzed_join, right_sample_block);
+
     if (algorithm == JoinAlgorithm::DIRECT || algorithm == JoinAlgorithm::DEFAULT)
     {
         JoinPtr direct_join = tryKeyValueJoin(analyzed_join, right_sample_block);

src/Interpreters/InterpreterSelectQuery.cpp

@@ -870,7 +870,38 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis()
     ASTSelectQuery & query = getSelectQuery();
 
     /// While only_analyze we don't know anything about parts, so any decision about how many parallel replicas to use would be wrong
-    if (!storage || options.only_analyze || !context->canUseParallelReplicasOnInitiator())
+    if (!storage || !context->canUseParallelReplicasOnInitiator())
+        return false;
+
+    /// check if IN operator with subquery is present in the query
+    /// if so, disable parallel replicas
+    if (query_analyzer->getPreparedSets()->hasSubqueries())
+    {
+        bool in_subqueries = false;
+        const auto & sets = query_analyzer->getPreparedSets();
+        const auto subqueries = sets->getSubqueries();
+        for (const auto & subquery : subqueries)
+        {
+            if (subquery->isINSubquery())
+            {
+                in_subqueries = true;
+                break;
+            }
+        }
+
+        if (in_subqueries)
+        {
+            if (settings.allow_experimental_parallel_reading_from_replicas == 2)
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "IN with subquery is not supported with parallel replicas");
+
+            context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0));
+            context->setSetting("max_parallel_replicas", UInt64{0});
+            LOG_DEBUG(log, "Disabling parallel replicas to execute a query with IN with subquery");
+            return true;
+        }
+    }
+
+    if (options.only_analyze)
         return false;
 
     if (getTrivialCount(0).has_value())
@@ -1698,7 +1729,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
                         return step_raw_ptr;
                     };
 
-                    if (expressions.join->pipelineType() == JoinPipelineType::YShaped)
+                    if (expressions.join->pipelineType() == JoinPipelineType::YShaped && expressions.join->getTableJoin().kind() != JoinKind::Paste)
                     {
                         const auto & table_join = expressions.join->getTableJoin();
                         const auto & join_clause = table_join.getOnlyClause();

src/Interpreters/JoinUtils.cpp

@@ -345,27 +345,6 @@ ColumnRawPtrs getRawPointers(const Columns & columns)
     return ptrs;
 }
 
-void convertToFullColumnsInplace(Block & block)
-{
-    for (size_t i = 0; i < block.columns(); ++i)
-    {
-        auto & col = block.getByPosition(i);
-        col.column = recursiveRemoveLowCardinality(recursiveRemoveSparse(col.column));
-        col.type = recursiveRemoveLowCardinality(col.type);
-    }
-}
-
-void convertToFullColumnsInplace(Block & block, const Names & names, bool change_type)
-{
-    for (const String & column_name : names)
-    {
-        auto & col = block.getByName(column_name);
-        col.column = recursiveRemoveLowCardinality(recursiveRemoveSparse(col.column));
-        if (change_type)
-            col.type = recursiveRemoveLowCardinality(col.type);
-    }
-}
-
 void restoreLowCardinalityInplace(Block & block, const Names & lowcard_keys)
 {
     for (const auto & column_name : lowcard_keys)
@@ -495,8 +474,8 @@ void addDefaultValues(IColumn & column, const DataTypePtr & type, size_t count)
 
 bool typesEqualUpToNullability(DataTypePtr left_type, DataTypePtr right_type)
 {
-    DataTypePtr left_type_strict = removeNullable(recursiveRemoveLowCardinality(left_type));
-    DataTypePtr right_type_strict = removeNullable(recursiveRemoveLowCardinality(right_type));
+    DataTypePtr left_type_strict = removeNullable(removeLowCardinality(left_type));
+    DataTypePtr right_type_strict = removeNullable(removeLowCardinality(right_type));
     return left_type_strict->equals(*right_type_strict);
 }
 

src/Interpreters/JoinUtils.h

@@ -71,8 +71,6 @@ ColumnPtr materializeColumn(const Block & block, const String & name);
 Columns materializeColumns(const Block & block, const Names & names);
 ColumnRawPtrs materializeColumnsInplace(Block & block, const Names & names);
 ColumnRawPtrs getRawPointers(const Columns & columns);
-void convertToFullColumnsInplace(Block & block);
-void convertToFullColumnsInplace(Block & block, const Names & names, bool change_type = true);
 void restoreLowCardinalityInplace(Block & block, const Names & lowcard_keys);
 
 ColumnRawPtrs extractKeysForJoin(const Block & block_keys, const Names & key_names_right);

src/Interpreters/MergeJoin.cpp

@@ -138,6 +138,9 @@ Block extractMinMax(const Block & block, const Block & keys)
     }
 
     min_max.setColumns(std::move(columns));
+
+    for (auto & column : min_max)
+        column.column = column.column->convertToFullColumnIfLowCardinality();
     return min_max;
 }
 
@@ -224,6 +227,16 @@ public:
     MergeJoinCursor(const Block & block, const SortDescription & desc_)
         : impl(block, desc_)
     {
+        for (auto *& column : impl.sort_columns)
+        {
+            const auto * lowcard_column = typeid_cast<const ColumnLowCardinality *>(column);
+            if (lowcard_column)
+            {
+                auto & new_col = column_holder.emplace_back(lowcard_column->convertToFullColumn());
+                column = new_col.get();
+            }
+        }
+
         /// SortCursorImpl can work with permutation, but MergeJoinCursor can't.
         if (impl.permutation)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error: MergeJoinCursor doesn't support permutation");
@@ -287,6 +300,7 @@ public:
 
 private:
     SortCursorImpl impl;
+    Columns column_holder;
     bool has_left_nullable = false;
     bool has_right_nullable = false;
 
@@ -537,9 +551,6 @@ MergeJoin::MergeJoin(std::shared_ptr<TableJoin> table_join_, const Block & right
             lowcard_right_keys.push_back(right_key);
     }
 
-    JoinCommon::convertToFullColumnsInplace(right_table_keys);
-    JoinCommon::convertToFullColumnsInplace(right_sample_block, key_names_right);
-
     for (const auto & column : right_table_keys)
         if (required_right_keys.contains(column.name))
             right_columns_to_add.insert(ColumnWithTypeAndName{nullptr, column.type, column.name});
@@ -662,9 +673,7 @@ bool MergeJoin::saveRightBlock(Block && block)
 
 Block MergeJoin::modifyRightBlock(const Block & src_block) const
 {
-    Block block = materializeBlock(src_block);
-    JoinCommon::convertToFullColumnsInplace(block, table_join->getOnlyClause().key_names_right);
-    return block;
+    return materializeBlock(src_block);
 }
 
 bool MergeJoin::addBlockToJoin(const Block & src_block, bool)
@@ -705,8 +714,6 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed)
                 lowcard_keys.push_back(column_name);
         }
 
-        JoinCommon::convertToFullColumnsInplace(block, key_names_left, false);
-
         sortBlock(block, left_sort_description);
     }
 
@@ -739,8 +746,6 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed)
 
     if (needConditionJoinColumn())
         block.erase(deriveTempName(mask_column_name_left, JoinTableSide::Left));
-
-    JoinCommon::restoreLowCardinalityInplace(block, lowcard_keys);
 }
 
 template <bool in_memory, bool is_all>

src/Interpreters/PartLog.cpp

@@ -245,6 +245,7 @@ bool PartLog::addNewParts(
             elem.part_type = part->getType();
 
             elem.bytes_compressed_on_disk = part->getBytesOnDisk();
+            elem.bytes_uncompressed = part->getBytesUncompressedOnDisk();
             elem.rows = part->rows_count;
 
             elem.error = static_cast<UInt16>(execution_status.code);

src/Interpreters/PasteJoin.h

@@ -0,0 +1,96 @@
+#pragma once
+
+#include <Interpreters/IJoin.h>
+#include <Interpreters/TableJoin.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeLowCardinality.h>
+#include <Common/logger_useful.h>
+#include <Poco/Logger.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int NOT_IMPLEMENTED;
+}
+
+/// Dummy class, actual joining is done by MergeTransform
+class PasteJoin : public IJoin
+{
+public:
+    explicit PasteJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block_)
+        : table_join(table_join_)
+        , right_sample_block(right_sample_block_)
+    {
+        LOG_TRACE(&Poco::Logger::get("PasteJoin"), "Will use paste join");
+    }
+
+    std::string getName() const override { return "PasteJoin"; }
+    const TableJoin & getTableJoin() const override { return *table_join; }
+
+    bool addBlockToJoin(const Block & /* block */, bool /* check_limits */) override
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "PasteJoin::addBlockToJoin should not be called");
+    }
+
+    static bool isSupported(const std::shared_ptr<TableJoin> & table_join)
+    {
+        bool support_storage = !table_join->isSpecialStorage();
+
+        /// Key column can change nullability and it's not handled on type conversion stage, so algorithm should be aware of it
+        bool support_using = !table_join->hasUsing();
+
+        bool check_strictness = table_join->strictness() == JoinStrictness::All;
+
+        bool if_has_keys = table_join->getClauses().empty();
+
+        return support_using && support_storage && check_strictness && if_has_keys;
+    }
+
+    void checkTypesOfKeys(const Block & /*left_block*/) const override
+    {
+        if (!isSupported(table_join))
+            throw DB::Exception(ErrorCodes::NOT_IMPLEMENTED, "PasteJoin doesn't support specified query");
+    }
+
+    /// Used just to get result header
+    void joinBlock(Block & block, std::shared_ptr<ExtraBlock> & /* not_processed */) override
+    {
+        for (const auto & col : right_sample_block)
+            block.insert(col);
+        block = materializeBlock(block).cloneEmpty();
+    }
+
+    void setTotals(const Block & block) override { totals = block; }
+    const Block & getTotals() const override { return totals; }
+
+    size_t getTotalRowCount() const override
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "PasteJoin::getTotalRowCount should not be called");
+    }
+
+    size_t getTotalByteCount() const override
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "PasteJoin::getTotalByteCount should not be called");
+    }
+
+    bool alwaysReturnsEmptySet() const override { return false; }
+
+    IBlocksStreamPtr
+    getNonJoinedBlocks(const Block & /* left_sample_block */, const Block & /* result_sample_block */, UInt64 /* max_block_size */) const override
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "PasteJoin::getNonJoinedBlocks should not be called");
+    }
+
+    /// Left and right streams have the same priority and are processed simultaneously
+    JoinPipelineType pipelineType() const override { return JoinPipelineType::YShaped; }
+
+private:
+    std::shared_ptr<TableJoin> table_join;
+    Block right_sample_block;
+    Block totals;
+};
+
+}

src/Interpreters/PreparedSets.cpp

@@ -98,10 +98,12 @@ FutureSetFromSubquery::FutureSetFromSubquery(
     std::unique_ptr<QueryPlan> source_,
     StoragePtr external_table_,
     FutureSetPtr external_table_set_,
-    const Settings & settings)
+    const Settings & settings,
+    bool in_subquery_)
     : external_table(std::move(external_table_))
     , external_table_set(std::move(external_table_set_))
     , source(std::move(source_))
+    , in_subquery(in_subquery_)
 {
     set_and_key = std::make_shared<SetAndKey>();
     set_and_key->key = std::move(key);
@@ -261,14 +263,16 @@ FutureSetPtr PreparedSets::addFromSubquery(
     std::unique_ptr<QueryPlan> source,
     StoragePtr external_table,
     FutureSetPtr external_table_set,
-    const Settings & settings)
+    const Settings & settings,
+    bool in_subquery)
 {
     auto from_subquery = std::make_shared<FutureSetFromSubquery>(
         toString(key, {}),
         std::move(source),
         std::move(external_table),
         std::move(external_table_set),
-        settings);
+        settings,
+        in_subquery);
 
     auto [it, inserted] = sets_from_subqueries.emplace(key, from_subquery);
 
@@ -318,6 +322,15 @@ std::shared_ptr<FutureSetFromSubquery> PreparedSets::findSubquery(const Hash & k
     return it->second;
 }
 
+void PreparedSets::markAsINSubquery(const Hash & key)
+{
+    auto it = sets_from_subqueries.find(key);
+    if (it == sets_from_subqueries.end())
+        return;
+
+    it->second->markAsINSubquery();
+}
+
 std::shared_ptr<FutureSetFromStorage> PreparedSets::findStorage(const Hash & key) const
 {
     auto it = sets_from_storage.find(key);
@@ -327,11 +340,11 @@ std::shared_ptr<FutureSetFromStorage> PreparedSets::findStorage(const Hash & key
     return it->second;
 }
 
-PreparedSets::Subqueries PreparedSets::getSubqueries()
+PreparedSets::Subqueries PreparedSets::getSubqueries() const
 {
     PreparedSets::Subqueries res;
     res.reserve(sets_from_subqueries.size());
-    for (auto & [_, set] : sets_from_subqueries)
+    for (const auto & [_, set] : sets_from_subqueries)
         res.push_back(set);
 
     return res;

src/Interpreters/PreparedSets.h

@@ -59,7 +59,7 @@ using FutureSetPtr = std::shared_ptr<FutureSet>;
 class FutureSetFromStorage final : public FutureSet
 {
 public:
-    FutureSetFromStorage(SetPtr set_);
+    explicit FutureSetFromStorage(SetPtr set_);
 
     SetPtr get() const override;
     DataTypes getTypes() const override;
@@ -97,7 +97,8 @@ public:
         std::unique_ptr<QueryPlan> source_,
         StoragePtr external_table_,
         FutureSetPtr external_table_set_,
-        const Settings & settings);
+        const Settings & settings,
+        bool in_subquery_);
 
     FutureSetFromSubquery(
         String key,
@@ -112,6 +113,8 @@ public:
 
     QueryTreeNodePtr detachQueryTree() { return std::move(query_tree); }
     void setQueryPlan(std::unique_ptr<QueryPlan> source_);
+    void markAsINSubquery() { in_subquery = true; }
+    bool isINSubquery() const { return in_subquery; }
 
 private:
     SetAndKeyPtr set_and_key;
@@ -120,6 +123,11 @@ private:
 
     std::unique_ptr<QueryPlan> source;
     QueryTreeNodePtr query_tree;
+    bool in_subquery = false; // subquery used in IN operator
+                              // the flag can be removed after enabling new analyzer and removing interpreter
+                              // or after enabling support IN operator with subqueries in parallel replicas
+                              // Note: it's necessary with interpreter since prepared sets used also for GLOBAL JOINs,
+                              //       with new analyzer it's not a case
 };
 
 /// Container for all the sets used in query.
@@ -145,7 +153,8 @@ public:
         std::unique_ptr<QueryPlan> source,
         StoragePtr external_table,
         FutureSetPtr external_table_set,
-        const Settings & settings);
+        const Settings & settings,
+        bool in_subquery = false);
 
     FutureSetPtr addFromSubquery(
         const Hash & key,
@@ -155,9 +164,11 @@ public:
     FutureSetPtr findTuple(const Hash & key, const DataTypes & types) const;
     std::shared_ptr<FutureSetFromStorage> findStorage(const Hash & key) const;
     std::shared_ptr<FutureSetFromSubquery> findSubquery(const Hash & key) const;
+    void markAsINSubquery(const Hash & key);
 
     using Subqueries = std::vector<std::shared_ptr<FutureSetFromSubquery>>;
-    Subqueries getSubqueries();
+    Subqueries getSubqueries() const;
+    bool hasSubqueries() const { return !sets_from_subqueries.empty(); }
 
     const SetsFromTuple & getSetsFromTuple() const { return sets_from_tuple; }
     // const SetsFromStorage & getSetsFromStorage() const { return sets_from_storage; }

src/Interpreters/TableJoin.cpp

@@ -34,6 +34,7 @@
 #include <type_traits>
 #include <vector>
 
+#include <DataTypes/DataTypeLowCardinality.h>
 
 namespace DB
 {
@@ -375,7 +376,7 @@ void TableJoin::addJoinedColumnsAndCorrectTypesImpl(TColumns & left_columns, boo
              * For `JOIN ON expr1 == expr2` we will infer common type later in makeTableJoin,
              *   when part of plan built and types of expression will be known.
              */
-            inferJoinKeyCommonType(left_columns, columns_from_joined_table, !isSpecialStorage(), isEnabledAlgorithm(JoinAlgorithm::FULL_SORTING_MERGE));
+            inferJoinKeyCommonType(left_columns, columns_from_joined_table, !isSpecialStorage());
 
             if (auto it = left_type_map.find(col.name); it != left_type_map.end())
             {
@@ -558,7 +559,8 @@ TableJoin::createConvertingActions(
       */
     NameToNameMap left_column_rename;
     NameToNameMap right_column_rename;
-    inferJoinKeyCommonType(left_sample_columns, right_sample_columns, !isSpecialStorage(), isEnabledAlgorithm(JoinAlgorithm::FULL_SORTING_MERGE));
+
+    inferJoinKeyCommonType(left_sample_columns, right_sample_columns, !isSpecialStorage());
     if (!left_type_map.empty() || !right_type_map.empty())
     {
         left_dag = applyKeyConvertToTable(left_sample_columns, left_type_map, JoinTableSide::Left, left_column_rename);
@@ -612,8 +614,11 @@ TableJoin::createConvertingActions(
 }
 
 template <typename LeftNamesAndTypes, typename RightNamesAndTypes>
-void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right, bool strict)
+void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right)
 {
+    /// FullSortingMerge and PartialMerge join algorithms don't support joining keys with different types
+    /// (e.g. String and LowCardinality(String))
+    bool require_strict_keys_match = isEnabledAlgorithm(JoinAlgorithm::FULL_SORTING_MERGE);
     if (!left_type_map.empty() || !right_type_map.empty())
         return;
 
@@ -645,7 +650,7 @@ void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const Rig
         const auto & ltype = ltypeit->second;
         const auto & rtype = rtypeit->second;
 
-        bool type_equals = strict ? ltype->equals(*rtype) : JoinCommon::typesEqualUpToNullability(ltype, rtype);
+        bool type_equals = require_strict_keys_match ? ltype->equals(*rtype) : JoinCommon::typesEqualUpToNullability(ltype, rtype);
         if (type_equals)
             return true;
 

src/Interpreters/TableJoin.h

@@ -218,7 +218,7 @@ private:
 
     /// Calculates common supertypes for corresponding join key columns.
     template <typename LeftNamesAndTypes, typename RightNamesAndTypes>
-    void inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right, bool strict);
+    void inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right);
 
     void deduplicateAndQualifyColumnNames(const NameSet & left_table_columns, const String & right_table_prefix);
 

src/Interpreters/executeDDLQueryOnCluster.cpp

@@ -41,12 +41,9 @@ static ZooKeeperRetriesInfo getRetriesInfo()
 {
     const auto & config_ref = Context::getGlobalContextInstance()->getConfigRef();
     return ZooKeeperRetriesInfo(
-        "DistributedDDL",
-        &Poco::Logger::get("DDLQueryStatusSource"),
         config_ref.getInt("distributed_ddl_keeper_max_retries", 5),
         config_ref.getInt("distributed_ddl_keeper_initial_backoff_ms", 100),
-        config_ref.getInt("distributed_ddl_keeper_max_backoff_ms", 5000)
-    );
+        config_ref.getInt("distributed_ddl_keeper_max_backoff_ms", 5000));
 }
 
 bool isSupportedAlterTypeForOnClusterDDLQuery(int type)
@@ -438,8 +435,8 @@ Chunk DDLQueryStatusSource::generate()
         Strings tmp_active_hosts;
 
         {
-            auto retries_info = getRetriesInfo();
-            auto retries_ctl = ZooKeeperRetriesControl("executeDDLQueryOnCluster", retries_info, context->getProcessListElement());
+            auto retries_ctl = ZooKeeperRetriesControl(
+                "executeDDLQueryOnCluster", &Poco::Logger::get("DDLQueryStatusSource"), getRetriesInfo(), context->getProcessListElement());
             retries_ctl.retryLoop([&]()
             {
                 auto zookeeper = context->getZooKeeper();
@@ -478,8 +475,11 @@ Chunk DDLQueryStatusSource::generate()
                 String status_data;
                 bool finished_exists = false;
 
-                auto retries_info = getRetriesInfo();
-                auto retries_ctl = ZooKeeperRetriesControl("executeDDLQueryOnCluster", retries_info, context->getProcessListElement());
+                auto retries_ctl = ZooKeeperRetriesControl(
+                    "executeDDLQueryOnCluster",
+                    &Poco::Logger::get("DDLQueryStatusSource"),
+                    getRetriesInfo(),
+                    context->getProcessListElement());
                 retries_ctl.retryLoop([&]()
                 {
                     finished_exists = context->getZooKeeper()->tryGet(fs::path(node_path) / "finished" / host_id, status_data);

src/Parsers/ASTTablesInSelectQuery.cpp

@@ -211,6 +211,9 @@ void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, Format
         case JoinKind::Comma:
             settings.ostr << ",";
             break;
+        case JoinKind::Paste:
+            settings.ostr << "PASTE JOIN";
+            break;
     }
 
     settings.ostr << (settings.hilite ? hilite_none : "");

src/Parsers/ParserTablesInSelectQuery.cpp

@@ -6,6 +6,7 @@
 #include <Parsers/ParserSelectQuery.h>
 #include <Parsers/ParserSampleRatio.h>
 #include <Parsers/ParserTablesInSelectQuery.h>
+#include <Core/Joins.h>
 
 
 namespace DB
@@ -166,6 +167,8 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec
                 table_join->kind = JoinKind::Full;
             else if (ParserKeyword("CROSS").ignore(pos))
                 table_join->kind = JoinKind::Cross;
+            else if (ParserKeyword("PASTE").ignore(pos))
+                table_join->kind = JoinKind::Paste;
             else
                 no_kind = true;
 
@@ -191,8 +194,8 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec
             }
 
             if (table_join->strictness != JoinStrictness::Unspecified
-                && table_join->kind == JoinKind::Cross)
-                throw Exception(ErrorCodes::SYNTAX_ERROR, "You must not specify ANY or ALL for CROSS JOIN.");
+                && (table_join->kind == JoinKind::Cross || table_join->kind == JoinKind::Paste))
+                throw Exception(ErrorCodes::SYNTAX_ERROR, "You must not specify ANY or ALL for {} JOIN.", toString(table_join->kind));
 
             if ((table_join->strictness == JoinStrictness::Semi || table_join->strictness == JoinStrictness::Anti) &&
                 (table_join->kind != JoinKind::Left && table_join->kind != JoinKind::Right))
@@ -206,7 +209,7 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec
             return false;
 
         if (table_join->kind != JoinKind::Comma
-            && table_join->kind != JoinKind::Cross)
+            && table_join->kind != JoinKind::Cross && table_join->kind != JoinKind::Paste)
         {
             if (ParserKeyword("USING").ignore(pos, expected))
             {

src/Parsers/obfuscateQueries.cpp

@@ -1095,7 +1095,11 @@ void obfuscateIdentifier(std::string_view src, WriteBuffer & result, WordMap & o
 }
 
 
-void obfuscateLiteral(std::string_view src, WriteBuffer & result, SipHash hash_func)
+void obfuscateLiteral(
+    std::string_view src,
+    WriteBuffer & result,
+    SipHash hash_func,
+    KnownIdentifierFunc known_identifier_func)
 {
     const char * src_pos = src.data();
     const char * src_end = src_pos + src.size();
@@ -1208,15 +1212,15 @@ void obfuscateLiteral(std::string_view src, WriteBuffer & result, SipHash hash_f
         }
         else if (isAlphaASCII(src_pos[0]))
         {
-            /// Alphabetial characters
+            /// Alphabetical characters
 
             const char * alpha_end = src_pos + 1;
             while (alpha_end < src_end && isAlphaASCII(*alpha_end))
                 ++alpha_end;
 
-            String wordcopy(src_pos, alpha_end);
-            Poco::toUpperInPlace(wordcopy);
-            if (keep_words.contains(wordcopy))
+            String word(src_pos, alpha_end);
+            String wordcopy = Poco::toUpper(word);
+            if (keep_words.contains(wordcopy) || known_identifier_func(word))
             {
                 result.write(src_pos, alpha_end - src_pos);
                 src_pos = alpha_end;
@@ -1337,14 +1341,14 @@ void obfuscateQueries(
         }
         else if (token.type == TokenType::Number)
         {
-            obfuscateLiteral(whole_token, result, hash_func);
+            obfuscateLiteral(whole_token, result, hash_func, known_identifier_func);
         }
         else if (token.type == TokenType::StringLiteral)
         {
             assert(token.size() >= 2);
 
             result.write(*token.begin);
-            obfuscateLiteral({token.begin + 1, token.size() - 2}, result, hash_func);
+            obfuscateLiteral({token.begin + 1, token.size() - 2}, result, hash_func, known_identifier_func);
             result.write(token.end[-1]);
         }
         else if (token.type == TokenType::Comment)
@@ -1360,4 +1364,3 @@ void obfuscateQueries(
 }
 
 }
-

src/Planner/Planner.cpp

@@ -1333,14 +1333,27 @@ void Planner::buildPlanForQueryNode()
     }
 
     collectSets(query_tree, *planner_context);
+
+    const auto & settings = query_context->getSettingsRef();
+    if (query_context->canUseTaskBasedParallelReplicas())
+    {
+        if (planner_context->getPreparedSets().hasSubqueries())
+        {
+            if (settings.allow_experimental_parallel_reading_from_replicas == 2)
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "IN with subquery is not supported with parallel replicas");
+
+            auto & mutable_context = planner_context->getMutableQueryContext();
+            mutable_context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0));
+            LOG_DEBUG(&Poco::Logger::get("Planner"), "Disabling parallel replicas to execute a query with IN with subquery");
+        }
+    }
+
     collectTableExpressionData(query_tree, planner_context);
     checkStoragesSupportTransactions(planner_context);
 
     if (!select_query_options.only_analyze)
         collectFiltersForAnalysis(query_tree, planner_context);
 
-    const auto & settings = query_context->getSettingsRef();
-
     if (query_context->canUseTaskBasedParallelReplicas())
     {
         const auto & table_expression_nodes = planner_context->getTableExpressionNodeToData();

src/Planner/PlannerJoinTree.cpp

@@ -955,6 +955,29 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
     };
 }
 
+void joinCastPlanColumnsToNullable(QueryPlan & plan_to_add_cast, PlannerContextPtr & planner_context, const FunctionOverloadResolverPtr & to_nullable_function)
+{
+    auto cast_actions_dag = std::make_shared<ActionsDAG>(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
+
+    for (auto & output_node : cast_actions_dag->getOutputs())
+    {
+        if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(output_node->result_name))
+        {
+            DataTypePtr type_to_check = output_node->result_type;
+            if (const auto * type_to_check_low_cardinality = typeid_cast<const DataTypeLowCardinality *>(type_to_check.get()))
+                type_to_check = type_to_check_low_cardinality->getDictionaryType();
+
+            if (type_to_check->canBeInsideNullable())
+                output_node = &cast_actions_dag->addFunction(to_nullable_function, {output_node}, output_node->result_name);
+        }
+    }
+
+    cast_actions_dag->projectInput();
+    auto cast_join_columns_step = std::make_unique<ExpressionStep>(plan_to_add_cast.getCurrentDataStream(), std::move(cast_actions_dag));
+    cast_join_columns_step->setStepDescription("Cast JOIN columns to Nullable");
+    plan_to_add_cast.addStep(std::move(cast_join_columns_step));
+}
+
 JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_expression,
     JoinTreeQueryPlan left_join_tree_query_plan,
     JoinTreeQueryPlan right_join_tree_query_plan,
@@ -1068,45 +1091,21 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
     const auto & query_context = planner_context->getQueryContext();
     const auto & settings = query_context->getSettingsRef();
 
-    auto to_nullable_function = FunctionFactory::instance().get("toNullable", query_context);
-
-    auto join_cast_plan_columns_to_nullable = [&](QueryPlan & plan_to_add_cast)
-    {
-        auto cast_actions_dag = std::make_shared<ActionsDAG>(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
-
-        for (auto & output_node : cast_actions_dag->getOutputs())
-        {
-            if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(output_node->result_name))
-            {
-                DataTypePtr type_to_check = output_node->result_type;
-                if (const auto * type_to_check_low_cardinality = typeid_cast<const DataTypeLowCardinality *>(type_to_check.get()))
-                    type_to_check = type_to_check_low_cardinality->getDictionaryType();
-
-                if (type_to_check->canBeInsideNullable())
-                    output_node = &cast_actions_dag->addFunction(to_nullable_function, {output_node}, output_node->result_name);
-            }
-        }
-
-        cast_actions_dag->projectInput();
-        auto cast_join_columns_step = std::make_unique<ExpressionStep>(plan_to_add_cast.getCurrentDataStream(), std::move(cast_actions_dag));
-        cast_join_columns_step->setStepDescription("Cast JOIN columns to Nullable");
-        plan_to_add_cast.addStep(std::move(cast_join_columns_step));
-    };
-
     if (settings.join_use_nulls)
     {
+        auto to_nullable_function = FunctionFactory::instance().get("toNullable", query_context);
         if (isFull(join_kind))
         {
-            join_cast_plan_columns_to_nullable(left_plan);
-            join_cast_plan_columns_to_nullable(right_plan);
+            joinCastPlanColumnsToNullable(left_plan, planner_context, to_nullable_function);
+            joinCastPlanColumnsToNullable(right_plan, planner_context, to_nullable_function);
         }
         else if (isLeft(join_kind))
         {
-            join_cast_plan_columns_to_nullable(right_plan);
+            joinCastPlanColumnsToNullable(right_plan, planner_context, to_nullable_function);
         }
         else if (isRight(join_kind))
         {
-            join_cast_plan_columns_to_nullable(left_plan);
+            joinCastPlanColumnsToNullable(left_plan, planner_context, to_nullable_function);
         }
     }
 
@@ -1312,7 +1311,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
             return step_raw_ptr;
         };
 
-        if (join_algorithm->pipelineType() == JoinPipelineType::YShaped)
+        if (join_algorithm->pipelineType() == JoinPipelineType::YShaped && join_kind != JoinKind::Paste)
         {
             const auto & join_clause = table_join->getOnlyClause();
 

src/Planner/PlannerJoins.cpp

@@ -35,6 +35,7 @@
 #include <Interpreters/JoinSwitcher.h>
 #include <Interpreters/ArrayJoinAction.h>
 #include <Interpreters/GraceHashJoin.h>
+#include <Interpreters/PasteJoin.h>
 
 #include <Planner/PlannerActionsVisitor.h>
 #include <Planner/PlannerContext.h>
@@ -653,6 +654,8 @@ static std::shared_ptr<IJoin> tryCreateJoin(JoinAlgorithm algorithm,
     const Block & right_table_expression_header,
     const PlannerContextPtr & planner_context)
 {
+    if (table_join->kind() == JoinKind::Paste)
+        return std::make_shared<PasteJoin>(table_join, right_table_expression_header);
     /// Direct JOIN with special storages that support key value access. For example JOIN with Dictionary
     if (algorithm == JoinAlgorithm::DIRECT || algorithm == JoinAlgorithm::DEFAULT)
     {

src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp

@@ -308,6 +308,41 @@ static inline void throwIfError(llvm::Error & e, const char * what)
     throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Failed to parse {}: {}", what, llvm::toString(std::move(e)));
 }
 
+llvm::DWARFFormValue DWARFBlockInputFormat::parseAttribute(
+    const llvm::DWARFAbbreviationDeclaration::AttributeSpec & attr, uint64_t * offset,
+    const UnitState & unit) const
+{
+    auto val = llvm::DWARFFormValue::createFromSValue(
+        attr.Form, attr.isImplicitConst() ? attr.getImplicitConstValue() : 0);
+    if (!val.extractValue(*extractor, offset, unit.dwarf_unit->getFormParams(), unit.dwarf_unit))
+        throw Exception(ErrorCodes::CANNOT_PARSE_DWARF,
+            "Failed to parse attribute {} of form {} at offset {}",
+                llvm::dwarf::AttributeString(attr.Attr), attr.Form, *offset);
+    return val;
+}
+
+void DWARFBlockInputFormat::skipAttribute(
+    const llvm::DWARFAbbreviationDeclaration::AttributeSpec & attr, uint64_t * offset,
+    const UnitState & unit) const
+{
+    if (!llvm::DWARFFormValue::skipValue(
+        attr.Form, *extractor, offset, unit.dwarf_unit->getFormParams()))
+            throw Exception(ErrorCodes::CANNOT_PARSE_DWARF,
+                "Failed to skip attribute {} of form {} at offset {}",
+                llvm::dwarf::AttributeString(attr.Attr), attr.Form, *offset);
+}
+
+uint64_t DWARFBlockInputFormat::parseAddress(llvm::dwarf::Attribute attr, const llvm::DWARFFormValue & val, const UnitState & unit)
+{
+    if (val.getForm() == llvm::dwarf::DW_FORM_addr)
+        return val.getRawUValue();
+    if (val.getForm() == llvm::dwarf::DW_FORM_addrx ||
+        (val.getForm() >= llvm::dwarf::DW_FORM_addrx1 &&
+         val.getForm() <= llvm::dwarf::DW_FORM_addrx4))
+        return fetchFromDebugAddr(unit.debug_addr_base, val.getRawUValue());
+    throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Form {} for {} is not supported", llvm::dwarf::FormEncodingString(val.getForm()), llvm::dwarf::AttributeString(attr));
+}
+
 Chunk DWARFBlockInputFormat::parseEntries(UnitState & unit)
 {
     const auto & header = getPort().getHeader();
@@ -315,7 +350,6 @@ Chunk DWARFBlockInputFormat::parseEntries(UnitState & unit)
     std::array<bool, COL_COUNT> need{};
     for (const std::string & name : header.getNames())
         need[column_name_to_idx.at(name)] = true;
-    auto form_params = unit.dwarf_unit->getFormParams();
 
     /// For parallel arrays, we nominate one of them to be responsible for populating the offsets vector.
     need[COL_ATTR_NAME] = need[COL_ATTR_NAME] || need[COL_ATTR_FORM] || need[COL_ATTR_INT] || need[COL_ATTR_STR];
@@ -390,6 +424,34 @@ Chunk DWARFBlockInputFormat::parseEntries(UnitState & unit)
             if (need[COL_TAG])
                 col_tag->insertValue(tag);
 
+            if (tag == llvm::dwarf::DW_TAG_compile_unit)
+            {
+                /// Pre-parse DW_AT_addr_base and DW_AT_rnglists_base because other attributes may
+                /// rely on them. (Why couldn't DWARF just promise that these attributes must appear
+                /// before any attributes that depend on them?)
+                uint64_t offset = unit.offset;
+                std::optional<llvm::DWARFFormValue> low_pc;
+                for (auto attr : abbrev->attributes())
+                {
+                    if (attr.Attr == llvm::dwarf::DW_AT_addr_base ||
+                        attr.Attr == llvm::dwarf::DW_AT_rnglists_base)
+                    {
+                        auto val = parseAttribute(attr, &offset, unit);
+                        if (attr.Attr == llvm::dwarf::DW_AT_addr_base)
+                            unit.debug_addr_base = val.getRawUValue();
+                        else
+                            unit.rnglists_base = val.getRawUValue();
+                    }
+                    else if (attr.Attr == llvm::dwarf::DW_AT_low_pc)
+                        low_pc = parseAttribute(attr, &offset, unit);
+                    else
+                        skipAttribute(attr, &offset, unit);
+                }
+                /// May use addr_base.
+                if (low_pc.has_value())
+                    unit.base_address = parseAddress(llvm::dwarf::DW_AT_low_pc, *low_pc, unit);
+            }
+
             bool need_name = need[COL_NAME];
             bool need_linkage_name = need[COL_LINKAGE_NAME];
             bool need_decl_file = need[COL_DECL_FILE];
@@ -410,11 +472,7 @@ Chunk DWARFBlockInputFormat::parseEntries(UnitState & unit)
 
             for (auto attr : abbrev->attributes())
             {
-                auto val = llvm::DWARFFormValue::createFromSValue(attr.Form, attr.isImplicitConst() ? attr.getImplicitConstValue() : 0);
-                /// This is relatively slow, maybe we should reimplement it.
-                if (!val.extractValue(*extractor, &unit.offset, form_params, unit.dwarf_unit))
-                    throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Failed to parse attribute {} of form {} at offset {}",
-                        llvm::dwarf::AttributeString(attr.Attr), attr.Form, unit.offset);
+                auto val = parseAttribute(attr, &unit.offset, unit);
 
                 if (need[COL_ATTR_NAME])
                     col_attr_name->insertValue(attr.Attr);
@@ -452,13 +510,6 @@ Chunk DWARFBlockInputFormat::parseEntries(UnitState & unit)
                         if (attr.Attr == llvm::dwarf::DW_AT_decl_line && std::exchange(need_decl_line, false))
                             col_decl_line->insertValue(static_cast<UInt32>(val.getRawUValue()));
 
-                        /// Starting offset of this unit's data in .debug_addr section.
-                        if (attr.Attr == llvm::dwarf::DW_AT_addr_base)
-                            unit.addr_base = val.getRawUValue();
-                        /// Same for .debug_rnglists section.
-                        if (attr.Attr == llvm::dwarf::DW_AT_rnglists_base)
-                            unit.rnglists_base = val.getRawUValue();
-
                         if (attr.Attr == llvm::dwarf::DW_AT_high_pc)
                         {
                             high_pc = val.getRawUValue();
@@ -515,16 +566,7 @@ Chunk DWARFBlockInputFormat::parseEntries(UnitState & unit)
 
                         if (need_ranges && (attr.Attr == llvm::dwarf::DW_AT_low_pc || attr.Attr == llvm::dwarf::DW_AT_high_pc))
                         {
-                            UInt64 addr;
-                            if (val.getForm() == llvm::dwarf::DW_FORM_addr)
-                                addr = val.getRawUValue();
-                            else if (val.getForm() == llvm::dwarf::DW_FORM_addrx ||
-                                (val.getForm() >= llvm::dwarf::DW_FORM_addrx1 &&
-                                 val.getForm() <= llvm::dwarf::DW_FORM_addrx4))
-                                addr = fetchFromDebugAddr(unit.addr_base, val.getRawUValue());
-                            else
-                                throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Form {} for {} is not supported", llvm::dwarf::FormEncodingString(val.getForm()), llvm::dwarf::AttributeString(attr.Attr));
-
+                            UInt64 addr = parseAddress(attr.Attr, val, unit);
                             if (attr.Attr == llvm::dwarf::DW_AT_low_pc)
                                 low_pc = addr;
                             else
@@ -618,7 +660,7 @@ Chunk DWARFBlockInputFormat::parseEntries(UnitState & unit)
             if (need_ranges)
             {
                 if (ranges.has_value())
-                    parseRanges(*ranges, ranges_rnglistx, low_pc, unit, col_ranges_start, col_ranges_end);
+                    parseRanges(*ranges, ranges_rnglistx, unit, col_ranges_start, col_ranges_end);
                 else if (low_pc.has_value())
                 {
                     UInt64 high;
@@ -740,7 +782,7 @@ void DWARFBlockInputFormat::parseFilenameTable(UnitState & unit, uint64_t offset
     auto error = prologue.parse(*debug_line_extractor, &offset, /*RecoverableErrorHandler*/ [&](auto e)
         {
             if (++seen_debug_line_warnings < 10)
-                LOG_INFO(&Poco::Logger::get("DWARF"), "{}", llvm::toString(std::move(e)));
+                LOG_INFO(&Poco::Logger::get("DWARF"), "Parsing error: {}", llvm::toString(std::move(e)));
         }, *dwarf_context, unit.dwarf_unit);
 
     if (error)
@@ -783,12 +825,12 @@ uint64_t DWARFBlockInputFormat::fetchFromDebugAddr(uint64_t addr_base, uint64_t
 }
 
 void DWARFBlockInputFormat::parseRanges(
-    uint64_t offset, bool form_rnglistx, std::optional<uint64_t> low_pc, const UnitState & unit, const ColumnVector<UInt64>::MutablePtr & col_ranges_start,
+    uint64_t offset, bool form_rnglistx, const UnitState & unit, const ColumnVector<UInt64>::MutablePtr & col_ranges_start,
     const ColumnVector<UInt64>::MutablePtr & col_ranges_end) const
 {
     llvm::Optional<llvm::object::SectionedAddress> base_addr;
-    if (low_pc.has_value())
-        base_addr = llvm::object::SectionedAddress{.Address = *low_pc};
+    if (unit.base_address != UINT64_MAX)
+        base_addr = llvm::object::SectionedAddress{.Address = unit.base_address};
 
     llvm::DWARFAddressRangesVector ranges;
 
@@ -833,7 +875,7 @@ void DWARFBlockInputFormat::parseRanges(
 
         auto lookup_addr = [&](uint32_t idx) -> llvm::Optional<llvm::object::SectionedAddress>
             {
-                uint64_t addr = fetchFromDebugAddr(unit.addr_base, idx);
+                uint64_t addr = fetchFromDebugAddr(unit.debug_addr_base, idx);
                 return llvm::object::SectionedAddress{.Address = addr};
             };
         ranges = list.getAbsoluteRanges(base_addr, /*AddressByteSize*/ 8, lookup_addr);

src/Processors/Formats/Impl/DWARFBlockInputFormat.h

@@ -53,8 +53,11 @@ private:
         std::string unit_name;
         ColumnPtr filename_table; // from .debug_line
         size_t filename_table_size = 0;
-        uint64_t addr_base = UINT64_MAX;
+        /// Starting offset of this unit's data in .debug_addr and .debug_rnglists sections.
+        uint64_t debug_addr_base = UINT64_MAX;
         uint64_t rnglists_base = UINT64_MAX;
+        /// "Base address" for parsing range lists. Not to be confused with "addr base".
+        uint64_t base_address = UINT64_MAX;
 
         uint64_t offset = 0;
         std::vector<StackEntry> stack;
@@ -102,11 +105,18 @@ private:
     void parseFilenameTable(UnitState & unit, uint64_t offset);
     Chunk parseEntries(UnitState & unit);
 
+    llvm::DWARFFormValue parseAttribute(
+        const llvm::DWARFAbbreviationDeclaration::AttributeSpec & attr, uint64_t * offset,
+        const UnitState & unit) const;
+    void skipAttribute(
+        const llvm::DWARFAbbreviationDeclaration::AttributeSpec & attr, uint64_t * offset,
+        const UnitState & unit) const;
+    uint64_t parseAddress(llvm::dwarf::Attribute attr, const llvm::DWARFFormValue & val, const UnitState & unit);
     /// Parse .debug_addr entry.
     uint64_t fetchFromDebugAddr(uint64_t addr_base, uint64_t idx) const;
     /// Parse .debug_ranges (DWARF4) or .debug_rnglists (DWARF5) entry.
     void parseRanges(
-        uint64_t offset, bool form_rnglistx, std::optional<uint64_t> low_pc, const UnitState & unit,
+        uint64_t offset, bool form_rnglistx, const UnitState & unit,
         const ColumnVector<UInt64>::MutablePtr & col_ranges_start,
         const ColumnVector<UInt64>::MutablePtr & col_ranges_end) const;
 };

src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp

@@ -284,23 +284,4 @@ void registerJSONCompactEachRowSchemaReader(FormatFactory & factory)
     }
 }
 
-void registerFileSegmentationEngineJSONCompactEachRow(FormatFactory & factory)
-{
-    auto register_func = [&](const String & format_name, bool with_names, bool with_types)
-    {
-        /// In case when we have names and/or types in the first two/one rows,
-        /// we need to read at least one more row of actual data. So, set
-        /// the minimum of rows for segmentation engine according to
-        /// parameters with_names and with_types.
-        size_t min_rows = 1 + int(with_names) + int(with_types);
-        factory.registerFileSegmentationEngine(format_name, [min_rows](ReadBuffer & in, DB::Memory<> & memory, size_t min_bytes, size_t max_rows)
-        {
-            return JSONUtils::fileSegmentationEngineJSONCompactEachRow(in, memory, min_bytes, min_rows, max_rows);
-        });
-    };
-
-    registerWithNamesAndTypes("JSONCompactEachRow", register_func);
-    registerWithNamesAndTypes("JSONCompactStringsEachRow", register_func);
-}
-
 }

src/Processors/Transforms/PasteJoinTransform.cpp

@@ -0,0 +1,127 @@
+#include <cassert>
+#include <cstddef>
+#include <limits>
+#include <memory>
+#include <type_traits>
+
+#include <base/defines.h>
+#include <base/types.h>
+
+#include <Common/logger_useful.h>
+#include <Columns/ColumnNullable.h>
+#include <Columns/ColumnsNumber.h>
+#include <Columns/IColumn.h>
+#include <IO/WriteHelpers.h>
+#include <Interpreters/TableJoin.h>
+#include <Parsers/ASTTablesInSelectQuery.h>
+#include <Processors/Transforms/PasteJoinTransform.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int NOT_IMPLEMENTED;
+}
+
+
+PasteJoinAlgorithm::PasteJoinAlgorithm(
+    JoinPtr table_join_,
+    const Blocks & input_headers,
+    size_t max_block_size_)
+    : table_join(table_join_)
+    , max_block_size(max_block_size_)
+    , log(&Poco::Logger::get("PasteJoinAlgorithm"))
+{
+    if (input_headers.size() != 2)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "PasteJoinAlgorithm requires exactly two inputs");
+
+    auto strictness = table_join->getTableJoin().strictness();
+    if (strictness != JoinStrictness::Any && strictness != JoinStrictness::All)
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "PasteJoinAlgorithm is not implemented for strictness {}", strictness);
+
+    auto kind = table_join->getTableJoin().kind();
+    if (!isPaste(kind))
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "PasteJoinAlgorithm is not implemented for kind {}", kind);
+}
+
+static void prepareChunk(Chunk & chunk)
+{
+    if (!chunk)
+        return;
+
+    auto num_rows = chunk.getNumRows();
+    auto columns = chunk.detachColumns();
+
+    chunk.setColumns(std::move(columns), num_rows);
+}
+
+void PasteJoinAlgorithm::initialize(Inputs inputs)
+{
+    if (inputs.size() != 2)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Two inputs are required, got {}", inputs.size());
+
+    for (size_t i = 0; i < inputs.size(); ++i)
+    {
+        consume(inputs[i], i);
+    }
+}
+
+void PasteJoinAlgorithm::consume(Input & input, size_t source_num)
+{
+    if (input.skip_last_row)
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "skip_last_row is not supported");
+
+    if (input.permutation)
+        throw DB::Exception(ErrorCodes::NOT_IMPLEMENTED, "permutation is not supported");
+
+    last_used_row[source_num] = 0;
+
+    prepareChunk(input.chunk);
+    chunks[source_num] = std::move(input.chunk);
+}
+
+IMergingAlgorithm::Status PasteJoinAlgorithm::merge()
+{
+    if (chunks[0].empty() || chunks[1].empty())
+        return Status({}, true);
+    if (last_used_row[0] >= chunks[0].getNumRows())
+        return Status(0);
+    if (last_used_row[1] >= chunks[1].getNumRows())
+        return Status(1);
+    /// We have unused rows from both inputs
+    size_t result_num_rows = std::min(chunks[0].getNumRows() - last_used_row[0], chunks[1].getNumRows() - last_used_row[1]);
+
+    Chunk result;
+    for (size_t source_num = 0; source_num < 2; ++source_num)
+        for (const auto & col : chunks[source_num].getColumns())
+            result.addColumn(col->cut(last_used_row[source_num], result_num_rows));
+    last_used_row[0] += result_num_rows;
+    last_used_row[1] += result_num_rows;
+    return Status(std::move(result));
+}
+
+PasteJoinTransform::PasteJoinTransform(
+        JoinPtr table_join,
+        const Blocks & input_headers,
+        const Block & output_header,
+        size_t max_block_size,
+        UInt64 limit_hint_)
+    : IMergingTransform<PasteJoinAlgorithm>(
+        input_headers,
+        output_header,
+        /* have_all_inputs_= */ true,
+        limit_hint_,
+        /* always_read_till_end_= */ false,
+        /* empty_chunk_on_finish_= */ true,
+        table_join, input_headers, max_block_size)
+    , log(&Poco::Logger::get("PasteJoinTransform"))
+{
+    LOG_TRACE(log, "Use PasteJoinTransform");
+}
+
+void PasteJoinTransform::onFinish() {};
+
+}

src/Processors/Transforms/PasteJoinTransform.h

@@ -0,0 +1,88 @@
+#pragma once
+#include <cassert>
+#include <cstddef>
+#include <memory>
+#include <mutex>
+#include <utility>
+
+#include <boost/core/noncopyable.hpp>
+
+#include <Common/PODArray.h>
+#include <IO/ReadBuffer.h>
+#include <Parsers/ASTTablesInSelectQuery.h>
+#include <Processors/Chunk.h>
+#include <Processors/Merges/Algorithms/IMergingAlgorithm.h>
+#include <Processors/Merges/IMergingTransform.h>
+
+namespace Poco { class Logger; }
+
+namespace DB
+{
+
+class IJoin;
+using JoinPtr = std::shared_ptr<IJoin>;
+
+/*
+ * This class is used to join chunks from two sorted streams.
+ * It is used in MergeJoinTransform.
+ */
+class PasteJoinAlgorithm final : public IMergingAlgorithm
+{
+public:
+    explicit PasteJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_);
+
+    const char * getName() const override { return "PasteJoinAlgorithm"; }
+    virtual void initialize(Inputs inputs) override;
+    virtual void consume(Input & input, size_t source_num) override;
+    virtual Status merge() override;
+
+    void logElapsed(double seconds);
+
+private:
+    Chunk createBlockWithDefaults(size_t source_num);
+    Chunk createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const;
+
+    /// For `USING` join key columns should have values from right side instead of defaults
+    std::unordered_map<size_t, size_t> left_to_right_key_remap;
+
+    std::array<Chunk, 2> chunks;
+
+    JoinPtr table_join;
+
+    size_t max_block_size;
+
+    struct Statistic
+    {
+        size_t num_blocks[2] = {0, 0};
+        size_t num_rows[2] = {0, 0};
+
+        size_t max_blocks_loaded = 0;
+    };
+
+    Statistic stat;
+
+    Poco::Logger * log;
+    UInt64 last_used_row[2] = {0, 0};
+};
+
+class PasteJoinTransform final : public IMergingTransform<PasteJoinAlgorithm>
+{
+    using Base = IMergingTransform<PasteJoinAlgorithm>;
+
+public:
+    PasteJoinTransform(
+        JoinPtr table_join,
+        const Blocks & input_headers,
+        const Block & output_header,
+        size_t max_block_size,
+        UInt64 limit_hint = 0);
+
+    String getName() const override { return "PasteJoinTransform"; }
+
+protected:
+    void onFinish() override;
+
+    Poco::Logger * log;
+};
+
+}

src/QueryPipeline/QueryPipelineBuilder.cpp

@@ -25,6 +25,7 @@
 #include <Processors/Transforms/ExtremesTransform.h>
 #include <Processors/Transforms/JoiningTransform.h>
 #include <Processors/Transforms/MergeJoinTransform.h>
+#include <Processors/Transforms/PasteJoinTransform.h>
 #include <Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h>
 #include <Processors/Transforms/PartialSortingTransform.h>
 #include <Processors/Transforms/TotalsHavingTransform.h>
@@ -36,6 +37,7 @@ namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
     extern const int NOT_IMPLEMENTED;
+    extern const int BAD_ARGUMENTS;
 }
 
 void QueryPipelineBuilder::checkInitialized()
@@ -354,7 +356,9 @@ std::unique_ptr<QueryPipelineBuilder> QueryPipelineBuilder::joinPipelinesYShaped
 
     left->pipe.dropExtremes();
     right->pipe.dropExtremes();
-    if (left->getNumStreams() != 1 || right->getNumStreams() != 1)
+    if ((left->getNumStreams() != 1 || right->getNumStreams() != 1) && join->getTableJoin().kind() == JoinKind::Paste)
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Paste JOIN requires sorted tables only");
+    else if (left->getNumStreams() != 1 || right->getNumStreams() != 1)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Join is supported only for pipelines with one output port");
 
     if (left->hasTotals() || right->hasTotals())
@@ -362,9 +366,16 @@ std::unique_ptr<QueryPipelineBuilder> QueryPipelineBuilder::joinPipelinesYShaped
 
     Blocks inputs = {left->getHeader(), right->getHeader()};
 
-    auto joining = std::make_shared<MergeJoinTransform>(join, inputs, out_header, max_block_size);
-
-    return mergePipelines(std::move(left), std::move(right), std::move(joining), collected_processors);
+    if (join->getTableJoin().kind() == JoinKind::Paste)
+    {
+        auto joining = std::make_shared<PasteJoinTransform>(join, inputs, out_header, max_block_size);
+        return mergePipelines(std::move(left), std::move(right), std::move(joining), collected_processors);
+    }
+    else
+    {
+        auto joining = std::make_shared<MergeJoinTransform>(join, inputs, out_header, max_block_size);
+        return mergePipelines(std::move(left), std::move(right), std::move(joining), collected_processors);
+    }
 }
 
 std::unique_ptr<QueryPipelineBuilder> QueryPipelineBuilder::joinPipelinesRightLeft(

src/Storages/MergeTree/MergeTreeData.cpp

@@ -2325,6 +2325,7 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa
             part_log_elem.partition_id = part->info.partition_id;
             part_log_elem.part_name = part->name;
             part_log_elem.bytes_compressed_on_disk = part->getBytesOnDisk();
+            part_log_elem.bytes_uncompressed = part->getBytesUncompressedOnDisk();
             part_log_elem.rows = part->rows_count;
             part_log_elem.part_type = part->getType();
 
@@ -7509,6 +7510,7 @@ try
         part_log_elem.disk_name = result_part->getDataPartStorage().getDiskName();
         part_log_elem.path_on_disk = result_part->getDataPartStorage().getFullPath();
         part_log_elem.bytes_compressed_on_disk = result_part->getBytesOnDisk();
+        part_log_elem.bytes_uncompressed = result_part->getBytesUncompressedOnDisk();
         part_log_elem.rows = result_part->rows_count;
         part_log_elem.part_type = result_part->getType();
     }
@@ -7523,7 +7525,6 @@ try
         part_log_elem.bytes_read_uncompressed = (*merge_entry)->bytes_read_uncompressed;
 
         part_log_elem.rows = (*merge_entry)->rows_written;
-        part_log_elem.bytes_uncompressed = (*merge_entry)->bytes_written_uncompressed;
         part_log_elem.peak_memory_usage = (*merge_entry)->getMemoryTracker().getPeak();
     }
 

src/Storages/MergeTree/MergeTreeSettings.cpp

@@ -212,4 +212,14 @@ void MergeTreeSettings::sanityCheck(size_t background_pool_tasks) const
             merge_selecting_sleep_slowdown_factor);
     }
 }
+
+
+std::vector<String> MergeTreeSettings::getAllRegisteredNames() const
+{
+    std::vector<String> all_settings;
+    for (const auto & setting_field : all())
+        all_settings.push_back(setting_field.getName());
+    return all_settings;
+}
+
 }

src/Storages/MergeTree/MergeTreeSettings.h

@@ -4,6 +4,7 @@
 #include <Core/Defines.h>
 #include <Core/BaseSettings.h>
 #include <Core/SettingsEnums.h>
+#include <Common/NamePrompter.h>
 #include <Interpreters/Context_fwd.h>
 #include <Storages/MergeTree/MergeTreeDataFormatVersion.h>
 
@@ -248,7 +249,7 @@ DECLARE_SETTINGS_TRAITS(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS)
 /** Settings for the MergeTree family of engines.
   * Could be loaded from config or from a CREATE TABLE query (SETTINGS clause).
   */
-struct MergeTreeSettings : public BaseSettings<MergeTreeSettingsTraits>
+struct MergeTreeSettings : public BaseSettings<MergeTreeSettingsTraits>, public IHints<2>
 {
     void loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config);
 
@@ -269,6 +270,8 @@ struct MergeTreeSettings : public BaseSettings<MergeTreeSettingsTraits>
 
     /// Check that the values are sane taking also query-level settings into account.
     void sanityCheck(size_t background_pool_tasks) const;
+
+    std::vector<String> getAllRegisteredNames() const override;
 };
 
 using MergeTreeSettingsPtr = std::shared_ptr<const MergeTreeSettings>;

src/Storages/MergeTree/MutateTask.cpp

@@ -879,6 +879,7 @@ void finalizeMutatedPart(
     /// All information about sizes is stored in checksums.
     /// It doesn't make sense to touch filesystem for sizes.
     new_data_part->setBytesOnDisk(new_data_part->checksums.getTotalSizeOnDisk());
+    new_data_part->setBytesUncompressedOnDisk(new_data_part->checksums.getTotalSizeUncompressedOnDisk());
     /// Also use information from checksums
     new_data_part->calculateColumnsAndSecondaryIndicesSizesOnDisk();
 

src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp

@@ -23,6 +23,11 @@
 #include <fmt/core.h>
 #include <fmt/format.h>
 
+namespace ProfileEvents
+{
+    extern const Event ParallelReplicasUsedCount;
+}
+
 namespace DB
 {
 struct Part
@@ -573,7 +578,15 @@ ParallelReadResponse ParallelReplicasReadingCoordinator::handleRequest(ParallelR
         initialize();
     }
 
-    return pimpl->handleRequest(std::move(request));
+    const auto replica_num = request.replica_num;
+    auto response = pimpl->handleRequest(std::move(request));
+    if (!response.finish)
+    {
+        if (replicas_used.insert(replica_num).second)
+            ProfileEvents::increment(ProfileEvents::ParallelReplicasUsedCount);
+    }
+
+    return response;
 }
 
 void ParallelReplicasReadingCoordinator::markReplicaAsUnavailable(size_t replica_number)

src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h

@@ -39,6 +39,7 @@ private:
     std::atomic<bool> initialized{false};
     std::unique_ptr<ImplInterface> pimpl;
     ProgressCallback progress_callback; // store the callback only to bypass it to coordinator implementation
+    std::set<size_t> replicas_used;
 
     /// To initialize `pimpl` we need to know the coordinator mode. We can know it only from initial announcement or regular request.
     /// The problem is `markReplicaAsUnavailable` might be called before any of these requests happened.

src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp

@@ -40,8 +40,6 @@ namespace ErrorCodes
     extern const int READONLY;
     extern const int UNKNOWN_STATUS_OF_INSERT;
     extern const int INSERT_WAS_DEDUPLICATED;
-    extern const int TIMEOUT_EXCEEDED;
-    extern const int NO_ACTIVE_REPLICAS;
     extern const int DUPLICATE_DATA_PART;
     extern const int PART_IS_TEMPORARILY_LOCKED;
     extern const int LOGICAL_ERROR;
@@ -160,7 +158,12 @@ size_t ReplicatedMergeTreeSinkImpl<async_insert>::checkQuorumPrecondition(const
 
     size_t replicas_number = 0;
 
-    ZooKeeperRetriesControl quorum_retries_ctl("checkQuorumPrecondition", zookeeper_retries_info, context->getProcessListElement());
+    const auto & settings = context->getSettingsRef();
+    ZooKeeperRetriesControl quorum_retries_ctl(
+        "checkQuorumPrecondition",
+        log,
+        {settings.insert_keeper_max_retries, settings.insert_keeper_retry_initial_backoff_ms, settings.insert_keeper_retry_max_backoff_ms},
+        context->getProcessListElement());
     quorum_retries_ctl.retryLoop(
         [&]()
         {
@@ -255,12 +258,6 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
     auto block = getHeader().cloneWithColumns(chunk.detachColumns());
 
     const auto & settings = context->getSettingsRef();
-    zookeeper_retries_info = ZooKeeperRetriesInfo(
-        "ReplicatedMergeTreeSink::consume",
-        settings.insert_keeper_max_retries ? log : nullptr,
-        settings.insert_keeper_max_retries,
-        settings.insert_keeper_retry_initial_backoff_ms,
-        settings.insert_keeper_retry_max_backoff_ms);
 
     ZooKeeperWithFaultInjectionPtr zookeeper = ZooKeeperWithFaultInjection::createInstance(
         settings.insert_keeper_fault_injection_probability,
@@ -636,7 +633,12 @@ std::pair<std::vector<String>, bool> ReplicatedMergeTreeSinkImpl<async_insert>::
 
     CommitRetryContext retry_context;
 
-    ZooKeeperRetriesControl retries_ctl("commitPart", zookeeper_retries_info, context->getProcessListElement());
+    const auto & settings = context->getSettingsRef();
+    ZooKeeperRetriesControl retries_ctl(
+        "commitPart",
+        log,
+        {settings.insert_keeper_max_retries, settings.insert_keeper_retry_initial_backoff_ms, settings.insert_keeper_retry_max_backoff_ms},
+        context->getProcessListElement());
 
     auto resolve_duplicate_stage = [&] () -> CommitRetryContext::Stages
     {
@@ -910,12 +912,8 @@ std::pair<std::vector<String>, bool> ReplicatedMergeTreeSinkImpl<async_insert>::
                         part->name, multi_code, MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER);
             });
 
-            /// Independently of how many retries we had left we want to do at least one check of this inner retry
-            /// so a) we try to verify at least once if metadata was written and b) we set the proper final error
-            /// (UNKNOWN_STATUS_OF_INSERT) if we fail to reconnect to keeper
-            new_retry_controller.requestUnconditionalRetry();
-
             bool node_exists = false;
+            /// The loop will be executed at least once
             new_retry_controller.retryLoop([&]
             {
                 fiu_do_on(FailPoints::replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault, { zookeeper->forceFailureBeforeOperation(); });
@@ -1073,7 +1071,26 @@ std::pair<std::vector<String>, bool> ReplicatedMergeTreeSinkImpl<async_insert>::
             if (quorum_parallel)
                 quorum_info.status_path = storage.zookeeper_path + "/quorum/parallel/" + retry_context.actual_part_name;
 
-            waitForQuorum(zookeeper, retry_context.actual_part_name, quorum_info.status_path, quorum_info.is_active_node_version, replicas_num);
+            ZooKeeperRetriesControl new_retry_controller = retries_ctl;
+            new_retry_controller.actionAfterLastFailedRetry([&]
+            {
+                /// We do not know whether or not data has been inserted in other replicas
+                new_retry_controller.setUserError(
+                    ErrorCodes::UNKNOWN_STATUS_OF_INSERT,
+                    "Unknown quorum status. The data was inserted in the local replica but we could not verify quorum. Reason: {}",
+                    new_retry_controller.getLastKeeperErrorMessage());
+            });
+
+            new_retry_controller.retryLoop([&]()
+            {
+                zookeeper->setKeeper(storage.getZooKeeper());
+                waitForQuorum(
+                    zookeeper,
+                    retry_context.actual_part_name,
+                    quorum_info.status_path,
+                    quorum_info.is_active_node_version,
+                    replicas_num);
+            });
         }
     }
 
@@ -1106,49 +1123,44 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::waitForQuorum(
     /// We are waiting for quorum to be satisfied.
     LOG_TRACE(log, "Waiting for quorum '{}' for part {}{}", quorum_path, part_name, quorumLogMessage(replicas_num));
 
-    try
+    fiu_do_on(FailPoints::replicated_merge_tree_insert_quorum_fail_0, { zookeeper->forceFailureBeforeOperation(); });
+
+    while (true)
     {
-        fiu_do_on(FailPoints::replicated_merge_tree_insert_quorum_fail_0, { zookeeper->forceFailureBeforeOperation(); });
+        zkutil::EventPtr event = std::make_shared<Poco::Event>();
 
-        while (true)
-        {
-            zkutil::EventPtr event = std::make_shared<Poco::Event>();
+        std::string value;
+        /// `get` instead of `exists` so that `watch` does not leak if the node is no longer there.
+        if (!zookeeper->tryGet(quorum_path, value, nullptr, event))
+            break;
 
-            std::string value;
-            /// `get` instead of `exists` so that `watch` does not leak if the node is no longer there.
-            if (!zookeeper->tryGet(quorum_path, value, nullptr, event))
-                break;
+        LOG_TRACE(log, "Quorum node {} still exists, will wait for updates", quorum_path);
 
-            LOG_TRACE(log, "Quorum node {} still exists, will wait for updates", quorum_path);
+        ReplicatedMergeTreeQuorumEntry quorum_entry(value);
 
-            ReplicatedMergeTreeQuorumEntry quorum_entry(value);
+        /// If the node has time to disappear, and then appear again for the next insert.
+        if (quorum_entry.part_name != part_name)
+            break;
 
-            /// If the node has time to disappear, and then appear again for the next insert.
-            if (quorum_entry.part_name != part_name)
-                break;
+        if (!event->tryWait(quorum_timeout_ms))
+            throw Exception(
+                ErrorCodes::UNKNOWN_STATUS_OF_INSERT,
+                "Unknown quorum status. The data was inserted in the local replica but we could not verify quorum. Reason: "
+                "Timeout while waiting for quorum");
 
-            if (!event->tryWait(quorum_timeout_ms))
-                throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout while waiting for quorum");
-
-            LOG_TRACE(log, "Quorum {} for part {} updated, will check quorum node still exists", quorum_path, part_name);
-        }
-
-        /// And what if it is possible that the current replica at this time has ceased to be active
-        /// and the quorum is marked as failed and deleted?
-        Coordination::Stat stat;
-        String value;
-        if (!zookeeper->tryGet(storage.replica_path + "/is_active", value, &stat)
-            || stat.version != is_active_node_version)
-            throw Exception(ErrorCodes::NO_ACTIVE_REPLICAS, "Replica become inactive while waiting for quorum");
-    }
-    catch (...)
-    {
-        /// We do not know whether or not data has been inserted
-        /// - whether other replicas have time to download the part and mark the quorum as done.
-        throw Exception(ErrorCodes::UNKNOWN_STATUS_OF_INSERT, "Unknown status, client must retry. Reason: {}",
-            getCurrentExceptionMessage(false));
+        LOG_TRACE(log, "Quorum {} for part {} updated, will check quorum node still exists", quorum_path, part_name);
     }
 
+    /// And what if it is possible that the current replica at this time has ceased to be active
+    /// and the quorum is marked as failed and deleted?
+    Coordination::Stat stat;
+    String value;
+    if (!zookeeper->tryGet(storage.replica_path + "/is_active", value, &stat) || stat.version != is_active_node_version)
+        throw Exception(
+            ErrorCodes::UNKNOWN_STATUS_OF_INSERT,
+            "Unknown quorum status. The data was inserted in the local replica but we could not verify quorum. Reason: "
+            "Replica became inactive while waiting for quorum");
+
     LOG_TRACE(log, "Quorum '{}' for part {} satisfied", quorum_path, part_name);
 }
 

src/Storages/MergeTree/ReplicatedMergeTreeSink.h

@@ -74,7 +74,6 @@ private:
 
     using BlockIDsType = std::conditional_t<async_insert, std::vector<String>, String>;
 
-    ZooKeeperRetriesInfo zookeeper_retries_info;
     struct QuorumInfo
     {
         String status_path;

src/Storages/MergeTree/ZooKeeperRetries.h

@@ -5,6 +5,8 @@
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/logger_useful.h>
 
+#include <memory>
+
 namespace DB
 {
 
@@ -15,29 +17,31 @@ namespace ErrorCodes
 
 struct ZooKeeperRetriesInfo
 {
-    ZooKeeperRetriesInfo() = default;
-    ZooKeeperRetriesInfo(std::string name_, Poco::Logger * logger_, UInt64 max_retries_, UInt64 initial_backoff_ms_, UInt64 max_backoff_ms_)
-        : name(std::move(name_))
-        , logger(logger_)
-        , max_retries(max_retries_)
-        , curr_backoff_ms(std::min(initial_backoff_ms_, max_backoff_ms_))
-        , max_backoff_ms(max_backoff_ms_)
+    ZooKeeperRetriesInfo(UInt64 max_retries_, UInt64 initial_backoff_ms_, UInt64 max_backoff_ms_)
+        : max_retries(max_retries_), initial_backoff_ms(std::min(initial_backoff_ms_, max_backoff_ms_)), max_backoff_ms(max_backoff_ms_)
     {
     }
 
-    std::string name;
-    Poco::Logger * logger = nullptr;
-    UInt64 max_retries = 0;
-    UInt64 curr_backoff_ms = 0;
-    UInt64 max_backoff_ms = 0;
-    UInt64 retry_count = 0;
+    UInt64 max_retries;
+    UInt64 initial_backoff_ms;
+    UInt64 max_backoff_ms;
 };
 
 class ZooKeeperRetriesControl
 {
 public:
-    ZooKeeperRetriesControl(std::string name_, ZooKeeperRetriesInfo & retries_info_, QueryStatusPtr elem)
-        : name(std::move(name_)), retries_info(retries_info_), process_list_element(elem)
+    ZooKeeperRetriesControl(std::string name_, Poco::Logger * logger_, ZooKeeperRetriesInfo retries_info_, QueryStatusPtr elem)
+        : name(std::move(name_)), logger(logger_), retries_info(retries_info_), process_list_element(elem)
+    {
+    }
+
+    ZooKeeperRetriesControl(const ZooKeeperRetriesControl & other)
+        : name(other.name)
+        , logger(other.logger)
+        , retries_info(other.retries_info)
+        , total_failures(other.total_failures)
+        , process_list_element(other.process_list_element)
+        , current_backoff_ms(other.current_backoff_ms)
     {
     }
 
@@ -46,7 +50,7 @@ public:
         retryLoop(f, []() {});
     }
 
-    /// retryLoop() executes f() until it succeeds/max_retries is reached/non-retrialable error is encountered
+    /// retryLoop() executes f() until it succeeds/max_retries is reached/non-retryable error is encountered
     ///
     /// the callable f() can provide feedback in terms of errors in two ways:
     /// 1. throw KeeperException exception:
@@ -56,10 +60,17 @@ public:
     ///     The idea is that if the caller has some semantics on top of non-hardware keeper errors,
     ///     then it can provide feedback to retries controller via user errors
     ///
+    /// It is possible to use it multiple times (it will share nº of errors over the total amount of calls)
+    /// Each retryLoop is independent and it will execute f at least once
     void retryLoop(auto && f, auto && iteration_cleanup)
     {
-        while (canTry())
+        current_iteration = 0;
+        current_backoff_ms = retries_info.initial_backoff_ms;
+
+        while (current_iteration == 0 || canTry())
         {
+            /// reset the flag, it will be set to false in case of error
+            iteration_succeeded = true;
             try
             {
                 f();
@@ -79,6 +90,7 @@ public:
                 iteration_cleanup();
                 throw;
             }
+            current_iteration++;
         }
     }
 
@@ -102,13 +114,11 @@ public:
 
     void setUserError(std::exception_ptr exception, int code, std::string message)
     {
-        if (retries_info.logger)
-            LOG_TRACE(
-                retries_info.logger, "ZooKeeperRetriesControl: {}/{}: setUserError: error={} message={}", retries_info.name, name, code, message);
+        if (logger)
+            LOG_TRACE(logger, "ZooKeeperRetriesControl: {}: setUserError: error={} message={}", name, code, message);
 
-        /// if current iteration is already failed, keep initial error
-        if (!iteration_succeeded)
-            return;
+        if (iteration_succeeded)
+            total_failures++;
 
         iteration_succeeded = false;
         user_error.code = code;
@@ -136,13 +146,11 @@ public:
 
     void setKeeperError(std::exception_ptr exception, Coordination::Error code, std::string message)
     {
-        if (retries_info.logger)
-            LOG_TRACE(
-                retries_info.logger, "ZooKeeperRetriesControl: {}/{}: setKeeperError: error={} message={}", retries_info.name, name, code, message);
+        if (logger)
+            LOG_TRACE(logger, "ZooKeeperRetriesControl: {}: setKeeperError: error={} message={}", name, code, message);
 
-        /// if current iteration is already failed, keep initial error
-        if (!iteration_succeeded)
-            return;
+        if (iteration_succeeded)
+            total_failures++;
 
         iteration_succeeded = false;
         keeper_error.code = code;
@@ -170,17 +178,19 @@ public:
 
     void stopRetries() { stop_retries = true; }
 
-    void requestUnconditionalRetry() { unconditional_retry = true; }
+    bool isLastRetry() const { return total_failures >= retries_info.max_retries; }
 
-    bool isLastRetry() const { return retries_info.retry_count >= retries_info.max_retries; }
+    bool isRetry() const { return current_iteration > 1; }
 
-    bool isRetry() const { return retries_info.retry_count > 0; }
-
-    Coordination::Error getLastKeeperErrorCode() const { return keeper_error.code; }
+    const std::string & getLastKeeperErrorMessage() const { return keeper_error.message; }
 
     /// action will be called only once and only after latest failed retry
     void actionAfterLastFailedRetry(std::function<void()> f) { action_after_last_failed_retry = std::move(f); }
 
+    const std::string & getName() const { return name; }
+
+    Poco::Logger * getLogger() const { return logger; }
+
 private:
     struct KeeperError
     {
@@ -199,59 +209,42 @@ private:
 
     bool canTry()
     {
-        ++iteration_count;
-        /// first iteration is ordinary execution, no further checks needed
-        if (0 == iteration_count)
-            return true;
-
-        if (process_list_element && !process_list_element->checkTimeLimitSoft())
-            return false;
-
-        if (unconditional_retry)
-        {
-            unconditional_retry = false;
-            return true;
-        }
-
-        /// iteration succeeded -> no need to retry
         if (iteration_succeeded)
         {
-            /// avoid unnecessary logs, - print something only in case of retries
-            if (retries_info.logger && iteration_count > 1)
+            if (logger && total_failures > 0)
                 LOG_DEBUG(
-                    retries_info.logger,
-                    "ZooKeeperRetriesControl: {}/{}: succeeded after: iterations={} total_retries={}",
-                    retries_info.name,
+                    logger,
+                    "ZooKeeperRetriesControl: {}: succeeded after: Iterations={} Total keeper failures={}/{}",
                     name,
-                    iteration_count,
-                    retries_info.retry_count);
+                    current_iteration,
+                    total_failures,
+                    retries_info.max_retries);
             return false;
         }
 
         if (stop_retries)
         {
-            logLastError("stop retries on request");
             action_after_last_failed_retry();
+            logLastError("stop retries on request");
             throwIfError();
             return false;
         }
 
-        if (retries_info.retry_count >= retries_info.max_retries)
+        if (total_failures > retries_info.max_retries)
         {
-            logLastError("retry limit is reached");
             action_after_last_failed_retry();
+            logLastError("retry limit is reached");
             throwIfError();
             return false;
         }
 
+        if (process_list_element && !process_list_element->checkTimeLimitSoft())
+            return false;
+
         /// retries
-        ++retries_info.retry_count;
         logLastError("will retry due to error");
-        sleepForMilliseconds(retries_info.curr_backoff_ms);
-        retries_info.curr_backoff_ms = std::min(retries_info.curr_backoff_ms * 2, retries_info.max_backoff_ms);
-
-        /// reset the flag, it will be set to false in case of error
-        iteration_succeeded = true;
+        sleepForMilliseconds(current_backoff_ms);
+        current_backoff_ms = std::min(current_backoff_ms * 2, retries_info.max_backoff_ms);
 
         return true;
     }
@@ -265,49 +258,52 @@ private:
             std::rethrow_exception(keeper_error.exception);
     }
 
-    void logLastError(std::string_view header)
+    void logLastError(const std::string_view & header)
     {
+        if (!logger)
+            return;
         if (user_error.code == ErrorCodes::OK)
         {
-            if (retries_info.logger)
-                LOG_DEBUG(
-                    retries_info.logger,
-                    "ZooKeeperRetriesControl: {}/{}: {}: retry_count={} timeout={}ms error={} message={}",
-                    retries_info.name,
-                    name,
-                    header,
-                    retries_info.retry_count,
-                    retries_info.curr_backoff_ms,
-                    keeper_error.code,
-                    keeper_error.message);
+            LOG_DEBUG(
+                logger,
+                "ZooKeeperRetriesControl: {}: {}: retry_count={}/{} timeout={}ms error={} message={}",
+                name,
+                header,
+                current_iteration,
+                retries_info.max_retries,
+                current_backoff_ms,
+                keeper_error.code,
+                keeper_error.message);
         }
         else
         {
-            if (retries_info.logger)
-                LOG_DEBUG(
-                    retries_info.logger,
-                    "ZooKeeperRetriesControl: {}/{}: {}: retry_count={} timeout={}ms error={} message={}",
-                    retries_info.name,
-                    name,
-                    header,
-                    retries_info.retry_count,
-                    retries_info.curr_backoff_ms,
-                    user_error.code,
-                    user_error.message);
+            LOG_DEBUG(
+                logger,
+                "ZooKeeperRetriesControl: {}: {}: retry_count={}/{} timeout={}ms error={} message={}",
+                name,
+                header,
+                current_iteration,
+                retries_info.max_retries,
+                current_backoff_ms,
+                user_error.code,
+                user_error.message);
         }
     }
 
 
     std::string name;
-    ZooKeeperRetriesInfo & retries_info;
-    Int64 iteration_count = -1;
+    Poco::Logger * logger = nullptr;
+    ZooKeeperRetriesInfo retries_info;
+    UInt64 total_failures = 0;
     UserError user_error;
     KeeperError keeper_error;
     std::function<void()> action_after_last_failed_retry = []() {};
-    bool unconditional_retry = false;
     bool iteration_succeeded = true;
     bool stop_retries = false;
     QueryStatusPtr process_list_element;
+
+    UInt64 current_iteration = 0;
+    UInt64 current_backoff_ms = 0;
 };
 
 }

src/Storages/StorageFile.cpp

@@ -1935,7 +1935,7 @@ void StorageFile::parseFileSource(String source, String & filename, String & pat
     }
 
     std::string_view path_to_archive_view = std::string_view{source}.substr(0, pos);
-    while (path_to_archive_view.back() == ' ')
+    while (path_to_archive_view.ends_with(' '))
         path_to_archive_view.remove_suffix(1);
 
     if (path_to_archive_view.empty())

src/Storages/StorageReplicatedMergeTree.cpp

@@ -10258,7 +10258,7 @@ void StorageReplicatedMergeTree::backupData(
         bool exists = false;
         Strings mutation_ids;
         {
-            ZooKeeperRetriesControl retries_ctl("getMutations", zookeeper_retries_info, nullptr);
+            ZooKeeperRetriesControl retries_ctl("getMutations", log, zookeeper_retries_info, nullptr);
             retries_ctl.retryLoop([&]()
             {
                 if (!zookeeper || zookeeper->expired())
@@ -10277,7 +10277,7 @@ void StorageReplicatedMergeTree::backupData(
                 bool mutation_id_exists = false;
                 String mutation;
 
-                ZooKeeperRetriesControl retries_ctl("getMutation", zookeeper_retries_info, nullptr);
+                ZooKeeperRetriesControl retries_ctl("getMutation", log, zookeeper_retries_info, nullptr);
                 retries_ctl.retryLoop([&]()
                 {
                     if (!zookeeper || zookeeper->expired())

src/Storages/System/StorageSystemBackups.cpp

@@ -20,6 +20,7 @@ NamesAndTypesList StorageSystemBackups::getNamesAndTypes()
     NamesAndTypesList names_and_types{
         {"id", std::make_shared<DataTypeString>()},
         {"name", std::make_shared<DataTypeString>()},
+        {"base_backup_name", std::make_shared<DataTypeString>()},
         {"status", std::make_shared<DataTypeEnum8>(getBackupStatusEnumValues())},
         {"error", std::make_shared<DataTypeString>()},
         {"start_time", std::make_shared<DataTypeDateTime>()},
@@ -42,6 +43,7 @@ void StorageSystemBackups::fillData(MutableColumns & res_columns, ContextPtr con
     size_t column_index = 0;
     auto & column_id = assert_cast<ColumnString &>(*res_columns[column_index++]);
     auto & column_name = assert_cast<ColumnString &>(*res_columns[column_index++]);
+    auto & column_base_backup_name = assert_cast<ColumnString &>(*res_columns[column_index++]);
     auto & column_status = assert_cast<ColumnInt8 &>(*res_columns[column_index++]);
     auto & column_error = assert_cast<ColumnString &>(*res_columns[column_index++]);
     auto & column_start_time = assert_cast<ColumnUInt32 &>(*res_columns[column_index++]);
@@ -59,6 +61,7 @@ void StorageSystemBackups::fillData(MutableColumns & res_columns, ContextPtr con
     {
         column_id.insertData(info.id.data(), info.id.size());
         column_name.insertData(info.name.data(), info.name.size());
+        column_base_backup_name.insertData(info.base_backup_name.data(), info.base_backup_name.size());
         column_status.insertValue(static_cast<Int8>(info.status));
         column_error.insertData(info.error_message.data(), info.error_message.size());
         column_start_time.insertValue(static_cast<UInt32>(std::chrono::system_clock::to_time_t(info.start_time)));

src/Storages/System/StorageSystemStackTrace.cpp

@@ -1,4 +1,4 @@
-#ifdef OS_LINUX /// Because of 'sigqueue' functions and RT signals.
+#ifdef OS_LINUX /// Because of 'rt_tgsigqueueinfo' functions and RT signals.
 
 #include <csignal>
 #include <poll.h>
@@ -6,6 +6,7 @@
 #include <mutex>
 #include <filesystem>
 #include <unordered_map>
+#include <memory>
 
 #include <base/scope_guard.h>
 
@@ -24,11 +25,16 @@
 #include <Common/HashTable/Hash.h>
 #include <Common/logger_useful.h>
 #include <Common/Stopwatch.h>
+#include <Core/ColumnsWithTypeAndName.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/TranslateQualifiedNamesVisitor.h>
 #include <Processors/Sources/SourceFromSingleChunk.h>
+#include <Processors/QueryPlan/SourceStepWithFilter.h>
+#include <Processors/QueryPlan/QueryPlan.h>
 #include <QueryPipeline/Pipe.h>
+#include <QueryPipeline/QueryPipelineBuilder.h>
 #include <base/getThreadId.h>
-
+#include <sys/syscall.h>
 
 namespace DB
 {
@@ -48,8 +54,8 @@ namespace
 {
 
 // Initialized in StorageSystemStackTrace's ctor and used in signalHandler.
-std::atomic<pid_t> expected_pid;
-const int sig = SIGRTMIN;
+std::atomic<pid_t> server_pid;
+const int STACK_TRACE_SERVICE_SIGNAL = SIGRTMIN;
 
 std::atomic<int> sequence_num = 0;    /// For messages sent via pipe.
 std::atomic<int> data_ready_num = 0;
@@ -73,6 +79,11 @@ size_t query_id_size = 0;
 
 LazyPipeFDs notification_pipe;
 
+int rt_tgsigqueueinfo(pid_t tgid, pid_t tid, int sig, siginfo_t *info)
+{
+    return static_cast<int>(syscall(__NR_rt_tgsigqueueinfo, tgid, tid, sig, info));
+}
+
 void signalHandler(int, siginfo_t * info, void * context)
 {
     DENY_ALLOCATIONS_IN_SCOPE;
@@ -80,7 +91,7 @@ void signalHandler(int, siginfo_t * info, void * context)
 
     /// In case malicious user is sending signals manually (for unknown reason).
     /// If we don't check - it may break our synchronization.
-    if (info->si_pid != expected_pid)
+    if (info->si_pid != server_pid)
         return;
 
     /// Signal received too late.
@@ -162,7 +173,7 @@ bool wait(int timeout_ms)
 }
 
 using ThreadIdToName = std::unordered_map<UInt64, String, DefaultHash<UInt64>>;
-ThreadIdToName getFilteredThreadNames(ASTPtr query, ContextPtr context, const PaddedPODArray<UInt64> & thread_ids, Poco::Logger * log)
+ThreadIdToName getFilteredThreadNames(const ActionsDAG::Node * predicate, ContextPtr context, const PaddedPODArray<UInt64> & thread_ids, Poco::Logger * log)
 {
     ThreadIdToName tid_to_name;
     MutableColumnPtr all_thread_names = ColumnString::create();
@@ -193,7 +204,7 @@ ThreadIdToName getFilteredThreadNames(ASTPtr query, ContextPtr context, const Pa
     LOG_TRACE(log, "Read {} thread names for {} threads, took {} ms", tid_to_name.size(), thread_ids.size(), watch.elapsedMilliseconds());
 
     Block block { ColumnWithTypeAndName(std::move(all_thread_names), std::make_shared<DataTypeString>(), "thread_name") };
-    VirtualColumnUtils::filterBlockWithQuery(query, block, context);
+    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
     ColumnPtr thread_names = std::move(block.getByPosition(0).column);
 
     std::unordered_set<String> filtered_thread_names;
@@ -214,24 +225,69 @@ ThreadIdToName getFilteredThreadNames(ASTPtr query, ContextPtr context, const Pa
     return tid_to_name;
 }
 
+bool parseHexNumber(std::string_view sv, UInt64 & res)
+{
+    errno = 0; /// Functions strto* don't clear errno.
+    char * pos_integer = const_cast<char *>(sv.begin());
+    res = std::strtoull(sv.begin(), &pos_integer, 16);
+    return (pos_integer == sv.begin() + sv.size() && errno != ERANGE);
+}
+bool isSignalBlocked(UInt64 tid, int signal)
+{
+    String buffer;
+
+    try
+    {
+        ReadBufferFromFile status(fmt::format("/proc/{}/status", tid));
+        while (!status.eof())
+        {
+            readEscapedStringUntilEOL(buffer, status);
+            if (!status.eof())
+                ++status.position();
+            if (buffer.starts_with("SigBlk:"))
+                break;
+        }
+        status.close();
+
+        std::string_view line(buffer);
+        line = line.substr(strlen("SigBlk:"));
+        line = line.substr(0, line.rend() - std::find_if_not(line.rbegin(), line.rend(), ::isspace));
+
+        UInt64 sig_blk;
+        if (parseHexNumber(line, sig_blk))
+            return sig_blk & signal;
+    }
+    catch (const Exception & e)
+    {
+        /// Ignore TOCTOU error
+        if (e.code() != ErrorCodes::FILE_DOESNT_EXIST)
+            throw;
+    }
+
+    return false;
+}
+
 /// Send a signal to every thread and wait for result.
 /// We must wait for every thread one by one sequentially,
 ///  because there is a limit on number of queued signals in OS and otherwise signals may get lost.
 /// Also, non-RT signals are not delivered if previous signal is handled right now (by default; but we use RT signals).
-class StorageSystemStackTraceSource : public ISource
+class StackTraceSource : public ISource
 {
 public:
-    StorageSystemStackTraceSource(const Names & column_names, Block header_, const ASTPtr query_, ContextPtr context_, UInt64 max_block_size_, Poco::Logger * log_)
+    StackTraceSource(const Names & column_names, Block header_, ASTPtr && query_, ActionsDAGPtr && filter_dag_, ContextPtr context_, UInt64 max_block_size_, Poco::Logger * log_)
         : ISource(header_)
         , context(context_)
         , header(std::move(header_))
-        , query(query_)
+        , query(std::move(query_))
+        , filter_dag(std::move(filter_dag_))
+        , predicate(filter_dag ? filter_dag->getOutputs().at(0) : nullptr)
         , max_block_size(max_block_size_)
         , pipe_read_timeout_ms(static_cast<int>(context->getSettingsRef().storage_system_stack_trace_pipe_read_timeout_ms.totalMilliseconds()))
         , log(log_)
         , proc_it("/proc/self/task")
         /// It shouldn't be possible to do concurrent reads from this table.
         , lock(mutex)
+        , signal_str(strsignal(STACK_TRACE_SERVICE_SIGNAL)) /// NOLINT(concurrency-mt-unsafe) // not thread-safe but ok in this context
     {
         /// Create a mask of what columns are needed in the result.
         NameSet names_set(column_names.begin(), column_names.end());
@@ -257,9 +313,10 @@ protected:
 
         const auto & thread_ids_data = assert_cast<const ColumnUInt64 &>(*thread_ids).getData();
 
+        /// NOTE: This is racy, so you may get incorrect thread_name.
         ThreadIdToName thread_names;
         if (read_thread_names)
-            thread_names = getFilteredThreadNames(query, context, thread_ids_data, log);
+            thread_names = getFilteredThreadNames(predicate, context, thread_ids_data, log);
 
         for (UInt64 tid : thread_ids_data)
         {
@@ -283,53 +340,71 @@ protected:
             }
             else
             {
-                ++signals_sent;
-                Stopwatch watch;
-                SCOPE_EXIT({ signals_sent_ms += watch.elapsedMilliseconds(); });
-
-                sigval sig_value{};
-
-                sig_value.sival_int = sequence_num.load(std::memory_order_acquire);
-                if (0 != ::sigqueue(static_cast<int>(tid), sig, sig_value))
+                /// NOTE: This check is racy (thread can be
+                /// destroyed/replaced/...), but it is OK, since only the
+                /// following could happen:
+                /// - it will incorrectly detect that the signal is blocked and
+                ///   will not send it this time
+                /// - it will incorrectly detect that the signal is not blocked
+                ///   then it will wait storage_system_stack_trace_pipe_read_timeout_ms
+                bool signal_blocked = isSignalBlocked(tid, STACK_TRACE_SERVICE_SIGNAL);
+                if (!signal_blocked)
                 {
-                    /// The thread may has been already finished.
-                    if (ESRCH == errno)
+                    ++signals_sent;
+                    Stopwatch watch;
+                    SCOPE_EXIT({
+                        signals_sent_ms += watch.elapsedMilliseconds();
+
+                        /// Signed integer overflow is undefined behavior in both C and C++. However, according to
+                        /// C++ standard, Atomic signed integer arithmetic is defined to use two's complement; there
+                        /// are no undefined results. See https://en.cppreference.com/w/cpp/atomic/atomic and
+                        /// http://eel.is/c++draft/atomics.types.generic#atomics.types.int-8
+                        ++sequence_num;
+                    });
+
+                    siginfo_t sig_info{};
+                    sig_info.si_code = SI_QUEUE; /// sigqueue()
+                    sig_info.si_pid = server_pid;
+                    sig_info.si_value.sival_int = sequence_num.load(std::memory_order_acquire);
+
+                    if (0 != rt_tgsigqueueinfo(server_pid, static_cast<pid_t>(tid), STACK_TRACE_SERVICE_SIGNAL, &sig_info))
+                    {
+                        /// The thread may has been already finished.
+                        if (ESRCH == errno)
+                            continue;
+
+                        throw ErrnoException(ErrorCodes::CANNOT_SIGQUEUE, "Cannot queue a signal");
+                    }
+
+                    /// Just in case we will wait for pipe with timeout. In case signal didn't get processed.
+                    if (wait(pipe_read_timeout_ms) && sig_info.si_value.sival_int == data_ready_num.load(std::memory_order_acquire))
+                    {
+                        size_t stack_trace_size = stack_trace.getSize();
+                        size_t stack_trace_offset = stack_trace.getOffset();
+
+                        Array arr;
+                        arr.reserve(stack_trace_size - stack_trace_offset);
+                        for (size_t i = stack_trace_offset; i < stack_trace_size; ++i)
+                            arr.emplace_back(reinterpret_cast<intptr_t>(stack_trace.getFramePointers()[i]));
+
+                        res_columns[res_index++]->insert(thread_name);
+                        res_columns[res_index++]->insert(tid);
+                        res_columns[res_index++]->insertData(query_id_data, query_id_size);
+                        res_columns[res_index++]->insert(arr);
+
                         continue;
-
-                    throw ErrnoException(ErrorCodes::CANNOT_SIGQUEUE, "Cannot send signal with sigqueue");
+                    }
                 }
 
-                /// Just in case we will wait for pipe with timeout. In case signal didn't get processed.
-                if (wait(pipe_read_timeout_ms) && sig_value.sival_int == data_ready_num.load(std::memory_order_acquire))
-                {
-                    size_t stack_trace_size = stack_trace.getSize();
-                    size_t stack_trace_offset = stack_trace.getOffset();
-
-                    Array arr;
-                    arr.reserve(stack_trace_size - stack_trace_offset);
-                    for (size_t i = stack_trace_offset; i < stack_trace_size; ++i)
-                        arr.emplace_back(reinterpret_cast<intptr_t>(stack_trace.getFramePointers()[i]));
-
-                    res_columns[res_index++]->insert(thread_name);
-                    res_columns[res_index++]->insert(tid);
-                    res_columns[res_index++]->insertData(query_id_data, query_id_size);
-                    res_columns[res_index++]->insert(arr);
-                }
+                if (signal_blocked)
+                    LOG_DEBUG(log, "Thread {} ({}) blocks SIG{} signal", tid, thread_name, signal_str);
                 else
-                {
-                    LOG_DEBUG(log, "Cannot obtain a stack trace for thread {}", tid);
+                    LOG_DEBUG(log, "Cannot obtain a stack trace for thread {} ({})", tid, thread_name);
 
-                    res_columns[res_index++]->insert(thread_name);
-                    res_columns[res_index++]->insert(tid);
-                    res_columns[res_index++]->insertDefault();
-                    res_columns[res_index++]->insertDefault();
-                }
-
-                /// Signed integer overflow is undefined behavior in both C and C++. However, according to
-                /// C++ standard, Atomic signed integer arithmetic is defined to use two's complement; there
-                /// are no undefined results. See https://en.cppreference.com/w/cpp/atomic/atomic and
-                /// http://eel.is/c++draft/atomics.types.generic#atomics.types.int-8
-                ++sequence_num;
+                res_columns[res_index++]->insert(thread_name);
+                res_columns[res_index++]->insert(tid);
+                res_columns[res_index++]->insertDefault();
+                res_columns[res_index++]->insertDefault();
             }
         }
         LOG_TRACE(log, "Send signal to {} threads (total), took {} ms", signals_sent, signals_sent_ms);
@@ -343,6 +418,8 @@ private:
     ContextPtr context;
     Block header;
     const ASTPtr query;
+    const ActionsDAGPtr filter_dag;
+    const ActionsDAG::Node * predicate;
 
     const size_t max_block_size;
     const int pipe_read_timeout_ms;
@@ -358,6 +435,7 @@ private:
     size_t signals_sent_ms = 0;
 
     std::unique_lock<std::mutex> lock;
+    const char * signal_str;
 
     ColumnPtr getFilteredThreadIds()
     {
@@ -372,11 +450,55 @@ private:
         }
 
         Block block { ColumnWithTypeAndName(std::move(all_thread_ids), std::make_shared<DataTypeUInt64>(), "thread_id") };
-        VirtualColumnUtils::filterBlockWithQuery(query, block, context);
+        VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+
         return block.getByPosition(0).column;
     }
 };
 
+class ReadFromSystemStackTrace : public SourceStepWithFilter
+{
+public:
+    std::string getName() const override { return "ReadFromSystemStackTrace"; }
+
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override
+    {
+        auto filter_actions_dag = ActionsDAG::buildFilterActionsDAG(filter_nodes.nodes, {}, context);
+        Pipe pipe(std::make_shared<StackTraceSource>(
+            column_names,
+            getOutputStream().header,
+            std::move(query),
+            std::move(filter_actions_dag),
+            context,
+            max_block_size,
+            log));
+        pipeline.init(std::move(pipe));
+    }
+
+    ReadFromSystemStackTrace(
+        const Names & column_names_,
+        Block sample_block,
+        ASTPtr && query_,
+        ContextPtr context_,
+        size_t max_block_size_,
+        Poco::Logger * log_)
+        : SourceStepWithFilter(DataStream{.header = std::move(sample_block)})
+        , column_names(column_names_)
+        , query(query_)
+        , context(std::move(context_))
+        , max_block_size(max_block_size_)
+        , log(log_)
+    {
+    }
+
+private:
+    Names column_names;
+    ASTPtr query;
+    ContextPtr context;
+    size_t max_block_size;
+    Poco::Logger * log;
+};
+
 }
 
 
@@ -396,7 +518,7 @@ StorageSystemStackTrace::StorageSystemStackTrace(const StorageID & table_id_)
     notification_pipe.open();
 
     /// Setup signal handler.
-    expected_pid = getpid();
+    server_pid = getpid();
     struct sigaction sa{};
     sa.sa_sigaction = signalHandler;
     sa.sa_flags = SA_SIGINFO;
@@ -404,31 +526,35 @@ StorageSystemStackTrace::StorageSystemStackTrace(const StorageID & table_id_)
     if (sigemptyset(&sa.sa_mask))
         throw ErrnoException(ErrorCodes::CANNOT_MANIPULATE_SIGSET, "Cannot set signal handler");
 
-    if (sigaddset(&sa.sa_mask, sig))
+    if (sigaddset(&sa.sa_mask, STACK_TRACE_SERVICE_SIGNAL))
         throw ErrnoException(ErrorCodes::CANNOT_MANIPULATE_SIGSET, "Cannot set signal handler");
 
-    if (sigaction(sig, &sa, nullptr))
+    if (sigaction(STACK_TRACE_SERVICE_SIGNAL, &sa, nullptr))
         throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
 }
 
 
-Pipe StorageSystemStackTrace::read(
+void StorageSystemStackTrace::read(
+    QueryPlan & query_plan,
     const Names & column_names,
     const StorageSnapshotPtr & storage_snapshot,
     SelectQueryInfo & query_info,
     ContextPtr context,
     QueryProcessingStage::Enum /*processed_stage*/,
-    const size_t max_block_size,
-    const size_t /*num_streams*/)
+    size_t max_block_size,
+    size_t /*num_streams*/)
 {
     storage_snapshot->check(column_names);
-    return Pipe(std::make_shared<StorageSystemStackTraceSource>(
+    Block sample_block = storage_snapshot->metadata->getSampleBlock();
+
+    auto reading = std::make_unique<ReadFromSystemStackTrace>(
         column_names,
-        storage_snapshot->metadata->getSampleBlock(),
+        sample_block,
         query_info.query->clone(),
         context,
         max_block_size,
-        log));
+        log);
+    query_plan.addStep(std::move(reading));
 }
 
 }

src/Storages/System/StorageSystemStackTrace.h

@@ -25,14 +25,15 @@ public:
 
     String getName() const override { return "SystemStackTrace"; }
 
-    Pipe read(
+    void read(
+        QueryPlan & query_plan,
         const Names & column_names,
         const StorageSnapshotPtr & storage_snapshot,
         SelectQueryInfo & query_info,
         ContextPtr context,
-        QueryProcessingStage::Enum processed_stage,
+        QueryProcessingStage::Enum /*processed_stage*/,
         size_t max_block_size,
-        size_t num_streams) override;
+        size_t /*num_streams*/) override;
 
     bool isSystemStorage() const override { return true; }
 

tests/ci/build_check.py

@@ -9,7 +9,7 @@ import sys
 import time
 
 from ci_config import CI_CONFIG, BuildConfig
-from ccache_utils import CargoCache
+from cache_utils import CargoCache
 
 from env_helper import (
     GITHUB_JOB_API_URL,

tests/ci/cache_utils.py

@@ -5,12 +5,10 @@ import os
 import shutil
 from pathlib import Path
 
-import requests  # type: ignore
-
-from build_download_helper import download_build_with_progress, DownloadException
-from compress_files import decompress_fast, compress_fast
+from build_download_helper import DownloadException, download_build_with_progress
+from compress_files import compress_fast, decompress_fast
 from digest_helper import digest_path
-from env_helper import S3_DOWNLOAD, S3_BUILDS_BUCKET
+from env_helper import S3_BUILDS_BUCKET, S3_DOWNLOAD
 from git_helper import git_runner
 from s3_helper import S3Helper
 
@@ -98,7 +96,67 @@ def upload_ccache(
     logging.info("Upload finished")
 
 
-class CargoCache:
+class CacheError(Exception):
+    pass
+
+
+class Cache:
+    """a generic class for all caches"""
+
+    def __init__(
+        self,
+        directory: Path,
+        temp_path: Path,
+        archive_name: str,
+        s3_helper: S3Helper,
+    ):
+        self.directory = directory
+        self.temp_path = temp_path
+        self.archive_name = archive_name
+        self.s3_helper = s3_helper
+
+    def _download(self, url: str, ignore_error: bool = False) -> None:
+        compressed_cache = self.temp_path / self.archive_name
+        try:
+            download_build_with_progress(url, compressed_cache)
+        except DownloadException as e:
+            if not ignore_error:
+                raise CacheError(f"Failed to download {url}") from e
+            logging.warning("Unable downloading cache, creating empty directory")
+            self.directory.mkdir(parents=True, exist_ok=True)
+            return
+
+        # decompress the cache and check if the necessary directory is there
+        self.directory.parent.mkdir(parents=True, exist_ok=True)
+        decompress_fast(compressed_cache, self.directory.parent)
+        if not self.directory.exists():
+            if not ignore_error:
+                raise CacheError(
+                    "The cache is downloaded and uncompressed, but directory "
+                    f"{self.directory} does not exist"
+                )
+            logging.warning(
+                "The cache archive was successfully downloaded and "
+                "decompressed, but %s does not exitst. Creating empty one",
+                self.directory,
+            )
+            self.directory.mkdir(parents=True, exist_ok=True)
+
+    def _upload(self, s3_path: str, force_upload: bool = False) -> None:
+        if not force_upload:
+            existing_cache = self.s3_helper.list_prefix_non_recursive(s3_path)
+            if existing_cache:
+                logging.info("Remote cache %s already exist, won't reupload", s3_path)
+                return
+
+        logging.info("Compressing cargo cache")
+        archive_path = self.temp_path / self.archive_name
+        compress_fast(self.directory, archive_path)
+        logging.info("Uploading %s to S3 path %s", archive_path, s3_path)
+        self.s3_helper.upload_build_file_to_s3(archive_path, s3_path)
+
+
+class CargoCache(Cache):
     PREFIX = "ccache/cargo_cache"
 
     def __init__(
@@ -107,51 +165,49 @@ class CargoCache:
         temp_path: Path,
         s3_helper: S3Helper,
     ):
-        self._cargo_lock_file = Path(git_runner.cwd) / "rust" / "Cargo.lock"
-        self.lock_hash = digest_path(self._cargo_lock_file).hexdigest()
-        self.directory = directory
-        self.archive_name = f"Cargo_cache_{self.lock_hash}.tar.zst"
-        self.temp_path = temp_path
-        self.s3_helper = s3_helper
-        self._url = (
-            f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/{self.PREFIX}/{self.archive_name}"
-        )
+        cargo_lock_file = Path(git_runner.cwd) / "rust" / "Cargo.lock"
+        self.lock_hash = digest_path(cargo_lock_file).hexdigest()
         self._force_upload_cache = False
+        super().__init__(
+            directory, temp_path, f"Cargo_cache_{self.lock_hash}.tar.zst", s3_helper
+        )
+        self._url = self.s3_helper.get_url(
+            S3_BUILDS_BUCKET, f"{self.PREFIX}/{self.archive_name}"
+        )
 
     def download(self):
         logging.info("Searching rust cache for Cargo.lock md5 %s", self.lock_hash)
-        compressed_cache = self.temp_path / self.archive_name
         try:
-            download_build_with_progress(self._url, compressed_cache)
-        except DownloadException:
+            self._download(self._url, False)
+        except CacheError:
             logging.warning("Unable downloading cargo cache, creating empty directory")
+            logging.info("Cache for Cargo.lock md5 %s will be uploaded", self.lock_hash)
+            self._force_upload_cache = True
             self.directory.mkdir(parents=True, exist_ok=True)
             return
 
-        # decompress the cache and check if the necessary directory is there
-        self.directory.parent.mkdir(parents=True, exist_ok=True)
-        decompress_fast(compressed_cache, self.directory.parent)
-        if not self.directory.exists():
-            logging.warning(
-                "The cargo cache archive was successfully downloaded and "
-                "decompressed, but %s does not exitst. Creating empty one",
-                self.directory,
-            )
-            logging.info("Cache for Cargo.lock md5 %s will be uploaded", self.lock_hash)
-            self.directory.mkdir(parents=True, exist_ok=True)
+    def upload(self):
+        self._upload(f"{self.PREFIX}/{self.archive_name}", self._force_upload_cache)
+
+
+class GitHubCache(Cache):
+    PREFIX = "ccache/github"
+
+    def __init__(
+        self,
+        directory: Path,
+        temp_path: Path,
+        s3_helper: S3Helper,
+    ):
+        self.force_upload = True
+        super().__init__(directory, temp_path, "GitHub.tar.zst", s3_helper)
+        self._url = self.s3_helper.get_url(
+            S3_BUILDS_BUCKET, f"{self.PREFIX}/{self.archive_name}"
+        )
+
+    def download(self):
+        logging.info("Searching cache for GitHub class")
+        self._download(self._url, True)
 
     def upload(self):
-        if not self._force_upload_cache:
-            cache_response = requests.head(self._url)
-            if cache_response.status_code == 200:
-                logging.info(
-                    "Remote cargo cache %s already exist, won't reupload", self._url
-                )
-                return
-
-        logging.info("Compressing cargo cache")
-        archive_path = self.directory.parent / self.archive_name
-        compress_fast(self.directory, archive_path)
-        s3_path = f"{self.PREFIX}/{self.archive_name}"
-        logging.info("Uploading %s to S3 path %s", archive_path, s3_path)
-        self.s3_helper.upload_build_file_to_s3(archive_path, s3_path)
+        self._upload(f"{self.PREFIX}/{self.archive_name}", True)

tests/ci/ci.py

@@ -397,6 +397,21 @@ def _configure_jobs(
         else:
             jobs_to_skip += (job,)
 
+    if pr_labels:
+        jobs_requested_by_label = []  # type: List[str]
+        ci_controlling_labels = []  # type: List[str]
+        for label in pr_labels:
+            label_config = CI_CONFIG.get_label_config(label)
+            if label_config:
+                jobs_requested_by_label += label_config.run_jobs
+                ci_controlling_labels += [label]
+        if ci_controlling_labels:
+            print(f"NOTE: CI controlling labels are set: [{ci_controlling_labels}]")
+            print(
+                f"    :   following jobs will be executed: [{jobs_requested_by_label}]"
+            )
+            jobs_to_do = jobs_requested_by_label
+
     if commit_tokens:
         requested_jobs = [
             token[len("#job_") :]
@@ -416,7 +431,7 @@ def _configure_jobs(
                     if parent in jobs_to_do and parent not in jobs_to_do_requested:
                         jobs_to_do_requested.append(parent)
             print(
-                f"NOTE: Only specific job(s) were requested: [{jobs_to_do_requested}]"
+                f"NOTE: Only specific job(s) were requested by commit message tokens: [{jobs_to_do_requested}]"
             )
             jobs_to_do = jobs_to_do_requested
 
@@ -608,7 +623,7 @@ def main() -> int:
         result["jobs_data"] = jobs_data
         result["docker_data"] = docker_data
         if pr_info.number != 0 and not args.docker_digest_or_latest:
-            #FIXME: it runs style check before docker build if possible (style-check images is not changed)
+            # FIXME: it runs style check before docker build if possible (style-check images is not changed)
             #    find a way to do style check always before docker build and others
             _check_and_update_for_early_style_check(result)
         if pr_info.has_changes_in_documentation_only():

tests/ci/ci_config.py

@@ -1,12 +1,18 @@
 #!/usr/bin/env python3
 
+from enum import Enum
 import logging
-
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Callable, Dict, Iterable, List, Literal, Optional, Union
 
+from integration_test_images import IMAGES
+
+
+class Labels(Enum):
+    DO_NOT_TEST_LABEL = "do not test"
+
 
 @dataclass
 class DigestConfig:
@@ -22,6 +28,15 @@ class DigestConfig:
     git_submodules: bool = False
 
 
+@dataclass
+class LabelConfig:
+    """
+    class to configure different CI scenarious per GH label
+    """
+
+    run_jobs: Iterable[str] = frozenset()
+
+
 @dataclass
 class JobConfig:
     """
@@ -95,7 +110,7 @@ class TestConfig:
 BuildConfigs = Dict[str, BuildConfig]
 BuildsReportConfig = Dict[str, BuildReportConfig]
 TestConfigs = Dict[str, TestConfig]
-
+LabelConfigs = Dict[str, LabelConfig]
 
 # common digests configs
 compatibility_check_digest = DigestConfig(
@@ -131,20 +146,7 @@ upgrade_check_digest = DigestConfig(
 integration_check_digest = DigestConfig(
     include_paths=["./tests/ci/integration_test_check.py", "./tests/integration"],
     exclude_files=[".md"],
-    docker=[
-        "clickhouse/dotnet-client",
-        "clickhouse/integration-helper",
-        "clickhouse/integration-test",
-        "clickhouse/integration-tests-runner",
-        "clickhouse/kerberized-hadoop",
-        "clickhouse/kerberos-kdc",
-        "clickhouse/mysql-golang-client",
-        "clickhouse/mysql-java-client",
-        "clickhouse/mysql-js-client",
-        "clickhouse/mysql-php-client",
-        "clickhouse/nginx-dav",
-        "clickhouse/postgresql-java-client",
-    ],
+    docker=IMAGES.copy(),
 )
 
 ast_fuzzer_check_digest = DigestConfig(
@@ -188,20 +190,9 @@ bugfix_validate_check = DigestConfig(
         "./tests/ci/bugfix_validate_check.py",
     ],
     exclude_files=[".md"],
-    docker=[
+    docker=IMAGES.copy()
+    + [
         "clickhouse/stateless-test",
-        "clickhouse/dotnet-client",
-        "clickhouse/integration-helper",
-        "clickhouse/integration-test",
-        "clickhouse/integration-tests-runner",
-        "clickhouse/kerberized-hadoop",
-        "clickhouse/kerberos-kdc",
-        "clickhouse/mysql-golang-client",
-        "clickhouse/mysql-java-client",
-        "clickhouse/mysql-js-client",
-        "clickhouse/mysql-php-client",
-        "clickhouse/nginx-dav",
-        "clickhouse/postgresql-java-client",
     ],
 )
 # common test params
@@ -268,6 +259,13 @@ class CiConfig:
     builds_report_config: BuildsReportConfig
     test_configs: TestConfigs
     other_jobs_configs: TestConfigs
+    label_configs: LabelConfigs
+
+    def get_label_config(self, label_name: str) -> Optional[LabelConfig]:
+        for label, config in self.label_configs.items():
+            if label_name == label:
+                return config
+        return None
 
     def get_job_config(self, check_name: str) -> JobConfig:
         res = None
@@ -417,6 +415,9 @@ class CiConfig:
 
 
 CI_CONFIG = CiConfig(
+    label_configs={
+        Labels.DO_NOT_TEST_LABEL.value: LabelConfig(run_jobs=["Style check"]),
+    },
     build_config={
         "package_release": BuildConfig(
             name="package_release",

tests/ci/github_helper.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 """Helper for GitHub API requests"""
 import logging
+import re
 from datetime import date, datetime, timedelta
-from pathlib import Path
 from os import path as p
+from pathlib import Path
 from time import sleep
 from typing import List, Optional, Tuple, Union
 
@@ -143,7 +144,9 @@ class GitHub(github.Github):
     def get_pull_cached(
         self, repo: Repository, number: int, obj_updated_at: Optional[datetime] = None
     ) -> PullRequest:
-        cache_file = self.cache_path / f"pr-{number}.pickle"
+        # clean any special symbol from the repo name, especially '/'
+        repo_name = re.sub(r"\W", "_", repo.full_name)
+        cache_file = self.cache_path / f"pr-{repo_name}-{number}.pickle"
 
         if cache_file.is_file():
             is_updated, cached_pr = self._is_cache_updated(cache_file, obj_updated_at)
@@ -192,6 +195,32 @@ class GitHub(github.Github):
         with open(path, "rb") as ob_fd:
             return self.load(ob_fd)  # type: ignore
 
+    # pylint: disable=protected-access
+    @staticmethod
+    def toggle_pr_draft(pr: PullRequest) -> None:
+        """GH rest API does not provide a way to toggle the draft status for PR"""
+        node_id = pr._rawData["node_id"]
+        if pr.draft:
+            action = (
+                "mutation PullRequestReadyForReview($input:MarkPullRequestReadyForReviewInput!)"
+                "{markPullRequestReadyForReview(input: $input){pullRequest{id}}}"
+            )
+        else:
+            action = (
+                "mutation ConvertPullRequestToDraft($input:ConvertPullRequestToDraftInput!)"
+                "{convertPullRequestToDraft(input: $input){pullRequest{id}}}"
+            )
+        query = {
+            "query": action,
+            "variables": {"input": {"pullRequestId": node_id}},
+        }
+        url = f"{pr._requester.base_url}/graphql"
+        _, data = pr._requester.requestJsonAndCheck("POST", url, input=query)
+        if data.get("data"):
+            pr._draft = pr._makeBoolAttribute(not pr.draft)
+
+    # pylint: enable=protected-access
+
     def _is_cache_updated(
         self, cache_file: Path, obj_updated_at: Optional[datetime]
     ) -> Tuple[bool, object]:

tests/ci/integration_test_check.py

@@ -10,13 +10,8 @@ import sys
 from pathlib import Path
 from typing import Dict, List, Tuple
 
-from github import Github
-
 from build_download_helper import download_all_deb_packages
-from clickhouse_helper import (
-    ClickHouseHelper,
-    prepare_tests_results_for_clickhouse,
-)
+from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
 from commit_status_helper import (
     RerunHelper,
     get_commit,
@@ -24,10 +19,12 @@ from commit_status_helper import (
     post_commit_status,
     post_commit_status_to_file,
 )
-from docker_images_helper import DockerImage, pull_image, get_docker_image
+from docker_images_helper import DockerImage, get_docker_image, pull_image
 from download_release_packages import download_last_release
-from env_helper import REPORT_PATH, TEMP_PATH, REPO_COPY
+from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
 from get_robot_token import get_best_robot_token
+from github_helper import GitHub
+from integration_test_images import IMAGES
 from pr_info import PRInfo
 from report import ERROR, TestResult, TestResults, read_test_results
 from s3_helper import S3Helper
@@ -36,24 +33,6 @@ from tee_popen import TeePopen
 from upload_result_helper import upload_results
 
 
-# When update, update
-# tests/integration/ci-runner.py:ClickhouseIntegrationTestsRunner.get_images_names too
-IMAGES = [
-    "clickhouse/dotnet-client",
-    "clickhouse/integration-helper",
-    "clickhouse/integration-test",
-    "clickhouse/integration-tests-runner",
-    "clickhouse/kerberized-hadoop",
-    "clickhouse/kerberos-kdc",
-    "clickhouse/mysql-golang-client",
-    "clickhouse/mysql-java-client",
-    "clickhouse/mysql-js-client",
-    "clickhouse/mysql-php-client",
-    "clickhouse/nginx-dav",
-    "clickhouse/postgresql-java-client",
-]
-
-
 def get_json_params_dict(
     check_name: str,
     pr_info: PRInfo,
@@ -210,7 +189,7 @@ def main():
         logging.info("Skipping '%s' (no pr-bugfix in '%s')", check_name, pr_info.labels)
         sys.exit(0)
 
-    gh = Github(get_best_robot_token(), per_page=100)
+    gh = GitHub(get_best_robot_token())
     commit = get_commit(gh, pr_info.sha)
 
     rerun_helper = RerunHelper(commit, check_name_with_group)

tests/ci/integration_test_images.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+IMAGES_ENV = {
+    "clickhouse/dotnet-client": "DOCKER_DOTNET_CLIENT_TAG",
+    "clickhouse/integration-helper": "DOCKER_HELPER_TAG",
+    "clickhouse/integration-test": "DOCKER_BASE_TAG",
+    "clickhouse/integration-tests-runner": "",
+    "clickhouse/kerberized-hadoop": "DOCKER_KERBERIZED_HADOOP_TAG",
+    "clickhouse/kerberos-kdc": "DOCKER_KERBEROS_KDC_TAG",
+    "clickhouse/mysql-golang-client": "DOCKER_MYSQL_GOLANG_CLIENT_TAG",
+    "clickhouse/mysql-java-client": "DOCKER_MYSQL_JAVA_CLIENT_TAG",
+    "clickhouse/mysql-js-client": "DOCKER_MYSQL_JS_CLIENT_TAG",
+    "clickhouse/mysql-php-client": "DOCKER_MYSQL_PHP_CLIENT_TAG",
+    "clickhouse/nginx-dav": "DOCKER_NGINX_DAV_TAG",
+    "clickhouse/postgresql-java-client": "DOCKER_POSTGRESQL_JAVA_CLIENT_TAG",
+    "clickhouse/python-bottle": "DOCKER_PYTHON_BOTTLE_TAG",
+}
+
+IMAGES = list(IMAGES_ENV.keys())
+
+
+def get_image_env(image: str) -> str:
+    return IMAGES_ENV.get(image, "")
+
+
+def get_docker_env(image: str, tag: str) -> str:
+    "if image belongs to IMAGES_ENV, return `-e` argument for docker command"
+    env = get_image_env(image)
+    if not env:
+        return env
+    return f"-e {env}={tag} "