Merge eb5b16686c into e0f8b8d351

ci/docker/fasttest/Dockerfile

@@ -105,5 +105,14 @@ RUN groupadd --system --gid 1000 clickhouse \
     && useradd --system --gid 1000 --uid 1000 -m clickhouse \
     && mkdir -p /.cache/sccache && chmod 777 /.cache/sccache
 
+
+# TODO move nfpm to docker that will do packaging
+ARG TARGETARCH
+ARG NFPM_VERSION=2.20.0
+RUN arch=${TARGETARCH:-amd64} \
+    && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
+    && dpkg -i /tmp/nfpm.deb \
+    && rm /tmp/nfpm.deb
+
 ENV PYTHONPATH="/wd"
 ENV PYTHONUNBUFFERED=1

ci/docker/stateless-test/Dockerfile

@@ -58,6 +58,7 @@ RUN apt-get update -y \
             curl \
             wget \
             xz-utils \
+            ripgrep \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
@@ -114,4 +115,5 @@ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoo
 RUN npm install -g azurite@3.30.0 \
     && npm install -g tslib && npm install -g node
 
+ENV PYTHONPATH=".:./ci"
 USER clickhouse

ci/docker/stateless-test/requirements.txt

@@ -4,3 +4,4 @@ requests==2.32.3
 pandas==1.5.3
 scipy==1.12.0
 pyarrow==18.0.0
+grpcio==1.47.0

ci/jobs/build_clickhouse.py

@@ -4,11 +4,15 @@ from praktika.result import Result
 from praktika.settings import Settings
 from praktika.utils import MetaClasses, Shell, Utils
 
+from ci.jobs.scripts.clickhouse_version import CHVersion
+
 
 class JobStages(metaclass=MetaClasses.WithIter):
     CHECKOUT_SUBMODULES = "checkout"
     CMAKE = "cmake"
+    UNSHALLOW = "unshallow"
     BUILD = "build"
+    PACKAGE = "package"
 
 
 def parse_args():
@@ -33,8 +37,7 @@ CMAKE_CMD = """cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA \
 -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \
 {AUX_DEFS} \
 -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 \
--DCOMPILER_CACHE={CACHE_TYPE} \
--DENABLE_BUILD_PROFILING=1 {DIR}"""
+-DCOMPILER_CACHE={CACHE_TYPE} -DENABLE_BUILD_PROFILING=1 {DIR}"""
 
 
 def main():
@@ -91,6 +94,27 @@ def main():
 
     res = True
     results = []
+    version = ""
+
+    if res and JobStages.UNSHALLOW in stages:
+        results.append(
+            Result.create_from_command_execution(
+                name="Repo Unshallow",
+                command="git rev-parse --is-shallow-repository | grep -q true && git fetch --depth 10000 --no-tags --filter=tree:0 origin $(git rev-parse --abbrev-ref HEAD)",
+                with_log=True,
+            )
+        )
+        res = results[-1].is_ok()
+        if res:
+            try:
+                version = CHVersion.get_version()
+                assert version
+                print(f"Got version from repo [{version}]")
+            except Exception as e:
+                results[-1].set_failed().set_info(
+                    f"Failed to get version from repo, ex [{e}]"
+                )
+                res = False
 
     if res and JobStages.CHECKOUT_SUBMODULES in stages:
         Shell.check(f"rm -rf {build_dir} && mkdir -p {build_dir}")
@@ -127,6 +151,38 @@ def main():
         Shell.check(f"ls -l {build_dir}/programs/")
         res = results[-1].is_ok()
 
+    if res and JobStages.PACKAGE in stages:
+        if "debug" in build_type:
+            package_type = "debug"
+        elif "release" in build_type:
+            package_type = "release"
+        elif "asan" in build_type:
+            package_type = "asan"
+        else:
+            assert False, "TODO"
+
+        if "amd" in build_type:
+            deb_arch = "amd64"
+        else:
+            deb_arch = "arm64"
+
+        output_dir = "/tmp/praktika/output/"
+        assert Shell.check(f"rm -f {output_dir}/*.deb")
+
+        results.append(
+            Result.create_from_command_execution(
+                name="Build Packages",
+                command=[
+                    f"DESTDIR={build_dir}/root ninja programs/install",
+                    f"ln -sf {build_dir}/root {Utils.cwd()}/packages/root",
+                    f"cd {Utils.cwd()}/packages/ && OUTPUT_DIR={output_dir} BUILD_TYPE={package_type} VERSION_STRING={version} DEB_ARCH={deb_arch} ./build --deb",
+                ],
+                workdir=build_dir,
+                with_log=True,
+            )
+        )
+        res = results[-1].is_ok()
+
     Result.create_from(results=results, stopwatch=stop_watch).complete_job()
 
 

ci/jobs/functional_stateful_tests.py

@@ -1,5 +1,4 @@
 import argparse
-import os
 import time
 from pathlib import Path
 
@@ -131,6 +130,10 @@ def main():
         )
         res = res and CH.start()
         res = res and CH.wait_ready()
+        # TODO: Use --database-replicated optionally
+        res = res and Shell.check(
+            f"./ci/jobs/scripts/functional_tests/setup_ch_cluster.sh"
+        )
         if res:
             print("ch started")
         logs_to_attach += [

ci/jobs/functional_stateless_tests.py

@@ -101,6 +101,7 @@ def main():
             f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client",
             f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor",
             f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local",
+            f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-disks",
             f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
             f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
             # TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
@@ -114,6 +115,7 @@ def main():
             f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
             f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|<path>local_disk|<path>{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done",
             f"clickhouse-server --version",
+            f"chmod +x /tmp/praktika/input/clickhouse-odbc-bridge",
         ]
         results.append(
             Result.create_from_command_execution(
@@ -138,6 +140,7 @@ def main():
         res = res and Shell.check(
             "aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True
         )
+        res = res and CH.log_cluster_config()
         res = res and CH.start()
         res = res and CH.wait_ready()
         if res:
@@ -170,6 +173,7 @@ def main():
             batch_total=total_batches,
             test=args.test,
         )
+        CH.log_cluster_stop_replication()
         results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
         results[-1].set_timing(stopwatch=stop_watch_)
         res = results[-1].is_ok()

ci/jobs/scripts/clickhouse_proc.py

@@ -66,6 +66,24 @@ class ClickHouseProc:
         print(f"Started setup_minio.sh asynchronously with PID {process.pid}")
         return True
 
+    def log_cluster_config(self):
+        return Shell.check(
+            f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --config-logs-export-cluster /tmp/praktika/etc/clickhouse-server/config.d/system_logs_export.yaml",
+            verbose=True,
+        )
+
+    def log_cluster_setup_replication(self):
+        return Shell.check(
+            f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --setup-logs-replication",
+            verbose=True,
+        )
+
+    def log_cluster_stop_replication(self):
+        return Shell.check(
+            f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --stop-log-replication",
+            verbose=True,
+        )
+
     def start(self):
         print("Starting ClickHouse server")
         Shell.check(f"rm {self.pid_file}")

ci/jobs/scripts/clickhouse_version.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+
+from praktika.utils import Shell
+
+
+class CHVersion:
+    FILE_WITH_VERSION_PATH = "./cmake/autogenerated_versions.txt"
+
+    @classmethod
+    def _get_tweak(cls):
+        tag = Shell.get_output("git describe --tags --abbrev=0")
+        assert tag.startswith("v24")
+        num = Shell.get_output(f"git rev-list --count {tag}..HEAD")
+        return int(num)
+
+    @classmethod
+    def get_version(cls):
+        versions = {}
+        for line in (
+            Path(cls.FILE_WITH_VERSION_PATH).read_text(encoding="utf-8").splitlines()
+        ):
+            line = line.strip()
+            if not line.startswith("SET("):
+                continue
+
+            name, value = line[4:-1].split(maxsplit=1)
+            name = name.removeprefix("VERSION_").lower()
+            try:
+                value = int(value)
+            except ValueError:
+                pass
+            versions[name] = value
+
+        version_sha = versions["githash"]
+        tweak = int(
+            Shell.get_output(f"git rev-list --count {version_sha}..HEAD", verbose=True)
+        )
+        return f"{versions['major']}.{versions['minor']}.{versions['patch']}.{tweak}"

ci/jobs/scripts/functional_tests/setup_ch_cluster.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+set -e -x
+
+clickhouse-client --query "SHOW DATABASES"
+clickhouse-client --query "CREATE DATABASE datasets"
+clickhouse-client < ./tests/docker_scripts/create.sql
+clickhouse-client --query "SHOW TABLES FROM datasets"
+
+USE_DATABASE_REPLICATED=0
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --database-replicated)
+            echo "Setup cluster for testing with Database Replicated"
+            USE_DATABASE_REPLICATED=1
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+    clickhouse-client --query "CREATE DATABASE test ON CLUSTER 'test_cluster_database_replicated'
+        ENGINE=Replicated('/test/clickhouse/db/test', '{shard}', '{replica}')"
+
+    clickhouse-client --query "CREATE TABLE test.hits AS datasets.hits_v1"
+    clickhouse-client --query "CREATE TABLE test.visits AS datasets.visits_v1"
+
+    clickhouse-client --max_memory_usage 10G --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1"
+    clickhouse-client --max_memory_usage 10G --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1"
+
+    clickhouse-client --query "DROP TABLE datasets.hits_v1"
+    clickhouse-client --query "DROP TABLE datasets.visits_v1"
+else
+    clickhouse-client --query "CREATE DATABASE test"
+    clickhouse-client --query "SHOW TABLES FROM test"
+    if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
+        clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64,  JavaEnable UInt8,  Title String,  GoodEvent Int16,
+            EventTime DateTime,  EventDate Date,  CounterID UInt32,  ClientIP UInt32,  ClientIP6 FixedString(16),  RegionID UInt32,
+            UserID UInt64,  CounterClass Int8,  OS UInt8,  UserAgent UInt8,  URL String,  Referer String,  URLDomain String,
+            RefererDomain String,  Refresh UInt8,  IsRobot UInt8,  RefererCategories Array(UInt16),  URLCategories Array(UInt16),
+            URLRegions Array(UInt32),  RefererRegions Array(UInt32),  ResolutionWidth UInt16,  ResolutionHeight UInt16,  ResolutionDepth UInt8,
+            FlashMajor UInt8, FlashMinor UInt8,  FlashMinor2 String,  NetMajor UInt8,  NetMinor UInt8, UserAgentMajor UInt16,
+            UserAgentMinor FixedString(2),  CookieEnable UInt8, JavascriptEnable UInt8,  IsMobile UInt8,  MobilePhone UInt8,
+            MobilePhoneModel String,  Params String,  IPNetworkID UInt32,  TraficSourceID Int8, SearchEngineID UInt16,
+            SearchPhrase String,  AdvEngineID UInt8,  IsArtifical UInt8,  WindowClientWidth UInt16,  WindowClientHeight UInt16,
+            ClientTimeZone Int16,  ClientEventTime DateTime,  SilverlightVersion1 UInt8, SilverlightVersion2 UInt8,  SilverlightVersion3 UInt32,
+            SilverlightVersion4 UInt16,  PageCharset String,  CodeVersion UInt32,  IsLink UInt8,  IsDownload UInt8,  IsNotBounce UInt8,
+            FUniqID UInt64,  HID UInt32,  IsOldCounter UInt8, IsEvent UInt8,  IsParameter UInt8,  DontCountHits UInt8,  WithHash UInt8,
+            HitColor FixedString(1),  UTCEventTime DateTime,  Age UInt8,  Sex UInt8,  Income UInt8,  Interests UInt16,  Robotness UInt8,
+            GeneralInterests Array(UInt16), RemoteIP UInt32,  RemoteIP6 FixedString(16),  WindowName Int32,  OpenerName Int32,
+            HistoryLength Int16,  BrowserLanguage FixedString(2),  BrowserCountry FixedString(2),  SocialNetwork String,  SocialAction String,
+            HTTPError UInt16, SendTiming Int32,  DNSTiming Int32,  ConnectTiming Int32,  ResponseStartTiming Int32,  ResponseEndTiming Int32,
+            FetchTiming Int32,  RedirectTiming Int32, DOMInteractiveTiming Int32,  DOMContentLoadedTiming Int32,  DOMCompleteTiming Int32,
+            LoadEventStartTiming Int32,  LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32,  FirstPaintTiming Int32,
+            RedirectCount Int8, SocialSourceNetworkID UInt8,  SocialSourcePage String,  ParamPrice Int64, ParamOrderID String,
+            ParamCurrency FixedString(3),  ParamCurrencyID UInt16, GoalsReached Array(UInt32),  OpenstatServiceName String,
+            OpenstatCampaignID String,  OpenstatAdID String,  OpenstatSourceID String,  UTMSource String, UTMMedium String,
+            UTMCampaign String,  UTMContent String,  UTMTerm String, FromTag String,  HasGCLID UInt8,  RefererHash UInt64,
+            URLHash UInt64,  CLID UInt32,  YCLID UInt64,  ShareService String,  ShareURL String,  ShareTitle String,
+            ParsedParams Nested(Key1 String,  Key2 String, Key3 String, Key4 String, Key5 String,  ValueDouble Float64),
+            IslandID FixedString(16),  RequestNum UInt32,  RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
+            ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
+        clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32,  StartDate Date,  Sign Int8,  IsNew UInt8,
+            VisitID UInt64,  UserID UInt64,  StartTime DateTime,  Duration UInt32,  UTCStartTime DateTime,  PageViews Int32,
+            Hits Int32,  IsBounce UInt8,  Referer String,  StartURL String,  RefererDomain String,  StartURLDomain String,
+            EndURL String,  LinkURL String,  IsDownload UInt8,  TraficSourceID Int8,  SearchEngineID UInt16,  SearchPhrase String,
+            AdvEngineID UInt8,  PlaceID Int32,  RefererCategories Array(UInt16),  URLCategories Array(UInt16),  URLRegions Array(UInt32),
+            RefererRegions Array(UInt32),  IsYandex UInt8,  GoalReachesDepth Int32,  GoalReachesURL Int32,  GoalReachesAny Int32,
+            SocialSourceNetworkID UInt8,  SocialSourcePage String,  MobilePhoneModel String,  ClientEventTime DateTime,  RegionID UInt32,
+            ClientIP UInt32,  ClientIP6 FixedString(16),  RemoteIP UInt32,  RemoteIP6 FixedString(16),  IPNetworkID UInt32,
+            SilverlightVersion3 UInt32,  CodeVersion UInt32,  ResolutionWidth UInt16,  ResolutionHeight UInt16,  UserAgentMajor UInt16,
+            UserAgentMinor UInt16,  WindowClientWidth UInt16,  WindowClientHeight UInt16,  SilverlightVersion2 UInt8,  SilverlightVersion4 UInt16,
+            FlashVersion3 UInt16,  FlashVersion4 UInt16,  ClientTimeZone Int16,  OS UInt8,  UserAgent UInt8,  ResolutionDepth UInt8,
+            FlashMajor UInt8,  FlashMinor UInt8,  NetMajor UInt8,  NetMinor UInt8,  MobilePhone UInt8,  SilverlightVersion1 UInt8,
+            Age UInt8,  Sex UInt8,  Income UInt8,  JavaEnable UInt8,  CookieEnable UInt8,  JavascriptEnable UInt8,  IsMobile UInt8,
+            BrowserLanguage UInt16,  BrowserCountry UInt16,  Interests UInt16,  Robotness UInt8,  GeneralInterests Array(UInt16),
+            Params Array(String),  Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime,  Price Int64,  OrderID String, CurrencyID UInt32),
+            WatchIDs Array(UInt64),  ParamSumPrice Int64,  ParamCurrency FixedString(3),  ParamCurrencyID UInt16,  ClickLogID UInt64,
+            ClickEventID Int32,  ClickGoodEvent Int32,  ClickEventTime DateTime,  ClickPriorityID Int32,  ClickPhraseID Int32,  ClickPageID Int32,
+            ClickPlaceID Int32,  ClickTypeID Int32,  ClickResourceID Int32,  ClickCost UInt32,  ClickClientIP UInt32,  ClickDomainID UInt32,
+            ClickURL String,  ClickAttempt UInt8,  ClickOrderID UInt32,  ClickBannerID UInt32,  ClickMarketCategoryID UInt32,  ClickMarketPP UInt32,
+            ClickMarketCategoryName String,  ClickMarketPPName String,  ClickAWAPSCampaignName String,  ClickPageName String,  ClickTargetType UInt16,
+            ClickTargetPhraseID UInt64,  ClickContextType UInt8,  ClickSelectType Int8,  ClickOptions String,  ClickGroupBannerID Int32,
+            OpenstatServiceName String,  OpenstatCampaignID String,  OpenstatAdID String,  OpenstatSourceID String,  UTMSource String,
+            UTMMedium String,  UTMCampaign String,  UTMContent String,  UTMTerm String,  FromTag String,  HasGCLID UInt8,  FirstVisit DateTime,
+            PredLastVisit Date,  LastVisit Date,  TotalVisits UInt32,  TraficSource    Nested(ID Int8,  SearchEngineID UInt16, AdvEngineID UInt8,
+            PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String),  Attendance FixedString(16),
+            CLID UInt32,  YCLID UInt64,  NormalizedRefererHash UInt64,  SearchPhraseHash UInt64,  RefererDomainHash UInt64,  NormalizedStartURLHash UInt64,
+            StartURLDomainHash UInt64,  NormalizedEndURLHash UInt64,  TopLevelDomain UInt64,  URLScheme UInt64,  OpenstatServiceNameHash UInt64,
+            OpenstatCampaignIDHash UInt64,  OpenstatAdIDHash UInt64,  OpenstatSourceIDHash UInt64,  UTMSourceHash UInt64,  UTMMediumHash UInt64,
+            UTMCampaignHash UInt64,  UTMContentHash UInt64,  UTMTermHash UInt64,  FromHash UInt64,  WebVisorEnabled UInt8,  WebVisorActivity UInt32,
+            ParsedParams    Nested(Key1 String,  Key2 String,  Key3 String,  Key4 String, Key5 String, ValueDouble    Float64),
+            Market Nested(Type UInt8, GoalID UInt32, OrderID String,  OrderPrice Int64,  PP UInt32,  DirectPlaceID UInt32,  DirectOrderID  UInt32,
+            DirectBannerID UInt32,  GoodID String, GoodName String, GoodQuantity Int32,  GoodPrice Int64),  IslandID FixedString(16))
+            ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
+            SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
+
+        clickhouse-client --max_memory_usage 10G --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
+        clickhouse-client --max_memory_usage 10G --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
+        clickhouse-client --query "DROP TABLE datasets.visits_v1 SYNC"
+        clickhouse-client --query "DROP TABLE datasets.hits_v1 SYNC"
+    else
+        clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
+        clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
+    fi
+    clickhouse-client --query "CREATE TABLE test.hits_s3  (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
+    # AWS S3 is very inefficient, so increase memory even further:
+    clickhouse-client --max_memory_usage 30G --max_memory_usage_for_user 30G --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
+fi
+
+clickhouse-client --query "SHOW TABLES FROM test"
+clickhouse-client --query "SELECT count() FROM test.hits"
+clickhouse-client --query "SELECT count() FROM test.visits"

ci/jobs/scripts/functional_tests/setup_log_cluster.sh

@@ -0,0 +1,261 @@
+#!/bin/bash
+
+set -e
+# This script sets up export of system log tables to a remote server.
+# Remote tables are created if not exist, and augmented with extra columns,
+# and their names will contain a hash of the table structure,
+# which allows exporting tables from servers of different versions.
+
+# Config file contains KEY=VALUE pairs with any necessary parameters like:
+# CLICKHOUSE_CI_LOGS_HOST - remote host
+# CLICKHOUSE_CI_LOGS_USER - password for user
+# CLICKHOUSE_CI_LOGS_PASSWORD - password for user
+CLICKHOUSE_CI_LOGS_CREDENTIALS=${CLICKHOUSE_CI_LOGS_CREDENTIALS:-/tmp/export-logs-config.sh}
+CLICKHOUSE_CI_LOGS_USER=${CLICKHOUSE_CI_LOGS_USER:-ci}
+
+# Pre-configured destination cluster, where to export the data
+CLICKHOUSE_CI_LOGS_CLUSTER=${CLICKHOUSE_CI_LOGS_CLUSTER:-system_logs_export}
+
+EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime('UTC'), check_name LowCardinality(String), instance_type LowCardinality(String), instance_id String, INDEX ix_pr (pull_request_number) TYPE set(100), INDEX ix_commit (commit_sha) TYPE set(100), INDEX ix_check_time (check_start_time) TYPE minmax, "}
+EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"CAST(0 AS UInt32) AS pull_request_number, '' AS commit_sha, now() AS check_start_time, toLowCardinality('') AS check_name, toLowCardinality('') AS instance_type, '' AS instance_id"}
+EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:-"check_name"}
+
+# trace_log needs more columns for symbolization
+EXTRA_COLUMNS_TRACE_LOG="${EXTRA_COLUMNS} symbols Array(LowCardinality(String)), lines Array(LowCardinality(String)), "
+EXTRA_COLUMNS_EXPRESSION_TRACE_LOG="${EXTRA_COLUMNS_EXPRESSION}, arrayMap(x -> demangle(addressToSymbol(x)), trace)::Array(LowCardinality(String)) AS symbols, arrayMap(x -> addressToLine(x), trace)::Array(LowCardinality(String)) AS lines"
+
+# coverage_log needs more columns for symbolization, but only symbol names (the line numbers are too heavy to calculate)
+EXTRA_COLUMNS_COVERAGE_LOG="${EXTRA_COLUMNS} symbols Array(LowCardinality(String)), "
+EXTRA_COLUMNS_EXPRESSION_COVERAGE_LOG="${EXTRA_COLUMNS_EXPRESSION}, arrayDistinct(arrayMap(x -> demangle(addressToSymbol(x)), coverage))::Array(LowCardinality(String)) AS symbols"
+
+
+function __set_connection_args
+{
+    # It's impossible to use a generic $CONNECTION_ARGS string, it's unsafe from word splitting perspective.
+    # That's why we must stick to the generated option
+    CONNECTION_ARGS=(
+        --receive_timeout=45 --send_timeout=45 --secure
+        --user "${CLICKHOUSE_CI_LOGS_USER}" --host "${CLICKHOUSE_CI_LOGS_HOST}"
+        --password "${CLICKHOUSE_CI_LOGS_PASSWORD}"
+    )
+}
+
+function __shadow_credentials
+{
+    # The function completely screws the output, it shouldn't be used in normal functions, only in ()
+    # The only way to substitute the env as a plain text is using perl 's/\Qsomething\E/another/
+    exec &> >(perl -pe '
+        s(\Q$ENV{CLICKHOUSE_CI_LOGS_HOST}\E)[CLICKHOUSE_CI_LOGS_HOST]g;
+        s(\Q$ENV{CLICKHOUSE_CI_LOGS_USER}\E)[CLICKHOUSE_CI_LOGS_USER]g;
+        s(\Q$ENV{CLICKHOUSE_CI_LOGS_PASSWORD}\E)[CLICKHOUSE_CI_LOGS_PASSWORD]g;
+    ')
+}
+
+function check_logs_credentials
+(
+    # The function connects with given credentials, and if it's unable to execute the simplest query, returns exit code
+
+    # First check, if all necessary parameters are set
+    set +x
+    for parameter in CLICKHOUSE_CI_LOGS_HOST CLICKHOUSE_CI_LOGS_USER CLICKHOUSE_CI_LOGS_PASSWORD; do
+      export -p | grep -q "$parameter" || {
+        echo "Credentials parameter $parameter is unset"
+        return 1
+      }
+    done
+
+    __shadow_credentials
+    __set_connection_args
+    local code
+    # Catch both success and error to not fail on `set -e`
+    clickhouse-client "${CONNECTION_ARGS[@]}" -q 'SELECT 1 FORMAT Null' && return 0 || code=$?
+    if [ "$code" != 0 ]; then
+        echo 'Failed to connect to CI Logs cluster'
+        return $code
+    fi
+)
+
+function config_logs_export_cluster
+(
+    # The function is launched in a separate shell instance to not expose the
+    # exported values from CLICKHOUSE_CI_LOGS_CREDENTIALS
+    set +x
+    if ! [ -r "${CLICKHOUSE_CI_LOGS_CREDENTIALS}" ]; then
+        echo "File $CLICKHOUSE_CI_LOGS_CREDENTIALS does not exist, do not setup"
+        return
+    fi
+    set -a
+    # shellcheck disable=SC1090
+    source "${CLICKHOUSE_CI_LOGS_CREDENTIALS}"
+    set +a
+    __shadow_credentials
+    echo "Checking if the credentials work"
+    check_logs_credentials || return 0
+    cluster_config="${1:-/etc/clickhouse-server/config.d/system_logs_export.yaml}"
+    mkdir -p "$(dirname "$cluster_config")"
+    echo "remote_servers:
+    ${CLICKHOUSE_CI_LOGS_CLUSTER}:
+        shard:
+            replica:
+                secure: 1
+                user: '${CLICKHOUSE_CI_LOGS_USER}'
+                host: '${CLICKHOUSE_CI_LOGS_HOST}'
+                port: 9440
+                password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
+" > "$cluster_config"
+    echo "Cluster ${CLICKHOUSE_CI_LOGS_CLUSTER} is confugured in ${cluster_config}"
+)
+
+function setup_logs_replication
+(
+    # The function is launched in a separate shell instance to not expose the
+    # exported values from CLICKHOUSE_CI_LOGS_CREDENTIALS
+    set +x
+    # disable output
+    if ! [ -r "${CLICKHOUSE_CI_LOGS_CREDENTIALS}" ]; then
+        echo "File $CLICKHOUSE_CI_LOGS_CREDENTIALS does not exist, do not setup"
+        return 0
+    fi
+    set -a
+    # shellcheck disable=SC1090
+    source "${CLICKHOUSE_CI_LOGS_CREDENTIALS}"
+    set +a
+    __shadow_credentials
+    echo "Checking if the credentials work"
+    check_logs_credentials || return 0
+    __set_connection_args
+
+    echo "My hostname is ${HOSTNAME}"
+
+    echo 'Create all configured system logs'
+    clickhouse-client --query "SYSTEM FLUSH LOGS"
+
+    debug_or_sanitizer_build=$(clickhouse-client -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%'")
+    echo "Build is debug or sanitizer: $debug_or_sanitizer_build"
+
+    # We will pre-create a table system.coverage_log.
+    # It is normally created by clickhouse-test rather than the server,
+    # so we will create it in advance to make it be picked up by the next commands:
+
+    clickhouse-client --query "
+        CREATE TABLE IF NOT EXISTS system.coverage_log
+        (
+            time DateTime COMMENT 'The time of test run',
+            test_name String COMMENT 'The name of the test',
+            coverage Array(UInt64) COMMENT 'An array of addresses of the code (a subset of addresses instrumented for coverage) that were encountered during the test run'
+        ) ENGINE = MergeTree ORDER BY test_name COMMENT 'Contains information about per-test coverage from the CI, but used only for exporting to the CI cluster'
+    "
+
+    # For each system log table:
+    echo 'Create %_log tables'
+    clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
+    do
+        if [[ "$table" = "trace_log" ]]
+        then
+            EXTRA_COLUMNS_FOR_TABLE="${EXTRA_COLUMNS_TRACE_LOG}"
+            # Do not try to resolve stack traces in case of debug/sanitizers
+            # build, since it is too slow (flushing of trace_log can take ~1min
+            # with such MV attached)
+            if [[ "$debug_or_sanitizer_build" = 1 ]]
+            then
+                EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION}"
+            else
+                EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION_TRACE_LOG}"
+            fi
+        elif [[ "$table" = "coverage_log" ]]
+        then
+            EXTRA_COLUMNS_FOR_TABLE="${EXTRA_COLUMNS_COVERAGE_LOG}"
+            EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION_COVERAGE_LOG}"
+        else
+            EXTRA_COLUMNS_FOR_TABLE="${EXTRA_COLUMNS}"
+            EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION}"
+        fi
+
+        # Calculate hash of its structure. Note: 4 is the version of extra columns - increment it if extra columns are changed:
+        hash=$(clickhouse-client --query "
+            SELECT sipHash64(9, groupArray((name, type)))
+            FROM (SELECT name, type FROM system.columns
+                WHERE database = 'system' AND table = '$table'
+                ORDER BY position)
+            ")
+
+        # Create the destination table with adapted name and structure:
+        statement=$(clickhouse-client --format TSVRaw --query "SHOW CREATE TABLE system.${table}" | sed -r -e '
+            s/^\($/('"$EXTRA_COLUMNS_FOR_TABLE"'/;
+            s/^ORDER BY (([^\(].+?)|\((.+?)\))$/ORDER BY ('"$EXTRA_ORDER_BY_COLUMNS"', \2\3)/;
+            s/^CREATE TABLE system\.\w+_log$/CREATE TABLE IF NOT EXISTS '"$table"'_'"$hash"'/;
+            /^TTL /d
+            ')
+
+        echo -e "Creating remote destination table ${table}_${hash} with statement:" >&2
+
+        echo "::group::${table}"
+        # there's the only way big "$statement" can be printed without causing EAGAIN error
+        # cat: write error: Resource temporarily unavailable
+        statement_print="${statement}"
+        if [ "${#statement_print}" -gt 4000 ]; then
+          statement_print="${statement::1999}\n…\n${statement:${#statement}-1999}"
+        fi
+        echo -e "$statement_print"
+        echo "::endgroup::"
+
+        echo "$statement" | clickhouse-client --database_replicated_initial_query_timeout_sec=10 \
+            --distributed_ddl_task_timeout=30 --distributed_ddl_output_mode=throw_only_active \
+            "${CONNECTION_ARGS[@]}" || continue
+
+        echo "Creating table system.${table}_sender" >&2
+
+        # Create Distributed table and materialized view to watch on the original table:
+        clickhouse-client --query "
+            CREATE TABLE system.${table}_sender
+            ENGINE = Distributed(${CLICKHOUSE_CI_LOGS_CLUSTER}, default, ${table}_${hash})
+            SETTINGS flush_on_detach=0
+            EMPTY AS
+            SELECT ${EXTRA_COLUMNS_EXPRESSION_FOR_TABLE}, *
+            FROM system.${table}
+        " || continue
+
+        echo "Creating materialized view system.${table}_watcher" >&2
+
+        clickhouse-client --query "
+            CREATE MATERIALIZED VIEW system.${table}_watcher TO system.${table}_sender AS
+            SELECT ${EXTRA_COLUMNS_EXPRESSION_FOR_TABLE}, *
+            FROM system.${table}
+        " || continue
+    done
+)
+
+function stop_logs_replication
+{
+    echo "Detach all logs replication"
+    clickhouse-client --query "select database||'.'||table from system.tables where database = 'system' and (table like '%_sender' or table like '%_watcher')" | {
+        tee /dev/stderr
+    } | {
+        timeout --preserve-status --signal TERM --kill-after 5m 15m xargs -n1 -r -i clickhouse-client --query "drop table {}"
+    }
+}
+
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --stop-log-replication)
+            echo "Stopping log replication..."
+            stop_logs_replication
+            ;;
+        --setup-logs-replication)
+            echo "Setting up log replication..."
+            setup_logs_replication
+            ;;
+        --config-logs-export-cluster)
+            echo "Configuring logs export for the cluster..."
+            config_logs_export_cluster "$2"
+            shift
+            ;;
+        *)
+            echo "Unknown option: $1"
+            echo "Usage: $0 [--stop-log-replication | --setup-logs-replication | --config-logs-export-cluster ]"
+            exit 1
+            ;;
+    esac
+    shift
+done

ci/praktika/_environment.py

@@ -179,7 +179,7 @@ class _Environment(MetaClasses.Serializable):
             if bucket in path:
                 path = path.replace(bucket, endpoint)
                 break
-        REPORT_URL = f"https://{path}/{Path(settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={'latest' if latest else self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}&name_1={urllib.parse.quote(self.JOB_NAME, safe='')}"
+        REPORT_URL = f"https://{path}/{Path(settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={'latest' if latest else self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}"
         return REPORT_URL
 
     def is_local_run(self):

ci/praktika/artifact.py

@@ -1,3 +1,4 @@
+import copy
 from dataclasses import dataclass
 
 
@@ -24,6 +25,14 @@ class Artifact:
         def is_s3_artifact(self):
             return self.type == Artifact.Type.S3
 
+        def parametrize(self, names):
+            res = []
+            for name in names:
+                obj = copy.deepcopy(self)
+                obj.name = name
+                res.append(obj)
+            return res
+
     @classmethod
     def define_artifact(cls, name, type, path):
         return cls.Config(name=name, type=type, path=path)

ci/praktika/hook_html.py

@@ -128,6 +128,9 @@ class HtmlRunnerHooks:
         for job in _workflow.jobs:
             if job.name not in skip_jobs:
                 result = Result.generate_pending(job.name)
+                # Preemptively add the general job log to the result directory to ensure
+                #   the post-job handler can upload it, even if the job is terminated unexpectedly
+                result.set_files([Settings.RUN_LOG])
             else:
                 result = Result.generate_skipped(job.name, job_cache_records[job.name])
             results.append(result)
@@ -137,14 +140,14 @@ class HtmlRunnerHooks:
         summary_result.start_time = Utils.timestamp()
 
         assert _ResultS3.copy_result_to_s3_with_version(summary_result, version=0)
-        page_url = env.get_report_url(settings=Settings)
+        page_url = env.get_report_url(settings=Settings, latest=True)
         print(f"CI Status page url [{page_url}]")
 
         res1 = GH.post_commit_status(
             name=_workflow.name,
             status=Result.Status.PENDING,
             description="",
-            url=env.get_report_url(settings=Settings, latest=True),
+            url=page_url,
         )
         res2 = GH.post_pr_comment(
             comment_body=f"Workflow [[{_workflow.name}]({page_url})], commit [{_Environment.get().SHA[:8]}]",

ci/praktika/json.html

@@ -601,7 +601,7 @@
                     td.classList.add('time-column');
                     td.textContent = value ? formatDuration(value) : '';
                 } else if (column === 'info') {
-                    td.textContent = value.includes('\n') ? '↵' : (value || '');
+                    td.textContent = value && value.includes('\n') ? '↵' : (value || '');
                     td.classList.add('info-column');
                 }
 

ci/praktika/native_jobs.py

@@ -310,7 +310,7 @@ def _finish_workflow(workflow, job_name):
     print(env.get_needs_statuses())
 
     print("Check Workflow results")
-    _ResultS3.copy_result_from_s3(
+    version = _ResultS3.copy_result_from_s3_with_version(
         Result.file_name_static(workflow.name),
     )
     workflow_result = Result.from_fs(workflow.name)
@@ -333,7 +333,7 @@ def _finish_workflow(workflow, job_name):
             # dump workflow result after update - to have an updated result in post
             workflow_result.dump()
             # add error into env - should apper in the report
-            env.add_info(ResultInfo.NOT_FINALIZED + f" [{result.name}]")
+            env.add_info(f"{result.name}: {ResultInfo.NOT_FINALIZED}")
             update_final_report = True
         job = workflow.get_job(result.name)
         if not job or not job.allow_merge_on_failure:
@@ -358,9 +358,7 @@ def _finish_workflow(workflow, job_name):
         env.add_info(ResultInfo.GH_STATUS_ERROR)
 
     if update_final_report:
-        _ResultS3.copy_result_to_s3(
-            workflow_result,
-        )
+        _ResultS3.copy_result_to_s3_with_version(workflow_result, version + 1)
 
     Result.from_fs(job_name).set_status(Result.Status.SUCCESS)
 

ci/praktika/result.py

@@ -121,6 +121,9 @@ class Result(MetaClasses.Serializable):
     def set_success(self) -> "Result":
         return self.set_status(Result.Status.SUCCESS)
 
+    def set_failed(self) -> "Result":
+        return self.set_status(Result.Status.FAILED)
+
     def set_results(self, results: List["Result"]) -> "Result":
         self.results = results
         self.dump()

ci/praktika/runner.py

@@ -1,3 +1,5 @@
+import glob
+import json
 import os
 import re
 import sys
@@ -58,6 +60,9 @@ class Runner:
             workflow_config.digest_dockers[docker.name] = Digest().calc_docker_digest(
                 docker, workflow.dockers
             )
+
+        # work around for old clickhouse jobs
+        os.environ["DOCKER_TAG"] = json.dumps(workflow_config.digest_dockers)
         workflow_config.dump()
 
         Result.generate_pending(job.name).dump()
@@ -119,8 +124,21 @@ class Runner:
         else:
             prefixes = [env.get_s3_prefix()] * len(required_artifacts)
         for artifact, prefix in zip(required_artifacts, prefixes):
+            recursive = False
+            include_pattern = ""
+            if "*" in artifact.path:
+                s3_path = f"{Settings.S3_ARTIFACT_PATH}/{prefix}/{Utils.normalize_string(artifact._provided_by)}/"
+                recursive = True
+                include_pattern = Path(artifact.path).name
+                assert "*" in include_pattern
+            else:
                 s3_path = f"{Settings.S3_ARTIFACT_PATH}/{prefix}/{Utils.normalize_string(artifact._provided_by)}/{Path(artifact.path).name}"
-            assert S3.copy_file_from_s3(s3_path=s3_path, local_path=Settings.INPUT_DIR)
+            assert S3.copy_file_from_s3(
+                s3_path=s3_path,
+                local_path=Settings.INPUT_DIR,
+                recursive=recursive,
+                include_pattern=include_pattern,
+            )
 
         return 0
 
@@ -239,9 +257,11 @@ class Runner:
             info = f"ERROR: {ResultInfo.KILLED}"
             print(info)
             result.set_info(info).set_status(Result.Status.ERROR).dump()
+        else:
+            # TODO: add setting with different ways of storing general praktika log: always, on error, never.
+            #   now let's store it on error only
+            result.files = [file for file in result.files if file != Settings.RUN_LOG]
 
-        if not result.is_ok():
-            result.set_files(files=[Settings.RUN_LOG])
         result.update_duration().dump()
 
         if run_exit_code == 0:
@@ -262,8 +282,9 @@ class Runner:
                             f"ls -l {artifact.path}", verbose=True
                         ), f"Artifact {artifact.path} not found"
                         s3_path = f"{Settings.S3_ARTIFACT_PATH}/{env.get_s3_prefix()}/{Utils.normalize_string(env.JOB_NAME)}"
+                        for file_path in glob.glob(artifact.path):
                             link = S3.copy_file_to_s3(
-                            s3_path=s3_path, local_path=artifact.path
+                                s3_path=s3_path, local_path=file_path
                             )
                             result.set_link(link)
                     except Exception as e:

ci/praktika/s3.py

@@ -2,6 +2,7 @@ import dataclasses
 import json
 from pathlib import Path
 from typing import Dict
+from urllib.parse import quote
 
 from praktika._environment import _Environment
 from praktika.settings import Settings
@@ -55,7 +56,7 @@ class S3:
         bucket = s3_path.split("/")[0]
         endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket]
         assert endpoint
-        return f"https://{s3_full_path}".replace(bucket, endpoint)
+        return quote(f"https://{s3_full_path}".replace(bucket, endpoint), safe=":/?&=")
 
     @classmethod
     def put(cls, s3_path, local_path, text=False, metadata=None, if_none_matched=False):
@@ -117,15 +118,21 @@ class S3:
         return res
 
     @classmethod
-    def copy_file_from_s3(cls, s3_path, local_path):
+    def copy_file_from_s3(
+        cls, s3_path, local_path, recursive=False, include_pattern=""
+    ):
         assert Path(s3_path), f"Invalid S3 Path [{s3_path}]"
         if Path(local_path).is_dir():
-            local_path = Path(local_path) / Path(s3_path).name
+            pass
         else:
             assert Path(
                 local_path
             ).parent.is_dir(), f"Parent path for [{local_path}] does not exist"
         cmd = f"aws s3 cp s3://{s3_path}  {local_path}"
+        if recursive:
+            cmd += " --recursive"
+        if include_pattern:
+            cmd += f" --include {include_pattern}"
         res = cls.run_command_with_retries(cmd)
         return res
 

ci/settings/definitions.py

@@ -242,3 +242,4 @@ class JobNames:
     BUILD = "Build"
     STATELESS = "Stateless tests"
     STATEFUL = "Stateful tests"
+    STRESS = "Stress tests"

ci/workflows/pull_request.py

@@ -16,6 +16,16 @@ class ArtifactNames:
     CH_ARM_RELEASE = "CH_ARM_RELEASE"
     CH_ARM_ASAN = "CH_ARM_ASAN"
 
+    CH_ODBC_B_AMD_DEBUG = "CH_ODBC_B_AMD_DEBUG"
+    CH_ODBC_B_AMD_RELEASE = "CH_ODBC_B_AMD_RELEASE"
+    CH_ODBC_B_ARM_RELEASE = "CH_ODBC_B_ARM_RELEASE"
+    CH_ODBC_B_ARM_ASAN = "CH_ODBC_B_ARM_ASAN"
+
+    DEB_AMD_DEBUG = "DEB_AMD_DEBUG"
+    DEB_AMD_RELEASE = "DEB_AMD_RELEASE"
+    DEB_ARM_RELEASE = "DEB_ARM_RELEASE"
+    DEB_ARM_ASAN = "DEB_ARM_ASAN"
+
 
 style_check_job = Job.Config(
     name=JobNames.STYLE_CHECK,
@@ -41,7 +51,7 @@ fast_test_job = Job.Config(
 build_jobs = Job.Config(
     name=JobNames.BUILD,
     runs_on=["...from params..."],
-    requires=[JobNames.FAST_TEST],
+    requires=[],
     command="python3 ./ci/jobs/build_clickhouse.py --build-type {PARAMETER}",
     run_in_docker="clickhouse/fasttest",
     timeout=3600 * 2,
@@ -63,10 +73,26 @@ build_jobs = Job.Config(
 ).parametrize(
     parameter=["amd_debug", "amd_release", "arm_release", "arm_asan"],
     provides=[
-        [ArtifactNames.CH_AMD_DEBUG],
-        [ArtifactNames.CH_AMD_RELEASE],
-        [ArtifactNames.CH_ARM_RELEASE],
-        [ArtifactNames.CH_ARM_ASAN],
+        [
+            ArtifactNames.CH_AMD_DEBUG,
+            ArtifactNames.DEB_AMD_DEBUG,
+            ArtifactNames.CH_ODBC_B_AMD_DEBUG,
+        ],
+        [
+            ArtifactNames.CH_AMD_RELEASE,
+            ArtifactNames.DEB_AMD_RELEASE,
+            ArtifactNames.CH_ODBC_B_AMD_RELEASE,
+        ],
+        [
+            ArtifactNames.CH_ARM_RELEASE,
+            ArtifactNames.DEB_ARM_RELEASE,
+            ArtifactNames.CH_ODBC_B_ARM_RELEASE,
+        ],
+        [
+            ArtifactNames.CH_ARM_ASAN,
+            ArtifactNames.DEB_ARM_ASAN,
+            ArtifactNames.CH_ODBC_B_ARM_ASAN,
+        ],
     ],
     runs_on=[
         [RunnerLabels.BUILDER_AMD],
@@ -105,12 +131,12 @@ stateless_tests_jobs = Job.Config(
         [RunnerLabels.FUNC_TESTER_ARM],
     ],
     requires=[
-        [ArtifactNames.CH_AMD_DEBUG],
-        [ArtifactNames.CH_AMD_DEBUG],
-        [ArtifactNames.CH_AMD_RELEASE],
-        [ArtifactNames.CH_AMD_RELEASE],
-        [ArtifactNames.CH_ARM_ASAN],
-        [ArtifactNames.CH_ARM_ASAN],
+        [ArtifactNames.CH_AMD_DEBUG, ArtifactNames.CH_ODBC_B_AMD_DEBUG],
+        [ArtifactNames.CH_AMD_DEBUG, ArtifactNames.CH_ODBC_B_AMD_DEBUG],
+        [ArtifactNames.CH_AMD_RELEASE, ArtifactNames.CH_ODBC_B_AMD_RELEASE],
+        [ArtifactNames.CH_AMD_RELEASE, ArtifactNames.CH_ODBC_B_AMD_RELEASE],
+        [ArtifactNames.CH_ARM_ASAN, ArtifactNames.CH_ODBC_B_ARM_ASAN],
+        [ArtifactNames.CH_ARM_ASAN, ArtifactNames.CH_ODBC_B_ARM_ASAN],
     ],
 )
 
@@ -128,7 +154,7 @@ stateful_tests_jobs = Job.Config(
     ),
 ).parametrize(
     parameter=[
-        "amd_debug,parallel",
+        "amd_release,parallel",
     ],
     runs_on=[
         [RunnerLabels.BUILDER_AMD],
@@ -138,6 +164,29 @@ stateful_tests_jobs = Job.Config(
     ],
 )
 
+# TODO: refactor job to be aligned with praktika style (remove wrappers, run in docker)
+stress_test_jobs = Job.Config(
+    name=JobNames.STRESS,
+    runs_on=[RunnerLabels.BUILDER_ARM],
+    command="python3 ./tests/ci/stress_check.py {PARAMETER}",
+    digest_config=Job.CacheDigestConfig(
+        include_paths=[
+            "./ci/jobs/functional_stateful_tests.py",
+        ],
+    ),
+).parametrize(
+    parameter=[
+        "arm_release",
+    ],
+    runs_on=[
+        [RunnerLabels.FUNC_TESTER_ARM],
+    ],
+    requires=[
+        [ArtifactNames.DEB_ARM_RELEASE],
+    ],
+)
+
+
 workflow = Workflow.Config(
     name="PR",
     event=Workflow.Event.PULL_REQUEST,
@@ -148,27 +197,52 @@ workflow = Workflow.Config(
         *build_jobs,
         *stateless_tests_jobs,
         *stateful_tests_jobs,
+        *stress_test_jobs,
     ],
     artifacts=[
-        Artifact.Config(
-            name=ArtifactNames.CH_AMD_DEBUG,
+        *Artifact.Config(
+            name="...",
             type=Artifact.Type.S3,
             path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
+        ).parametrize(
+            names=[
+                ArtifactNames.CH_AMD_DEBUG,
+                ArtifactNames.CH_AMD_RELEASE,
+                ArtifactNames.CH_ARM_RELEASE,
+                ArtifactNames.CH_ARM_ASAN,
+            ]
+        ),
+        *Artifact.Config(
+            name="...",
+            type=Artifact.Type.S3,
+            path=f"{Settings.TEMP_DIR}/build/programs/clickhouse-odbc-bridge",
+        ).parametrize(
+            names=[
+                ArtifactNames.CH_ODBC_B_AMD_DEBUG,
+                ArtifactNames.CH_ODBC_B_AMD_RELEASE,
+                ArtifactNames.CH_ODBC_B_ARM_RELEASE,
+                ArtifactNames.CH_ODBC_B_ARM_ASAN,
+            ]
         ),
         Artifact.Config(
-            name=ArtifactNames.CH_AMD_RELEASE,
+            name=ArtifactNames.DEB_AMD_DEBUG,
             type=Artifact.Type.S3,
-            path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
+            path=f"{Settings.TEMP_DIR}/output/*.deb",
         ),
         Artifact.Config(
-            name=ArtifactNames.CH_ARM_RELEASE,
+            name=ArtifactNames.DEB_AMD_RELEASE,
             type=Artifact.Type.S3,
-            path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
+            path=f"{Settings.TEMP_DIR}/output/*.deb",
         ),
         Artifact.Config(
-            name=ArtifactNames.CH_ARM_ASAN,
+            name=ArtifactNames.DEB_ARM_RELEASE,
             type=Artifact.Type.S3,
-            path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
+            path=f"{Settings.TEMP_DIR}/output/*.deb",
+        ),
+        Artifact.Config(
+            name=ArtifactNames.DEB_ARM_ASAN,
+            type=Artifact.Type.S3,
+            path=f"{Settings.TEMP_DIR}/output/*.deb",
         ),
     ],
     dockers=DOCKERS,

packages/build

@@ -5,24 +5,14 @@ set -e
 # Avoid dependency on locale
 LC_ALL=C
 
-# Normalize output directory
-if [ -n "$OUTPUT_DIR" ]; then
-    OUTPUT_DIR=$(realpath -m "$OUTPUT_DIR")
-fi
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-cd "$CUR_DIR"
-
-ROOT_DIR=$(readlink -f "$(git rev-parse --show-cdup)")
-
 PKG_ROOT='root'
 
 DEB_ARCH=${DEB_ARCH:-amd64}
-OUTPUT_DIR=${OUTPUT_DIR:-$ROOT_DIR}
-[ -d "${OUTPUT_DIR}" ] || mkdir -p "${OUTPUT_DIR}"
 SANITIZER=${SANITIZER:-""}
 SOURCE=${SOURCE:-$PKG_ROOT}
 
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
 HELP="${0} [--test] [--rpm] [-h|--help]
   --test - adds '+test' prefix to version
   --apk - build APK packages
@@ -40,12 +30,7 @@ Used envs:
   VERSION_STRING='${VERSION_STRING}' - the package version to overwrite
 "
 
-if [ -z "${VERSION_STRING}" ]; then
-    # Get CLICKHOUSE_VERSION_STRING from the current git repo
-    eval "$("$ROOT_DIR/tests/ci/version_helper.py" -e)"
-else
 CLICKHOUSE_VERSION_STRING=${VERSION_STRING}
-fi
 export CLICKHOUSE_VERSION_STRING
 
 
@@ -144,28 +129,29 @@ CLICKHOUSE_VERSION_STRING+=$VERSION_POSTFIX
 echo -e "\nCurrent version is $CLICKHOUSE_VERSION_STRING"
 
 for config in clickhouse*.yaml; do
+    if [[ $BUILD_TYPE != 'release' ]] && [[ "$config" == "clickhouse-keeper-dbg.yaml" ]]; then
+        continue
+    fi
     if [ -n "$MAKE_DEB" ] || [ -n "$MAKE_TGZ" ]; then
         echo "Building deb package for $config"
-
-        # Preserve package path
-        exec 9>&1
-        PKG_PATH=$(nfpm package --target "$OUTPUT_DIR" --config "$config" --packager deb | tee /dev/fd/9)
-        PKG_PATH=${PKG_PATH##*created package: }
-        exec 9>&-
+        PKG_PATH=$(nfpm package --target "$OUTPUT_DIR" --config "$config" --packager deb | tee /dev/stderr | grep "created package:" | sed 's/.*created package: //')
     fi
 
     if [ -n "$MAKE_APK" ]; then
         echo "Building apk package for $config"
         nfpm package --target "$OUTPUT_DIR" --config "$config" --packager apk
     fi
+
     if [ -n "$MAKE_ARCHLINUX" ]; then
         echo "Building archlinux package for $config"
         nfpm package --target "$OUTPUT_DIR" --config "$config" --packager archlinux
     fi
+
     if [ -n "$MAKE_RPM" ]; then
         echo "Building rpm package for $config"
         nfpm package --target "$OUTPUT_DIR" --config "$config" --packager rpm
     fi
+
     if [ -n "$MAKE_TGZ" ]; then
         echo "Building tarball for $config"
         deb2tgz "$PKG_PATH"

tests/ci/report.py

@@ -4,6 +4,7 @@ import datetime
 import json
 import logging
 import os
+import sys
 from ast import literal_eval
 from dataclasses import asdict, dataclass
 from html import escape
@@ -414,6 +415,42 @@ class JobReport:
     dummy: bool = False
     exit_code: int = -1
 
+    def to_praktika_result(self, job_name):
+        sys.path.append("./ci")
+
+        # ugly WA to exclude ci.py file form import
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        if current_dir in sys.path:
+            sys.path.remove(current_dir)
+        from praktika.result import Result
+
+        if self.start_time:
+            dt = datetime.datetime.strptime(self.start_time, "%Y-%m-%d %H:%M:%S")
+            timestamp = dt.timestamp()
+        else:
+            timestamp = None
+
+        sub_results = []
+        for r in self.test_results:
+            sub_results.append(
+                Result(
+                    name=r.name,
+                    status=r.status,
+                    info=r.raw_logs,
+                    links=list(r.log_urls) if r.log_urls else [],
+                    duration=r.time,
+                )
+            )
+
+        return Result(
+            name=job_name,
+            status=self.status,
+            start_time=timestamp,
+            duration=self.duration,
+            results=sub_results,
+            files=[str(f) for f in self.additional_files] if self.additional_files else [],
+        )
+
     @staticmethod
     def get_start_time_from_current():
         return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
@@ -474,6 +511,7 @@ class JobReport:
         to_file = to_file or JOB_REPORT_FILE
         with open(to_file, "w", encoding="utf-8") as json_file:
             json.dump(asdict(self), json_file, default=path_converter, indent=2)
+        return self
 
 
 def read_test_results(results_path: Path, with_raw_logs: bool = True) -> TestResults:

tests/ci/stress_check.py

@@ -15,6 +15,7 @@ from docker_images_helper import DockerImage, get_docker_image, pull_image
 from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
 from get_robot_token import get_parameter_from_ssm
 from pr_info import PRInfo
+from praktika.utils import Shell
 from report import ERROR, JobReport, TestResults, read_test_results
 from stopwatch import Stopwatch
 from tee_popen import TeePopen
@@ -154,11 +155,18 @@ def run_stress_test(upgrade_check: bool = False) -> None:
 
     pr_info = PRInfo()
 
-    docker_image = pull_image(get_docker_image("clickhouse/stress-test"))
+    docker_image = pull_image(get_docker_image("clickhouse/stateful-test"))
 
     packages_path = temp_path / "packages"
     packages_path.mkdir(parents=True, exist_ok=True)
 
+    if check_name in ("amd_release", "amd_debug", "arm_release"):
+        # this is praktika based CI
+        print("Copy input *.deb artifacts")
+        assert Shell.check(
+            f"cp /tmp/praktika/input/*.deb {packages_path}", verbose=True
+        )
+    else:
         download_all_deb_packages(check_name, reports_path, packages_path)
 
     server_log_path = temp_path / "server_log"
@@ -201,6 +209,7 @@ def run_stress_test(upgrade_check: bool = False) -> None:
         result_path, server_log_path, run_log_path
     )
 
+    Shell.check("pwd", verbose=True)
     JobReport(
         description=description,
         test_results=test_results,
@@ -208,7 +217,7 @@ def run_stress_test(upgrade_check: bool = False) -> None:
         start_time=stopwatch.start_time_str,
         duration=stopwatch.duration_seconds,
         additional_files=additional_logs,
-    ).dump()
+    ).dump().to_praktika_result(job_name=f"Stress tests ({check_name})").dump()
 
     if state == "failure":
         sys.exit(1)

tests/config/install.sh

@@ -21,7 +21,7 @@ while [[ "$#" -gt 0 ]]; do
         --fast-test) FAST_TEST=1 ;;
         --s3-storage) EXPORT_S3_STORAGE_POLICIES=1 ;;
         --no-azure) NO_AZURE=1 ;;
-        *) echo "Unknown option: $1" ; exit 1 ;;
+        *) echo "Unknown option: $1" ;;
     esac
     shift
 done