improve ci_utils' Shell

.github/workflows/pull_request.yml

@@ -142,8 +142,13 @@ jobs:
   # Reports should run even if Builds_1/2 fail - run them separately (not in Tests_1/2/3)
   Builds_Report:
     # run report check for failed builds to indicate the CI error
-    if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
-    needs: [RunConfig, StyleCheck, Builds_1, Builds_2]
+    if: ${{ !cancelled()
+      && needs.RunConfig.result == 'success'
+      && needs.StyleCheck.result != 'failure'
+      && needs.FastTest.result != 'failure'
+      && needs.BuildDockers.result != 'failure'
+      && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2]
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Builds

.github/workflows/release.yml

@@ -55,12 +55,14 @@ jobs:
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
         export CHECK_NAME="Docker server image"
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$(git rev-list -n 1 $GITHUB_TAG)" --check-name "$CHECK_NAME" --push
+        SHA=$(git rev-list -n 1 "$GITHUB_TAG")
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$SHA"  --check-name "$CHECK_NAME" --push
     - name: Check docker clickhouse/clickhouse-keeper building
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
         export CHECK_NAME="Docker keeper image"
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$(git rev-list -n 1 $GITHUB_TAG)" --check-name "$CHECK_NAME" --push
+        SHA=$(git rev-list -n 1 "$GITHUB_TAG")
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$SHA" --check-name "$CHECK_NAME" --push
     - name: Cleanup
       if: always()
       run: |

pyproject.toml

@@ -39,6 +39,7 @@ disable = '''
           no-else-return,
           global-statement,
           f-string-without-interpolation,
+          consider-using-with,
           '''
 
 [tool.pylint.SIMILARITIES]

tests/ci/artifactory.py

@@ -13,7 +13,6 @@ from ci_utils import WithIter, Shell
 
 
 class MountPointApp(metaclass=WithIter):
-    RCLONE = "rclone"
     S3FS = "s3fs"
     GEESEFS = "geesefs"
 
@@ -31,9 +30,6 @@ class R2MountPoint:
     DEBUG = True
     # enable cache for mountpoint
     CACHE_ENABLED = False
-    # TODO: which mode is better: minimal/writes/full/off
-    _RCLONE_CACHE_MODE = "minimal"
-    UMASK = "0000"
 
     def __init__(self, app: str, dry_run: bool) -> None:
         assert app in MountPointApp
@@ -59,18 +55,6 @@ class R2MountPoint:
                 )
             # without -o nomultipart there are errors like "Error 5 writing to /home/ubuntu/***.deb: Input/output error"
             self.mount_cmd = f"s3fs {self.bucket_name} {self.MOUNT_POINT} -o url={self.API_ENDPOINT} -o use_path_request_style -o umask=0000 -o nomultipart -o logfile={self.LOG_FILE} {self.aux_mount_options}"
-        elif self.app == MountPointApp.RCLONE:
-            # run rclone mount process asynchronously, otherwise subprocess.run(daemonized command) will not return
-            self.cache_dir = "/home/ubuntu/rclone_cache"
-            self.aux_mount_options += "--no-modtime " if self.NOMODTIME else ""
-            self.aux_mount_options += "-v " if self.DEBUG else ""  # -vv too verbose
-            self.aux_mount_options += (
-                f"--vfs-cache-mode {self._RCLONE_CACHE_MODE} --vfs-cache-max-size {self._CACHE_MAX_SIZE_GB}G"
-                if self.CACHE_ENABLED
-                else "--vfs-cache-mode off"
-            )
-            # Use --no-modtime to try to avoid: ERROR : rpm/lts/clickhouse-client-24.3.6.5.x86_64.rpm: Failed to apply pending mod time
-            self.mount_cmd = f"rclone mount remote:{self.bucket_name} {self.MOUNT_POINT} --daemon --cache-dir {self.cache_dir} --umask 0000 --log-file {self.LOG_FILE} {self.aux_mount_options}"
         elif self.app == MountPointApp.GEESEFS:
             self.cache_dir = "/home/ubuntu/geesefs_cache"
             self.aux_mount_options += (
@@ -98,22 +82,17 @@ class R2MountPoint:
         )
 
         _TEST_MOUNT_CMD = f"mount | grep -q {self.MOUNT_POINT}"
-        Shell.run(_CLEAN_LOG_FILE_CMD)
-        Shell.run(_UNMOUNT_CMD)
-        Shell.run(_MKDIR_CMD)
-        Shell.run(_MKDIR_FOR_CACHE)
-        if self.app != MountPointApp.RCLONE:
-            Shell.run(self.mount_cmd, check=True)
-        else:
-            # didn't manage to use simple run() and without blocking or failure
-            Shell.run_as_daemon(self.mount_cmd)
+        Shell.check(_CLEAN_LOG_FILE_CMD, verbose=True)
+        Shell.check(_UNMOUNT_CMD, verbose=True)
+        Shell.check(_MKDIR_CMD, verbose=True)
+        Shell.check(_MKDIR_FOR_CACHE, verbose=True)
+        Shell.check(self.mount_cmd, strict=True, verbose=True)
         time.sleep(3)
-        Shell.run(_TEST_MOUNT_CMD, check=True)
+        Shell.check(_TEST_MOUNT_CMD, strict=True, verbose=True)
 
     @classmethod
     def teardown(cls):
-        print(f"Unmount [{cls.MOUNT_POINT}]")
-        Shell.run(f"umount {cls.MOUNT_POINT}")
+        Shell.check(f"umount {cls.MOUNT_POINT}", verbose=True)
 
 
 class RepoCodenames(metaclass=WithIter):
@@ -148,10 +127,9 @@ class DebianArtifactory:
         ]
         REPREPRO_CMD_PREFIX = f"reprepro --basedir {R2MountPoint.MOUNT_POINT}/configs/deb --outdir {R2MountPoint.MOUNT_POINT}/deb --verbose"
         cmd = f"{REPREPRO_CMD_PREFIX} includedeb {self.codename} {' '.join(paths)}"
-        print("Running export command:")
-        print(f"  {cmd}")
-        Shell.run(cmd, check=True)
-        Shell.run("sync")
+        print("Running export commands:")
+        Shell.check(cmd, strict=True, verbose=True)
+        Shell.check("sync")
 
         if self.codename == RepoCodenames.LTS:
             packages_with_version = [
@@ -163,11 +141,11 @@ class DebianArtifactory:
             cmd = f"{REPREPRO_CMD_PREFIX} copy {RepoCodenames.STABLE} {RepoCodenames.LTS} {' '.join(packages_with_version)}"
             print("Running copy command:")
             print(f"  {cmd}")
-            Shell.run(cmd, check=True)
-            Shell.run("sync")
+            Shell.check(cmd, strict=True)
+            Shell.check("sync")
 
     def test_packages(self):
-        Shell.run("docker pull ubuntu:latest")
+        Shell.check("docker pull ubuntu:latest", strict=True)
         print(f"Test packages installation, version [{self.version}]")
         debian_command = f"echo 'deb {self.repo_url} stable main' | tee /etc/apt/sources.list.d/clickhouse.list; apt update -y; apt-get install -y clickhouse-common-static={self.version} clickhouse-client={self.version}"
         cmd = f'docker run --rm ubuntu:latest bash -c "apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; {debian_command}"'
@@ -236,20 +214,18 @@ class RpmArtifactory:
         print(f"Exporting RPM packages into [{codename}]")
 
         for command in commands:
-            print("Running command:")
-            print(f"    {command}")
-            Shell.run(command, check=True)
+            Shell.check(command, strict=True, verbose=True)
 
         update_public_key = f"gpg --armor --export {self._SIGN_KEY}"
         pub_key_path = dest_dir / "repodata" / "repomd.xml.key"
         print("Updating repomd.xml.key")
-        pub_key_path.write_text(Shell.run(update_public_key, check=True))
+        pub_key_path.write_text(Shell.get_output_or_raise(update_public_key))
         if codename == RepoCodenames.LTS:
             self.export_packages(RepoCodenames.STABLE)
-        Shell.run("sync")
+        Shell.check("sync")
 
     def test_packages(self):
-        Shell.run("docker pull fedora:latest")
+        Shell.check("docker pull fedora:latest", strict=True)
         print(f"Test package installation, version [{self.version}]")
         rpm_command = f"dnf config-manager --add-repo={self.repo_url} && dnf makecache && dnf -y install clickhouse-client-{self.version}-1"
         cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && {rpm_command}"'
@@ -302,26 +278,30 @@ class TgzArtifactory:
 
         if codename == RepoCodenames.LTS:
             self.export_packages(RepoCodenames.STABLE)
-        Shell.run("sync")
+        Shell.check("sync")
 
     def test_packages(self):
         tgz_file = "/tmp/tmp.tgz"
         tgz_sha_file = "/tmp/tmp.tgz.sha512"
         cmd = f"curl -o {tgz_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz"
-        Shell.run(
+        Shell.check(
             cmd,
-            check=True,
+            strict=True,
+            verbose=True,
         )
-        Shell.run(
+        Shell.check(
             f"curl -o {tgz_sha_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz.sha512",
-            check=True,
+            strict=True,
+            verbose=True,
+        )
+        expected_checksum = Shell.get_output_or_raise(f"cut -d ' ' -f 1 {tgz_sha_file}")
+        actual_checksum = Shell.get_output_or_raise(
+            f"sha512sum {tgz_file} | cut -d ' ' -f 1"
         )
-        expected_checksum = Shell.run(f"cut -d ' ' -f 1 {tgz_sha_file}", check=True)
-        actual_checksum = Shell.run(f"sha512sum {tgz_file} | cut -d ' ' -f 1")
         assert (
             expected_checksum == actual_checksum
         ), f"[{actual_checksum} != {expected_checksum}]"
-        Shell.run("rm /tmp/tmp.tgz*")
+        Shell.check("rm /tmp/tmp.tgz*", verbose=True)
         self.release_info.tgz_command = cmd
         self.release_info.dump()
 
@@ -373,9 +353,9 @@ if __name__ == "__main__":
     args = parse_args()
 
     """
-    Use S3FS. RCLONE has some errors with r2 remote which I didn't figure out how to resolve:
-           ERROR : IO error: NotImplemented: versionId not implemented
-           Failed to copy: NotImplemented: versionId not implemented
+    S3FS - very slow with a big repo
+    RCLONE - fuse had many different errors with r2 remote and completely removed
+    GEESEFS ? 
     """
     mp = R2MountPoint(MountPointApp.GEESEFS, dry_run=args.dry_run)
     if args.export_debian:

tests/ci/auto_release.py

@@ -85,7 +85,7 @@ class AutoReleaseInfo:
 def _prepare(token):
     assert len(token) > 10
     os.environ["GH_TOKEN"] = token
-    Shell.run("gh auth status", check=True)
+    Shell.check("gh auth status")
 
     gh = GitHub(token)
     prs = gh.get_release_pulls(GITHUB_REPOSITORY)
@@ -106,9 +106,8 @@ def _prepare(token):
         latest_release_tag_ref = refs[-1]
         latest_release_tag = repo.get_git_tag(latest_release_tag_ref.object.sha)
 
-        commits = Shell.run(
+        commits = Shell.get_output_or_raise(
             f"git rev-list --first-parent {latest_release_tag.tag}..origin/{pr.head.ref}",
-            check=True,
         ).split("\n")
         commit_num = len(commits)
         print(

tests/ci/ci_buddy.py

@@ -120,8 +120,10 @@ class CIBuddy:
     ) -> None:
         instance_id, instance_type = "unknown", "unknown"
         if with_instance_info:
-            instance_id = Shell.run("ec2metadata --instance-id") or instance_id
-            instance_type = Shell.run("ec2metadata --instance-type") or instance_type
+            instance_id = Shell.get_output("ec2metadata --instance-id") or instance_id
+            instance_type = (
+                Shell.get_output("ec2metadata --instance-type") or instance_type
+            )
         if not job_name:
             job_name = os.getenv("CHECK_NAME", "unknown")
         sign = ":red_circle:" if not critical else ":black_circle:"

tests/ci/ci_definitions.py

@@ -554,7 +554,7 @@ class CommonJobConfigs:
         run_command="sqllogic_test.py",
         timeout=10800,
         release_only=True,
-        runner_type=Runners.STYLE_CHECKER,
+        runner_type=Runners.FUNC_TESTER,
     )
     SQL_TEST = JobConfig(
         job_name_keyword="sqltest",
@@ -582,6 +582,7 @@ class CommonJobConfigs:
         digest=DigestConfig(
             include_paths=[
                 "tests/ci/docker_server.py",
+                "tests/ci/docker_images_helper.py",
                 "./docker/server",
             ]
         ),

tests/ci/ci_utils.py

@@ -2,6 +2,7 @@ import json
 import os
 import re
 import subprocess
+import sys
 import time
 from contextlib import contextmanager
 from pathlib import Path
@@ -192,7 +193,7 @@ class GHActions:
         get_url_cmd = (
             f"gh pr list --repo {repo} --head {branch} --json url --jq '.[0].url'"
         )
-        url = Shell.run(get_url_cmd)
+        url = Shell.get_output(get_url_cmd)
         if not url:
             print(f"ERROR: PR nor found, branch [{branch}]")
         return url
@@ -200,59 +201,56 @@ class GHActions:
 
 class Shell:
     @classmethod
-    def run_strict(cls, command):
+    def get_output_or_raise(cls, command):
+        return cls.get_output(command, strict=True)
+
+    @classmethod
+    def get_output(cls, command, strict=False):
         res = subprocess.run(
             command,
             shell=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
-            check=True,
+            check=strict,
         )
         return res.stdout.strip()
 
     @classmethod
-    def run(cls, command, check=False, dry_run=False, **kwargs):
+    def check(
+        cls,
+        command,
+        strict=False,
+        verbose=False,
+        dry_run=False,
+        stdin_str=None,
+        **kwargs,
+    ):
         if dry_run:
             print(f"Dry-ryn. Would run command [{command}]")
-            return ""
-        print(f"Run command [{command}]")
-        res = ""
-        result = subprocess.run(
-            command,
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-            check=False,
-            **kwargs,
-        )
-        if result.returncode == 0:
-            print(f"stdout: {result.stdout.strip()}")
-            res = result.stdout
-        else:
-            print(
-                f"ERROR: stdout: {result.stdout.strip()}, stderr: {result.stderr.strip()}"
-            )
-            if check:
-                assert result.returncode == 0
-        return res.strip()
-
-    @classmethod
-    def run_as_daemon(cls, command):
-        print(f"Run daemon command [{command}]")
-        subprocess.Popen(command.split(" "))  # pylint:disable=consider-using-with
-        return 0, ""
-
-    @classmethod
-    def check(cls, command):
+            return 0
+        if verbose:
+            print(f"Run command [{command}]")
         proc = subprocess.Popen(
             command,
             shell=True,
-            stdout=subprocess.STDOUT,
             stderr=subprocess.STDOUT,
+            stdout=subprocess.PIPE,
+            stdin=subprocess.PIPE if stdin_str else None,
+            universal_newlines=True,
+            start_new_session=True,
+            bufsize=1,
+            errors="backslashreplace",
+            **kwargs,
         )
+        if stdin_str:
+            proc.communicate(input=stdin_str)
+        elif proc.stdout:
+            for line in proc.stdout:
+                sys.stdout.write(line)
         proc.wait()
+        if strict:
+            assert proc.returncode == 0
         return proc.returncode == 0
 
 
@@ -277,7 +275,7 @@ class Utils:
 
     @staticmethod
     def clear_dmesg():
-        Shell.run("sudo dmesg --clear ||:")
+        Shell.check("sudo dmesg --clear", verbose=True)
 
     @staticmethod
     def check_pr_description(pr_body: str, repo_name: str) -> Tuple[str, str]:

tests/ci/create_release.py

@@ -137,12 +137,13 @@ class ReleaseInfo:
         assert release_type in ("patch", "new")
         if release_type == "new":
             # check commit_ref is right and on a right branch
-            Shell.run(
+            Shell.check(
                 f"git merge-base --is-ancestor {commit_ref} origin/master",
-                check=True,
+                strict=True,
+                verbose=True,
             )
             with checkout(commit_ref):
-                commit_sha = Shell.run(f"git rev-parse {commit_ref}", check=True)
+                commit_sha = Shell.get_output_or_raise(f"git rev-parse {commit_ref}")
                 # Git() must be inside "with checkout" contextmanager
                 git = Git()
                 version = get_version_from_repo(git=git)
@@ -154,13 +155,13 @@ class ReleaseInfo:
                 ), f"BUG: latest tag [{git.latest_tag}], expected [{expected_prev_tag}]"
                 release_tag = version.describe
                 previous_release_tag = expected_prev_tag
-                previous_release_sha = Shell.run_strict(
+                previous_release_sha = Shell.get_output_or_raise(
                     f"git rev-parse {previous_release_tag}"
                 )
                 assert previous_release_sha
         if release_type == "patch":
             with checkout(commit_ref):
-                commit_sha = Shell.run(f"git rev-parse {commit_ref}", check=True)
+                commit_sha = Shell.get_output_or_raise(f"git rev-parse {commit_ref}")
                 # Git() must be inside "with checkout" contextmanager
                 git = Git()
                 version = get_version_from_repo(git=git)
@@ -168,11 +169,16 @@ class ReleaseInfo:
                 version.with_description(codename)
                 release_branch = f"{version.major}.{version.minor}"
                 release_tag = version.describe
-            Shell.run(f"{GIT_PREFIX} fetch origin {release_branch} --tags", check=True)
+            Shell.check(
+                f"{GIT_PREFIX} fetch origin {release_branch} --tags",
+                strict=True,
+                verbose=True,
+            )
             # check commit is right and on a right branch
-            Shell.run(
+            Shell.check(
                 f"git merge-base --is-ancestor {commit_ref} origin/{release_branch}",
-                check=True,
+                strict=True,
+                verbose=True,
             )
             if version.patch == 1:
                 expected_version = copy(version)
@@ -197,7 +203,7 @@ class ReleaseInfo:
                     False
                 ), f"BUG: Unexpected latest tag [{git.latest_tag}] expected [{expected_tag_prefix}*{expected_tag_suffix}]"
 
-            previous_release_sha = Shell.run_strict(
+            previous_release_sha = Shell.get_output_or_raise(
                 f"git rev-parse {previous_release_tag}"
             )
             assert previous_release_sha
@@ -226,25 +232,26 @@ class ReleaseInfo:
     def push_release_tag(self, dry_run: bool) -> None:
         if dry_run:
             # remove locally created tag from prev run
-            Shell.run(
-                f"{GIT_PREFIX} tag -l | grep -q {self.release_tag} && git tag -d {self.release_tag} ||:"
+            Shell.check(
+                f"{GIT_PREFIX} tag -l | grep -q {self.release_tag} && git tag -d {self.release_tag}"
             )
         # Create release tag
         print(
             f"Create and push release tag [{self.release_tag}], commit [{self.commit_sha}]"
         )
         tag_message = f"Release {self.release_tag}"
-        Shell.run(
+        Shell.check(
             f"{GIT_PREFIX} tag -a -m '{tag_message}' {self.release_tag} {self.commit_sha}",
-            check=True,
+            strict=True,
+            verbose=True,
         )
         cmd_push_tag = f"{GIT_PREFIX} push origin {self.release_tag}:{self.release_tag}"
-        Shell.run(cmd_push_tag, dry_run=dry_run, check=True)
+        Shell.check(cmd_push_tag, dry_run=dry_run, strict=True, verbose=True)
 
     @staticmethod
     def _create_gh_label(label: str, color_hex: str, dry_run: bool) -> None:
         cmd = f"gh api repos/{GITHUB_REPOSITORY}/labels -f name={label} -f color={color_hex}"
-        Shell.run(cmd, dry_run=dry_run, check=True)
+        Shell.check(cmd, dry_run=dry_run, strict=True)
 
     def push_new_release_branch(self, dry_run: bool) -> None:
         assert (
@@ -261,7 +268,7 @@ class ReleaseInfo:
         ), f"Unexpected current version in git, must precede [{self.version}] by one step, actual [{version.string}]"
         if dry_run:
             # remove locally created branch from prev run
-            Shell.run(
+            Shell.check(
                 f"{GIT_PREFIX} branch -l | grep -q {new_release_branch} && git branch -d {new_release_branch}"
             )
         print(
@@ -275,7 +282,7 @@ class ReleaseInfo:
                 cmd_push_branch = (
                     f"{GIT_PREFIX} push --set-upstream origin {new_release_branch}"
                 )
-                Shell.run(cmd_push_branch, dry_run=dry_run, check=True)
+                Shell.check(cmd_push_branch, dry_run=dry_run, strict=True, verbose=True)
 
         print("Create and push backport tags for new release branch")
         ReleaseInfo._create_gh_label(
@@ -284,13 +291,14 @@ class ReleaseInfo:
         ReleaseInfo._create_gh_label(
             f"v{new_release_branch}-affected", "c2bfff", dry_run=dry_run
         )
-        Shell.run(
+        Shell.check(
             f"""gh pr create --repo {GITHUB_REPOSITORY} --title 'Release pull request for branch {new_release_branch}'
             --head {new_release_branch} {pr_labels}
             --body 'This PullRequest is a part of ClickHouse release cycle. It is used by CI system only. Do not perform any changes with it.'
             """,
             dry_run=dry_run,
-            check=True,
+            strict=True,
+            verbose=True,
         )
 
     def update_version_and_contributors_list(self, dry_run: bool) -> None:
@@ -316,13 +324,19 @@ class ReleaseInfo:
                 body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")
                 actor = os.getenv("GITHUB_ACTOR", "") or "me"
                 cmd_create_pr = f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body-file {body_file} --label 'do not test' --assignee {actor}"
-                Shell.run(cmd_commit_version_upd, check=True, dry_run=dry_run)
-                Shell.run(cmd_push_branch, check=True, dry_run=dry_run)
-                Shell.run(cmd_create_pr, check=True, dry_run=dry_run)
+                Shell.check(
+                    cmd_commit_version_upd, strict=True, dry_run=dry_run, verbose=True
+                )
+                Shell.check(cmd_push_branch, strict=True, dry_run=dry_run, verbose=True)
+                Shell.check(cmd_create_pr, strict=True, dry_run=dry_run, verbose=True)
                 if dry_run:
-                    Shell.run(f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'")
-                    Shell.run(
-                        f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'"
+                    Shell.check(
+                        f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'",
+                        verbose=True,
+                    )
+                    Shell.check(
+                        f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'",
+                        verbose=True,
                     )
                     self.version_bump_pr = "dry-run"
                 else:
@@ -358,7 +372,7 @@ class ReleaseInfo:
             cmds.append(f"gh release upload {self.release_tag} {file}")
         if not dry_run:
             for cmd in cmds:
-                Shell.run(cmd, check=True)
+                Shell.check(cmd, strict=True, verbose=True)
             self.release_url = f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
         else:
             print("Dry-run, would run commands:")
@@ -424,7 +438,7 @@ class PackageDownloader:
         self.macos_package_files = ["clickhouse-macos", "clickhouse-macos-aarch64"]
         self.file_to_type = {}
 
-        Shell.run(f"mkdir -p {self.LOCAL_DIR}")
+        Shell.check(f"mkdir -p {self.LOCAL_DIR}")
 
         for package_type in self.PACKAGE_TYPES:
             for package in self.package_names:
@@ -474,7 +488,7 @@ class PackageDownloader:
         return res
 
     def run(self):
-        Shell.run(f"rm -rf {self.LOCAL_DIR}/*")
+        Shell.check(f"rm -rf {self.LOCAL_DIR}/*")
         for package_file in (
             self.deb_package_files + self.rpm_package_files + self.tgz_package_files
         ):
@@ -549,33 +563,33 @@ class PackageDownloader:
 
 @contextmanager
 def checkout(ref: str) -> Iterator[None]:
-    orig_ref = Shell.run(f"{GIT_PREFIX} symbolic-ref --short HEAD", check=True)
+    orig_ref = Shell.get_output_or_raise(f"{GIT_PREFIX} symbolic-ref --short HEAD")
     rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
     assert orig_ref
     if ref not in (orig_ref,):
-        Shell.run(f"{GIT_PREFIX} checkout {ref}")
+        Shell.check(f"{GIT_PREFIX} checkout {ref}", strict=True, verbose=True)
     try:
         yield
     except (Exception, KeyboardInterrupt) as e:
         print(f"ERROR: Exception [{e}]")
-        Shell.run(rollback_cmd)
+        Shell.check(rollback_cmd, verbose=True)
         raise
-    Shell.run(rollback_cmd)
+    Shell.check(rollback_cmd, verbose=True)
 
 
 @contextmanager
 def checkout_new(ref: str) -> Iterator[None]:
-    orig_ref = Shell.run(f"{GIT_PREFIX} symbolic-ref --short HEAD", check=True)
+    orig_ref = Shell.get_output_or_raise(f"{GIT_PREFIX} symbolic-ref --short HEAD")
     rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
     assert orig_ref
-    Shell.run(f"{GIT_PREFIX} checkout -b {ref}", check=True)
+    Shell.check(f"{GIT_PREFIX} checkout -b {ref}", strict=True, verbose=True)
     try:
         yield
     except (Exception, KeyboardInterrupt) as e:
         print(f"ERROR: Exception [{e}]")
-        Shell.run(rollback_cmd)
+        Shell.check(rollback_cmd, verbose=True)
         raise
-    Shell.run(rollback_cmd)
+    Shell.check(rollback_cmd, verbose=True)
 
 
 def parse_args() -> argparse.Namespace:

tests/ci/docker_images_helper.py

@@ -19,11 +19,11 @@ def docker_login(relogin: bool = True) -> None:
     if relogin or not Shell.check(
         "docker system info | grep --quiet -E 'Username|Registry'"
     ):
-        Shell.run(  # pylint: disable=unexpected-keyword-arg
+        Shell.check(  # pylint: disable=unexpected-keyword-arg
             "docker login --username 'robotclickhouse' --password-stdin",
-            input=get_parameter_from_ssm("dockerhub_robot_password"),
+            strict=True,
+            stdin_str=get_parameter_from_ssm("dockerhub_robot_password"),
             encoding="utf-8",
-            check=True,
         )
 
 
@@ -42,7 +42,7 @@ class DockerImage:
 def pull_image(image: DockerImage) -> DockerImage:
     try:
         logging.info("Pulling image %s - start", image)
-        Shell.run(f"docker pull {image}", check=True)
+        Shell.check(f"docker pull {image}", strict=True)
         logging.info("Pulling image %s - done", image)
     except Exception as ex:
         logging.info("Got exception pulling docker %s", ex)