add geesfs

.github/workflows/create_release.yml

@@ -62,15 +62,14 @@ jobs:
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
-      - name: Bump CH Version and Update Contributors' List
-        shell: bash
-        run: |
-          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
+#      - name: Bump CH Version and Update Contributors' List
+#        shell: bash
+#        run: |
+#          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Bump Docker versions, Changelog, Security
         if: ${{ inputs.type == 'patch' }}
         shell: bash
         run: |
-          git checkout master
           python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
           echo "List versions"
           ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
@@ -96,6 +95,7 @@ jobs:
           committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
           commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
           branch: auto/${{ env.RELEASE_TAG }}
+          base: master
           assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
           delete-branch: true
           title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }}

.github/workflows/release.yml

@@ -55,12 +55,12 @@ jobs:
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
         export CHECK_NAME="Docker server image"
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --check-name "$CHECK_NAME" --push
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$(git rev-list -n 1 $GITHUB_TAG)" --check-name "$CHECK_NAME" --push
     - name: Check docker clickhouse/clickhouse-keeper building
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
         export CHECK_NAME="Docker keeper image"
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --check-name "$CHECK_NAME" --push
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$(git rev-list -n 1 $GITHUB_TAG)" --check-name "$CHECK_NAME" --push
     - name: Cleanup
       if: always()
       run: |

tests/ci/artifactory.py

@@ -15,6 +15,7 @@ from ci_utils import WithIter, Shell
 class MountPointApp(metaclass=WithIter):
     RCLONE = "rclone"
     S3FS = "s3fs"
+    GEESEFS = "geesefs"
 
 
 class R2MountPoint:
@@ -70,6 +71,20 @@ class R2MountPoint:
             )
             # Use --no-modtime to try to avoid: ERROR : rpm/lts/clickhouse-client-24.3.6.5.x86_64.rpm: Failed to apply pending mod time
             self.mount_cmd = f"rclone mount remote:{self.bucket_name} {self.MOUNT_POINT} --daemon --cache-dir {self.cache_dir} --umask 0000 --log-file {self.LOG_FILE} {self.aux_mount_options}"
+        elif self.app == MountPointApp.GEESEFS:
+            self.cache_dir = "/home/ubuntu/geesefs_cache"
+            self.aux_mount_options += (
+                f" --cache={self.cache_dir} " if self.CACHE_ENABLED else ""
+            )
+            if not dry_run:
+                self.aux_mount_options += f" --shared-config=/home/ubuntu/.r2_auth "
+            else:
+                self.aux_mount_options += (
+                    f" --shared-config=/home/ubuntu/.r2_auth_test "
+                )
+            if self.DEBUG:
+                self.aux_mount_options += " --debug_s3 --debug_fuse "
+            self.mount_cmd = f"geesefs --endpoint={self.API_ENDPOINT} --cheap --memory-limit=2050 --gc-interval=100 --max-flushers=5 --max-parallel-parts=1 --max-parallel-copy=2 --log-file={self.LOG_FILE} {self.aux_mount_options} {self.bucket_name} {self.MOUNT_POINT}"
         else:
             assert False
 
@@ -87,7 +102,7 @@ class R2MountPoint:
         Shell.run(_UNMOUNT_CMD)
         Shell.run(_MKDIR_CMD)
         Shell.run(_MKDIR_FOR_CACHE)
-        if self.app == MountPointApp.S3FS:
+        if self.app != MountPointApp.RCLONE:
             Shell.run(self.mount_cmd, check=True)
         else:
             # didn't manage to use simple run() and without blocking or failure
@@ -158,7 +173,13 @@ class DebianArtifactory:
         cmd = f'docker run --rm ubuntu:latest bash -c "apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; {debian_command}"'
         print("Running test command:")
         print(f"  {cmd}")
-        Shell.run(cmd, check=True)
+        assert Shell.check(cmd)
+        print(f"Test packages installation, version [latest]")
+        debian_command_2 = f"echo 'deb {self.repo_url} stable main' | tee /etc/apt/sources.list.d/clickhouse.list; apt update -y; apt-get install -y clickhouse-common-static clickhouse-client"
+        cmd = f'docker run --rm ubuntu:latest bash -c "apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; {debian_command_2}"'
+        print("Running test command:")
+        print(f"  {cmd}")
+        assert Shell.check(cmd)
         self.release_info.debian_command = debian_command
         self.release_info.dump()
 
@@ -234,7 +255,13 @@ class RpmArtifactory:
         cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && {rpm_command}"'
         print("Running test command:")
         print(f"  {cmd}")
-        Shell.run(cmd, check=True)
+        assert Shell.check(cmd)
+        print(f"Test package installation, version [latest]")
+        rpm_command_2 = f"dnf config-manager --add-repo={self.repo_url} && dnf makecache && dnf -y install clickhouse-client"
+        cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && {rpm_command_2}"'
+        print("Running test command:")
+        print(f"  {cmd}")
+        assert Shell.check(cmd)
         self.release_info.rpm_command = rpm_command
         self.release_info.dump()
 
@@ -350,7 +377,7 @@ if __name__ == "__main__":
            ERROR : IO error: NotImplemented: versionId not implemented
            Failed to copy: NotImplemented: versionId not implemented
     """
-    mp = R2MountPoint(MountPointApp.S3FS, dry_run=args.dry_run)
+    mp = R2MountPoint(MountPointApp.GEESEFS, dry_run=args.dry_run)
     if args.export_debian:
         with ReleaseContextManager(
             release_progress=ReleaseProgress.EXPORT_DEB

tests/ci/changelog.py

@@ -7,7 +7,7 @@ import re
 from datetime import date, timedelta
 from pathlib import Path
 from subprocess import DEVNULL
-from typing import Any, Dict, List, Optional, TextIO
+from typing import Any, Dict, List, Optional, TextIO, Tuple
 
 import tqdm  # type: ignore
 from github.GithubException import RateLimitExceededException, UnknownObjectException
@@ -400,13 +400,19 @@ def get_year(prs: PullRequests) -> int:
     return max(pr.created_at.year for pr in prs)
 
 
-def get_branch_by_tag(tag: str) -> Optional[str]:
-    tag.removeprefix("v")
+def get_branch_and_patch_by_tag(tag: str) -> Tuple[Optional[str], Optional[int]]:
+    tag = tag.removeprefix("v")
     versions = tag.split(".")
-    if len(versions) < 3:
+    if len(versions) < 4:
         print("ERROR: Can't get branch by tag")
-        return None
-    return f"{versions[0]}.{versions[1]}"
+        return None, None
+    try:
+        patch_version = int(versions[2])
+        branch = f"{int(versions[0])}.{int(versions[1])}"
+        print(f"Branch [{branch}], patch version [{patch_version}]")
+    except ValueError:
+        return None, None
+    return branch, patch_version
 
 
 def main():
@@ -458,17 +464,22 @@ def main():
     gh_cache = GitHubCache(gh.cache_path, temp_path, S3Helper())
     gh_cache.download()
     query = f"type:pr repo:{args.repo} is:merged"
-    branch = get_branch_by_tag(TO_REF)
-    if branch and Shell.check(f"git show-ref --quiet {branch}"):
-        try:
-            if int(branch.split(".")[-1]) > 1:
-                query += f" base:{branch}"
-            print(f"NOTE: will use base branch to filter PRs {branch}")
-        except ValueError:
-            print(f"ERROR: cannot get minor version from branch {branch} - pass")
-            pass
+
+    branch, patch = get_branch_and_patch_by_tag(TO_REF)
+    if branch and patch and Shell.check(f"git show-ref --quiet {branch}"):
+        if patch > 1:
+            query += f" base:{branch}"
+            print(
+                f"NOTE: It's a patch [{patch}]. will use base branch to filter PRs [{branch}]"
+            )
+        else:
+            print(
+                f"NOTE: It's a first patch version. should count PRs merged on master - won't filter PRs by branch"
+            )
     else:
         print(f"ERROR: invalid branch {branch} - pass")
+
+    print(f"Fetch PRs with query {query}")
     prs = gh.get_pulls_from_search(
         query=query, merged=merged, sort="created", progress_func=tqdm.tqdm
     )

tests/ci/ci_utils.py

@@ -246,15 +246,14 @@ class Shell:
 
     @classmethod
     def check(cls, command):
-        result = subprocess.run(
+        proc = subprocess.Popen(
             command,
             shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-            check=False,
+            stdout=subprocess.STDOUT,
+            stderr=subprocess.STDOUT,
         )
-        return result.returncode == 0
+        proc.wait()
+        return proc.returncode == 0
 
 
 class Utils: