Merge pull request #36029 from ClickHouse/fix-release-workflow

Fix release workflow

.github/workflows/backport_branches.yml

@@ -9,6 +9,18 @@ on: # yamllint disable-line rule:truthy
     branches:
       - 'backport/**'
 jobs:
+  PythonUnitTests:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Python unit tests
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 -m unittest discover -s . -p '*_test.py'
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:

.github/workflows/release.yml

@@ -52,8 +52,8 @@ jobs:
     - name: Check docker clickhouse/clickhouse-server building
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
-        python3 docker_server.py --release-type auto
-        python3 docker_server.py --release-type auto --no-ubuntu \
+        python3 docker_server.py --release-type auto --version "${{ github.ref }}"
+        python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \
           --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
     - name: Cleanup
       if: always()

tests/ci/docker_server.py

@@ -24,7 +24,7 @@ from version_helper import (
     ClickHouseVersion,
     get_tagged_versions,
     get_version_from_repo,
-    get_version_from_string,
+    version_arg,
 )
 
 TEMP_PATH = p.join(RUNNER_TEMP, "docker_images_check")
@@ -49,7 +49,8 @@ def parse_args() -> argparse.Namespace:
         "--version",
         type=version_arg,
         default=get_version_from_repo().string,
-        help="a version to build",
+        help="a version to build, automaticaly got from version_helper, accepts either "
+        "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format",
     )
     parser.add_argument(
         "--release-type",
@@ -111,13 +112,6 @@ def parse_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def version_arg(version: str) -> ClickHouseVersion:
-    try:
-        return get_version_from_string(version)
-    except ValueError as e:
-        raise argparse.ArgumentTypeError(e)
-
-
 def auto_release_type(version: ClickHouseVersion, release_type: str) -> str:
     if release_type != "auto":
         return release_type
@@ -125,7 +119,7 @@ def auto_release_type(version: ClickHouseVersion, release_type: str) -> str:
     git_versions = get_tagged_versions()
     reference_version = git_versions[0]
     for i in reversed(range(len(git_versions))):
-        if git_versions[i] < version:
+        if git_versions[i] <= version:
             if i == len(git_versions) - 1:
                 return "latest"
             reference_version = git_versions[i + 1]
@@ -209,7 +203,7 @@ def build_and_push_image(
     result = []
     if os != "ubuntu":
         tag += f"-{os}"
-    init_args = ["docker", "buildx", "build"]
+    init_args = ["docker", "buildx", "build", "--build-arg BUILDKIT_INLINE_CACHE=1"]
     if push:
         init_args.append("--push")
         init_args.append("--output=type=image,push-by-digest=true")

tests/ci/docker_test.py

@@ -9,7 +9,7 @@ from pr_info import PRInfo
 import docker_images_check as di
 
 with patch("git_helper.Git"):
-    from version_helper import get_version_from_string, get_tagged_versions
+    from version_helper import get_version_from_string
     import docker_server as ds
 
 # di.logging.basicConfig(level=di.logging.INFO)
@@ -254,7 +254,8 @@ class TestDockerServer(unittest.TestCase):
             get_version_from_string("2.2.1.1"),
             get_version_from_string("2.2.2.1"),
         ]
-        cases = (
+
+        cases_less = (
             (get_version_from_string("1.0.1.1"), "minor"),
             (get_version_from_string("1.1.2.1"), "minor"),
             (get_version_from_string("1.3.1.1"), "major"),
@@ -263,8 +264,18 @@ class TestDockerServer(unittest.TestCase):
             (get_version_from_string("2.2.3.1"), "latest"),
             (get_version_from_string("2.3.1.1"), "latest"),
         )
-        _ = get_tagged_versions()
-        for case in cases:
+        for case in cases_less:
+            release = ds.auto_release_type(case[0], "auto")
+            self.assertEqual(case[1], release)
+
+        cases_equal = (
+            (get_version_from_string("1.1.1.1"), "minor"),
+            (get_version_from_string("1.2.1.1"), "major"),
+            (get_version_from_string("2.1.1.1"), "minor"),
+            (get_version_from_string("2.2.1.1"), "patch"),
+            (get_version_from_string("2.2.2.1"), "latest"),
+        )
+        for case in cases_equal:
             release = ds.auto_release_type(case[0], "auto")
             self.assertEqual(case[1], release)
 

tests/ci/git_helper.py

@@ -93,7 +93,7 @@ class Git:
         if value == "":
             return
         if not self._tag_pattern.match(value):
-            raise Exception(f"last tag {value} doesn't match the pattern")
+            raise ValueError(f"last tag {value} doesn't match the pattern")
 
     @property
     def latest_tag(self) -> str:

tests/ci/push_to_artifactory.py

@@ -40,13 +40,12 @@ class Packages:
             "_".join((name, version, arch + ".deb")) for name, arch in self.packages
         )
 
-        rev = "2"
         self.rpm = tuple(
-            "-".join((name, version, rev + "." + self.rpm_arch[arch] + ".rpm"))
+            "-".join((name, version + "." + self.rpm_arch[arch] + ".rpm"))
             for name, arch in self.packages
         )
 
-        self.tgz = tuple(f"{name}-{version}.tgz" for name, _ in self.packages)
+        self.tgz = tuple(f"{name}-{version}-amd64.tgz" for name, _ in self.packages)
 
     def arch(self, deb_pkg: str) -> str:
         if deb_pkg not in self.deb:

tests/ci/version_helper.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import logging
 import os.path as p
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError
 from typing import Dict, List, Tuple, Union
 
 from git_helper import Git, removeprefix
@@ -150,6 +150,9 @@ class ClickHouseVersion:
 
         return False
 
+    def __le__(self, other: "ClickHouseVersion") -> bool:
+        return self == other or self < other
+
 
 class VersionType:
     LTS = "lts"
@@ -217,6 +220,20 @@ def get_version_from_tag(tag: str) -> ClickHouseVersion:
     return get_version_from_string(tag)
 
 
+def version_arg(version: str) -> ClickHouseVersion:
+    version = removeprefix(version, "refs/tags/")
+    try:
+        return get_version_from_string(version)
+    except ValueError:
+        pass
+    try:
+        return get_version_from_tag(version)
+    except ValueError:
+        pass
+
+    raise ArgumentTypeError(f"version {version} does not match tag of plain version")
+
+
 def get_tagged_versions() -> List[ClickHouseVersion]:
     versions = []
     for tag in git.get_tags():

tests/ci/version_test.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+import unittest
+from argparse import ArgumentTypeError
+
+import version_helper as vh
+
+
+class TestFunctions(unittest.TestCase):
+    def test_version_arg(self):
+        cases = (
+            ("0.0.0.0", vh.get_version_from_string("0.0.0.0")),
+            ("1.1.1.2", vh.get_version_from_string("1.1.1.2")),
+            ("v1.1.1.2-lts", vh.get_version_from_string("1.1.1.2")),
+            ("v1.1.1.2-prestable", vh.get_version_from_string("1.1.1.2")),
+            ("v1.1.1.2-stable", vh.get_version_from_string("1.1.1.2")),
+            ("v1.1.1.2-testing", vh.get_version_from_string("1.1.1.2")),
+            ("refs/tags/v1.1.1.2-testing", vh.get_version_from_string("1.1.1.2")),
+        )
+        for case in cases:
+            version = vh.version_arg(case[0])
+            self.assertEqual(case[1], version)
+        error_cases = (
+            "0.0.0",
+            "1.1.1.a",
+            "1.1.1.1.1",
+            "1.1.1.2-testing",
+            "v1.1.1.2-testin",
+            "refs/tags/v1.1.1.2-testin",
+        )
+        for case in error_cases:
+            with self.assertRaises(ArgumentTypeError):
+                version = vh.version_arg(case[0])