Merge pull request #66498 from ClickHouse/ci_create_release_workflow_updates

CI: Create release workflow updates

.github/workflows/backport_branches.yml

@@ -62,7 +62,7 @@ jobs:
   BuildDockers:
     needs: [RunConfig]
     if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_docker.yml
+    uses: ./.github/workflows/docker_test_images.yml
     with:
       data: ${{ needs.RunConfig.outputs.data }}
   CompatibilityCheckX86:

.github/workflows/create_release.yml

@@ -94,7 +94,7 @@ jobs:
           echo "Generate Security"
           python3 ./utils/security-generator/generate_security.py > SECURITY.md
           git diff HEAD
-      - name: Generate ChangeLog
+      - name: Create ChangeLog PR
         if: ${{ inputs.type == 'patch' && ! inputs.dry-run }}
         uses: peter-evans/create-pull-request@v6
         with:

.github/workflows/master.yml

@@ -58,7 +58,7 @@ jobs:
 #  BuildDockers:
 #    needs: [RunConfig]
 #    if: ${{ !failure() && !cancelled() }}
-#    uses: ./.github/workflows/reusable_docker.yml
+#    uses: ./.github/workflows/docker_test_images.yml
 #    with:
 #      data: ${{ needs.RunConfig.outputs.data }}
   # StyleCheck:

.github/workflows/merge_queue.yml

@@ -51,7 +51,7 @@ jobs:
   BuildDockers:
     needs: [RunConfig]
     if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }}
-    uses: ./.github/workflows/reusable_docker.yml
+    uses: ./.github/workflows/docker_test_images.yml
     with:
       data: ${{ needs.RunConfig.outputs.data }}
   StyleCheck:

.github/workflows/nightly.yml

@@ -40,7 +40,7 @@ jobs:
             } >> "$GITHUB_OUTPUT"
   BuildDockers:
     needs: [RunConfig]
-    uses: ./.github/workflows/reusable_docker.yml
+    uses: ./.github/workflows/docker_test_images.yml
     with:
       data: "${{ needs.RunConfig.outputs.data }}"
       set_latest: true

.github/workflows/pull_request.yml

@@ -72,7 +72,7 @@ jobs:
   BuildDockers:
     needs: [RunConfig]
     if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }}
-    uses: ./.github/workflows/reusable_docker.yml
+    uses: ./.github/workflows/docker_test_images.yml
     with:
       data: ${{ needs.RunConfig.outputs.data }}
   StyleCheck:

.github/workflows/release_branches.yml

@@ -57,7 +57,7 @@ jobs:
   BuildDockers:
     needs: [RunConfig]
     if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_docker.yml
+    uses: ./.github/workflows/docker_test_images.yml
     with:
       data: ${{ needs.RunConfig.outputs.data }}
   CompatibilityCheckX86:

tests/ci/ci_utils.py

@@ -49,14 +49,15 @@ class GHActions:
 class Shell:
     @classmethod
     def run_strict(cls, command):
-        subprocess.run(
-            command + " 2>&1",
+        res = subprocess.run(
+            command,
             shell=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
             check=True,
         )
+        return res.stdout.strip()
 
     @classmethod
     def run(cls, command):

tests/ci/create_release.py

@@ -14,6 +14,7 @@ from ssh import SSHAgent
 from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET
 from s3_helper import S3Helper
 from autoscale_runners_lambda.lambda_shared.pr import Labels
+from ci_utils import Shell
 from version_helper import (
     FILE_WITH_VERSION_PATH,
     GENERATED_CONTRIBUTORS,
@@ -65,6 +66,8 @@ class ReleaseInfo:
     commit_sha: str
     # lts or stable
     codename: str
+    previous_release_tag: str
+    previous_release_sha: str
 
     @staticmethod
     def from_file(file_path: str) -> "ReleaseInfo":
@@ -79,6 +82,8 @@ class ReleaseInfo:
         version = None
         release_branch = None
         release_tag = None
+        previous_release_tag = None
+        previous_release_sha = None
         codename = None
         assert release_type in ("patch", "new")
         if release_type == "new":
@@ -101,6 +106,11 @@ class ReleaseInfo:
                 codename = (
                     VersionType.STABLE
                 )  # dummy value (artifactory won't be updated for new release)
+                previous_release_tag = expected_prev_tag
+                previous_release_sha = Shell.run_strict(
+                    f"git rev-parse {previous_release_tag}"
+                )
+                assert previous_release_sha
         if release_type == "patch":
             with checkout(commit_ref):
                 _, commit_sha = ShellRunner.run(f"git rev-parse {commit_ref}")
@@ -118,9 +128,10 @@ class ReleaseInfo:
             )
             if version.patch == 1:
                 expected_version = copy(version)
+                previous_release_tag = f"v{version.major}.{version.minor}.1.1-new"
                 expected_version.bump()
                 expected_tag_prefix = (
-                    f"v{expected_version.major}.{expected_version.minor}-"
+                    f"v{expected_version.major}.{expected_version.minor}."
                 )
                 expected_tag_suffix = "-new"
             else:
@@ -128,6 +139,7 @@ class ReleaseInfo:
                     f"v{version.major}.{version.minor}.{version.patch-1}."
                 )
                 expected_tag_suffix = f"-{version.get_stable_release_type()}"
+                previous_release_tag = git.latest_tag
             if git.latest_tag.startswith(
                 expected_tag_prefix
             ) and git.latest_tag.endswith(expected_tag_suffix):
@@ -137,8 +149,15 @@ class ReleaseInfo:
                     False
                 ), f"BUG: Unexpected latest tag [{git.latest_tag}] expected [{expected_tag_prefix}*{expected_tag_suffix}]"
 
+            previous_release_sha = Shell.run_strict(
+                f"git rev-parse {previous_release_tag}"
+            )
+            assert previous_release_sha
+
         assert (
             release_branch
+            and previous_release_tag
+            and previous_release_sha
             and commit_sha
             and release_tag
             and version
@@ -150,6 +169,8 @@ class ReleaseInfo:
             release_tag=release_tag,
             version=version.string,
             codename=codename,
+            previous_release_tag=previous_release_tag,
+            previous_release_sha=previous_release_sha,
         )
         with open(outfile, "w", encoding="utf-8") as f:
             print(json.dumps(dataclasses.asdict(res), indent=2), file=f)
@@ -618,6 +639,8 @@ sudo apt install --yes --no-install-recommends python3-dev python3-pip gh unzip
 sudo apt install --yes python3-boto3
 sudo apt install --yes python3-github
 sudo apt install --yes python3-unidiff
+sudo apt install --yes python3-tqdm # cloud changelog
+sudo apt install --yes python3-thefuzz # cloud changelog
 sudo apt install --yes s3fs
 
 ### INSTALL AWS CLI