Merge 5378b49088 into 4e56c026cd

.github/workflows/create_release.yml

@@ -25,6 +25,11 @@ env:
         required: false
         default: false
         type: boolean
+      only-docker:
+        description: 'Run only docker builds (repo-recovery, tests)'
+        required: false
+        default: false
+        type: boolean
       dry-run:
         description: 'Dry run'
         required: false
@@ -45,6 +50,11 @@ env:
         required: false
         default: false
         type: boolean
+      only-docker:
+        description: 'Run only docker builds (repo-recovery, tests)'
+        required: false
+        default: false
+        type: boolean
       dry-run:
         description: 'Dry run'
         required: false
@@ -69,13 +79,13 @@ jobs:
       - name: Prepare Release Info
         shell: bash
         run: |
-          if [ ${{ inputs.only-repo }} == "true" ]; then
+          if [ ${{ inputs.only-repo }} == "true" ] || [ ${{ inputs.only-docker }} == "true" ]; then
-              git tag -l ${{ inputs.ref }} || { echo "With only-repo option ref must be a valid release tag"; exit 1; }
+              git tag -l ${{ inputs.ref }} || { echo "With only-repo/docker option ref must be a valid release tag"; exit 1; }
           fi
           python3 ./tests/ci/create_release.py --prepare-release-info \
             --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \
             ${{ inputs.dry-run == true && '--dry-run' || '' }} \
-            ${{ inputs.only-repo == true && '--skip-tag-check' || '' }}
+            ${{ (inputs.only-repo == true || inputs.only-docker == true) && '--skip-tag-check' || '' }}
           echo "::group::Release Info"
           python3 -m json.tool /tmp/release_info.json
           echo "::endgroup::"
@@ -87,31 +97,33 @@ jobs:
           echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
           if [ "$is_latest" == "true" ]; then
             echo "DOCKER_TAG_TYPE=release-latest" >> "$GITHUB_ENV"
+            echo "IS_LATEST=1" >> "$GITHUB_ENV"
           else
             echo "DOCKER_TAG_TYPE=release" >> "$GITHUB_ENV"
+            echo "IS_LATEST=0" >> "$GITHUB_ENV"
           fi
       - name: Download All Release Artifacts
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Push Git Tag for the Release
-        if: ${{ ! inputs.only-repo }}
+        if: ${{ ! inputs.only-repo &&  ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Push New Release Branch
-        if: ${{ inputs.type == 'new' && ! inputs.only-repo }}
+        if: ${{ inputs.type == 'new' && ! inputs.only-repo && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Bump CH Version and Update Contributors' List
-        if: ${{ ! inputs.only-repo }}
+        if: ${{ ! inputs.only-repo && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Bump Docker versions, Changelog, Security
-        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
@@ -135,7 +147,7 @@ jobs:
           python3 ./utils/security-generator/generate_security.py > SECURITY.md
           git diff HEAD
       - name: Create ChangeLog PR
-        if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo }}
+        if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo && ! inputs.only-docker }}
         uses: peter-evans/create-pull-request@v6
         with:
           author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
@@ -153,65 +165,176 @@ jobs:
             ### Changelog category (leave one):
             - Not for changelog (changelog entry is not required)
       - name: Complete previous steps and Restore git state
-        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker }}
         shell: bash
         run: |
           git reset --hard HEAD
           git checkout "$GITHUB_REF_NAME"
           python3 ./tests/ci/create_release.py --set-progress-completed
       - name: Create GH Release
-        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker}}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Export TGZ Packages
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Test TGZ Packages
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Export RPM Packages
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Test RPM Packages
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Export Debian Packages
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Test Debian Packages
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
         shell: bash
         run: |
           python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Docker clickhouse/clickhouse-server building
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && inputs.dry-run != true }}
         shell: bash
         run: |
           cd "./tests/ci"
           python3 ./create_release.py --set-progress-started --progress "docker server release"
-          export CHECK_NAME="Docker server image"
+          export DOCKER_IMAGE="clickhouse/clickhouse-server"
-          python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
+
+          # We must use docker file from the release commit
+          git checkout "${{ env.RELEASE_TAG }}"
+          python3 ./version_helper.py --export > /tmp/version.sh
+          . /tmp/version.sh
+
+          if [[ $CLICKHOUSE_VERSION_STRING =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "ClickHouse version: $CLICKHOUSE_VERSION_STRING"
+          else
+            echo "Invalid version string: $CLICKHOUSE_VERSION_STRING"
+            exit 1
+          fi
+          CLICKHOUSE_VERSION_MINOR=${CLICKHOUSE_VERSION_STRING%.*}
+          CLICKHOUSE_VERSION_MAJOR=${CLICKHOUSE_VERSION_MINOR%.*}
+
+          # Define build configurations
+          configs=(
+            "ubuntu:../../docker/server/Dockerfile.ubuntu"
+            "alpine:../../docker/server/Dockerfile.alpine"
+          )
+
+          for config in "${configs[@]}"; do
+            # Split the config into variant and Dockerfile path
+            variant=${config%%:*}
+            dockerfile=${config##*:}
+
+            VERSION_SUFFIX=$([ "$variant" = "ubuntu" ] && echo "" || echo "-$variant")
+            LABEL_VERSION="${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}"
+            TAGS=(
+              "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}"
+              "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MINOR}${VERSION_SUFFIX}"
+              "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MAJOR}${VERSION_SUFFIX}"
+            )
+
+            if [ "$IS_LATEST" = "1" ]; then
+              TAGS+=("--tag=${DOCKER_IMAGE}:latest${VERSION_SUFFIX}")
+            fi
+
+            echo "Following tags will be created: ${TAGS[*]}"
+
+            # shellcheck disable=SC2086,SC2048
+            docker buildx build \
+              --platform=linux/amd64,linux/arm64 \
+              --output=type=registry \
+              --label=com.clickhouse.build.version="$LABEL_VERSION" \
+              ${TAGS[*]} \
+              --build-arg=VERSION="$CLICKHOUSE_VERSION_STRING" \
+              --progress=plain \
+              --file="$dockerfile" \
+              ../../docker/server
+          done
+
+          git checkout -
           python3 ./create_release.py --set-progress-completed
       - name: Docker clickhouse/clickhouse-keeper building
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && inputs.dry-run != true }}
         shell: bash
         run: |
           cd "./tests/ci"
           python3 ./create_release.py --set-progress-started --progress "docker keeper release"
-          export CHECK_NAME="Docker keeper image"
+          
-          python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
+          export DOCKER_IMAGE="clickhouse/clickhouse-keeper"
+
+          # We must use docker file from the release commit
+          git checkout "${{ env.RELEASE_TAG }}"
+          python3 ./version_helper.py --export > /tmp/version.sh
+          . /tmp/version.sh
+
+          if [[ $CLICKHOUSE_VERSION_STRING =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "ClickHouse version: $CLICKHOUSE_VERSION_STRING"
+          else
+            echo "Invalid version string: $CLICKHOUSE_VERSION_STRING"
+            exit 1
+          fi
+          CLICKHOUSE_VERSION_MINOR=${CLICKHOUSE_VERSION_STRING%.*}
+          CLICKHOUSE_VERSION_MAJOR=${CLICKHOUSE_VERSION_MINOR%.*}
+
+          # Define build configurations
+          configs=(
+            "ubuntu:../../docker/keeper/Dockerfile.ubuntu"
+            "alpine:../../docker/keeper/Dockerfile.alpine"
+          )
+
+          for config in "${configs[@]}"; do
+            # Split the config into variant and Dockerfile path
+            variant=${config%%:*}
+            dockerfile=${config##*:}
+
+            VERSION_SUFFIX=$([ "$variant" = "ubuntu" ] && echo "" || echo "-$variant")
+            LABEL_VERSION="${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}"
+            TAGS=(
+              "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}"
+              "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MINOR}${VERSION_SUFFIX}"
+              "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MAJOR}${VERSION_SUFFIX}"
+            )
+
+            if [ "$IS_LATEST" = "1" ]; then
+              TAGS+=("--tag=${DOCKER_IMAGE}:latest${VERSION_SUFFIX}")
+            fi
+
+            echo "Following tags will be created: ${TAGS[*]}"
+
+            # shellcheck disable=SC2086,SC2048
+            docker buildx build \
+              --platform=linux/amd64,linux/arm64 \
+              --output=type=registry \
+              --label=com.clickhoghuse.build.version="$LABEL_VERSION" \
+              ${TAGS[*]} \
+              --build-arg=VERSION="$CLICKHOUSE_VERSION_STRING" \
+              --progress=plain \
+              --file="$dockerfile" \
+              ../../docker/keeper
+          done
+
+          git checkout -
           python3 ./create_release.py --set-progress-completed
+      # check out back if previous steps failed
+      - name: Checkout back
+        if: ${{ ! cancelled() }}
+        shell: bash
+        run: |
+          git checkout ${{ github.ref }}
       - name: Update release info. Merge created PRs
         shell: bash
         run: |