From 5378b49088070a6d2c88431dbb87902f5dccaced Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Nov 2024 17:32:20 +0100 Subject: [PATCH] CI: Enable Stress Tests with praktika --- .github/workflows/create_release.yml | 169 +++++++++++++++++++++++---- 1 file changed, 146 insertions(+), 23 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 9d2ee70365e..b53bf172096 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -25,6 +25,11 @@ env: required: false default: false type: boolean + only-docker: + description: 'Run only docker builds (repo-recovery, tests)' + required: false + default: false + type: boolean dry-run: description: 'Dry run' required: false @@ -45,6 +50,11 @@ env: required: false default: false type: boolean + only-docker: + description: 'Run only docker builds (repo-recovery, tests)' + required: false + default: false + type: boolean dry-run: description: 'Dry run' required: false @@ -69,13 +79,13 @@ jobs: - name: Prepare Release Info shell: bash run: | - if [ ${{ inputs.only-repo }} == "true" ]; then - git tag -l ${{ inputs.ref }} || { echo "With only-repo option ref must be a valid release tag"; exit 1; } + if [ ${{ inputs.only-repo }} == "true" ] || [ ${{ inputs.only-docker }} == "true" ]; then + git tag -l ${{ inputs.ref }} || { echo "With only-repo/docker option ref must be a valid release tag"; exit 1; } fi python3 ./tests/ci/create_release.py --prepare-release-info \ --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \ ${{ inputs.dry-run == true && '--dry-run' || '' }} \ - ${{ inputs.only-repo == true && '--skip-tag-check' || '' }} + ${{ (inputs.only-repo == true || inputs.only-docker == true) && '--skip-tag-check' || '' }} echo "::group::Release Info" python3 -m json.tool /tmp/release_info.json echo "::endgroup::" @@ -87,31 +97,33 @@ jobs: echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV" if [ "$is_latest" == "true" ]; then echo "DOCKER_TAG_TYPE=release-latest" >> "$GITHUB_ENV" + echo "IS_LATEST=1" >> "$GITHUB_ENV" else echo "DOCKER_TAG_TYPE=release" >> "$GITHUB_ENV" + echo "IS_LATEST=0" >> "$GITHUB_ENV" fi - name: Download All Release Artifacts - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Push Git Tag for the Release - if: ${{ ! inputs.only-repo }} + if: ${{ ! inputs.only-repo && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Push New Release Branch - if: ${{ inputs.type == 'new' && ! inputs.only-repo }} + if: ${{ inputs.type == 'new' && ! inputs.only-repo && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Bump CH Version and Update Contributors' List - if: ${{ ! inputs.only-repo }} + if: ${{ ! inputs.only-repo && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Bump Docker versions, Changelog, Security - if: ${{ inputs.type == 'patch' && ! inputs.only-repo }} + if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security" @@ -135,7 +147,7 @@ jobs: python3 ./utils/security-generator/generate_security.py > SECURITY.md git diff HEAD - name: Create ChangeLog PR - if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo }} + if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo && ! inputs.only-docker }} uses: peter-evans/create-pull-request@v6 with: author: "robot-clickhouse " @@ -153,65 +165,176 @@ jobs: ### Changelog category (leave one): - Not for changelog (changelog entry is not required) - name: Complete previous steps and Restore git state - if: ${{ inputs.type == 'patch' && ! inputs.only-repo }} + if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker }} shell: bash run: | git reset --hard HEAD git checkout "$GITHUB_REF_NAME" python3 ./tests/ci/create_release.py --set-progress-completed - name: Create GH Release - if: ${{ inputs.type == 'patch' && ! inputs.only-repo }} + if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker}} shell: bash run: | python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Export TGZ Packages - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Test TGZ Packages - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Export RPM Packages - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Test RPM Packages - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Export Debian Packages - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Test Debian Packages - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && ! inputs.only-docker }} shell: bash run: | python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run == true && '--dry-run' || '' }} - name: Docker clickhouse/clickhouse-server building - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && inputs.dry-run != true }} shell: bash run: | cd "./tests/ci" python3 ./create_release.py --set-progress-started --progress "docker server release" - export CHECK_NAME="Docker server image" - python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} + export DOCKER_IMAGE="clickhouse/clickhouse-server" + + # We must use docker file from the release commit + git checkout "${{ env.RELEASE_TAG }}" + python3 ./version_helper.py --export > /tmp/version.sh + . /tmp/version.sh + + if [[ $CLICKHOUSE_VERSION_STRING =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ClickHouse version: $CLICKHOUSE_VERSION_STRING" + else + echo "Invalid version string: $CLICKHOUSE_VERSION_STRING" + exit 1 + fi + CLICKHOUSE_VERSION_MINOR=${CLICKHOUSE_VERSION_STRING%.*} + CLICKHOUSE_VERSION_MAJOR=${CLICKHOUSE_VERSION_MINOR%.*} + + # Define build configurations + configs=( + "ubuntu:../../docker/server/Dockerfile.ubuntu" + "alpine:../../docker/server/Dockerfile.alpine" + ) + + for config in "${configs[@]}"; do + # Split the config into variant and Dockerfile path + variant=${config%%:*} + dockerfile=${config##*:} + + VERSION_SUFFIX=$([ "$variant" = "ubuntu" ] && echo "" || echo "-$variant") + LABEL_VERSION="${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}" + TAGS=( + "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}" + "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MINOR}${VERSION_SUFFIX}" + "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MAJOR}${VERSION_SUFFIX}" + ) + + if [ "$IS_LATEST" = "1" ]; then + TAGS+=("--tag=${DOCKER_IMAGE}:latest${VERSION_SUFFIX}") + fi + + echo "Following tags will be created: ${TAGS[*]}" + + # shellcheck disable=SC2086,SC2048 + docker buildx build \ + --platform=linux/amd64,linux/arm64 \ + --output=type=registry \ + --label=com.clickhouse.build.version="$LABEL_VERSION" \ + ${TAGS[*]} \ + --build-arg=VERSION="$CLICKHOUSE_VERSION_STRING" \ + --progress=plain \ + --file="$dockerfile" \ + ../../docker/server + done + + git checkout - python3 ./create_release.py --set-progress-completed - name: Docker clickhouse/clickhouse-keeper building - if: ${{ inputs.type == 'patch' }} + if: ${{ inputs.type == 'patch' && inputs.dry-run != true }} shell: bash run: | cd "./tests/ci" python3 ./create_release.py --set-progress-started --progress "docker keeper release" - export CHECK_NAME="Docker keeper image" - python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} + + export DOCKER_IMAGE="clickhouse/clickhouse-keeper" + + # We must use docker file from the release commit + git checkout "${{ env.RELEASE_TAG }}" + python3 ./version_helper.py --export > /tmp/version.sh + . /tmp/version.sh + + if [[ $CLICKHOUSE_VERSION_STRING =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ClickHouse version: $CLICKHOUSE_VERSION_STRING" + else + echo "Invalid version string: $CLICKHOUSE_VERSION_STRING" + exit 1 + fi + CLICKHOUSE_VERSION_MINOR=${CLICKHOUSE_VERSION_STRING%.*} + CLICKHOUSE_VERSION_MAJOR=${CLICKHOUSE_VERSION_MINOR%.*} + + # Define build configurations + configs=( + "ubuntu:../../docker/keeper/Dockerfile.ubuntu" + "alpine:../../docker/keeper/Dockerfile.alpine" + ) + + for config in "${configs[@]}"; do + # Split the config into variant and Dockerfile path + variant=${config%%:*} + dockerfile=${config##*:} + + VERSION_SUFFIX=$([ "$variant" = "ubuntu" ] && echo "" || echo "-$variant") + LABEL_VERSION="${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}" + TAGS=( + "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}" + "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MINOR}${VERSION_SUFFIX}" + "--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MAJOR}${VERSION_SUFFIX}" + ) + + if [ "$IS_LATEST" = "1" ]; then + TAGS+=("--tag=${DOCKER_IMAGE}:latest${VERSION_SUFFIX}") + fi + + echo "Following tags will be created: ${TAGS[*]}" + + # shellcheck disable=SC2086,SC2048 + docker buildx build \ + --platform=linux/amd64,linux/arm64 \ + --output=type=registry \ + --label=com.clickhoghuse.build.version="$LABEL_VERSION" \ + ${TAGS[*]} \ + --build-arg=VERSION="$CLICKHOUSE_VERSION_STRING" \ + --progress=plain \ + --file="$dockerfile" \ + ../../docker/keeper + done + + git checkout - python3 ./create_release.py --set-progress-completed + # check out back if previous steps failed + - name: Checkout back + if: ${{ ! cancelled() }} + shell: bash + run: | + git checkout ${{ github.ref }} - name: Update release info. Merge created PRs shell: bash run: |