thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

CI: Enable Stress Tests with praktika

Browse Source
This commit is contained in:
Max Kainov 2024-11-20 17:32:20 +01:00
parent 0f4990d2e7
commit 5378b49088
1 changed files with 146 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
.github/workflows/create_release.yml vendored
View File

@ -25,6 +25,11 @@ env:
required: false required: false
default: false default: false
type: boolean type: boolean
only-docker:
description: 'Run only docker builds (repo-recovery, tests)'
required: false
default: false
type: boolean
dry-run: dry-run:
description: 'Dry run' description: 'Dry run'
required: false required: false
@ -45,6 +50,11 @@ env:
required: false required: false
default: false default: false
type: boolean type: boolean
only-docker:
description: 'Run only docker builds (repo-recovery, tests)'
required: false
default: false
type: boolean
dry-run: dry-run:
description: 'Dry run' description: 'Dry run'
required: false required: false
@ -69,13 +79,13 @@ jobs:
- name: Prepare Release Info - name: Prepare Release Info
shell: bash shell: bash
run: | run: |
if [ ${{ inputs.only-repo }} == "true" ]; then if [ ${{ inputs.only-repo }} == "true" ] || [ ${{ inputs.only-docker }} == "true" ]; then
git tag -l ${{ inputs.ref }} || { echo "With only-repo option ref must be a valid release tag"; exit 1; } git tag -l ${{ inputs.ref }} || { echo "With only-repo/docker option ref must be a valid release tag"; exit 1; }
fi fi
python3 ./tests/ci/create_release.py --prepare-release-info \ python3 ./tests/ci/create_release.py --prepare-release-info \
--ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \ --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \
${{ inputs.dry-run == true && '--dry-run' || '' }} \ ${{ inputs.dry-run == true && '--dry-run' || '' }} \
${{ inputs.only-repo == true && '--skip-tag-check' || '' }} ${{ (inputs.only-repo == true || inputs.only-docker == true) && '--skip-tag-check' || '' }}
echo "::group::Release Info" echo "::group::Release Info"
python3 -m json.tool /tmp/release_info.json python3 -m json.tool /tmp/release_info.json
echo "::endgroup::" echo "::endgroup::"
@ -87,31 +97,33 @@ jobs:
echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV" echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
if [ "$is_latest" == "true" ]; then if [ "$is_latest" == "true" ]; then
echo "DOCKER_TAG_TYPE=release-latest" >> "$GITHUB_ENV" echo "DOCKER_TAG_TYPE=release-latest" >> "$GITHUB_ENV"
echo "IS_LATEST=1" >> "$GITHUB_ENV"
else else
echo "DOCKER_TAG_TYPE=release" >> "$GITHUB_ENV" echo "DOCKER_TAG_TYPE=release" >> "$GITHUB_ENV"
echo "IS_LATEST=0" >> "$GITHUB_ENV"
fi fi
- name: Download All Release Artifacts - name: Download All Release Artifacts
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Push Git Tag for the Release - name: Push Git Tag for the Release
if: ${{ ! inputs.only-repo }} if: ${{ ! inputs.only-repo && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Push New Release Branch - name: Push New Release Branch
if: ${{ inputs.type == 'new' && ! inputs.only-repo }} if: ${{ inputs.type == 'new' && ! inputs.only-repo && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Bump CH Version and Update Contributors' List - name: Bump CH Version and Update Contributors' List
if: ${{ ! inputs.only-repo }} if: ${{ ! inputs.only-repo && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Bump Docker versions, Changelog, Security - name: Bump Docker versions, Changelog, Security
if: ${{ inputs.type == 'patch' && ! inputs.only-repo }} if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security" python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
@ -135,7 +147,7 @@ jobs:
python3 ./utils/security-generator/generate_security.py > SECURITY.md python3 ./utils/security-generator/generate_security.py > SECURITY.md
git diff HEAD git diff HEAD
- name: Create ChangeLog PR - name: Create ChangeLog PR
if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo }} if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo && ! inputs.only-docker }}
uses: peter-evans/create-pull-request@v6 uses: peter-evans/create-pull-request@v6
with: with:
author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>" author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
@ -153,65 +165,176 @@ jobs:
### Changelog category (leave one): ### Changelog category (leave one):
- Not for changelog (changelog entry is not required) - Not for changelog (changelog entry is not required)
- name: Complete previous steps and Restore git state - name: Complete previous steps and Restore git state
if: ${{ inputs.type == 'patch' && ! inputs.only-repo }} if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
git reset --hard HEAD git reset --hard HEAD
git checkout "$GITHUB_REF_NAME" git checkout "$GITHUB_REF_NAME"
python3 ./tests/ci/create_release.py --set-progress-completed python3 ./tests/ci/create_release.py --set-progress-completed
- name: Create GH Release - name: Create GH Release
if: ${{ inputs.type == 'patch' && ! inputs.only-repo }} if: ${{ inputs.type == 'patch' && ! inputs.only-repo && ! inputs.only-docker}}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Export TGZ Packages - name: Export TGZ Packages
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Test TGZ Packages - name: Test TGZ Packages
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Export RPM Packages - name: Export RPM Packages
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Test RPM Packages - name: Test RPM Packages
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Export Debian Packages - name: Export Debian Packages
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Test Debian Packages - name: Test Debian Packages
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && ! inputs.only-docker }}
shell: bash shell: bash
run: | run: |
python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run == true && '--dry-run' || '' }} python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
- name: Docker clickhouse/clickhouse-server building - name: Docker clickhouse/clickhouse-server building
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && inputs.dry-run != true }}
shell: bash shell: bash
run: | run: |
cd "./tests/ci" cd "./tests/ci"
python3 ./create_release.py --set-progress-started --progress "docker server release" python3 ./create_release.py --set-progress-started --progress "docker server release"
export CHECK_NAME="Docker server image" export DOCKER_IMAGE="clickhouse/clickhouse-server"
python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
# We must use docker file from the release commit
git checkout "${{ env.RELEASE_TAG }}"
python3 ./version_helper.py --export > /tmp/version.sh
. /tmp/version.sh
if [[ $CLICKHOUSE_VERSION_STRING =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "ClickHouse version: $CLICKHOUSE_VERSION_STRING"
else
echo "Invalid version string: $CLICKHOUSE_VERSION_STRING"
exit 1
fi
CLICKHOUSE_VERSION_MINOR=${CLICKHOUSE_VERSION_STRING%.*}
CLICKHOUSE_VERSION_MAJOR=${CLICKHOUSE_VERSION_MINOR%.*}
# Define build configurations
configs=(
"ubuntu:../../docker/server/Dockerfile.ubuntu"
"alpine:../../docker/server/Dockerfile.alpine"
)
for config in "${configs[@]}"; do
# Split the config into variant and Dockerfile path
variant=${config%%:*}
dockerfile=${config##*:}
VERSION_SUFFIX=$([ "$variant" = "ubuntu" ] && echo "" || echo "-$variant")
LABEL_VERSION="${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}"
TAGS=(
"--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}"
"--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MINOR}${VERSION_SUFFIX}"
"--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MAJOR}${VERSION_SUFFIX}"
)
if [ "$IS_LATEST" = "1" ]; then
TAGS+=("--tag=${DOCKER_IMAGE}:latest${VERSION_SUFFIX}")
fi
echo "Following tags will be created: ${TAGS[*]}"
# shellcheck disable=SC2086,SC2048
docker buildx build \
--platform=linux/amd64,linux/arm64 \
--output=type=registry \
--label=com.clickhouse.build.version="$LABEL_VERSION" \
${TAGS[*]} \
--build-arg=VERSION="$CLICKHOUSE_VERSION_STRING" \
--progress=plain \
--file="$dockerfile" \
../../docker/server
done
git checkout -
python3 ./create_release.py --set-progress-completed python3 ./create_release.py --set-progress-completed
- name: Docker clickhouse/clickhouse-keeper building - name: Docker clickhouse/clickhouse-keeper building
if: ${{ inputs.type == 'patch' }} if: ${{ inputs.type == 'patch' && inputs.dry-run != true }}
shell: bash shell: bash
run: | run: |
cd "./tests/ci" cd "./tests/ci"
python3 ./create_release.py --set-progress-started --progress "docker keeper release" python3 ./create_release.py --set-progress-started --progress "docker keeper release"
export CHECK_NAME="Docker keeper image"
python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }} export DOCKER_IMAGE="clickhouse/clickhouse-keeper"
# We must use docker file from the release commit
git checkout "${{ env.RELEASE_TAG }}"
python3 ./version_helper.py --export > /tmp/version.sh
. /tmp/version.sh
if [[ $CLICKHOUSE_VERSION_STRING =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "ClickHouse version: $CLICKHOUSE_VERSION_STRING"
else
echo "Invalid version string: $CLICKHOUSE_VERSION_STRING"
exit 1
fi
CLICKHOUSE_VERSION_MINOR=${CLICKHOUSE_VERSION_STRING%.*}
CLICKHOUSE_VERSION_MAJOR=${CLICKHOUSE_VERSION_MINOR%.*}
# Define build configurations
configs=(
"ubuntu:../../docker/keeper/Dockerfile.ubuntu"
"alpine:../../docker/keeper/Dockerfile.alpine"
)
for config in "${configs[@]}"; do
# Split the config into variant and Dockerfile path
variant=${config%%:*}
dockerfile=${config##*:}
VERSION_SUFFIX=$([ "$variant" = "ubuntu" ] && echo "" || echo "-$variant")
LABEL_VERSION="${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}"
TAGS=(
"--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_STRING}${VERSION_SUFFIX}"
"--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MINOR}${VERSION_SUFFIX}"
"--tag=${DOCKER_IMAGE}:${CLICKHOUSE_VERSION_MAJOR}${VERSION_SUFFIX}"
)
if [ "$IS_LATEST" = "1" ]; then
TAGS+=("--tag=${DOCKER_IMAGE}:latest${VERSION_SUFFIX}")
fi
echo "Following tags will be created: ${TAGS[*]}"
# shellcheck disable=SC2086,SC2048
docker buildx build \
--platform=linux/amd64,linux/arm64 \
--output=type=registry \
--label=com.clickhoghuse.build.version="$LABEL_VERSION" \
${TAGS[*]} \
--build-arg=VERSION="$CLICKHOUSE_VERSION_STRING" \
--progress=plain \
--file="$dockerfile" \
../../docker/keeper
done
git checkout -
python3 ./create_release.py --set-progress-completed python3 ./create_release.py --set-progress-completed
# check out back if previous steps failed
- name: Checkout back
if: ${{ ! cancelled() }}
shell: bash
run: |
git checkout ${{ github.ref }}
- name: Update release info. Merge created PRs - name: Update release info. Merge created PRs
shell: bash shell: bash
run: | run: |