thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-30 11:32:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

add compatibility check

Browse Source
This commit is contained in:
Max Kainov 2024-11-25 16:50:32 +01:00
parent 3bb6b579be
commit 978b423b5b
10 changed files with 84 additions and 924 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

901
.github/workflows/pr.yaml vendored
View File

@ -1,901 +0,0 @@
# generated by praktika
name: PR
on:
pull_request:
branches: ['master']
# Cancel the previous wf run in PRs.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
# Force the stdout and stderr streams to be unbuffered
PYTHONUNBUFFERED: 1
GH_TOKEN: ${{ github.token }}
# Allow updating GH commit statuses and PR comments to post an actual job reports link
permissions: write-all
jobs:
config_workflow:
runs-on: [ci_services]
needs: []
name: "Config Workflow"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Config Workflow''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Config Workflow''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
docker_builds:
runs-on: [ci_services_ebs]
needs: [config_workflow]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIEJ1aWxkcw==') }}
name: "Docker Builds"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Docker Builds''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Docker Builds''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
style_check:
runs-on: [ci_services]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3R5bGUgQ2hlY2s=') }}
name: "Style Check"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Style Check''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Style Check''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
fast_test:
runs-on: [builder]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RmFzdCB0ZXN0') }}
name: "Fast test"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Fast test''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Fast test''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_amd_debug:
runs-on: [builder]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9kZWJ1Zyk=') }}
name: "Build (amd_debug)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (amd_debug)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (amd_debug)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_amd_release:
runs-on: [builder]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
name: "Build (amd_release)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (amd_release)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (amd_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_amd_asan:
runs-on: [builder]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9hc2FuKQ==') }}
name: "Build (amd_asan)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (amd_asan)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (amd_asan)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_amd_tsan:
runs-on: [builder]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF90c2FuKQ==') }}
name: "Build (amd_tsan)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (amd_tsan)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (amd_tsan)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_amd_msan:
runs-on: [builder]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9tc2FuKQ==') }}
name: "Build (amd_msan)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (amd_msan)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (amd_msan)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_amd_ubsan:
runs-on: [builder]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF91YnNhbik=') }}
name: "Build (amd_ubsan)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (amd_ubsan)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (amd_ubsan)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_amd_binary:
runs-on: [builder]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9iaW5hcnkp') }}
name: "Build (amd_binary)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (amd_binary)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (amd_binary)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_arm_release:
runs-on: [builder-aarch64]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
name: "Build (arm_release)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (arm_release)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (arm_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
build_arm_asan:
runs-on: [builder-aarch64]
needs: [config_workflow, docker_builds]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9hc2FuKQ==') }}
name: "Build (arm_asan)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Build (arm_asan)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Build (arm_asan)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
stateless_tests_amd_debugparallel:
runs-on: [builder]
needs: [config_workflow, docker_builds, build_amd_debug]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfZGVidWcscGFyYWxsZWwp') }}
name: "Stateless tests (amd_debug,parallel)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Stateless tests (amd_debug,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Stateless tests (amd_debug,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
stateless_tests_amd_debugnon_parallel:
runs-on: [func-tester]
needs: [config_workflow, docker_builds, build_amd_debug]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfZGVidWcsbm9uLXBhcmFsbGVsKQ==') }}
name: "Stateless tests (amd_debug,non-parallel)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Stateless tests (amd_debug,non-parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Stateless tests (amd_debug,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
stateless_tests_amd_releaseparallel:
runs-on: [builder]
needs: [config_workflow, docker_builds, build_amd_release]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfcmVsZWFzZSxwYXJhbGxlbCk=') }}
name: "Stateless tests (amd_release,parallel)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Stateless tests (amd_release,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Stateless tests (amd_release,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
stateless_tests_amd_releasenon_parallel:
runs-on: [func-tester]
needs: [config_workflow, docker_builds, build_amd_release]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfcmVsZWFzZSxub24tcGFyYWxsZWwp') }}
name: "Stateless tests (amd_release,non-parallel)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Stateless tests (amd_release,non-parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Stateless tests (amd_release,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
stateless_tests_arm_asanparallel:
runs-on: [builder-aarch64]
needs: [config_workflow, docker_builds, build_arm_asan]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbixwYXJhbGxlbCk=') }}
name: "Stateless tests (arm_asan,parallel)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Stateless tests (arm_asan,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Stateless tests (arm_asan,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
stateless_tests_arm_asannon_parallel:
runs-on: [func-tester-aarch64]
needs: [config_workflow, docker_builds, build_arm_asan]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbixub24tcGFyYWxsZWwp') }}
name: "Stateless tests (arm_asan,non-parallel)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Stateless tests (arm_asan,non-parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Stateless tests (arm_asan,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
stateful_tests_amd_releaseparallel:
runs-on: [builder]
needs: [config_workflow, docker_builds, build_amd_debug]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVmdWwgdGVzdHMgKGFtZF9yZWxlYXNlLHBhcmFsbGVsKQ==') }}
name: "Stateful tests (amd_release,parallel)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Stateful tests (amd_release,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Stateful tests (amd_release,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
stress_tests_arm_release:
runs-on: [func-tester-aarch64]
needs: [config_workflow, docker_builds, build_arm_release]
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RyZXNzIHRlc3RzIChhcm1fcmVsZWFzZSk=') }}
name: "Stress tests (arm_release)"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Stress tests (arm_release)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Stress tests (arm_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi
finish_workflow:
runs-on: [ci_services]
needs: [config_workflow, docker_builds, style_check, fast_test, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan, stateless_tests_amd_debugparallel, stateless_tests_amd_debugnon_parallel, stateless_tests_amd_releaseparallel, stateless_tests_amd_releasenon_parallel, stateless_tests_arm_asanparallel, stateless_tests_arm_asannon_parallel, stateful_tests_amd_releaseparallel, stress_tests_arm_release]
if: ${{ !cancelled() }}
name: "Finish Workflow"
outputs:
data: ${{ steps.run.outputs.DATA }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Prepare env script
run: |
cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
export PYTHONPATH=./ci:.
cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
${{ needs.config_workflow.outputs.data }}
EOF
cat > /tmp/praktika/workflow_status.json << 'EOF'
${{ toJson(needs) }}
EOF
ENV_SETUP_SCRIPT_EOF
rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
- name: Run
id: run
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run '''Finish Workflow''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run '''Finish Workflow''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
fi

5
ci/docker/compatibility/centos/Dockerfile Normal file
View File

@ -0,0 +1,5 @@
# docker build -t clickhouse/test-old-centos .
FROM centos:5
CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \
sleep 5 && /clickhouse client --query \"select 'OK'\" 2> /var/log/clickhouse-server/clientstderr.log || echo 'FAIL'"

5
ci/docker/compatibility/ubuntu/Dockerfile Normal file
View File

@ -0,0 +1,5 @@
# docker build -t clickhouse/test-old-ubuntu .
FROM ubuntu:12.04
CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \
sleep 5 && /clickhouse client --query \"select 'OK'\" 2> /var/log/clickhouse-server/clientstderr.log || echo 'FAIL'"

7
ci/jobs/build_clickhouse.py
View File

@ -1,4 +1,5 @@
import argparse
import os
from praktika.result import Result
from praktika.settings import Settings
@ -6,6 +7,7 @@ from praktika.utils import MetaClasses, Shell, Utils
from ci.jobs.scripts.clickhouse_version import CHVersion
from ci.workflows.defs import CIFiles, ToolSet
from ci.workflows.pull_request import S3_BUILDS_BUCKET
class JobStages(metaclass=MetaClasses.WithIter):
@ -49,6 +51,11 @@ CMAKE_CMD = """cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA \
def main():
args = parse_args()
# # for sccache
# os.environ["SCCACHE_BUCKET"] = S3_BUILDS_BUCKET
# os.environ["SCCACHE_S3_KEY_PREFIX"] = "ccache/sccache"
# TODO: check with SCCACHE_LOG=debug SCCACHE_NO_DAEMON=1
stop_watch = Utils.Stopwatch()
stages = list(JobStages)

12
ci/praktika/native_jobs.py
View File

@ -144,7 +144,7 @@ def _config_workflow(workflow: Workflow.Config, job_name):
f"git diff-index HEAD -- {Settings.WORKFLOW_PATH_PREFIX}"
)
info = ""
status = Result.Status.SUCCESS
status = Result.Status.FAILED
if exit_code != 0:
info = f"workspace has uncommitted files unexpectedly [{output}]"
status = Result.Status.ERROR
@ -154,10 +154,14 @@ def _config_workflow(workflow: Workflow.Config, job_name):
exit_code, output, err = Shell.get_res_stdout_stderr(
f"git diff-index HEAD -- {Settings.WORKFLOW_PATH_PREFIX}"
)
if exit_code != 0:
info = f"workspace has outdated workflows [{output}] - regenerate with [python -m praktika --generate]"
status = Result.Status.ERROR
if output:
info = f"workflows are outdated: [{output}]"
status = Result.Status.FAILED
print("ERROR: ", info)
elif exit_code == 0 and not err:
status = Result.Status.SUCCESS
else:
print(f"ERROR: exit code [{exit_code}], err [{err}]")
return (
Result(

10
ci/praktika/runner.py
View File

@ -61,8 +61,6 @@ class Runner:
docker, workflow.dockers
)
# work around for old clickhouse jobs
os.environ["DOCKER_TAG"] = json.dumps(workflow_config.digest_dockers)
workflow_config.dump()
Result.generate_pending(job.name).dump()
@ -149,6 +147,14 @@ class Runner:
env.JOB_NAME = job.name
env.dump()
# work around for old clickhouse jobs
try:
os.environ["DOCKER_TAG"] = json.dumps(
RunConfig.from_fs(workflow.name).digest_dockers
)
except Exception as e:
print(f"WARNING: Failed to set DOCKER_TAG, ex [{e}]")
if param:
if not isinstance(param, str):
Utils.raise_with_error(

49
ci/workflows/defs.py
View File

@ -9,6 +9,9 @@ class RunnerLabels:
BUILDER_ARM = "builder-aarch64"
FUNC_TESTER_AMD = "func-tester"
FUNC_TESTER_ARM = "func-tester-aarch64"
STYLE_CHECK_AMD = "style-checker"
STYLE_CHECK_ARM = "style-checker-aarch64"
CI_SERVICES = "ci_services"
class CIFiles:
@ -52,18 +55,18 @@ DOCKERS = [
# platforms=Docker.Platforms.arm_amd,
# depends_on=[],
# ),
# Docker.Config(
# name="clickhouse/test-old-centos",
# path="./ci/docker/test/compatibility/centos",
# platforms=Docker.Platforms.arm_amd,
# depends_on=[],
# ),
# Docker.Config(
# name="clickhouse/test-old-ubuntu",
# path="./ci/docker/test/compatibility/ubuntu",
# platforms=Docker.Platforms.arm_amd,
# depends_on=[],
# ),
Docker.Config(
name="clickhouse/test-old-centos",
path="./ci/docker/compatibility/centos",
platforms=Docker.Platforms.arm_amd,
depends_on=[],
),
Docker.Config(
name="clickhouse/test-old-ubuntu",
path="./ci/docker/compatibility/ubuntu",
platforms=Docker.Platforms.arm_amd,
depends_on=[],
),
# Docker.Config(
# name="clickhouse/test-util",
# path="./ci/docker/test/util",
@ -247,6 +250,7 @@ class JobNames:
STATEFUL = "Stateful tests"
STRESS = "Stress tests"
PERFORMANCE = "Performance tests"
COMPATIBILITY = "Compatibility check"
class ToolSet:
@ -576,7 +580,7 @@ class Jobs:
runs_on=[RunnerLabels.FUNC_TESTER_ARM],
command="./ci/jobs/scripts/performance_test.sh",
run_in_docker="clickhouse/stateless-test",
requires=[ArtifactNames.CH_ARM_RELEASE]
requires=[ArtifactNames.CH_ARM_RELEASE],
# digest_config=Job.CacheDigestConfig(
# include_paths=[
# "./ci/jobs/fast_test.py",
@ -585,3 +589,22 @@ class Jobs:
# ],
# ),
)
compatibility_test_jobs = Job.Config(
name=JobNames.COMPATIBILITY,
runs_on=["#from param"],
command="python3 ./tests/ci/compatibility_check.py --check-name {PARAMETER}",
digest_config=Job.CacheDigestConfig(
include_paths=[
"./tests/ci/compatibility_check.py",
"./docker/test/compatibility",
],
),
).parametrize(
parameter=["amd_release", "arm_release"],
runs_on=[
[RunnerLabels.STYLE_CHECK_AMD],
[RunnerLabels.STYLE_CHECK_ARM],
],
requires=[[ArtifactNames.DEB_AMD_RELEASE], [ArtifactNames.DEB_ARM_RELEASE]],
)

3
ci/workflows/pull_request.py
View File

@ -2,6 +2,8 @@ from praktika import Workflow
from ci.workflows.defs import ARTIFACTS, BASE_BRANCH, DOCKERS, SECRETS, Jobs
S3_BUILDS_BUCKET = "clickhouse-builds"
workflow = Workflow.Config(
name="PR",
event=Workflow.Event.PULL_REQUEST,
@ -14,6 +16,7 @@ workflow = Workflow.Config(
*Jobs.stateful_tests_jobs,
*Jobs.stress_test_jobs,
Jobs.performance_test_job,
*Jobs.compatibility_test_jobs,
],
artifacts=ARTIFACTS,
dockers=DOCKERS,

14
tests/ci/compatibility_check.py
View File

@ -11,6 +11,7 @@ from typing import List, Tuple
from pip._vendor.packaging.version import Version
from build_download_helper import download_builds_filter
from ci_utils import Shell
from docker_images_helper import DockerImage, get_docker_image, pull_image
from env_helper import REPORT_PATH, TEMP_PATH
from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults
@ -151,7 +152,14 @@ def main():
"clickhouse-common-static_" in url or "clickhouse-server_" in url
)
download_builds_filter(check_name, reports_path, packages_path, url_filter)
if check_name in ("amd_release", "amd_debug", "arm_release"):
# this is praktika based CI
print("Copy input *.deb artifacts")
assert Shell.check(
f"cp /tmp/praktika/input/*.deb {packages_path}", verbose=True
)
else:
download_builds_filter(check_name, reports_path, packages_path, url_filter)
for package in packages_path.iterdir():
if package.suffix == ".deb":
@ -196,9 +204,9 @@ def main():
# See https://sourceware.org/glibc/wiki/Glibc%20Timeline
max_glibc_version = ""
if "amd64" in check_name or "release" in check_name:
if "amd" in check_name or "(release)" in check_name:
max_glibc_version = "2.4"
elif "aarch64" in check_name:
elif "aarch" in check_name or "arm" in check_name:
max_glibc_version = "2.18" # because of build with newer sysroot?
else:
raise RuntimeError("Can't determine max glibc version")

2
tests/ci/stress_check.py
View File

@ -10,6 +10,7 @@ from pathlib import Path
from typing import List, Tuple
from build_download_helper import download_all_deb_packages
from ci_utils import Shell
from clickhouse_helper import CiLogsCredentials
from docker_images_helper import DockerImage, get_docker_image, pull_image
from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
@ -18,7 +19,6 @@ from pr_info import PRInfo
from report import ERROR, JobReport, TestResults, read_test_results
from stopwatch import Stopwatch
from tee_popen import TeePopen
from ci_utils import Shell
class SensitiveFormatter(logging.Formatter):