From 941c0f0c3b5794571361e5fb27780d656a106786 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 13 Nov 2024 13:24:32 +0100 Subject: [PATCH] add stateful tests --- .github/workflows/pr.yaml | 621 ------------------ .github/workflows/pull_request.yml | 212 ++++++ ci/docker/stateful-test/Dockerfile | 14 + ci/docker/stateless-test/Dockerfile | 7 +- ci/jobs/functional_stateful_tests.py | 170 +++++ ci/jobs/functional_stateless_tests.py | 11 +- ci/jobs/scripts/clickhouse_proc.py | 12 +- .../scripts/functional_tests/setup_minio.sh | 162 +++++ ci/praktika/json.html | 3 +- ci/settings/definitions.py | 13 +- ci/workflows/pull_request.py | 31 +- tests/config/config.d/ssl_certs.xml | 4 +- tests/config/install.sh | 11 +- tests/docker_scripts/setup_minio.sh | 32 +- 14 files changed, 634 insertions(+), 669 deletions(-) delete mode 100644 .github/workflows/pr.yaml create mode 100644 .github/workflows/pull_request.yml create mode 100644 ci/docker/stateful-test/Dockerfile create mode 100644 ci/jobs/functional_stateful_tests.py create mode 100755 ci/jobs/scripts/functional_tests/setup_minio.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml deleted file mode 100644 index a70ff0cfe23..00000000000 --- a/.github/workflows/pr.yaml +++ /dev/null @@ -1,621 +0,0 @@ -# generated by praktika - -name: PR - -on: - pull_request: - branches: ['master'] - -# Cancel the previous wf run in PRs. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 - GH_TOKEN: ${{ github.token }} - -# Allow updating GH commit statuses and PR comments to post an actual job reports link -permissions: write-all - -jobs: - - config_workflow: - runs-on: [ci_services] - needs: [] - name: "Config Workflow" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Config Workflow''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Config Workflow''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - docker_builds: - runs-on: [ci_services_ebs] - needs: [config_workflow] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIEJ1aWxkcw==') }} - name: "Docker Builds" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Docker Builds''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Docker Builds''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - style_check: - runs-on: [ci_services] - needs: [config_workflow, docker_builds] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3R5bGUgQ2hlY2s=') }} - name: "Style Check" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Style Check''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Style Check''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - fast_test: - runs-on: [builder] - needs: [config_workflow, docker_builds] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RmFzdCB0ZXN0') }} - name: "Fast test" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Fast test''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Fast test''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - build_amd_debug: - runs-on: [builder] - needs: [config_workflow, docker_builds, fast_test] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9kZWJ1Zyk=') }} - name: "Build (amd_debug)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Build (amd_debug)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Build (amd_debug)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - build_amd_release: - runs-on: [builder] - needs: [config_workflow, docker_builds, fast_test] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} - name: "Build (amd_release)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Build (amd_release)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Build (amd_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - build_arm_release: - runs-on: [builder-aarch64] - needs: [config_workflow, docker_builds, fast_test] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} - name: "Build (arm_release)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Build (arm_release)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Build (arm_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - build_arm_asan: - runs-on: [builder-aarch64] - needs: [config_workflow, docker_builds, fast_test] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9hc2FuKQ==') }} - name: "Build (arm_asan)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Build (arm_asan)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Build (arm_asan)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - stateless_tests_amd_debugparallel: - runs-on: [builder] - needs: [config_workflow, docker_builds, build_amd_debug] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfZGVidWcscGFyYWxsZWwp') }} - name: "Stateless tests (amd_debug,parallel)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Stateless tests (amd_debug,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Stateless tests (amd_debug,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - stateless_tests_amd_debugnon_parallel: - runs-on: [func-tester] - needs: [config_workflow, docker_builds, build_amd_debug] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfZGVidWcsbm9uLXBhcmFsbGVsKQ==') }} - name: "Stateless tests (amd_debug,non-parallel)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Stateless tests (amd_debug,non-parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Stateless tests (amd_debug,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - stateless_tests_amd_releaseparallel: - runs-on: [builder] - needs: [config_workflow, docker_builds, build_amd_release] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfcmVsZWFzZSxwYXJhbGxlbCk=') }} - name: "Stateless tests (amd_release,parallel)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Stateless tests (amd_release,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Stateless tests (amd_release,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - stateless_tests_amd_releasenon_parallel: - runs-on: [func-tester] - needs: [config_workflow, docker_builds, build_amd_release] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfcmVsZWFzZSxub24tcGFyYWxsZWwp') }} - name: "Stateless tests (amd_release,non-parallel)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Stateless tests (amd_release,non-parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Stateless tests (amd_release,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - stateless_tests_arm_asanparallel: - runs-on: [builder-aarch64] - needs: [config_workflow, docker_builds, build_arm_asan] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbixwYXJhbGxlbCk=') }} - name: "Stateless tests (arm_asan,parallel)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Stateless tests (arm_asan,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Stateless tests (arm_asan,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - stateless_tests_arm_asannon_parallel: - runs-on: [func-tester-aarch64] - needs: [config_workflow, docker_builds, build_arm_asan] - if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbixub24tcGFyYWxsZWwp') }} - name: "Stateless tests (arm_asan,non-parallel)" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Stateless tests (arm_asan,non-parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Stateless tests (arm_asan,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi - - finish_workflow: - runs-on: [ci_services] - needs: [config_workflow, docker_builds, style_check, fast_test, build_amd_debug, build_amd_release, build_arm_release, build_arm_asan, stateless_tests_amd_debugparallel, stateless_tests_amd_debugnon_parallel, stateless_tests_amd_releaseparallel, stateless_tests_amd_releasenon_parallel, stateless_tests_arm_asanparallel, stateless_tests_arm_asannon_parallel] - if: ${{ !cancelled() }} - name: "Finish Workflow" - outputs: - data: ${{ steps.run.outputs.DATA }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Prepare env script - run: | - cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:. - - cat > /tmp/praktika/workflow_config_pr.json << 'EOF' - ${{ needs.config_workflow.outputs.data }} - EOF - cat > /tmp/praktika/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika - mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output - - - name: Run - id: run - run: | - . /tmp/praktika_setup_env.sh - set -o pipefail - if command -v ts &> /dev/null; then - python3 -m praktika run --job '''Finish Workflow''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log - else - python3 -m praktika run --job '''Finish Workflow''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log - fi diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 00000000000..e4eb44b2774 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,212 @@ +# yamllint disable rule:comments-indentation +name: PullRequestCI + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + pull_request: + types: + - synchronize + - reopened + - opened + branches: + - master + +# Cancel the previous wf run in PRs. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + RunConfig: + runs-on: [self-hosted, style-checker-aarch64] + outputs: + data: ${{ steps.runconfig.outputs.CI_DATA }} + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true # to ensure correct digests + fetch-depth: 0 # to get a version + filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug + - name: Set pending Sync status + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --set-pending-status + - name: Labels check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 run_check.py + - name: Python unit tests + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + echo "Testing the main ci directory" + python3 -m unittest discover -s . -p 'test_*.py' + - name: PrepareRunConfig + id: runconfig + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json + + echo "::group::CI configuration" + python3 -m json.tool ${{ runner.temp }}/ci_run_data.json + echo "::endgroup::" + + { + echo 'CI_DATA<> "$GITHUB_OUTPUT" + - name: Re-create GH statuses for skipped jobs if any + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses + BuildDockers: + needs: [RunConfig] + if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }} + uses: ./.github/workflows/docker_test_images.yml + with: + data: ${{ needs.RunConfig.outputs.data }} + StyleCheck: + needs: [RunConfig, BuildDockers] + if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Style check + runner_type: style-checker-aarch64 + run_command: | + python3 style_check.py + data: ${{ needs.RunConfig.outputs.data }} + secrets: + secret_envs: | + ROBOT_CLICKHOUSE_SSH_KEY< "$WORKFLOW_RESULT_FILE" << 'EOF' + ${{ toJson(needs) }} + EOF + python3 merge_pr.py --set-ci-status + - name: Check Workflow results + uses: ./.github/actions/check_workflow + with: + needs: ${{ toJson(needs) }} + + ################################# Stage Final ################################# + # + FinishCheck: + if: ${{ !failure() && !cancelled() }} + needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2_ww, Tests_2] + runs-on: [self-hosted, style-checker-aarch64] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + filter: tree:0 + - name: Finish label + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }} + +############################################################################################# +###################################### JEPSEN TESTS ######################################### +############################################################################################# + # This is special test NOT INCLUDED in FinishCheck + # When it's skipped, all dependent tasks will be skipped too. + # DO NOT add it there + Jepsen: + # we need concurrency as the job uses dedicated instances in the cloud + concurrency: + group: jepsen + if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }} + needs: [RunConfig, Builds_1] + uses: ./.github/workflows/reusable_test.yml + with: + test_name: ClickHouse Keeper Jepsen + runner_type: style-checker-aarch64 + data: ${{ needs.RunConfig.outputs.data }} diff --git a/ci/docker/stateful-test/Dockerfile b/ci/docker/stateful-test/Dockerfile new file mode 100644 index 00000000000..e21aec4a48f --- /dev/null +++ b/ci/docker/stateful-test/Dockerfile @@ -0,0 +1,14 @@ +ARG FROM_TAG=latest +FROM clickhouse/stateless-test:$FROM_TAG + +USER root + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + nodejs \ + npm \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \ + +USER clickhouse diff --git a/ci/docker/stateless-test/Dockerfile b/ci/docker/stateless-test/Dockerfile index cd02fc27f8e..dcfaa5f6267 100644 --- a/ci/docker/stateless-test/Dockerfile +++ b/ci/docker/stateless-test/Dockerfile @@ -8,10 +8,10 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz" -RUN mkdir /etc/clickhouse-server /etc/clickhouse-keeper /etc/clickhouse-server && chmod 777 /etc/clickhouse-* +RUN mkdir /etc/clickhouse-server /etc/clickhouse-keeper /etc/clickhouse-client && chmod 777 /etc/clickhouse-* \ + && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server && chmod 777 /var/log/clickhouse-server /var/lib/clickhouse RUN addgroup --gid 1001 clickhouse && adduser --uid 1001 --gid 1001 --disabled-password clickhouse -USER clickhouse # moreutils - provides ts fo FT # expect, bzip2 - requried by FT @@ -57,6 +57,7 @@ RUN apt-get update -y \ p7zip-full \ curl \ wget \ + xz-utils \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* @@ -112,3 +113,5 @@ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoo RUN npm install -g azurite@3.30.0 \ && npm install -g tslib && npm install -g node + +USER clickhouse diff --git a/ci/jobs/functional_stateful_tests.py b/ci/jobs/functional_stateful_tests.py new file mode 100644 index 00000000000..0faba4383ec --- /dev/null +++ b/ci/jobs/functional_stateful_tests.py @@ -0,0 +1,170 @@ +import argparse +import os +import time +from pathlib import Path + +from praktika.result import Result +from praktika.settings import Settings +from praktika.utils import MetaClasses, Shell, Utils + +from ci.jobs.scripts.clickhouse_proc import ClickHouseProc +from ci.jobs.scripts.functional_tests_results import FTResultsProcessor + + +class JobStages(metaclass=MetaClasses.WithIter): + INSTALL_CLICKHOUSE = "install" + START = "start" + TEST = "test" + + +def parse_args(): + parser = argparse.ArgumentParser(description="ClickHouse Build Job") + parser.add_argument( + "--ch-path", help="Path to clickhouse binary", default=f"{Settings.INPUT_DIR}" + ) + parser.add_argument( + "--test-options", + help="Comma separated option(s): parallel|non-parallel|BATCH_NUM/BTATCH_TOT|..", + default="", + ) + parser.add_argument("--param", help="Optional job start stage", default=None) + parser.add_argument("--test", help="Optional test name pattern", default="") + return parser.parse_args() + + +def run_test( + no_parallel: bool, no_sequiential: bool, batch_num: int, batch_total: int, test="" +): + test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt" + + test_command = f"clickhouse-test --jobs 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless \ + --hung-check --print-time \ + --capture-client-stacktrace --queries ./tests/queries -- '{test}' \ + | ts '%Y-%m-%d %H:%M:%S' | tee -a \"{test_output_file}\"" + if Path(test_output_file).exists(): + Path(test_output_file).unlink() + Shell.run(test_command, verbose=True) + + +def main(): + + args = parse_args() + test_options = args.test_options.split(",") + no_parallel = "non-parallel" in test_options + no_sequential = "parallel" in test_options + batch_num, total_batches = 0, 0 + for to in test_options: + if "/" in to: + batch_num, total_batches = map(int, to.split("/")) + + # os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output( + # f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value", + # verbose=True, + # strict=True + # ) + + ch_path = args.ch_path + assert Path( + ch_path + "/clickhouse" + ).is_file(), f"clickhouse binary not found under [{ch_path}]" + + stop_watch = Utils.Stopwatch() + + stages = list(JobStages) + + logs_to_attach = [] + + stage = args.param or JobStages.INSTALL_CLICKHOUSE + if stage: + assert stage in JobStages, f"--param must be one of [{list(JobStages)}]" + print(f"Job will start from stage [{stage}]") + while stage in stages: + stages.pop(0) + stages.insert(0, stage) + + res = True + results = [] + + Utils.add_to_PATH(f"{ch_path}:tests") + + if res and JobStages.INSTALL_CLICKHOUSE in stages: + commands = [ + f"rm -rf /tmp/praktika/var/log/clickhouse-server/clickhouse-server.*", + f"chmod +x {ch_path}/clickhouse", + f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-server", + f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client", + f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor", + f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local", + f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server", + f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/", + f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage", + # clickhouse benchmark segfaults with --config-path, so provide client config by its default location + f"cp {Settings.TEMP_DIR}/etc/clickhouse-client/* /etc/clickhouse-client/", + # update_path_ch_config, + # f"sed -i 's|>/var/|>{Settings.TEMP_DIR}/var/|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.xml", + # f"sed -i 's|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/ssl_certs.xml", + f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done", + f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done", + f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|local_disk|{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done", + f"clickhouse-server --version", + ] + results.append( + Result.create_from_command_execution( + name="Install ClickHouse", command=commands, with_log=True + ) + ) + res = results[-1].is_ok() + + CH = ClickHouseProc() + if res and JobStages.START in stages: + stop_watch_ = Utils.Stopwatch() + step_name = "Start ClickHouse Server" + print(step_name) + minio_log = "/tmp/praktika/output/minio.log" + res = res and CH.start_minio(test_type="stateful", log_file_path=minio_log) + logs_to_attach += [minio_log] + time.sleep(10) + Shell.check("ps -ef | grep minio", verbose=True) + res = res and Shell.check( + "aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True + ) + res = res and CH.start() + res = res and CH.wait_ready() + if res: + print("ch started") + logs_to_attach += [ + "/tmp/praktika/var/log/clickhouse-server/clickhouse-server.log", + "/tmp/praktika/var/log/clickhouse-server/clickhouse-server.err.log", + ] + results.append( + Result.create_from( + name=step_name, + status=res, + stopwatch=stop_watch_, + ) + ) + res = results[-1].is_ok() + + if res and JobStages.TEST in stages: + stop_watch_ = Utils.Stopwatch() + step_name = "Tests" + print(step_name) + # assert Shell.check("clickhouse-client -q \"insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')\"", verbose=True) + run_test( + no_parallel=no_parallel, + no_sequiential=no_sequential, + batch_num=batch_num, + batch_total=total_batches, + test=args.test, + ) + results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run()) + results[-1].set_timing(stopwatch=stop_watch_) + res = results[-1].is_ok() + + Result.create_from( + results=results, stopwatch=stop_watch, files=logs_to_attach if not res else [] + ).complete_job() + + +if __name__ == "__main__": + main() diff --git a/ci/jobs/functional_stateless_tests.py b/ci/jobs/functional_stateless_tests.py index c2d374aa51b..0d73312bd9e 100644 --- a/ci/jobs/functional_stateless_tests.py +++ b/ci/jobs/functional_stateless_tests.py @@ -99,9 +99,12 @@ def main(): f"chmod +x {ch_path}/clickhouse", f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-server", f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client", + f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor", + f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local", f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server", f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/", - f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage", + # TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled + f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage --no-azure", # clickhouse benchmark segfaults with --config-path, so provide client config by its default location f"cp {Settings.TEMP_DIR}/etc/clickhouse-client/* /etc/clickhouse-client/", # update_path_ch_config, @@ -127,7 +130,7 @@ def main(): hdfs_log = "/tmp/praktika/output/hdfs_mini.log" minio_log = "/tmp/praktika/output/minio.log" res = res and CH.start_hdfs(log_file_path=hdfs_log) - res = res and CH.start_minio(log_file_path=minio_log) + res = res and CH.start_minio(test_type="stateful", log_file_path=minio_log) logs_to_attach += [minio_log, hdfs_log] time.sleep(10) Shell.check("ps -ef | grep minio", verbose=True) @@ -156,6 +159,10 @@ def main(): stop_watch_ = Utils.Stopwatch() step_name = "Tests" print(step_name) + assert Shell.check( + "clickhouse-client -q \"insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')\"", + verbose=True, + ) run_stateless_test( no_parallel=no_parallel, no_sequiential=no_sequential, diff --git a/ci/jobs/scripts/clickhouse_proc.py b/ci/jobs/scripts/clickhouse_proc.py index 8f9bef57083..6108563605f 100644 --- a/ci/jobs/scripts/clickhouse_proc.py +++ b/ci/jobs/scripts/clickhouse_proc.py @@ -31,12 +31,10 @@ class ClickHouseProc: self.info = "" self.info_file = "" - self.minio_cmd = f"tests/docker_scripts/setup_minio.sh stateless 2>&1 > {Settings.OUTPUT_DIR}/minio.log" - Utils.set_env("CLICKHOUSE_CONFIG_DIR", self.ch_config_dir) Utils.set_env("CLICKHOUSE_CONFIG", self.config_file) Utils.set_env("CLICKHOUSE_USER_FILES", self.user_files_path) - Utils.set_env("CLICKHOUSE_SCHEMA_FILES", f"{self.ch_config_dir}/format_schemas") + # Utils.set_env("CLICKHOUSE_SCHEMA_FILES", f"{self.ch_config_dir}/format_schemas") # if not fast_test: # with open(f"{self.ch_config_dir}/config.d/backups.xml", "w") as file: @@ -55,8 +53,12 @@ class ClickHouseProc: ) return True - def start_minio(self, log_file_path): - command = ["tests/docker_scripts/setup_minio.sh", "stateless", "./tests"] + def start_minio(self, test_type, log_file_path): + command = [ + "./ci/jobs/scripts/functional_tests/setup_minio.sh", + test_type, + "./tests", + ] with open(log_file_path, "w") as log_file: process = subprocess.Popen( command, stdout=log_file, stderr=subprocess.STDOUT diff --git a/ci/jobs/scripts/functional_tests/setup_minio.sh b/ci/jobs/scripts/functional_tests/setup_minio.sh new file mode 100755 index 00000000000..88839c39674 --- /dev/null +++ b/ci/jobs/scripts/functional_tests/setup_minio.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +set -euxf -o pipefail + +export MINIO_ROOT_USER=${MINIO_ROOT_USER:-clickhouse} +export MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-clickhouse} +TEST_DIR=${2:-/repo/tests/} + +if [ -d "$TEMP_DIR" ]; then + TEST_DIR=$(readlink -f $TEST_DIR) + cd "$TEMP_DIR" + # add / for minio mc in docker + PATH="/:.:$PATH" +fi + +usage() { + echo $"Usage: $0 (default path: /usr/share/clickhouse-test)" + exit 1 +} + +check_arg() { + local query_dir + if [ ! $# -eq 1 ]; then + if [ ! $# -eq 2 ]; then + echo "ERROR: need either one or two arguments, (default path: /usr/share/clickhouse-test)" + usage + fi + fi + case "$1" in + stateless) + query_dir="0_stateless" + ;; + stateful) + query_dir="1_stateful" + ;; + *) + echo "unknown test type ${test_type}" + usage + ;; + esac + echo ${query_dir} +} + +find_arch() { + local arch + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unknown architecture $(uname -m)"; + exit 1 + ;; + esac + echo ${arch} +} + +find_os() { + local os + os=$(uname -s | tr '[:upper:]' '[:lower:]') + echo "${os}" +} + +download_minio() { + local os + local arch + local minio_server_version=${MINIO_SERVER_VERSION:-2024-08-03T04-33-23Z} + local minio_client_version=${MINIO_CLIENT_VERSION:-2024-07-31T15-58-33Z} + + os=$(find_os) + arch=$(find_arch) + wget "https://dl.min.io/server/minio/release/${os}-${arch}/archive/minio.RELEASE.${minio_server_version}" -O ./minio + wget "https://dl.min.io/client/mc/release/${os}-${arch}/archive/mc.RELEASE.${minio_client_version}" -O ./mc + chmod +x ./mc ./minio +} + +start_minio() { + pwd + mkdir -p ./minio_data + minio --version + nohup minio server --address ":11111" ./minio_data & + wait_for_it + lsof -i :11111 + sleep 5 +} + +setup_minio() { + local test_type=$1 + echo "setup_minio(), test_type=$test_type" + mc alias set clickminio http://localhost:11111 clickhouse clickhouse + mc admin user add clickminio test testtest + mc admin policy attach clickminio readwrite --user=test ||: + mc mb --ignore-existing clickminio/test + if [ "$test_type" = "stateless" ]; then + echo "Create @test bucket in minio" + mc anonymous set public clickminio/test + fi +} + +# uploads data to minio, by default after unpacking all tests +# will be in /usr/share/clickhouse-test/queries +upload_data() { + local query_dir=$1 + local test_path=$2 + local data_path=${test_path}/queries/${query_dir}/data_minio + echo "upload_data() data_path=$data_path" + + # iterating over globs will cause redundant file variable to be + # a path to a file, not a filename + # shellcheck disable=SC2045 + if [ -d "${data_path}" ]; then + mc cp --recursive "${data_path}"/ clickminio/test/ + fi +} + +setup_aws_credentials() { + local minio_root_user=${MINIO_ROOT_USER:-clickhouse} + local minio_root_password=${MINIO_ROOT_PASSWORD:-clickhouse} + mkdir -p ~/.aws + cat <> ~/.aws/credentials +[default] +aws_access_key_id=${minio_root_user} +aws_secret_access_key=${minio_root_password} +EOT +} + +wait_for_it() { + local counter=0 + local max_counter=60 + local url="http://localhost:11111" + local params=( + --silent + --verbose + ) + while ! curl "${params[@]}" "${url}" 2>&1 | grep AccessDenied + do + if [[ ${counter} == "${max_counter}" ]]; then + echo "failed to setup minio" + exit 0 + fi + echo "trying to connect to minio" + sleep 1 + counter=$((counter + 1)) + done +} + +main() { + local query_dir + query_dir=$(check_arg "$@") + if ! (minio --version && mc --version); then + download_minio + fi + start_minio + setup_minio "$1" + upload_data "${query_dir}" "$TEST_DIR" + setup_aws_credentials +} + +main "$@" diff --git a/ci/praktika/json.html b/ci/praktika/json.html index 544fd6e68d4..b11106719cd 100644 --- a/ci/praktika/json.html +++ b/ci/praktika/json.html @@ -103,12 +103,13 @@ #result-container { background-color: var(--tile-background); margin-left: calc(var(--status-width) + 20px); - padding: 20px; + padding: 0; box-sizing: border-box; text-align: center; font-size: 18px; font-weight: normal; flex-grow: 1; + margin-bottom: 40px; } #footer { diff --git a/ci/settings/definitions.py b/ci/settings/definitions.py index 9f529798830..8ebf79231ac 100644 --- a/ci/settings/definitions.py +++ b/ci/settings/definitions.py @@ -133,12 +133,12 @@ DOCKERS = [ platforms=Docker.Platforms.arm_amd, depends_on=[], ), - # Docker.Config( - # name="clickhouse/stateful-test", - # path="./ci/docker/test/stateful", - # platforms=Docker.Platforms.arm_amd, - # depends_on=["clickhouse/stateless-test"], - # ), + Docker.Config( + name="clickhouse/stateful-test", + path="./ci/docker/stateful-test", + platforms=Docker.Platforms.arm_amd, + depends_on=["clickhouse/stateless-test"], + ), # Docker.Config( # name="clickhouse/stress-test", # path="./ci/docker/test/stress", @@ -241,3 +241,4 @@ class JobNames: FAST_TEST = "Fast test" BUILD = "Build" STATELESS = "Stateless tests" + STATEFUL = "Stateful tests" diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index 94dcc2ab722..761ab8a6ebc 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -1,5 +1,3 @@ -from typing import List - from praktika import Artifact, Job, Workflow from praktika.settings import Settings @@ -83,7 +81,7 @@ stateless_tests_jobs = Job.Config( runs_on=[RunnerLabels.BUILDER_AMD], command="python3 ./ci/jobs/functional_stateless_tests.py --test-options {PARAMETER}", # many tests expect to see "/var/lib/clickhouse" in various output lines - add mount for now, consider creating this dir in docker file - run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined+--volume=/tmp/praktika:/var/lib/clickhouse", + run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined", digest_config=Job.CacheDigestConfig( include_paths=[ "./ci/jobs/functional_stateless_tests.py", @@ -116,6 +114,30 @@ stateless_tests_jobs = Job.Config( ], ) +stateful_tests_jobs = Job.Config( + name=JobNames.STATEFUL, + runs_on=[RunnerLabels.BUILDER_AMD], + command="python3 ./ci/jobs/functional_stateful_tests.py --test-options {PARAMETER}", + # many tests expect to see "/var/lib/clickhouse" + # some tests expect to see "/var/log/clickhouse" + run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined", + digest_config=Job.CacheDigestConfig( + include_paths=[ + "./ci/jobs/functional_stateful_tests.py", + ], + ), +).parametrize( + parameter=[ + "amd_debug,parallel", + ], + runs_on=[ + [RunnerLabels.BUILDER_AMD], + ], + requires=[ + [ArtifactNames.CH_AMD_DEBUG], + ], +) + workflow = Workflow.Config( name="PR", event=Workflow.Event.PULL_REQUEST, @@ -125,6 +147,7 @@ workflow = Workflow.Config( fast_test_job, *build_jobs, *stateless_tests_jobs, + *stateful_tests_jobs, ], artifacts=[ Artifact.Config( @@ -157,7 +180,7 @@ workflow = Workflow.Config( WORKFLOWS = [ workflow, -] # type: List[Workflow.Config] +] # if __name__ == "__main__": diff --git a/tests/config/config.d/ssl_certs.xml b/tests/config/config.d/ssl_certs.xml index c20fef89e00..26b679f39df 100644 --- a/tests/config/config.d/ssl_certs.xml +++ b/tests/config/config.d/ssl_certs.xml @@ -1,8 +1,8 @@ - /tmp/praktika/etc/clickhouse-server/server.crt - /tmp/praktika/etc/clickhouse-server/server.key + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key diff --git a/tests/config/install.sh b/tests/config/install.sh index 9630977b9c1..b9d8092c74f 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -13,12 +13,13 @@ shift # DEST_SERVER_PATH shift # DEST_CLIENT_PATH FAST_TEST=0 -S3_STORAGE=0 +NO_AZURE=0 while [[ "$#" -gt 0 ]]; do case $1 in --fast-test) FAST_TEST=1 ;; - --s3-storage) S3_STORAGE=1 ;; + --s3-storage) EXPORT_S3_STORAGE_POLICIES=1 ;; + --no-azure) NO_AZURE=1 ;; *) echo "Unknown option: $1" ; exit 1 ;; esac shift @@ -199,8 +200,10 @@ elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then ln -sf $SRC_PATH/config.d/azure_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/ fi -if [[ "$EXPORT_S3_STORAGE_POLICIES" == "1" ]] || [[ "$S3_STORAGE" = "1" ]]; then - #ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/ +if [[ "$EXPORT_S3_STORAGE_POLICIES" == "1" ]]; then + if [[ "$NO_AZURE" != "1" ]]; then + ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/ + fi ln -sf $SRC_PATH/config.d/storage_conf.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf_02944.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf_02963.xml $DEST_SERVER_PATH/config.d/ diff --git a/tests/docker_scripts/setup_minio.sh b/tests/docker_scripts/setup_minio.sh index 88839c39674..40e93e713a1 100755 --- a/tests/docker_scripts/setup_minio.sh +++ b/tests/docker_scripts/setup_minio.sh @@ -4,14 +4,6 @@ set -euxf -o pipefail export MINIO_ROOT_USER=${MINIO_ROOT_USER:-clickhouse} export MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-clickhouse} -TEST_DIR=${2:-/repo/tests/} - -if [ -d "$TEMP_DIR" ]; then - TEST_DIR=$(readlink -f $TEST_DIR) - cd "$TEMP_DIR" - # add / for minio mc in docker - PATH="/:.:$PATH" -fi usage() { echo $"Usage: $0 (default path: /usr/share/clickhouse-test)" @@ -78,10 +70,9 @@ download_minio() { } start_minio() { - pwd mkdir -p ./minio_data - minio --version - nohup minio server --address ":11111" ./minio_data & + ./minio --version + ./minio server --address ":11111" ./minio_data & wait_for_it lsof -i :11111 sleep 5 @@ -89,14 +80,12 @@ start_minio() { setup_minio() { local test_type=$1 - echo "setup_minio(), test_type=$test_type" - mc alias set clickminio http://localhost:11111 clickhouse clickhouse - mc admin user add clickminio test testtest - mc admin policy attach clickminio readwrite --user=test ||: - mc mb --ignore-existing clickminio/test + ./mc alias set clickminio http://localhost:11111 clickhouse clickhouse + ./mc admin user add clickminio test testtest + ./mc admin policy attach clickminio readwrite --user=test + ./mc mb --ignore-existing clickminio/test if [ "$test_type" = "stateless" ]; then - echo "Create @test bucket in minio" - mc anonymous set public clickminio/test + ./mc anonymous set public clickminio/test fi } @@ -106,13 +95,12 @@ upload_data() { local query_dir=$1 local test_path=$2 local data_path=${test_path}/queries/${query_dir}/data_minio - echo "upload_data() data_path=$data_path" # iterating over globs will cause redundant file variable to be # a path to a file, not a filename # shellcheck disable=SC2045 if [ -d "${data_path}" ]; then - mc cp --recursive "${data_path}"/ clickminio/test/ + ./mc cp --recursive "${data_path}"/ clickminio/test/ fi } @@ -150,12 +138,12 @@ wait_for_it() { main() { local query_dir query_dir=$(check_arg "$@") - if ! (minio --version && mc --version); then + if [ ! -f ./minio ]; then download_minio fi start_minio setup_minio "$1" - upload_data "${query_dir}" "$TEST_DIR" + upload_data "${query_dir}" "${2:-/repo/tests/}" setup_aws_credentials }