Merge branch 'master' into remove_most_of_old_checks

2024-11-22 07:31:57 +00:00 · 2021-12-01 09:45:19 +00:00 · 2021-12-01 09:45:19 +00:00 · ffc1f4eb92
commit ffc1f4eb92
parent b421d07434 e80ff2c0c5
6 changed files with 211 additions and 39 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -824,10 +824,68 @@ jobs:
        uses: actions/checkout@v2
      - name: Functional test
        env:
-          TEMP_PATH: ${{runner.temp}}/stateless_debug
+          TEMP_PATH: ${{runner.temp}}/stateless_release
          REPORTS_PATH: ${{runner.temp}}/reports_dir
          CHECK_NAME: 'Stateless tests (release, actions)'
-          REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
+          REPO_COPY: ${{runner.temp}}/stateless_release/ClickHouse
+          KILL_TIMEOUT: 10800
+        run: |
+          sudo rm -fr $TEMP_PATH
+          mkdir -p $TEMP_PATH
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci
+          python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH
+  FunctionalStatelessTestReleaseDatabaseReplicated:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{runner.temp}}/reports_dir
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        env:
+          TEMP_PATH: ${{runner.temp}}/stateless_database_replicated
+          REPORTS_PATH: ${{runner.temp}}/reports_dir
+          CHECK_NAME: 'Stateless tests (release, DatabaseReplicated, actions)'
+          REPO_COPY: ${{runner.temp}}/stateless_database_replicated/ClickHouse
+          KILL_TIMEOUT: 10800
+        run: |
+          sudo rm -fr $TEMP_PATH
+          mkdir -p $TEMP_PATH
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci
+          python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH
+  FunctionalStatelessTestReleaseWideParts:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{runner.temp}}/reports_dir
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        env:
+          TEMP_PATH: ${{runner.temp}}/stateless_wide_parts
+          REPORTS_PATH: ${{runner.temp}}/reports_dir
+          CHECK_NAME: 'Stateless tests (release, wide parts enabled, actions)'
+          REPO_COPY: ${{runner.temp}}/stateless_wide_parts/ClickHouse
          KILL_TIMEOUT: 10800
        run: |
          sudo rm -fr $TEMP_PATH
@ -1032,10 +1090,10 @@ jobs:
        uses: actions/checkout@v2
      - name: Functional test
        env:
-          TEMP_PATH: ${{runner.temp}}/stateful_debug
+          TEMP_PATH: ${{runner.temp}}/stateful_release
          REPORTS_PATH: ${{runner.temp}}/reports_dir
          CHECK_NAME: 'Stateful tests (release, actions)'
-          REPO_COPY: ${{runner.temp}}/stateful_debug/ClickHouse
+          REPO_COPY: ${{runner.temp}}/stateful_release/ClickHouse
          KILL_TIMEOUT: 3600
        run: |
          sudo rm -fr $TEMP_PATH
@ -1747,6 +1805,8 @@ jobs:
      - FastTest
      - FunctionalStatelessTestDebug
      - FunctionalStatelessTestRelease
+      - FunctionalStatelessTestReleaseDatabaseReplicated
+      - FunctionalStatelessTestReleaseWideParts
      - FunctionalStatelessTestAsan
      - FunctionalStatelessTestTsan
      - FunctionalStatelessTestMsan
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@ -431,7 +431,7 @@ jobs:
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr $TEMP_PATH
  BuilderBinTidy:
-    needs: [DockerHubPush, FastTest]
+    needs: [DockerHubPush]
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
    runs-on: [self-hosted, builder]
    steps:
@ -470,7 +470,7 @@ jobs:
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr $TEMP_PATH
  BuilderBinDarwin:
-    needs: [DockerHubPush, FastTest]
+    needs: [DockerHubPush]
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
    runs-on: [self-hosted, builder]
    steps:
@ -509,7 +509,7 @@ jobs:
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr $TEMP_PATH
  BuilderBinAarch64:
-    needs: [DockerHubPush, FastTest]
+    needs: [DockerHubPush]
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
    runs-on: [self-hosted, builder]
    steps:
@ -548,7 +548,7 @@ jobs:
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr $TEMP_PATH
  BuilderBinFreeBSD:
-    needs: [DockerHubPush, FastTest]
+    needs: [DockerHubPush]
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
    runs-on: [self-hosted, builder]
    steps:
@ -587,7 +587,7 @@ jobs:
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr $TEMP_PATH
  BuilderBinDarwinAarch64:
-    needs: [DockerHubPush, FastTest]
+    needs: [DockerHubPush]
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
    runs-on: [self-hosted, builder]
    steps:
@ -626,7 +626,7 @@ jobs:
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr $TEMP_PATH
  BuilderBinPPC64:
-    needs: [DockerHubPush, FastTest]
+    needs: [DockerHubPush]
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
    runs-on: [self-hosted, builder]
    steps:
@ -748,10 +748,39 @@ jobs:
        uses: actions/checkout@v2
      - name: Functional test
        env:
-          TEMP_PATH: ${{runner.temp}}/stateless_debug
+          TEMP_PATH: ${{runner.temp}}/stateless_release
          REPORTS_PATH: ${{runner.temp}}/reports_dir
          CHECK_NAME: 'Stateless tests (release, actions)'
-          REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
+          REPO_COPY: ${{runner.temp}}/stateless_release/ClickHouse
+          KILL_TIMEOUT: 10800
+        run: |
+          sudo rm -fr $TEMP_PATH
+          mkdir -p $TEMP_PATH
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci
+          python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH
+  FunctionalStatelessTestReleaseDatabaseOrdinary:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{runner.temp}}/reports_dir
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        env:
+          TEMP_PATH: ${{runner.temp}}/stateless_release_database_ordinary
+          REPORTS_PATH: ${{runner.temp}}/reports_dir
+          CHECK_NAME: 'Stateless tests (release, DatabaseOrdinary, actions)'
+          REPO_COPY: ${{runner.temp}}/stateless_release_database_ordinary/ClickHouse
          KILL_TIMEOUT: 10800
        run: |
          sudo rm -fr $TEMP_PATH
@ -925,10 +954,39 @@ jobs:
        uses: actions/checkout@v2
      - name: Functional test
        env:
-          TEMP_PATH: ${{runner.temp}}/stateful_debug
+          TEMP_PATH: ${{runner.temp}}/stateful_release
          REPORTS_PATH: ${{runner.temp}}/reports_dir
          CHECK_NAME: 'Stateful tests (release, actions)'
-          REPO_COPY: ${{runner.temp}}/stateful_debug/ClickHouse
+          REPO_COPY: ${{runner.temp}}/stateful_release/ClickHouse
+          KILL_TIMEOUT: 3600
+        run: |
+          sudo rm -fr $TEMP_PATH
+          mkdir -p $TEMP_PATH
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci
+          python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH
+  FunctionalStatefulTestReleaseDatabaseOrdinary:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, func-tester]
+    steps:
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{runner.temp}}/reports_dir
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Functional test
+        env:
+          TEMP_PATH: ${{runner.temp}}/stateful_release_database_ordinary
+          REPORTS_PATH: ${{runner.temp}}/reports_dir
+          CHECK_NAME: 'Stateful tests (release, DatabaseOrdinary, actions)'
+          REPO_COPY: ${{runner.temp}}/stateful_release_database_ordinary/ClickHouse
          KILL_TIMEOUT: 3600
        run: |
          sudo rm -fr $TEMP_PATH
@ -1609,12 +1667,14 @@ jobs:
      - BuilderReport
      - FunctionalStatelessTestDebug
      - FunctionalStatelessTestRelease
+      - FunctionalStatelessTestReleaseDatabaseOrdinary
      - FunctionalStatelessTestAsan
      - FunctionalStatelessTestTsan
      - FunctionalStatelessTestMsan
      - FunctionalStatelessTestUBsan
      - FunctionalStatefulTestDebug
      - FunctionalStatefulTestRelease
+      - FunctionalStatefulTestReleaseDatabaseOrdinary
      - FunctionalStatefulTestAsan
      - FunctionalStatefulTestTsan
      - FunctionalStatefulTestMsan
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@ -18,6 +18,15 @@ from commit_status_helper import post_commit_status, get_commit
 from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse
 from stopwatch import Stopwatch

+def get_additional_envs(check_name):
+    if 'DatabaseReplicated' in check_name:
+        return ["USE_DATABASE_REPLICATED=1"]
+    if 'DatabaseOrdinary' in check_name:
+        return ["USE_DATABASE_ORDINARY=1"]
+    if 'wide parts enabled' in check_name:
+        return ["USE_POLYMORPHIC_PARTS=1"]
+
+    return []

 def get_image_name(check_name):
    if 'stateless' in check_name.lower():
@ -141,7 +150,8 @@ if __name__ == "__main__":

    run_log_path = os.path.join(result_path, "runlog.log")

-    run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, [], docker_image, flaky_check, tests_to_run)
+    additional_envs = get_additional_envs(check_name)
+    run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, additional_envs, docker_image, flaky_check, tests_to_run)
    logging.info("Going to run func tests: %s", run_command)

    with open(run_log_path, 'w', encoding='utf-8') as log:
--- a/tests/ci/worker/init_builder.sh
+++ b/tests/ci/worker/init_builder.sh
@ -1,20 +1,34 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -uo pipefail

 echo "Running init script"
 export DEBIAN_FRONTEND=noninteractive
 export RUNNER_HOME=/home/ubuntu/actions-runner

-echo "Receiving token"
-export RUNNER_TOKEN=`/usr/local/bin/aws ssm  get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`
 export RUNNER_URL="https://github.com/ClickHouse"
 # Funny fact, but metadata service has fixed IP
 export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`

-cd $RUNNER_HOME
+while true; do
+    runner_pid=`pgrep run.sh`
+    echo "Got runner pid $runner_pid"

-echo "Going to configure runner"
-sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,builder' --work _work
+    cd $RUNNER_HOME
+    if [ -z "$runner_pid" ]; then
+        echo "Receiving token"
+        RUNNER_TOKEN=`/usr/local/bin/aws ssm  get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`

-echo "Run"
-sudo -u ubuntu ./run.sh
+        echo "Will try to remove runner"
+        sudo -u ubuntu ./config.sh remove --token $RUNNER_TOKEN ||:
+
+        echo "Going to configure runner"
+        sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,builder' --work _work
+
+        echo "Run"
+        sudo -u ubuntu ./run.sh &
+        sleep 15
+    else
+        echo "Runner is working with pid $runner_pid, nothing to do"
+        sleep 10
+    fi
+done
--- a/tests/ci/worker/init_func_tester.sh
+++ b/tests/ci/worker/init_func_tester.sh
@ -1,20 +1,34 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -uo pipefail

 echo "Running init script"
 export DEBIAN_FRONTEND=noninteractive
 export RUNNER_HOME=/home/ubuntu/actions-runner

-echo "Receiving token"
-export RUNNER_TOKEN=`/usr/local/bin/aws ssm  get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`
 export RUNNER_URL="https://github.com/ClickHouse"
 # Funny fact, but metadata service has fixed IP
 export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`

-cd $RUNNER_HOME
+while true; do
+    runner_pid=`pgrep run.sh`
+    echo "Got runner pid $runner_pid"

-echo "Going to configure runner"
-sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,func-tester' --work _work
+    cd $RUNNER_HOME
+    if [ -z "$runner_pid" ]; then
+        echo "Receiving token"
+        RUNNER_TOKEN=`/usr/local/bin/aws ssm  get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`

-echo "Run"
-sudo -u ubuntu ./run.sh
+        echo "Will try to remove runner"
+        sudo -u ubuntu ./config.sh remove --token $RUNNER_TOKEN ||:
+
+        echo "Going to configure runner"
+        sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,func-tester' --work _work
+
+        echo "Run"
+        sudo -u ubuntu ./run.sh &
+        sleep 15
+    else
+        echo "Runner is working with pid $runner_pid, nothing to do"
+        sleep 10
+    fi
+done
--- a/tests/ci/worker/init_stress_tester.sh
+++ b/tests/ci/worker/init_stress_tester.sh
@ -1,20 +1,34 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -uo pipefail

 echo "Running init script"
 export DEBIAN_FRONTEND=noninteractive
 export RUNNER_HOME=/home/ubuntu/actions-runner

-echo "Receiving token"
-export RUNNER_TOKEN=`/usr/local/bin/aws ssm  get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`
 export RUNNER_URL="https://github.com/ClickHouse"
 # Funny fact, but metadata service has fixed IP
 export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`

-cd $RUNNER_HOME
+while true; do
+    runner_pid=`pgrep run.sh`
+    echo "Got runner pid $runner_pid"

-echo "Going to configure runner"
-sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,stress-tester' --work _work
+    cd $RUNNER_HOME
+    if [ -z "$runner_pid" ]; then
+        echo "Receiving token"
+        RUNNER_TOKEN=`/usr/local/bin/aws ssm  get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`

-echo "Run"
-sudo -u ubuntu ./run.sh
+        echo "Will try to remove runner"
+        sudo -u ubuntu ./config.sh remove --token $RUNNER_TOKEN ||:
+
+        echo "Going to configure runner"
+        sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64,stress-tester' --work _work
+
+        echo "Run"
+        sudo -u ubuntu ./run.sh &
+        sleep 15
+    else
+        echo "Runner is working with pid $runner_pid, nothing to do"
+        sleep 10
+    fi
+done