From 8e1a4bcf9a6376a36509e09efd9df7486346f791 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 7 Nov 2022 08:49:26 +0000 Subject: [PATCH] Update CI config --- .github/workflows/pull_request.yml | 68 +------------------ docker/test/server-jepsen/Dockerfile | 2 +- tests/ci/ci_config.py | 3 + tests/ci/jepsen_check.py | 2 +- tests/jepsen.clickhouse/resources/config.xml | 2 +- .../resources/keeper_config.xml | 2 +- .../resources/keeper_config_solo.xml | 2 +- .../src/jepsen/clickhouse/server/client.clj | 4 +- .../src/jepsen/clickhouse/server/utils.clj | 2 +- .../src/jepsen/clickhouse/utils.clj | 1 + 10 files changed, 14 insertions(+), 74 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5358d476636..199a5d71f92 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -32,70 +32,6 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 run_check.py - DockerHubPushAarch64: - needs: CheckLabels - runs-on: [self-hosted, style-checker-aarch64] - steps: - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v2 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - needs: CheckLabels - runs-on: [self-hosted, style-checker] - steps: - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v2 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, style-checker] - steps: - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Download changed aarch64 images - uses: actions/download-artifact@v2 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v2 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v2 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json ############################################################################################# ###################################### JEPSEN TESTS ######################################### ############################################################################################# @@ -104,11 +40,11 @@ jobs: # When it's skipped, all dependent tasks will be skipped too. # DO NOT add it there if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') - needs: [DockerHubPush] + needs: [CheckLabels] uses: ./.github/workflows/jepsen.yml FinishCheck: needs: - - DockerHubPush + - CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Clear repository diff --git a/docker/test/server-jepsen/Dockerfile b/docker/test/server-jepsen/Dockerfile index 0335186cf1d..974da67e853 100644 --- a/docker/test/server-jepsen/Dockerfile +++ b/docker/test/server-jepsen/Dockerfile @@ -12,7 +12,7 @@ ENV SHA_TO_TEST="" ENV NODES_USERNAME="root" ENV NODES_PASSWORD="" -ENV TESTS_TO_RUN="30" +ENV TESTS_TO_RUN="1" ENV TIME_LIMIT="30" ENV KEEPER_NODE="" diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 93322b69669..bfc7e45812b 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -347,6 +347,9 @@ CI_CONFIG = { "ClickHouse Keeper Jepsen": { "required_build": "binary_release", }, + "ClickHouse Server Jepsen": { + "required_build": "binary_release", + }, "Performance Comparison": { "required_build": "package_release", "test_grep_exclude_filter": "", diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index 87963a158c2..8b898a34e67 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -194,7 +194,7 @@ if __name__ == "__main__": os.makedirs(result_path) instances = prepare_autoscaling_group_and_get_hostnames(KEEPER_DESIRED_INSTANCE_COUNT if args.program == 'keeper' else SERVER_DESIRED_INSTANCE_COUNT) - nodes_path = save_nodes_to_file(instances[:SERVER_DESIRED_INSTANCE_COUNT - 1], TEMP_PATH) + nodes_path = save_nodes_to_file(instances[:KEEPER_DESIRED_INSTANCE_COUNT], TEMP_PATH) # always use latest docker_image = KEEPER_IMAGE_NAME if args.program == 'keeper' else SERVER_IMAGE_NAME diff --git a/tests/jepsen.clickhouse/resources/config.xml b/tests/jepsen.clickhouse/resources/config.xml index 25e38a58e63..ae0570a3a5f 100644 --- a/tests/jepsen.clickhouse/resources/config.xml +++ b/tests/jepsen.clickhouse/resources/config.xml @@ -1,5 +1,5 @@ - 0.0.0.0 + :: 9000 8123 diff --git a/tests/jepsen.clickhouse/resources/keeper_config.xml b/tests/jepsen.clickhouse/resources/keeper_config.xml index 20ca74c1cf5..2ab747fbd71 100644 --- a/tests/jepsen.clickhouse/resources/keeper_config.xml +++ b/tests/jepsen.clickhouse/resources/keeper_config.xml @@ -1,5 +1,5 @@ - 0.0.0.0 + :: trace diff --git a/tests/jepsen.clickhouse/resources/keeper_config_solo.xml b/tests/jepsen.clickhouse/resources/keeper_config_solo.xml index dfbc098b3dd..b20592545cd 100644 --- a/tests/jepsen.clickhouse/resources/keeper_config_solo.xml +++ b/tests/jepsen.clickhouse/resources/keeper_config_solo.xml @@ -1,5 +1,5 @@ - 0.0.0.0 + :: trace diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/server/client.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/server/client.clj index e051d361c73..9335a005149 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/server/client.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/server/client.clj @@ -22,12 +22,12 @@ (util/timeout 30000 (throw (RuntimeException. (str "Connection to " node " timed out"))) - (util/retry 0.1 + ;(util/retry 0.1 (let [spec (db-spec node) connection (j/get-connection spec) added-connection (j/add-connection spec connection)] (assert added-connection) - added-connection)))) + added-connection))) (defn close-connection "Close connection" diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/server/utils.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/server/utils.clj index 6e99da769a4..a61c825681a 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/server/utils.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/server/utils.clj @@ -11,7 +11,7 @@ (let [c (chc/open-connection node)] (jdbc/query c "SELECT 1") (chc/close-connection c)) - (catch Exception _ false))) + (catch Exception e (do (info e) false)))) (defn start-clickhouse! [node test] diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/utils.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/utils.clj index cbf18129d88..c29e11daadb 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/utils.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/utils.clj @@ -132,6 +132,7 @@ :chdir data-dir} binary-path binary-args) + (info "Waiting for server") (wait-clickhouse-alive! node test clickhouse-alive?))) (defn prepare-dirs