From 349f266c1a629d4c02aa28c9cc0175546e7e8171 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Nov 2021 16:49:19 +0300 Subject: [PATCH 1/4] Add flaky check --- .github/workflows/main.yml | 29 +++++++++++++++++++++++ tests/ci/functional_test_check.py | 39 +++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fda43824fc8..c355e2a6d0e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -246,6 +246,35 @@ jobs: docker kill $(docker ps -q) ||: docker rm -f $(docker ps -a -q) ||: sudo rm -fr $TEMP_PATH + FunctionalStatelessTestFlakyCheck: + needs: [BuilderDebAsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{runner.temp}}/reports_dir + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + env: + TEMP_PATH: ${{runner.temp}}/stateless_flaky_asan + REPORTS_PATH: ${{runner.temp}}/reports_dir + CHECK_NAME: 'Stateless tests flaky check (address, actions)' + REPO_COPY: ${{runner.temp}}/stateless_flaky_asan/ClickHouse + REQUIRED_BUILD_NUMBER: 3 + run: | + sudo rm -fr $TEMP_PATH + mkdir -p $TEMP_PATH + cp -r $GITHUB_WORKSPACE $TEMP_PATH + cd $REPO_COPY/tests/ci + python3 functional_test_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER $KILL_TIMEOUT + - name: Cleanup + if: always() + run: | + docker kill $(docker ps -q) ||: + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr $TEMP_PATH StressTestDebug: needs: [BuilderDebDebug] runs-on: [self-hosted, stress-tester] diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 75a0b31a221..01e765349fc 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -147,13 +147,22 @@ def build_config_to_string(build_config): build_config['package-type'], ]) -def get_run_command(builds_path, result_path, server_log_path, kill_timeout, additional_envs, image): +def get_run_command(builds_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): additional_options = ['--hung-check'] additional_options.append('--print-time') + + if tests_to_run: + additional_options += tests_to_run + additional_options_str = '-e ADDITIONAL_OPTIONS="' + ' '.join(additional_options) + '"' envs = [f'-e MAX_RUN_TIME={int(0.9 * kill_timeout)}', '-e S3_URL="https://clickhouse-datasets.s3.amazonaws.com"'] + + if flaky_check: + envs += ['-e NUM_TRIES=100', '-e MAX_RUN_TIME=1800'] + envs += [f'-e {e}' for e in additional_envs] + env_str = ' '.join(envs) return f"docker run --volume={builds_path}:/package_folder " \ @@ -161,6 +170,20 @@ def get_run_command(builds_path, result_path, server_log_path, kill_timeout, add f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" +def get_tests_to_run(pr_info): + result = set([]) + + if pr_info.changed_files is None: + return [] + + for fpath in pr_info.changed_files: + if 'tests/queries/0_stateless/0' in fpath: + logging.info('File %s changed and seems like stateless test', fpath) + fname = fpath.split('/')[3] + fname_without_ext = os.path.splitext(fname)[0] + result.add(fname_without_ext + '.') + return list(result) + def process_results(result_folder, server_log_path): test_results = [] additional_files = [] @@ -201,6 +224,7 @@ if __name__ == "__main__": check_name = sys.argv[1] build_number = int(sys.argv[2]) kill_timeout = int(sys.argv[3]) + flaky_check = 'flaky' in check_name.lower() if not os.path.exists(temp_path): os.makedirs(temp_path) @@ -208,10 +232,15 @@ if __name__ == "__main__": with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file: event = json.load(event_file) - pr_info = PRInfo(event) - gh = Github(get_best_robot_token()) - + pr_info = PRInfo(event, need_changed_files=flaky_check) + tests_to_run = [] + if flaky_check: + tests_to_run = get_tests_to_run(pr_info) + if not tests_to_run: + commit = get_commit(gh, pr_info.sha) + commit.create_status(context=check_name, description='Not found changed stateless tests', state='success') + sys.exit(0) for root, _, files in os.walk(reports_path): for f in files: @@ -264,7 +293,7 @@ if __name__ == "__main__": run_log_path = os.path.join(result_path, "runlog.log") download_builds(packages_path, urls) - run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, [], docker_image) + run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, [], docker_image, flaky_check, tests_to_run) logging.info("Going to run func tests: %s", run_command) with open(run_log_path, 'w', encoding='utf-8') as log: From f48f8f3f7e1346d4c31d0e3dfbf05bfd7edefeb1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Nov 2021 18:59:03 +0300 Subject: [PATCH 2/4] Better --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c355e2a6d0e..3d8ee9ca896 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -345,10 +345,10 @@ jobs: uses: actions/checkout@v2 - name: Integration test env: - TEMP_PATH: ${{runner.temp}}/integration_tests_debug + TEMP_PATH: ${{runner.temp}}/integration_tests_asan REPORTS_PATH: ${{runner.temp}}/reports_dir CHECK_NAME: 'Integration tests (asan, actions)' - REPO_COPY: ${{runner.temp}}/integration_tests_debug/ClickHouse + REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse REQUIRED_BUILD_NUMBER: 3 run: | sudo rm -fr $TEMP_PATH From a8ce97c6c41f55af2d1e1ed6d7a9d2e0cfa5e00d Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Nov 2021 22:22:54 +0300 Subject: [PATCH 3/4] fix yml --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3d8ee9ca896..60e78822cd3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -263,6 +263,7 @@ jobs: CHECK_NAME: 'Stateless tests flaky check (address, actions)' REPO_COPY: ${{runner.temp}}/stateless_flaky_asan/ClickHouse REQUIRED_BUILD_NUMBER: 3 + KILL_TIMEOUT: 3600 run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH From 2f42a5137d70218d15d1311d4106388081571354 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 11 Nov 2021 11:44:53 +0300 Subject: [PATCH 4/4] Trying flaky check --- .../0_stateless/02097_default_dict_get_add_database.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02097_default_dict_get_add_database.sql b/tests/queries/0_stateless/02097_default_dict_get_add_database.sql index af177566476..4e1ff4d6e67 100644 --- a/tests/queries/0_stateless/02097_default_dict_get_add_database.sql +++ b/tests/queries/0_stateless/02097_default_dict_get_add_database.sql @@ -40,5 +40,4 @@ DROP DICTIONARY test_dictionary; DROP TABLE test_table; DROP TABLE test_table_default; -DROP DATABASE 02097_db; - +DROP DATABASE IF EXISTS 02097_db;