diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 090a8912649..8114f9b2f85 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -285,6 +285,36 @@ jobs: docker kill $(docker ps -q) ||: docker rm -f $(docker ps -a -q) ||: sudo rm -fr $TEMP_PATH + FunctionalStatelessTestFlakyCheck: + needs: [BuilderDebAsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{runner.temp}}/reports_dir + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + env: + TEMP_PATH: ${{runner.temp}}/stateless_flaky_asan + REPORTS_PATH: ${{runner.temp}}/reports_dir + CHECK_NAME: 'Stateless tests flaky check (address, actions)' + REPO_COPY: ${{runner.temp}}/stateless_flaky_asan/ClickHouse + REQUIRED_BUILD_NUMBER: 3 + KILL_TIMEOUT: 3600 + run: | + sudo rm -fr $TEMP_PATH + mkdir -p $TEMP_PATH + cp -r $GITHUB_WORKSPACE $TEMP_PATH + cd $REPO_COPY/tests/ci + python3 functional_test_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER $KILL_TIMEOUT + - name: Cleanup + if: always() + run: | + docker kill $(docker ps -q) ||: + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr $TEMP_PATH StressTestDebug: needs: [BuilderDebDebug] runs-on: [self-hosted, stress-tester] @@ -355,10 +385,10 @@ jobs: uses: actions/checkout@v2 - name: Integration test env: - TEMP_PATH: ${{runner.temp}}/integration_tests_debug + TEMP_PATH: ${{runner.temp}}/integration_tests_asan REPORTS_PATH: ${{runner.temp}}/reports_dir CHECK_NAME: 'Integration tests (asan, actions)' - REPO_COPY: ${{runner.temp}}/integration_tests_debug/ClickHouse + REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse REQUIRED_BUILD_NUMBER: 3 run: | sudo rm -fr $TEMP_PATH diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 75a0b31a221..01e765349fc 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -147,13 +147,22 @@ def build_config_to_string(build_config): build_config['package-type'], ]) -def get_run_command(builds_path, result_path, server_log_path, kill_timeout, additional_envs, image): +def get_run_command(builds_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): additional_options = ['--hung-check'] additional_options.append('--print-time') + + if tests_to_run: + additional_options += tests_to_run + additional_options_str = '-e ADDITIONAL_OPTIONS="' + ' '.join(additional_options) + '"' envs = [f'-e MAX_RUN_TIME={int(0.9 * kill_timeout)}', '-e S3_URL="https://clickhouse-datasets.s3.amazonaws.com"'] + + if flaky_check: + envs += ['-e NUM_TRIES=100', '-e MAX_RUN_TIME=1800'] + envs += [f'-e {e}' for e in additional_envs] + env_str = ' '.join(envs) return f"docker run --volume={builds_path}:/package_folder " \ @@ -161,6 +170,20 @@ def get_run_command(builds_path, result_path, server_log_path, kill_timeout, add f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" +def get_tests_to_run(pr_info): + result = set([]) + + if pr_info.changed_files is None: + return [] + + for fpath in pr_info.changed_files: + if 'tests/queries/0_stateless/0' in fpath: + logging.info('File %s changed and seems like stateless test', fpath) + fname = fpath.split('/')[3] + fname_without_ext = os.path.splitext(fname)[0] + result.add(fname_without_ext + '.') + return list(result) + def process_results(result_folder, server_log_path): test_results = [] additional_files = [] @@ -201,6 +224,7 @@ if __name__ == "__main__": check_name = sys.argv[1] build_number = int(sys.argv[2]) kill_timeout = int(sys.argv[3]) + flaky_check = 'flaky' in check_name.lower() if not os.path.exists(temp_path): os.makedirs(temp_path) @@ -208,10 +232,15 @@ if __name__ == "__main__": with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file: event = json.load(event_file) - pr_info = PRInfo(event) - gh = Github(get_best_robot_token()) - + pr_info = PRInfo(event, need_changed_files=flaky_check) + tests_to_run = [] + if flaky_check: + tests_to_run = get_tests_to_run(pr_info) + if not tests_to_run: + commit = get_commit(gh, pr_info.sha) + commit.create_status(context=check_name, description='Not found changed stateless tests', state='success') + sys.exit(0) for root, _, files in os.walk(reports_path): for f in files: @@ -264,7 +293,7 @@ if __name__ == "__main__": run_log_path = os.path.join(result_path, "runlog.log") download_builds(packages_path, urls) - run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, [], docker_image) + run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, [], docker_image, flaky_check, tests_to_run) logging.info("Going to run func tests: %s", run_command) with open(run_log_path, 'w', encoding='utf-8') as log: diff --git a/tests/queries/0_stateless/02097_default_dict_get_add_database.sql b/tests/queries/0_stateless/02097_default_dict_get_add_database.sql index af177566476..4e1ff4d6e67 100644 --- a/tests/queries/0_stateless/02097_default_dict_get_add_database.sql +++ b/tests/queries/0_stateless/02097_default_dict_get_add_database.sql @@ -40,5 +40,4 @@ DROP DICTIONARY test_dictionary; DROP TABLE test_table; DROP TABLE test_table_default; -DROP DATABASE 02097_db; - +DROP DATABASE IF EXISTS 02097_db;