From e2acd0e93f2a5819bdb22a28b87640935bcee308 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Fri, 12 Apr 2024 10:18:43 +0000 Subject: [PATCH 1/6] CI: add FT to MQ remove Style from master --- .github/workflows/master.yml | 21 ++++++++-------- .github/workflows/pull_request.yml | 4 +-- tests/ci/ci.py | 39 ++++++++++++++---------------- tests/ci/sync_pr.py | 2 ++ 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 9a719a205d4..384141b78e7 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -59,16 +59,17 @@ jobs: uses: ./.github/workflows/reusable_docker.yml with: data: ${{ needs.RunConfig.outputs.data }} - StyleCheck: - needs: [RunConfig, BuildDockers] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_test.yml - with: - test_name: Style check - runner_type: style-checker - data: ${{ needs.RunConfig.outputs.data }} - run_command: | - python3 style_check.py --no-push + # Tested in MQ + # StyleCheck: + # needs: [RunConfig, BuildDockers] + # if: ${{ !failure() && !cancelled() }} + # uses: ./.github/workflows/reusable_test.yml + # with: + # test_name: Style check + # runner_type: style-checker + # data: ${{ needs.RunConfig.outputs.data }} + # run_command: | + # python3 style_check.py --no-push CompatibilityCheckX86: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 0eac9a9a722..c2e76de5e14 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -83,7 +83,7 @@ jobs: ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}} RCSK FastTest: - needs: [RunConfig, StyleCheck] + needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }} uses: ./.github/workflows/reusable_test.yml with: @@ -97,7 +97,7 @@ jobs: # for main CI chain # Builds_1: - needs: [RunConfig, FastTest] + needs: [RunConfig, StyleCheck, FastTest] if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }} # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab uses: ./.github/workflows/reusable_build_stage.yml diff --git a/tests/ci/ci.py b/tests/ci/ci.py index f11d62e9136..bd6117dc858 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1261,18 +1261,6 @@ def _print_results(result: Any, outfile: Optional[str], pretty: bool = False) -> raise AssertionError(f"Unexpected type for 'res': {type(result)}") -def _update_config_for_docs_only(jobs_data: dict) -> None: - DOCS_CHECK_JOBS = [JobNames.DOCS_CHECK, JobNames.STYLE_CHECK] - print(f"NOTE: Will keep only docs related jobs: [{DOCS_CHECK_JOBS}]") - jobs_to_do = jobs_data.get("jobs_to_do", []) - jobs_data["jobs_to_do"] = [job for job in jobs_to_do if job in DOCS_CHECK_JOBS] - jobs_data["jobs_to_wait"] = { - job: params - for job, params in jobs_data["jobs_to_wait"].items() - if job in DOCS_CHECK_JOBS - } - - def _configure_docker_jobs(docker_digest_or_latest: bool) -> Dict: print("::group::Docker images check") # generate docker jobs data @@ -1332,8 +1320,20 @@ def _configure_jobs( jobs_to_skip: List[str] = [] digests: Dict[str, str] = {} + # FIXME: find better place for these config variables + DOCS_CHECK_JOBS = [JobNames.DOCS_CHECK, JobNames.STYLE_CHECK] + MQ_JOBS = [JobNames.STYLE_CHECK, JobNames.FAST_TEST] + if pr_info.has_changes_in_documentation_only(): + print(f"WARNING: Only docs are changed - will run only [{DOCS_CHECK_JOBS}]") + if pr_info.is_merge_queue(): + print(f"WARNING: It's a MQ run - will run only [{MQ_JOBS}]") + print("::group::Job Digests") for job in CI_CONFIG.job_generator(pr_info.head_ref if CI else "dummy_branch_name"): + if pr_info.is_merge_queue() and job not in MQ_JOBS: + continue + if pr_info.has_changes_in_documentation_only() and job not in DOCS_CHECK_JOBS: + continue digest = job_digester.get_job_digest(CI_CONFIG.get_digest_config(job)) digests[job] = digest print(f" job [{job.rjust(50)}] has digest [{digest}]") @@ -1436,10 +1436,11 @@ def _configure_jobs( ) if pr_info.is_merge_queue(): - # FIXME: Quick support for MQ workflow which is only StyleCheck for now - jobs_to_do = [JobNames.STYLE_CHECK] - jobs_to_skip = [] - print(f"NOTE: This is Merge Queue CI: set jobs to do: [{jobs_to_do}]") + # no need to run pending job in MQ, since it's pending - it's not affected by current checnge + for job_to_wait in jobs_to_wait: + if job_to_wait in jobs_to_do: + print(f"Remove pending job [{job_to_wait}] from MQ workflow") + jobs_to_do.remove(job_to_wait) return { "digests": digests, @@ -1902,9 +1903,6 @@ def main() -> int: else {} ) - if not args.skip_jobs and pr_info.has_changes_in_documentation_only(): - _update_config_for_docs_only(jobs_data) - if not args.skip_jobs: ci_cache = CiCache(s3, jobs_data["digests"]) @@ -1928,8 +1926,7 @@ def main() -> int: jobs_to_skip.append(job) del jobs_params[job] - # set planned jobs as pending in the CI cache if on the master - if pr_info.is_master(): + # set planned jobs as pending in the CI cache if on the master for job in jobs_data["jobs_to_do"]: config = CI_CONFIG.get_job_config(job) if config.run_always or config.run_by_label: diff --git a/tests/ci/sync_pr.py b/tests/ci/sync_pr.py index acff7ba541b..639436ab3ac 100644 --- a/tests/ci/sync_pr.py +++ b/tests/ci/sync_pr.py @@ -54,10 +54,12 @@ def merge_sync_pr(gh, sync_pr): def set_sync_status(gh, pr_info, sync_pr): if not sync_pr or not sync_pr.mergeable: + print("Sync PR is not mergeable") post_commit_status( get_commit(gh, pr_info.sha), FAILURE, "", "Sync PR failure", "A Sync" ) else: + print("Sync PR is mergeable") post_commit_status(get_commit(gh, pr_info.sha), SUCCESS, "", "", "A Sync") From 5ebdcf06ca97543c9262c4271b48bfe3e0321062 Mon Sep 17 00:00:00 2001 From: Artur Malchanau Date: Sat, 20 Apr 2024 12:33:52 +0300 Subject: [PATCH 2/6] Fix taking full part if part contains less than 'limit' rows Do not take full part if part contains less than 'limit' rows. --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 6bdd060513c..662e833f090 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -987,7 +987,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( /// We take full part if it contains enough marks or /// if we know limit and part contains less than 'limit' rows. - bool take_full_part = marks_in_part <= need_marks || (input_order_info->limit && input_order_info->limit < part.getRowsCount()); + bool take_full_part = marks_in_part <= need_marks || (input_order_info->limit && input_order_info->limit > part.getRowsCount()); /// We take the whole part if it is small enough. if (take_full_part) From 0575d02b81de8f4efb59e9ddca88ac91833d96b9 Mon Sep 17 00:00:00 2001 From: Artur Malchanau Date: Sat, 20 Apr 2024 23:20:20 +0300 Subject: [PATCH 3/6] Fix 02499_monotonicity_toUnixTimestamp64 test Fix 02499_monotonicity_toUnixTimestamp64 test by using max_threads=1 to make read_rows more predictable. --- .../0_stateless/02499_monotonicity_toUnixTimestamp64.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh b/tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh index 59b6e2abb06..aa1f635f380 100755 --- a/tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh +++ b/tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh @@ -13,8 +13,7 @@ $CLICKHOUSE_CLIENT -q "create table t(ts DateTime64) engine=MergeTree order by t max_block_size=8192 query_id="${CLICKHOUSE_DATABASE}_02499_$RANDOM$RANDOM" -$CLICKHOUSE_CLIENT --query_id="$query_id" -q "select ts from t order by toUnixTimestamp64Nano(ts) limit 10 format Null settings max_block_size = $max_block_size, optimize_read_in_order = 1;" +$CLICKHOUSE_CLIENT --query_id="$query_id" -q "select ts from t order by toUnixTimestamp64Nano(ts) limit 10 format Null settings max_block_size = $max_block_size, optimize_read_in_order = 1, max_threads = 1;" $CLICKHOUSE_CLIENT -q "system flush logs;" $CLICKHOUSE_CLIENT --param_query_id="$query_id" -q "select read_rows <= $max_block_size from system.query_log where event_date >= yesterday() and current_database = '$CLICKHOUSE_DATABASE' and query_id = {query_id:String} and type = 'QueryFinish';" - From d3d0d4f6c02a79f40da6c4ae31c23ae7892a0e89 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 22 Apr 2024 11:36:27 +0100 Subject: [PATCH 4/6] empty commit From 15594c58ad49f0c46ff8e48678749289700d438d Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 24 Apr 2024 18:54:07 +0200 Subject: [PATCH 5/6] empty commit From 049c429b1f8c1e4edec077043870b9203e29aef7 Mon Sep 17 00:00:00 2001 From: Max K Date: Wed, 24 Apr 2024 19:19:27 +0200 Subject: [PATCH 6/6] Revert "CI: add FT to MQ remove Style from master" --- .github/workflows/master.yml | 21 ++++++++++----------- tests/ci/ci.py | 30 +++++++++++++++++------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index c5acd183751..100ec3b3b2c 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -59,17 +59,16 @@ jobs: uses: ./.github/workflows/reusable_docker.yml with: data: ${{ needs.RunConfig.outputs.data }} - # Tested in MQ - # StyleCheck: - # needs: [RunConfig, BuildDockers] - # if: ${{ !failure() && !cancelled() }} - # uses: ./.github/workflows/reusable_test.yml - # with: - # test_name: Style check - # runner_type: style-checker - # data: ${{ needs.RunConfig.outputs.data }} - # run_command: | - # python3 style_check.py --no-push + StyleCheck: + needs: [RunConfig, BuildDockers] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Style check + runner_type: style-checker + data: ${{ needs.RunConfig.outputs.data }} + run_command: | + python3 style_check.py --no-push CompatibilityCheckX86: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 7a7082c9c4a..24d6d95bd0d 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1261,6 +1261,18 @@ def _print_results(result: Any, outfile: Optional[str], pretty: bool = False) -> raise AssertionError(f"Unexpected type for 'res': {type(result)}") +def _update_config_for_docs_only(jobs_data: dict) -> None: + DOCS_CHECK_JOBS = [JobNames.DOCS_CHECK, JobNames.STYLE_CHECK] + print(f"NOTE: Will keep only docs related jobs: [{DOCS_CHECK_JOBS}]") + jobs_to_do = jobs_data.get("jobs_to_do", []) + jobs_data["jobs_to_do"] = [job for job in jobs_to_do if job in DOCS_CHECK_JOBS] + jobs_data["jobs_to_wait"] = { + job: params + for job, params in jobs_data["jobs_to_wait"].items() + if job in DOCS_CHECK_JOBS + } + + def _configure_docker_jobs(docker_digest_or_latest: bool) -> Dict: print("::group::Docker images check") # generate docker jobs data @@ -1320,20 +1332,8 @@ def _configure_jobs( jobs_to_skip: List[str] = [] digests: Dict[str, str] = {} - # FIXME: find better place for these config variables - DOCS_CHECK_JOBS = [JobNames.DOCS_CHECK, JobNames.STYLE_CHECK] - MQ_JOBS = [JobNames.STYLE_CHECK, JobNames.FAST_TEST] - if pr_info.has_changes_in_documentation_only(): - print(f"WARNING: Only docs are changed - will run only [{DOCS_CHECK_JOBS}]") - if pr_info.is_merge_queue(): - print(f"WARNING: It's a MQ run - will run only [{MQ_JOBS}]") - print("::group::Job Digests") for job in CI_CONFIG.job_generator(pr_info.head_ref if CI else "dummy_branch_name"): - if pr_info.is_merge_queue() and job not in MQ_JOBS: - continue - if pr_info.has_changes_in_documentation_only() and job not in DOCS_CHECK_JOBS: - continue digest = job_digester.get_job_digest(CI_CONFIG.get_digest_config(job)) digests[job] = digest print(f" job [{job.rjust(50)}] has digest [{digest}]") @@ -1902,6 +1902,9 @@ def main() -> int: else {} ) + if not args.skip_jobs and pr_info.has_changes_in_documentation_only(): + _update_config_for_docs_only(jobs_data) + if not args.skip_jobs: ci_cache = CiCache(s3, jobs_data["digests"]) @@ -1925,7 +1928,8 @@ def main() -> int: jobs_to_skip.append(job) del jobs_params[job] - # set planned jobs as in-progress in CI cache + # set planned jobs as pending in the CI cache if on the master + if pr_info.is_master: for job in jobs_data["jobs_to_do"]: config = CI_CONFIG.get_job_config(job) if config.run_always or config.run_by_label: