thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #50926 from ClickHouse/analyzer-integration-tests

Browse Source 
Add integration test check with enabled analyzer
This commit is contained in:
Dmitry Novik 2023-07-13 10:59:39 +02:00 committed by GitHub
commit 5c73a0b19c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 502 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
.github/workflows/pull_request.yml vendored
View File

@ -3903,6 +3903,216 @@ jobs:
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAnalyzerAsan0:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, analyzer)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAnalyzerAsan1:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, analyzer)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAnalyzerAsan2:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, analyzer)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAnalyzerAsan3:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, analyzer)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAnalyzerAsan4:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, analyzer)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=4
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsAnalyzerAsan5:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/integration_tests_asan
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Integration tests (asan, analyzer)
REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse
RUN_BY_HASH_NUM=5
RUN_BY_HASH_TOTAL=6
EOF
- name: Download json reports
uses: actions/download-artifact@v3
with:
path: ${{ env.REPORTS_PATH }}
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
clear-repository: true
- name: Integration test
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker ps --quiet | xargs --no-run-if-empty docker kill ||:
docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
sudo rm -fr "$TEMP_PATH"
IntegrationTestsTsan0:
needs: [BuilderDebTsan]
runs-on: [self-hosted, stress-tester]

198
tests/analyzer_integration_broken_tests.txt Normal file
View File

@ -0,0 +1,198 @@
test_access_for_functions/test.py::test_access_rights_for_function
test_backward_compatibility/test_normalized_count_comparison.py::test_select_aggregate_alias_column
test_concurrent_backups_s3/test.py::test_concurrent_backups
test_distributed_ddl/test.py::test_default_database[configs]
test_distributed_ddl/test.py::test_default_database[configs_secure]
test_distributed_ddl/test.py::test_on_server_fail[configs]
test_distributed_ddl/test.py::test_on_server_fail[configs_secure]
test_distributed_insert_backward_compatibility/test.py::test_distributed_in_tuple
test_distributed_inter_server_secret/test.py::test_per_user_inline_settings_secure_cluster[default-]
test_distributed_inter_server_secret/test.py::test_per_user_inline_settings_secure_cluster[nopass-]
test_distributed_inter_server_secret/test.py::test_per_user_inline_settings_secure_cluster[pass-foo]
test_distributed_inter_server_secret/test.py::test_per_user_protocol_settings_secure_cluster[default-]
test_distributed_inter_server_secret/test.py::test_per_user_protocol_settings_secure_cluster[nopass-]
test_distributed_inter_server_secret/test.py::test_per_user_protocol_settings_secure_cluster[pass-foo]
test_distributed_inter_server_secret/test.py::test_user_insecure_cluster[default-]
test_distributed_inter_server_secret/test.py::test_user_insecure_cluster[nopass-]
test_distributed_inter_server_secret/test.py::test_user_insecure_cluster[pass-foo]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster[default-]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster[nopass-]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster[pass-foo]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster_from_backward[default-]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster_from_backward[nopass-]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster_from_backward[pass-foo]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster_with_backward[default-]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster_with_backward[nopass-]
test_distributed_inter_server_secret/test.py::test_user_secure_cluster_with_backward[pass-foo]
test_distributed_load_balancing/test.py::test_distributed_replica_max_ignored_errors
test_distributed_load_balancing/test.py::test_load_balancing_default
test_distributed_load_balancing/test.py::test_load_balancing_priority_round_robin[dist_priority]
test_distributed_load_balancing/test.py::test_load_balancing_priority_round_robin[dist_priority_negative]
test_distributed_load_balancing/test.py::test_load_balancing_round_robin
test_backward_compatibility/test.py::test_backward_compatability1
test_backward_compatibility/test_aggregate_fixed_key.py::test_two_level_merge
test_backward_compatibility/test_aggregate_function_state.py::test_backward_compatability_for_avg
test_backward_compatibility/test_aggregate_function_state.py::test_backward_compatability_for_uniq_exact[1000]
test_backward_compatibility/test_aggregate_function_state.py::test_backward_compatability_for_uniq_exact[500000]
test_backward_compatibility/test_aggregate_function_state.py::test_backward_compatability_for_uniq_exact_variadic[1000]
test_backward_compatibility/test_aggregate_function_state.py::test_backward_compatability_for_uniq_exact_variadic[500000]
test_backward_compatibility/test_ip_types_binary_compatibility.py::test_ip_types_binary_compatibility
test_backward_compatibility/test_select_aggregate_alias_column.py::test_select_aggregate_alias_column
test_backward_compatibility/test_short_strings_aggregation.py::test_backward_compatability
test_mask_sensitive_info/test.py::test_encryption_functions
test_merge_table_over_distributed/test.py::test_global_in
test_merge_table_over_distributed/test.py::test_select_table_name_from_merge_over_distributed
test_mutations_with_merge_tree/test.py::test_mutations_with_merge_background_task
test_passing_max_partitions_to_read_remotely/test.py::test_default_database_on_cluster
test_row_policy/test.py::test_change_of_users_xml_changes_row_policies
test_row_policy/test.py::test_change_of_users_xml_changes_row_policies
test_row_policy/test.py::test_dcl_introspection
test_row_policy/test.py::test_dcl_introspection
test_row_policy/test.py::test_dcl_management
test_row_policy/test.py::test_dcl_management
test_row_policy/test.py::test_dcl_users_with_policies_from_users_xml
test_row_policy/test.py::test_dcl_users_with_policies_from_users_xml
test_row_policy/test.py::test_grant_create_row_policy
test_row_policy/test.py::test_grant_create_row_policy
test_row_policy/test.py::test_introspection
test_row_policy/test.py::test_introspection
test_row_policy/test.py::test_join
test_row_policy/test.py::test_join
test_row_policy/test.py::test_miscellaneous_engines
test_row_policy/test.py::test_miscellaneous_engines
test_row_policy/test.py::test_policy_from_users_xml_affects_only_user_assigned
test_row_policy/test.py::test_policy_from_users_xml_affects_only_user_assigned
test_row_policy/test.py::test_policy_on_distributed_table_via_role
test_row_policy/test.py::test_policy_on_distributed_table_via_role
test_row_policy/test.py::test_reload_users_xml_by_timer
test_row_policy/test.py::test_reload_users_xml_by_timer
test_row_policy/test.py::test_row_policy_filter_with_subquery
test_row_policy/test.py::test_row_policy_filter_with_subquery
test_row_policy/test.py::test_smoke
test_row_policy/test.py::test_smoke
test_row_policy/test.py::test_some_users_without_policies
test_row_policy/test.py::test_some_users_without_policies
test_row_policy/test.py::test_tags_with_db_and_table_names
test_row_policy/test.py::test_tags_with_db_and_table_names
test_row_policy/test.py::test_throwif_error_in_prewhere_with_same_condition_as_filter
test_row_policy/test.py::test_throwif_error_in_prewhere_with_same_condition_as_filter
test_row_policy/test.py::test_throwif_error_in_where_with_same_condition_as_filter
test_row_policy/test.py::test_throwif_error_in_where_with_same_condition_as_filter
test_row_policy/test.py::test_throwif_in_prewhere_doesnt_expose_restricted_data
test_row_policy/test.py::test_throwif_in_prewhere_doesnt_expose_restricted_data
test_row_policy/test.py::test_throwif_in_where_doesnt_expose_restricted_data
test_row_policy/test.py::test_throwif_in_where_doesnt_expose_restricted_data
test_row_policy/test.py::test_users_xml_is_readonly
test_row_policy/test.py::test_users_xml_is_readonly
test_row_policy/test.py::test_with_prewhere
test_row_policy/test.py::test_with_prewhere
test_settings_constraints_distributed/test.py::test_select_clamps_settings
test_backward_compatibility/test_cte_distributed.py::test_cte_distributed
test_compression_codec_read/test.py::test_default_codec_read
test_dictionaries_update_and_reload/test.py::test_reload_after_fail_in_cache_dictionary
test_distributed_type_object/test.py::test_distributed_type_object
test_materialized_mysql_database/test.py::test_select_without_columns_5_7
test_materialized_mysql_database/test.py::test_select_without_columns_8_0
test_shard_level_const_function/test.py::test_remote
test_storage_postgresql/test.py::test_postgres_select_insert
test_storage_rabbitmq/test.py::test_rabbitmq_materialized_view
test_system_merges/test.py::test_mutation_simple[]
test_system_merges/test.py::test_mutation_simple[replicated]
test_backward_compatibility/test_insert_profile_events.py::test_new_client_compatible
test_backward_compatibility/test_insert_profile_events.py::test_old_client_compatible
test_backward_compatibility/test_vertical_merges_from_compact_parts.py::test_vertical_merges_from_compact_parts
test_disk_over_web_server/test.py::test_cache[node2]
test_disk_over_web_server/test.py::test_incorrect_usage
test_disk_over_web_server/test.py::test_replicated_database
test_disk_over_web_server/test.py::test_unavailable_server
test_disk_over_web_server/test.py::test_usage[node2]
test_distributed_backward_compatability/test.py::test_distributed_in_tuple
test_executable_table_function/test.py::test_executable_function_input_python
test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py::test_groupBitmapAndState_on_different_version_nodes
test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py::test_groupBitmapAndState_on_distributed_table
test_settings_profile/test.py::test_show_profiles
test_sql_user_defined_functions_on_cluster/test.py::test_sql_user_defined_functions_on_cluster
test_backward_compatibility/test_functions.py::test_aggregate_states
test_backward_compatibility/test_functions.py::test_string_functions
test_default_compression_codec/test.py::test_default_codec_for_compact_parts
test_default_compression_codec/test.py::test_default_codec_multiple
test_default_compression_codec/test.py::test_default_codec_single
test_default_compression_codec/test.py::test_default_codec_version_update
test_postgresql_protocol/test.py::test_python_client
test_quota/test.py::test_add_remove_interval
test_quota/test.py::test_add_remove_quota
test_quota/test.py::test_consumption_of_show_clusters
test_quota/test.py::test_consumption_of_show_databases
test_quota/test.py::test_consumption_of_show_privileges
test_quota/test.py::test_consumption_of_show_processlist
test_quota/test.py::test_consumption_of_show_tables
test_quota/test.py::test_dcl_introspection
test_quota/test.py::test_dcl_management
test_quota/test.py::test_exceed_quota
test_quota/test.py::test_query_inserts
test_quota/test.py::test_quota_from_users_xml
test_quota/test.py::test_reload_users_xml_by_timer
test_quota/test.py::test_simpliest_quota
test_quota/test.py::test_tracking_quota
test_quota/test.py::test_users_xml_is_readonly
test_replicated_merge_tree_compatibility/test.py::test_replicated_merge_tree_defaults_compatibility
test_polymorphic_parts/test.py::test_different_part_types_on_replicas[polymorphic_table_wide-Wide]
test_old_versions/test.py::test_client_is_older_than_server
test_polymorphic_parts/test.py::test_polymorphic_parts_non_adaptive
test_old_versions/test.py::test_server_is_older_than_client
test_polymorphic_parts/test.py::test_compact_parts_only
test_polymorphic_parts/test.py::test_different_part_types_on_replicas[polymorphic_table_compact-Compact]
test_polymorphic_parts/test.py::test_polymorphic_parts_index
test_old_versions/test.py::test_distributed_query_initiator_is_older_than_shard
test_polymorphic_parts/test.py::test_polymorphic_parts_basics[first_node1-second_node1]
test_polymorphic_parts/test.py::test_polymorphic_parts_basics[first_node0-second_node0]
test_ttl_replicated/test.py::test_ttl_table[DELETE]
test_ttl_replicated/test.py::test_ttl_columns
test_ttl_replicated/test.py::test_ttl_compatibility[node_left2-node_right2-2]
test_ttl_replicated/test.py::test_ttl_table[]
test_version_update/test.py::test_aggregate_function_versioning_server_upgrade
test_version_update/test.py::test_aggregate_function_versioning_fetch_data_from_old_to_new_server
test_ttl_replicated/test.py::test_ttl_double_delete_rule_returns_error
test_ttl_replicated/test.py::test_ttl_alter_delete[test_ttl_alter_delete]
test_ttl_replicated/test.py::test_ttl_alter_delete[test_ttl_alter_delete_replicated]
test_ttl_replicated/test.py::test_ttl_compatibility[node_left0-node_right0-0]
test_version_update/test.py::test_modulo_partition_key_issue_23508
test_ttl_replicated/test.py::test_ttl_many_columns
test_ttl_replicated/test.py::test_modify_column_ttl
test_ttl_replicated/test.py::test_merge_with_ttl_timeout
test_ttl_replicated/test.py::test_ttl_empty_parts
test_ttl_replicated/test.py::test_ttl_compatibility[node_left1-node_right1-1]
test_version_update/test.py::test_aggregate_function_versioning_persisting_metadata
test_version_update/test.py::test_aggregate_function_versioning_issue_16587
test_ttl_replicated/test.py::test_modify_ttl
test_mysql_database_engine/test.py::test_mysql_ddl_for_mysql_database
test_profile_events_s3/test.py::test_profile_events
test_version_update_after_mutation/test.py::test_upgrade_while_mutation
test_version_update_after_mutation/test.py::test_mutate_and_upgrade
test_system_flush_logs/test.py::test_system_logs[system.text_log-0]
test_user_defined_object_persistence/test.py::test_persistence
test_settings_profile/test.py::test_show_profiles
test_sql_user_defined_functions_on_cluster/test.py::test_sql_user_defined_functions_on_cluster
test_select_access_rights/test_main.py::test_alias_columns
test_select_access_rights/test_main.py::test_select_count
test_select_access_rights/test_main.py::test_select_join
test_replicated_merge_tree_compatibility/test.py::test_replicated_merge_tree_defaults_compatibility
test_postgresql_protocol/test.py::test_python_client
test_quota/test.py::test_add_remove_interval
test_quota/test.py::test_add_remove_quota
test_quota/test.py::test_consumption_of_show_clusters
test_quota/test.py::test_consumption_of_show_databases
test_quota/test.py::test_consumption_of_show_privileges
test_quota/test.py::test_consumption_of_show_processlist
test_quota/test.py::test_consumption_of_show_tables
test_quota/test.py::test_dcl_introspection
test_quota/test.py::test_dcl_management
test_quota/test.py::test_exceed_quota
test_quota/test.py::test_query_inserts
test_quota/test.py::test_quota_from_users_xml
test_quota/test.py::test_reload_users_xml_by_timer
test_quota/test.py::test_simpliest_quota
test_quota/test.py::test_tracking_quota
test_quota/test.py::test_users_xml_is_readonly
test_replicating_constants/test.py::test_different_versions
test_merge_tree_s3/test.py::test_heavy_insert_select_check_memory[node]

3
tests/ci/ci_config.py
View File

@ -336,6 +336,9 @@ CI_CONFIG = {
"Integration tests (asan)": {
"required_build": "package_asan",
},
"Integration tests (asan, analyzer)": {
"required_build": "package_asan",
},
"Integration tests (tsan)": {
"required_build": "package_tsan",
},

9
tests/ci/integration_test_check.py
View File

@ -71,7 +71,7 @@ def get_json_params_dict(
}
def get_env_for_runner(build_path, repo_path, result_path, work_path):
def get_env_for_runner(check_name, build_path, repo_path, result_path, work_path):
binary_path = os.path.join(build_path, "clickhouse")
odbc_bridge_path = os.path.join(build_path, "clickhouse-odbc-bridge")
library_bridge_path = os.path.join(build_path, "clickhouse-library-bridge")
@ -88,6 +88,9 @@ def get_env_for_runner(build_path, repo_path, result_path, work_path):
my_env["CLICKHOUSE_TESTS_JSON_PARAMS_PATH"] = os.path.join(work_path, "params.json")
my_env["CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER"] = "0"
if "analyzer" in check_name.lower():
my_env["CLICKHOUSE_USE_NEW_ANALYZER"] = "1"
return my_env
@ -225,7 +228,9 @@ def main():
else:
download_all_deb_packages(check_name, reports_path, build_path)
my_env = get_env_for_runner(build_path, repo_path, result_path, work_path)
my_env = get_env_for_runner(
check_name, build_path, repo_path, result_path, work_path
)
json_path = os.path.join(work_path, "params.json")
with open(json_path, "w", encoding="utf-8") as json_params:

67
tests/integration/ci-runner.py
View File

@ -239,6 +239,8 @@ class ClickhouseIntegrationTestsRunner:
self.start_time = time.time()
self.soft_deadline_time = self.start_time + (TASK_TIMEOUT - MAX_TIME_IN_SANDBOX)
self.use_analyzer = os.environ.get("CLICKHOUSE_USE_NEW_ANALYZER") is not None
if "run_by_hash_total" in self.params:
self.run_by_hash_total = self.params["run_by_hash_total"]
self.run_by_hash_num = self.params["run_by_hash_num"]
@ -398,6 +400,9 @@ class ClickhouseIntegrationTestsRunner:
result.append("--tmpfs")
if self.disable_net_host:
result.append("--disable-net-host")
if self.use_analyzer:
result.append("--analyzer")
return " ".join(result)
def _get_all_tests(self, repo_path):
@ -480,7 +485,7 @@ class ClickhouseIntegrationTestsRunner:
result[test_file].append(test)
return result
def _update_counters(self, main_counters, current_counters):
def _update_counters(self, main_counters, current_counters, broken_tests):
for test in current_counters["PASSED"]:
if (
test not in main_counters["PASSED"]
@ -493,10 +498,17 @@ class ClickhouseIntegrationTestsRunner:
if test in main_counters["ERROR"]:
main_counters["ERROR"].remove(test)
is_flaky = True
if test in main_counters["BROKEN"]:
main_counters["BROKEN"].remove(test)
is_flaky = True
if is_flaky:
main_counters["FLAKY"].append(test)
else:
if test not in broken_tests:
main_counters["PASSED"].append(test)
else:
main_counters["NOT_FAILED"].append(test)
for state in ("ERROR", "FAILED"):
for test in current_counters[state]:
@ -506,8 +518,12 @@ class ClickhouseIntegrationTestsRunner:
main_counters["PASSED"].remove(test)
main_counters["FLAKY"].append(test)
continue
if test not in broken_tests:
if test not in main_counters[state]:
main_counters[state].append(test)
else:
if test not in main_counters["BROKEN"]:
main_counters["BROKEN"].append(test)
for state in ("SKIPPED",):
for test in current_counters[state]:
@ -565,11 +581,22 @@ class ClickhouseIntegrationTestsRunner:
return res
def try_run_test_group(
self, repo_path, test_group, tests_in_group, num_tries, num_workers
self,
repo_path,
test_group,
tests_in_group,
num_tries,
num_workers,
broken_tests,
):
try:
return self.run_test_group(
repo_path, test_group, tests_in_group, num_tries, num_workers
repo_path,
test_group,
tests_in_group,
num_tries,
num_workers,
broken_tests,
)
except Exception as e:
logging.info("Failed to run {}:\n{}".format(str(test_group), str(e)))
@ -587,7 +614,13 @@ class ClickhouseIntegrationTestsRunner:
return counters, tests_times, []
def run_test_group(
self, repo_path, test_group, tests_in_group, num_tries, num_workers
self,
repo_path,
test_group,
tests_in_group,
num_tries,
num_workers,
broken_tests,
):
counters = {
"ERROR": [],
@ -595,6 +628,8 @@ class ClickhouseIntegrationTestsRunner:
"FAILED": [],
"SKIPPED": [],
"FLAKY": [],
"BROKEN": [],
"NOT_FAILED": [],
}
tests_times = defaultdict(float)
@ -700,7 +735,7 @@ class ClickhouseIntegrationTestsRunner:
)
times_lines = parse_test_times(info_path)
new_tests_times = get_test_times(times_lines)
self._update_counters(counters, new_counters)
self._update_counters(counters, new_counters, broken_tests)
for test_name, test_time in new_tests_times.items():
tests_times[test_name] = test_time
@ -773,7 +808,7 @@ class ClickhouseIntegrationTestsRunner:
final_retry += 1
logging.info("Running tests for the %s time", i)
counters, tests_times, log_paths = self.try_run_test_group(
repo_path, "bugfix" if should_fail else "flaky", tests_to_run, 1, 1
repo_path, "bugfix" if should_fail else "flaky", tests_to_run, 1, 1, []
)
logs += log_paths
if counters["FAILED"]:
@ -894,6 +929,8 @@ class ClickhouseIntegrationTestsRunner:
"FAILED": [],
"SKIPPED": [],
"FLAKY": [],
"BROKEN": [],
"NOT_FAILED": [],
}
tests_times = defaultdict(float)
tests_log_paths = defaultdict(list)
@ -905,10 +942,16 @@ class ClickhouseIntegrationTestsRunner:
logging.info("Shuffling test groups")
random.shuffle(items_to_run)
broken_tests = list()
if self.use_analyzer:
with open(f"{repo_path}/tests/analyzer_integration_broken_tests.txt") as f:
broken_tests = f.read().splitlines()
logging.info(f"Broken tests in the list: {len(broken_tests)}")
for group, tests in items_to_run:
logging.info("Running test group %s containing %s tests", group, len(tests))
group_counters, group_test_times, log_paths = self.try_run_test_group(
repo_path, group, tests, MAX_RETRY, NUM_WORKERS
repo_path, group, tests, MAX_RETRY, NUM_WORKERS, broken_tests
)
total_tests = 0
for counter, value in group_counters.items():
@ -940,7 +983,15 @@ class ClickhouseIntegrationTestsRunner:
result_state = "success"
test_result = []
for state in ("ERROR", "FAILED", "PASSED", "SKIPPED", "FLAKY"):
for state in (
"ERROR",
"FAILED",
"PASSED",
"SKIPPED",
"FLAKY",
"BROKEN",
"NOT_FAILED",
):
if state == "PASSED":
text_state = "OK"
elif state == "FAILED":

7
tests/integration/helpers/0_common_enable_analyzer.xml Normal file
View File

@ -0,0 +1,7 @@
<clickhouse>
<profiles>
<default>
<allow_experimental_analyzer>1</allow_experimental_analyzer>
</default>
</profiles>
</clickhouse>

2
tests/integration/helpers/cluster.py
View File

@ -4190,6 +4190,8 @@ class ClickHouseInstance:
)
write_embedded_config("0_common_instance_users.xml", users_d_dir)
if os.environ.get("CLICKHOUSE_USE_NEW_ANALYZER") is not None:
write_embedded_config("0_common_enable_analyzer.xml", users_d_dir)
if len(self.custom_dictionaries_paths):
write_embedded_config("0_common_enable_dictionaries.xml", self.config_d_dir)

14
tests/integration/runner
View File

@ -283,6 +283,14 @@ if __name__ == "__main__":
help="Use tmpfs for dockerd files",
)
parser.add_argument(
"--analyzer",
action="store_true",
default=False,
dest="analyzer",
help="Use new analyzer infrastructure",
)
parser.add_argument(
"--cleanup-containers",
action="store_true",
@ -395,6 +403,10 @@ if __name__ == "__main__":
if args.keyword_expression:
args.pytest_args += ["-k", args.keyword_expression]
use_analyzer = ""
if args.analyzer:
use_analyzer = "-e CLICKHOUSE_USE_NEW_ANALYZER=1"
pytest_opts = " ".join(args.pytest_args).replace("'", "\\'")
tests_list = " ".join(args.tests_list)
cmd_base = (
@ -407,7 +419,7 @@ if __name__ == "__main__":
f"--volume={args.cases_dir}:/ClickHouse/tests/integration "
f"--volume={args.src_dir}/Server/grpc_protos:/ClickHouse/src/Server/grpc_protos "
f"--volume=/run:/run/host:ro {dockerd_internal_volume} {env_tags} {env_cleanup} "
"-e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 -e PYTHONUNBUFFERED=1 "
f"-e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 {use_analyzer} -e PYTHONUNBUFFERED=1 "
f"-e PYTEST_OPTS='{parallel_args} {pytest_opts} {tests_list} {rand_args} -vvv'"
f" {DIND_INTEGRATION_TESTS_IMAGE_NAME}:{args.docker_image_version}"
)