mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Merge pull request #63069 from azat/tests/integration-old-analyzer
Fix integration tests with old analyzer (and fix some leftovers of enabling it)
This commit is contained in:
Alexey Milovidov
GitHub
commit
d9fa3fdceb
@ -16,7 +16,7 @@
|
||||
#ci_set_reduced
|
||||
#ci_set_arm
|
||||
#ci_set_integration
|
||||
#ci_set_analyzer
|
||||
#ci_set_old_analyzer
|
||||
|
||||
## To run specified job in CI:
|
||||
#job_<JOB NAME>
|
||||
|
||||
@ -49,7 +49,7 @@ class CILabels(metaclass=WithIter):
|
||||
CI_SET_FAST = "ci_set_fast"
|
||||
CI_SET_ARM = "ci_set_arm"
|
||||
CI_SET_INTEGRATION = "ci_set_integration"
|
||||
CI_SET_ANALYZER = "ci_set_analyzer"
|
||||
CI_SET_OLD_ANALYZER = "ci_set_old_analyzer"
|
||||
CI_SET_STATLESS = "ci_set_stateless"
|
||||
CI_SET_STATEFUL = "ci_set_stateful"
|
||||
CI_SET_STATLESS_ASAN = "ci_set_stateless_asan"
|
||||
@ -98,11 +98,11 @@ class JobNames(metaclass=WithIter):
|
||||
STATELESS_TEST_TSAN = "Stateless tests (tsan)"
|
||||
STATELESS_TEST_MSAN = "Stateless tests (msan)"
|
||||
STATELESS_TEST_UBSAN = "Stateless tests (ubsan)"
|
||||
STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE = (
|
||||
STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE = (
|
||||
"Stateless tests (release, old analyzer, s3, DatabaseReplicated)"
|
||||
)
|
||||
# merged into STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE:
|
||||
# STATELESS_TEST_ANALYZER_RELEASE = "Stateless tests (release, analyzer)"
|
||||
# merged into STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE:
|
||||
# STATELESS_TEST_OLD_ANALYZER_RELEASE = "Stateless tests (release, analyzer)"
|
||||
# STATELESS_TEST_DB_REPL_RELEASE = "Stateless tests (release, DatabaseReplicated)"
|
||||
# STATELESS_TEST_S3_RELEASE = "Stateless tests (release, s3 storage)"
|
||||
STATELESS_TEST_S3_DEBUG = "Stateless tests (debug, s3 storage)"
|
||||
@ -134,7 +134,7 @@ class JobNames(metaclass=WithIter):
|
||||
|
||||
INTEGRATION_TEST = "Integration tests (release)"
|
||||
INTEGRATION_TEST_ASAN = "Integration tests (asan)"
|
||||
INTEGRATION_TEST_ASAN_ANALYZER = "Integration tests (asan, old analyzer)"
|
||||
INTEGRATION_TEST_ASAN_OLD_ANALYZER = "Integration tests (asan, old analyzer)"
|
||||
INTEGRATION_TEST_TSAN = "Integration tests (tsan)"
|
||||
INTEGRATION_TEST_ARM = "Integration tests (aarch64)"
|
||||
INTEGRATION_TEST_FLAKY = "Integration tests flaky check (asan)"
|
||||
@ -848,14 +848,14 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.INTEGRATION_TEST,
|
||||
]
|
||||
),
|
||||
CILabels.CI_SET_ANALYZER: LabelConfig(
|
||||
CILabels.CI_SET_OLD_ANALYZER: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
JobNames.FAST_TEST,
|
||||
Build.PACKAGE_RELEASE,
|
||||
Build.PACKAGE_ASAN,
|
||||
JobNames.STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE,
|
||||
JobNames.INTEGRATION_TEST_ASAN_ANALYZER,
|
||||
JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE,
|
||||
JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER,
|
||||
]
|
||||
),
|
||||
CILabels.CI_SET_STATLESS: LabelConfig(
|
||||
@ -1195,7 +1195,7 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.STATELESS_TEST_AARCH64: TestConfig(
|
||||
Build.PACKAGE_AARCH64, job_config=JobConfig(**statless_test_common_params) # type: ignore
|
||||
),
|
||||
JobNames.STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE: TestConfig(
|
||||
JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE: TestConfig(
|
||||
Build.PACKAGE_RELEASE,
|
||||
job_config=JobConfig(num_batches=4, **statless_test_common_params), # type: ignore
|
||||
),
|
||||
@ -1245,7 +1245,7 @@ CI_CONFIG = CIConfig(
|
||||
Build.PACKAGE_ASAN,
|
||||
job_config=JobConfig(num_batches=4, **integration_test_common_params, release_only=True), # type: ignore
|
||||
),
|
||||
JobNames.INTEGRATION_TEST_ASAN_ANALYZER: TestConfig(
|
||||
JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: TestConfig(
|
||||
Build.PACKAGE_ASAN,
|
||||
job_config=JobConfig(num_batches=6, **integration_test_common_params), # type: ignore
|
||||
),
|
||||
@ -1377,8 +1377,8 @@ REQUIRED_CHECKS = [
|
||||
JobNames.UNIT_TEST,
|
||||
JobNames.UNIT_TEST_TSAN,
|
||||
JobNames.UNIT_TEST_UBSAN,
|
||||
JobNames.INTEGRATION_TEST_ASAN_ANALYZER,
|
||||
JobNames.STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE,
|
||||
JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER,
|
||||
JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE,
|
||||
]
|
||||
|
||||
|
||||
|
||||
@ -265,7 +265,9 @@ class ClickhouseIntegrationTestsRunner:
|
||||
self.start_time = time.time()
|
||||
self.soft_deadline_time = self.start_time + (TASK_TIMEOUT - MAX_TIME_IN_SANDBOX)
|
||||
|
||||
self.use_analyzer = os.environ.get("CLICKHOUSE_USE_OLD_ANALYZER") is not None
|
||||
self.use_old_analyzer = (
|
||||
os.environ.get("CLICKHOUSE_USE_OLD_ANALYZER") is not None
|
||||
)
|
||||
|
||||
if "run_by_hash_total" in self.params:
|
||||
self.run_by_hash_total = self.params["run_by_hash_total"]
|
||||
@ -414,8 +416,8 @@ class ClickhouseIntegrationTestsRunner:
|
||||
result.append("--tmpfs")
|
||||
if self.disable_net_host:
|
||||
result.append("--disable-net-host")
|
||||
if self.use_analyzer:
|
||||
result.append("--analyzer")
|
||||
if self.use_old_analyzer:
|
||||
result.append("--old-analyzer")
|
||||
|
||||
return " ".join(result)
|
||||
|
||||
|
||||
@ -1601,7 +1601,7 @@ class ClickHouseCluster:
|
||||
with_jdbc_bridge=False,
|
||||
with_hive=False,
|
||||
with_coredns=False,
|
||||
use_old_analyzer=False,
|
||||
use_old_analyzer=None,
|
||||
hostname=None,
|
||||
env_variables=None,
|
||||
instance_env_variables=False,
|
||||
@ -4405,8 +4405,18 @@ class ClickHouseInstance:
|
||||
)
|
||||
|
||||
write_embedded_config("0_common_instance_users.xml", users_d_dir)
|
||||
if self.use_old_analyzer:
|
||||
write_embedded_config("0_common_enable_analyzer.xml", users_d_dir)
|
||||
|
||||
use_old_analyzer = os.environ.get("CLICKHOUSE_USE_OLD_ANALYZER") is not None
|
||||
# If specific version was used there can be no
|
||||
# allow_experimental_analyzer setting, so do this only if it was
|
||||
# explicitly requested.
|
||||
if self.tag:
|
||||
use_old_analyzer = False
|
||||
# Prefer specified in the test option:
|
||||
if self.use_old_analyzer is not None:
|
||||
use_old_analyzer = self.use_old_analyzer
|
||||
if use_old_analyzer:
|
||||
write_embedded_config("0_common_enable_old_analyzer.xml", users_d_dir)
|
||||
|
||||
if len(self.custom_dictionaries_paths):
|
||||
write_embedded_config("0_common_enable_dictionaries.xml", self.config_d_dir)
|
||||
|
||||
@ -285,11 +285,11 @@ if __name__ == "__main__":
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--analyzer",
|
||||
"--old-analyzer",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="analyzer",
|
||||
help="Use new analyzer infrastructure",
|
||||
dest="old_analyzer",
|
||||
help="Use old analyzer infrastructure",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
@ -385,9 +385,9 @@ if __name__ == "__main__":
|
||||
if args.keyword_expression:
|
||||
args.pytest_args += ["-k", args.keyword_expression]
|
||||
|
||||
use_analyzer = ""
|
||||
if args.analyzer:
|
||||
use_analyzer = "-e CLICKHOUSE_USE_OLD_ANALYZER=1"
|
||||
use_old_analyzer = ""
|
||||
if args.old_analyzer:
|
||||
use_old_analyzer = "-e CLICKHOUSE_USE_OLD_ANALYZER=1"
|
||||
|
||||
# NOTE: since pytest options is in the argument value already we need to additionally escape '"'
|
||||
pytest_opts = " ".join(
|
||||
@ -408,7 +408,7 @@ if __name__ == "__main__":
|
||||
f"--volume={args.utils_dir}/backupview:/ClickHouse/utils/backupview "
|
||||
f"--volume={args.utils_dir}/grpc-client/pb2:/ClickHouse/utils/grpc-client/pb2 "
|
||||
f"--volume=/run:/run/host:ro {dockerd_internal_volume} {env_tags} {env_cleanup} "
|
||||
f"-e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 {use_analyzer} -e PYTHONUNBUFFERED=1 "
|
||||
f"-e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 {use_old_analyzer} -e PYTHONUNBUFFERED=1 "
|
||||
f'-e PYTEST_ADDOPTS="{parallel_args} {pytest_opts} {tests_list} {rand_args} -vvv"'
|
||||
f" {DIND_INTEGRATION_TESTS_IMAGE_NAME}:{args.docker_image_version}"
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user