From 9b55180184c0f38cf88bf5902687603cf903ee40 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Fri, 9 Aug 2024 14:22:33 +0000 Subject: [PATCH] CI: Move out scripts from dockers --- docker/images.json | 7 +---- docker/test/fasttest/Dockerfile | 3 -- docker/test/stateful/Dockerfile | 4 --- docker/test/stateful/setup_minio.sh | 1 - docker/test/stateless/Dockerfile | 12 -------- docker/test/stress/Dockerfile | 3 -- docker/test/upgrade/Dockerfile | 29 ------------------- docker/test/util/Dockerfile | 2 -- tests/ci/.mypy.ini | 3 +- tests/ci/ci_config.py | 5 +++- tests/ci/ci_definitions.py | 7 +++-- tests/ci/docker_images_check.py | 2 +- tests/ci/fast_test_check.py | 7 ++--- tests/ci/functional_test_check.py | 20 ++++++------- tests/ci/stress_check.py | 17 +++++++---- tests/ci/upgrade_check.py | 2 +- .../docker_scripts}/attach_gdb.lib | 2 +- .../docker_scripts}/create.sql | 0 .../docker_scripts/fasttest_runner.sh | 2 +- .../process_functional_tests_result.py | 24 +++++++-------- .../docker_scripts}/setup_hdfs_minicluster.sh | 2 +- .../docker_scripts}/setup_minio.sh | 2 +- .../docker_scripts/stateful_runner.sh | 13 +++++---- .../docker_scripts/stateless_runner.sh | 22 +++++++++----- .../docker_scripts/stress_runner.sh | 21 +++++++------- .../docker_scripts}/stress_tests.lib | 2 +- .../docker_scripts/upgrade_runner.sh | 14 ++++----- .../docker_scripts}/utils.lib | 0 28 files changed, 94 insertions(+), 134 deletions(-) delete mode 120000 docker/test/stateful/setup_minio.sh delete mode 100644 docker/test/upgrade/Dockerfile rename {docker/test/stateless => tests/docker_scripts}/attach_gdb.lib (98%) rename {docker/test/stateful => tests/docker_scripts}/create.sql (100%) rename docker/test/fasttest/run.sh => tests/docker_scripts/fasttest_runner.sh (99%) rename {docker/test/util => tests/docker_scripts}/process_functional_tests_result.py (92%) rename {docker/test/stateless => tests/docker_scripts}/setup_hdfs_minicluster.sh (95%) rename {docker/test/stateless => tests/docker_scripts}/setup_minio.sh (98%) rename docker/test/stateful/run.sh => tests/docker_scripts/stateful_runner.sh (98%) rename docker/test/stateless/run.sh => tests/docker_scripts/stateless_runner.sh (97%) rename docker/test/stress/run.sh => tests/docker_scripts/stress_runner.sh (97%) mode change 100644 => 100755 rename {docker/test/stateless => tests/docker_scripts}/stress_tests.lib (99%) rename docker/test/upgrade/run.sh => tests/docker_scripts/upgrade_runner.sh (96%) mode change 100644 => 100755 rename {docker/test/stateless => tests/docker_scripts}/utils.lib (100%) diff --git a/docker/images.json b/docker/images.json index 716b76ee217..055394b69e6 100644 --- a/docker/images.json +++ b/docker/images.json @@ -47,8 +47,7 @@ "docker/test/stateful": { "name": "clickhouse/stateful-test", "dependent": [ - "docker/test/stress", - "docker/test/upgrade" + "docker/test/stress" ] }, "docker/test/unit": { @@ -59,10 +58,6 @@ "name": "clickhouse/stress-test", "dependent": [] }, - "docker/test/upgrade": { - "name": "clickhouse/upgrade-check", - "dependent": [] - }, "docker/test/integration/runner": { "name": "clickhouse/integration-tests-runner", "dependent": [] diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 5d311c673a4..ca93b24f66e 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -93,6 +93,3 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ENV COMMIT_SHA='' ENV PULL_REQUEST_NUMBER='' ENV COPY_CLICKHOUSE_BINARY_TO_OUTPUT=0 - -COPY run.sh / -CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 0daf88cad7e..9aa936cb069 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -10,7 +10,3 @@ RUN apt-get update -y \ npm \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* - -COPY create.sql / -COPY run.sh / -CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateful/setup_minio.sh b/docker/test/stateful/setup_minio.sh deleted file mode 120000 index 0d539f72cb3..00000000000 --- a/docker/test/stateful/setup_minio.sh +++ /dev/null @@ -1 +0,0 @@ -../stateless/setup_minio.sh \ No newline at end of file diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index b0c4914a4e8..69f81b35a95 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -85,18 +85,6 @@ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoo ENV MINIO_ROOT_USER="clickhouse" ENV MINIO_ROOT_PASSWORD="clickhouse" ENV EXPORT_S3_STORAGE_POLICIES=1 -ENV CLICKHOUSE_GRPC_CLIENT="/usr/share/clickhouse-utils/grpc-client/clickhouse-grpc-client.py" RUN npm install -g azurite@3.30.0 \ && npm install -g tslib && npm install -g node - -COPY run.sh / -COPY setup_minio.sh / -COPY setup_hdfs_minicluster.sh / -COPY attach_gdb.lib / -COPY utils.lib / - -# We store stress_tests.lib in stateless image to avoid duplication of this file in stress and upgrade tests -COPY stress_tests.lib / - -CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 0f81a1cd07f..ecb98a4e3ed 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -22,8 +22,5 @@ RUN apt-get update -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* -COPY run.sh / - ENV EXPORT_S3_STORAGE_POLICIES=1 -CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/upgrade/Dockerfile b/docker/test/upgrade/Dockerfile deleted file mode 100644 index 78d912fd031..00000000000 --- a/docker/test/upgrade/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# rebuild in #33610 -# docker build -t clickhouse/upgrade-check . -ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG - -RUN apt-get update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get install --yes --no-install-recommends \ - bash \ - tzdata \ - parallel \ - expect \ - python3 \ - python3-lxml \ - python3-termcolor \ - python3-requests \ - curl \ - sudo \ - openssl \ - netcat-openbsd \ - brotli \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* - -COPY run.sh / - -ENV EXPORT_S3_STORAGE_POLICIES=1 - -CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index 8b949ed95db..6b9fb94a4c6 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -56,7 +56,5 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* -COPY process_functional_tests_result.py / - COPY --from=clickhouse/cctools:0d6b90a7a490 /opt/gdb /opt/gdb ENV PATH="/opt/gdb/bin:${PATH}" diff --git a/tests/ci/.mypy.ini b/tests/ci/.mypy.ini index f12d27979ce..ecb4aef87dd 100644 --- a/tests/ci/.mypy.ini +++ b/tests/ci/.mypy.ini @@ -15,4 +15,5 @@ warn_return_any = True no_implicit_reexport = True strict_equality = True extra_checks = True -ignore_missing_imports = True \ No newline at end of file +ignore_missing_imports = True +logging-fstring-interpolation = False \ No newline at end of file diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 5453bffd9c6..58de25f039f 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -535,7 +535,10 @@ class CI: JobNames.FAST_TEST: JobConfig( pr_only=True, digest=DigestConfig( - include_paths=["./tests/queries/0_stateless/"], + include_paths=[ + "./tests/queries/0_stateless/", + "./tests/docker_scripts/", + ], exclude_files=[".md"], docker=["clickhouse/fasttest"], ), diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index 13c222b10b9..1cdb3f1487e 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -415,6 +415,7 @@ class CommonJobConfigs: "./tests/clickhouse-test", "./tests/config", "./tests/*.txt", + "./tests/docker_scripts/", ], exclude_files=[".md"], docker=["clickhouse/stateless-test"], @@ -431,6 +432,7 @@ class CommonJobConfigs: "./tests/clickhouse-test", "./tests/config", "./tests/*.txt", + "./tests/docker_scripts/", ], exclude_files=[".md"], docker=["clickhouse/stateful-test"], @@ -448,6 +450,7 @@ class CommonJobConfigs: "./tests/clickhouse-test", "./tests/config", "./tests/*.txt", + "./tests/docker_scripts/", ], exclude_files=[".md"], docker=["clickhouse/stress-test"], @@ -459,9 +462,9 @@ class CommonJobConfigs: UPGRADE_TEST = JobConfig( job_name_keyword="upgrade", digest=DigestConfig( - include_paths=["./tests/ci/upgrade_check.py"], + include_paths=["./tests/ci/upgrade_check.py", "./tests/docker_scripts/"], exclude_files=[".md"], - docker=["clickhouse/upgrade-check"], + docker=["clickhouse/stress-test"], ), run_command="upgrade_check.py", runner_type=Runners.STRESS_TESTER, diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 786a529e0a9..c8dbcd10245 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -93,7 +93,7 @@ def process_single_image( results = [] # type: TestResults for ver in versions: stopwatch = Stopwatch() - for i in range(5): + for i in range(2): success, build_log = build_and_push_one_image( image, ver, additional_cache, push, from_tag ) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index ed727dd3659..55eefcf9714 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -31,15 +31,14 @@ def get_fasttest_cmd( "--security-opt seccomp=unconfined " # required to issue io_uring sys-calls "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " - f"-e FASTTEST_SOURCE=/ClickHouse " + f"-e FASTTEST_SOURCE=/repo " f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " "-e stage=clone_submodules " - f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt " - f"--volume={output_path}:/test_output {image}" + f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/repo " + f"--volume={output_path}:/test_output {image} /repo/tests/docker_scripts/fasttest_runner.sh" ) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index d08f98fa05f..ce2ead59d1a 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -119,24 +119,24 @@ def get_run_command( envs += [f"-e {e}" for e in additional_envs] env_str = " ".join(envs) - volume_with_broken_test = ( - f"--volume={repo_path}/tests/analyzer_tech_debt.txt:/analyzer_tech_debt.txt " - if "analyzer" not in check_name - else "" - ) + + if "stateful" in check_name.lower(): + run_script = "/repo/tests/docker_scripts/stateful_runner.sh" + elif "stateless" in check_name.lower(): + run_script = "/repo/tests/docker_scripts/stateless_runner.sh" + else: + assert False return ( f"docker run --rm --name func-tester --volume={builds_path}:/package_folder " # For dmesg and sysctl "--privileged " - f"{ci_logs_args}" - f"--volume={repo_path}/tests:/usr/share/clickhouse-test " - f"--volume={repo_path}/utils/grpc-client:/usr/share/clickhouse-utils/grpc-client " - f"{volume_with_broken_test}" + f"{ci_logs_args} " + f"--volume={repo_path}:/repo " f"--volume={result_path}:/test_output " f"--volume={server_log_path}:/var/log/clickhouse-server " "--security-opt seccomp=unconfined " # required to issue io_uring sys-calls - f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" + f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image} {run_script}" ) diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 85da601e379..f9656e60448 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -57,10 +57,16 @@ def get_run_command( additional_envs: List[str], ci_logs_args: str, image: DockerImage, + upgrade_check: bool, ) -> str: envs = [f"-e {e}" for e in additional_envs] env_str = " ".join(envs) + if upgrade_check: + run_script = "/repo/tests/docker_scripts/upgrade_runner.sh" + else: + run_script = "/repo/tests/docker_scripts/stress_runner.sh" + cmd = ( "docker run --cap-add=SYS_PTRACE " # For dmesg and sysctl @@ -70,8 +76,8 @@ def get_run_command( f"{ci_logs_args}" f"--volume={build_path}:/package_folder " f"--volume={result_path}:/test_output " - f"--volume={repo_tests_path}:/usr/share/clickhouse-test " - f"--volume={server_log_path}:/var/log/clickhouse-server {env_str} {image} " + f"--volume={repo_tests_path}/..:/repo " + f"--volume={server_log_path}:/var/log/clickhouse-server {env_str} {image} {run_script}" ) return cmd @@ -128,7 +134,7 @@ def process_results( return state, description, test_results, additional_files -def run_stress_test(docker_image_name: str) -> None: +def run_stress_test(upgrade_check: bool = False) -> None: logging.basicConfig(level=logging.INFO) for handler in logging.root.handlers: # pylint: disable=protected-access @@ -148,7 +154,7 @@ def run_stress_test(docker_image_name: str) -> None: pr_info = PRInfo() - docker_image = pull_image(get_docker_image(docker_image_name)) + docker_image = pull_image(get_docker_image("clickhouse/stress-test")) packages_path = temp_path / "packages" packages_path.mkdir(parents=True, exist_ok=True) @@ -177,6 +183,7 @@ def run_stress_test(docker_image_name: str) -> None: additional_envs, ci_logs_args, docker_image, + upgrade_check, ) logging.info("Going to run stress test: %s", run_command) @@ -208,4 +215,4 @@ def run_stress_test(docker_image_name: str) -> None: if __name__ == "__main__": - run_stress_test("clickhouse/stress-test") + run_stress_test() diff --git a/tests/ci/upgrade_check.py b/tests/ci/upgrade_check.py index 83b6f9e299f..8662611dffe 100644 --- a/tests/ci/upgrade_check.py +++ b/tests/ci/upgrade_check.py @@ -1,4 +1,4 @@ import stress_check if __name__ == "__main__": - stress_check.run_stress_test("clickhouse/upgrade-check") + stress_check.run_stress_test(upgrade_check=True) diff --git a/docker/test/stateless/attach_gdb.lib b/tests/docker_scripts/attach_gdb.lib similarity index 98% rename from docker/test/stateless/attach_gdb.lib rename to tests/docker_scripts/attach_gdb.lib index 2f1375a2f0f..4170a19176c 100644 --- a/docker/test/stateless/attach_gdb.lib +++ b/tests/docker_scripts/attach_gdb.lib @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=./utils.lib -source /utils.lib +source /repo/tests/docker_scripts/utils.lib function attach_gdb_to_clickhouse() { diff --git a/docker/test/stateful/create.sql b/tests/docker_scripts/create.sql similarity index 100% rename from docker/test/stateful/create.sql rename to tests/docker_scripts/create.sql diff --git a/docker/test/fasttest/run.sh b/tests/docker_scripts/fasttest_runner.sh similarity index 99% rename from docker/test/fasttest/run.sh rename to tests/docker_scripts/fasttest_runner.sh index 9920326b11c..1eaba2c7cdf 100755 --- a/docker/test/fasttest/run.sh +++ b/tests/docker_scripts/fasttest_runner.sh @@ -325,7 +325,7 @@ case "$stage" in ;& "run_tests") run_tests ||: - /process_functional_tests_result.py --in-results-dir "$FASTTEST_OUTPUT/" \ + /repo/tests/docker_scripts/process_functional_tests_result.py --in-results-dir "$FASTTEST_OUTPUT/" \ --out-results-file "$FASTTEST_OUTPUT/test_results.tsv" \ --out-status-file "$FASTTEST_OUTPUT/check_status.tsv" || echo -e "failure\tCannot parse results" > "$FASTTEST_OUTPUT/check_status.tsv" ;; diff --git a/docker/test/util/process_functional_tests_result.py b/tests/docker_scripts/process_functional_tests_result.py similarity index 92% rename from docker/test/util/process_functional_tests_result.py rename to tests/docker_scripts/process_functional_tests_result.py index ec9e14b1430..1dc3090484c 100755 --- a/docker/test/util/process_functional_tests_result.py +++ b/tests/docker_scripts/process_functional_tests_result.py @@ -32,7 +32,7 @@ def process_test_log(log_path, broken_tests): success_finish = False test_results = [] test_end = True - with open(log_path, "r") as test_file: + with open(log_path, "r", encoding="utf-8") as test_file: for line in test_file: original_line = line line = line.strip() @@ -150,7 +150,7 @@ def process_result(result_path, broken_tests): if result_path and os.path.exists(result_path): ( - total, + _total, skipped, unknown, failed, @@ -191,11 +191,11 @@ def process_result(result_path, broken_tests): else: description = "" - description += "fail: {}, passed: {}".format(failed, success) + description += f"fail: {failed}, passed: {success}" if skipped != 0: - description += ", skipped: {}".format(skipped) + description += f", skipped: {skipped}" if unknown != 0: - description += ", unknown: {}".format(unknown) + description += f", unknown: {unknown}" else: state = "failure" description = "Output log doesn't exist" @@ -205,10 +205,10 @@ def process_result(result_path, broken_tests): def write_results(results_file, status_file, results, status): - with open(results_file, "w") as f: + with open(results_file, "w", encoding="utf-8") as f: out = csv.writer(f, delimiter="\t") out.writerows(results) - with open(status_file, "w") as f: + with open(status_file, "w", encoding="utf-8") as f: out = csv.writer(f, delimiter="\t") out.writerow(status) @@ -221,15 +221,15 @@ if __name__ == "__main__": parser.add_argument("--in-results-dir", default="/test_output/") parser.add_argument("--out-results-file", default="/test_output/test_results.tsv") parser.add_argument("--out-status-file", default="/test_output/check_status.tsv") - parser.add_argument("--broken-tests", default="/analyzer_tech_debt.txt") + parser.add_argument("--broken-tests", default="/repo/tests/analyzer_tech_debt.txt") args = parser.parse_args() - broken_tests = list() + broken_tests = [] if os.path.exists(args.broken_tests): - logging.info(f"File {args.broken_tests} with broken tests found") - with open(args.broken_tests) as f: + print(f"File {args.broken_tests} with broken tests found") + with open(args.broken_tests, encoding="utf-8") as f: broken_tests = f.read().splitlines() - logging.info(f"Broken tests in the list: {len(broken_tests)}") + print(f"Broken tests in the list: {len(broken_tests)}") state, description, test_results = process_result(args.in_results_dir, broken_tests) logging.info("Result parsed") diff --git a/docker/test/stateless/setup_hdfs_minicluster.sh b/tests/docker_scripts/setup_hdfs_minicluster.sh similarity index 95% rename from docker/test/stateless/setup_hdfs_minicluster.sh rename to tests/docker_scripts/setup_hdfs_minicluster.sh index 15a54f59096..622270ba5d5 100755 --- a/docker/test/stateless/setup_hdfs_minicluster.sh +++ b/tests/docker_scripts/setup_hdfs_minicluster.sh @@ -5,7 +5,7 @@ set -e -x -a -u ls -lha -cd hadoop-3.3.1 +cd /hadoop-3.3.1 export JAVA_HOME=/usr mkdir -p target/test/data diff --git a/docker/test/stateless/setup_minio.sh b/tests/docker_scripts/setup_minio.sh similarity index 98% rename from docker/test/stateless/setup_minio.sh rename to tests/docker_scripts/setup_minio.sh index c1508df7e82..40e93e713a1 100755 --- a/docker/test/stateless/setup_minio.sh +++ b/tests/docker_scripts/setup_minio.sh @@ -143,7 +143,7 @@ main() { fi start_minio setup_minio "$1" - upload_data "${query_dir}" "${2:-/usr/share/clickhouse-test}" + upload_data "${query_dir}" "${2:-/repo/tests/}" setup_aws_credentials } diff --git a/docker/test/stateful/run.sh b/tests/docker_scripts/stateful_runner.sh similarity index 98% rename from docker/test/stateful/run.sh rename to tests/docker_scripts/stateful_runner.sh index a489a4b6e14..86f6a299ad3 100755 --- a/docker/test/stateful/run.sh +++ b/tests/docker_scripts/stateful_runner.sh @@ -14,17 +14,17 @@ dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test +ln -s /repo/tests/clickhouse-test /usr/bin/clickhouse-test # shellcheck disable=SC1091 -source /utils.lib +source /repo/tests/docker_scripts/utils.lib # install test configs -/usr/share/clickhouse-test/config/install.sh +/repo/tests/config/install.sh azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & -./setup_minio.sh stateful +/repo/tests/docker_scripts/setup_minio.sh stateful ./mc admin trace clickminio > /test_output/minio.log & MC_ADMIN_PID=$! @@ -105,7 +105,7 @@ setup_logs_replication clickhouse-client --query "SHOW DATABASES" clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client --multiquery < create.sql +clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then @@ -237,6 +237,7 @@ function run_tests() --hung-check --print-time --capture-client-stacktrace + --queries "/repo/tests/queries" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" ) @@ -259,7 +260,7 @@ ls -la ./ echo "Files in root directory" ls -la / -/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv +/repo/tests/docker_scripts/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv sudo clickhouse stop ||: if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then diff --git a/docker/test/stateless/run.sh b/tests/docker_scripts/stateless_runner.sh similarity index 97% rename from docker/test/stateless/run.sh rename to tests/docker_scripts/stateless_runner.sh index 6973eebbac3..671b1f5ca71 100755 --- a/docker/test/stateless/run.sh +++ b/tests/docker_scripts/stateless_runner.sh @@ -1,10 +1,13 @@ #!/bin/bash +# fail on errors, verbose and export all env variables +set -e -x -a + # shellcheck disable=SC1091 source /setup_export_logs.sh # shellcheck source=../stateless/stress_tests.lib -source /stress_tests.lib +source /repo/tests/docker_scripts/stress_tests.lib # Avoid overlaps with previous runs dmesg --clear @@ -39,20 +42,22 @@ if [[ -z "$BUGFIX_VALIDATE_CHECK" ]]; then chc --version || exit 1 fi -ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test +ln -sf /repo/tests/clickhouse-test /usr/bin/clickhouse-test + +export CLICKHOUSE_GRPC_CLIENT="/repo/utils/grpc-client/clickhouse-grpc-client.py" # shellcheck disable=SC1091 -source /attach_gdb.lib +source /repo/tests/docker_scripts/attach_gdb.lib # shellcheck disable=SC1091 -source /utils.lib +source /repo/tests/docker_scripts/utils.lib # install test configs -/usr/share/clickhouse-test/config/install.sh +/repo/tests/config/install.sh -./setup_minio.sh stateless +/repo/tests/docker_scripts/setup_minio.sh stateless -./setup_hdfs_minicluster.sh +/repo/tests/docker_scripts/setup_hdfs_minicluster.sh config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml @@ -316,6 +321,7 @@ function run_tests() --print-time --no-drop-if-fail --capture-client-stacktrace + --queries "/repo/tests/queries" --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" ) @@ -341,7 +347,7 @@ ls -la ./ echo "Files in root directory" ls -la / -/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv +/repo/tests/docker_scripts/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv clickhouse-client -q "system flush logs" ||: diff --git a/docker/test/stress/run.sh b/tests/docker_scripts/stress_runner.sh old mode 100644 new mode 100755 similarity index 97% rename from docker/test/stress/run.sh rename to tests/docker_scripts/stress_runner.sh index b21114e456f..7666398e10b --- a/docker/test/stress/run.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -3,26 +3,25 @@ # shellcheck disable=SC2086 # shellcheck disable=SC2024 +set -x + # Avoid overlaps with previous runs dmesg --clear # shellcheck disable=SC1091 source /setup_export_logs.sh -set -x - -# we mount tests folder from repo to /usr/share -ln -s /usr/share/clickhouse-test/ci/stress.py /usr/bin/stress -ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test +ln -s /repo/tests/clickhouse-test/ci/stress.py /usr/bin/stress +ln -s /repo/tests/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test # Stress tests and upgrade check uses similar code that was placed # in a separate bash library. See tests/ci/stress_tests.lib # shellcheck source=../stateless/attach_gdb.lib -source /attach_gdb.lib +source /repo/tests/docker_scripts/attach_gdb.lib # shellcheck source=../stateless/stress_tests.lib -source /stress_tests.lib +source /repo/tests/docker_scripts/stress_tests.lib # shellcheck disable=SC1091 -source /utils.lib +source /repo/tests/docker_scripts/utils.lib install_packages package_folder @@ -55,7 +54,7 @@ export ZOOKEEPER_FAULT_INJECTION=1 # available for dump via clickhouse-local configure -./setup_minio.sh stateless # to have a proper environment +/repo/tests/docker_scripts/setup_minio.sh stateless # to have a proper environment config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml @@ -64,7 +63,7 @@ start_server setup_logs_replication clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client --multiquery < create.sql +clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test" @@ -267,7 +266,7 @@ fi start_server -stress --hung-check --drop-databases --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" --global-time-limit 1200 \ +python3 /repo/tests/ci/stress.py --hung-check --drop-databases --output-folder /test_output --skip-func-tests "$SKIP_TESTS_OPTION" --global-time-limit 1200 \ && echo -e "Test script exit code$OK" >> /test_output/test_results.tsv \ || echo -e "Test script failed$FAIL script exit code: $?" >> /test_output/test_results.tsv diff --git a/docker/test/stateless/stress_tests.lib b/tests/docker_scripts/stress_tests.lib similarity index 99% rename from docker/test/stateless/stress_tests.lib rename to tests/docker_scripts/stress_tests.lib index 51aa299f7a6..e2b5d983fcb 100644 --- a/docker/test/stateless/stress_tests.lib +++ b/tests/docker_scripts/stress_tests.lib @@ -42,7 +42,7 @@ function configure() # install test configs export USE_DATABASE_ORDINARY=1 export EXPORT_S3_STORAGE_POLICIES=1 - /usr/share/clickhouse-test/config/install.sh + /repo/tests/config/install.sh # avoid too slow startup sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ diff --git a/docker/test/upgrade/run.sh b/tests/docker_scripts/upgrade_runner.sh old mode 100644 new mode 100755 similarity index 96% rename from docker/test/upgrade/run.sh rename to tests/docker_scripts/upgrade_runner.sh index e9768873f6a..ece75ebf782 --- a/docker/test/upgrade/run.sh +++ b/tests/docker_scripts/upgrade_runner.sh @@ -9,20 +9,20 @@ dmesg --clear set -x # we mount tests folder from repo to /usr/share -ln -s /usr/share/clickhouse-test/ci/stress.py /usr/bin/stress -ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test -ln -s /usr/share/clickhouse-test/ci/download_release_packages.py /usr/bin/download_release_packages -ln -s /usr/share/clickhouse-test/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag +ln -s /repo/tests/ci/stress.py /usr/bin/stress +ln -s /repo/tests/clickhouse-test /usr/bin/clickhouse-test +ln -s /repo/tests/ci/download_release_packages.py /usr/bin/download_release_packages +ln -s /repo/tests/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag # Stress tests and upgrade check uses similar code that was placed # in a separate bash library. See tests/ci/stress_tests.lib # shellcheck source=../stateless/attach_gdb.lib -source /attach_gdb.lib +source /repo/tests/docker_scripts/attach_gdb.lib # shellcheck source=../stateless/stress_tests.lib -source /stress_tests.lib +source /repo/tests/docker_scripts/stress_tests.lib azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & -./setup_minio.sh stateless # to have a proper environment +/repo/tests/docker_scripts/setup_minio.sh stateless # to have a proper environment echo "Get previous release tag" # shellcheck disable=SC2016 diff --git a/docker/test/stateless/utils.lib b/tests/docker_scripts/utils.lib similarity index 100% rename from docker/test/stateless/utils.lib rename to tests/docker_scripts/utils.lib