diff --git a/debian/.gitignore b/debian/.gitignore index 7d927d3a70b..b4432556de7 100644 --- a/debian/.gitignore +++ b/debian/.gitignore @@ -10,7 +10,6 @@ clickhouse-common-static/ clickhouse-server-base/ clickhouse-server-common/ clickhouse-server/ -clickhouse-test/ debhelper-build-stamp files *.debhelper.log diff --git a/debian/clickhouse-test.install b/debian/clickhouse-test.install deleted file mode 100644 index 042a4e02be1..00000000000 --- a/debian/clickhouse-test.install +++ /dev/null @@ -1,2 +0,0 @@ -usr/bin/clickhouse-test -usr/share/clickhouse-test/* diff --git a/debian/control b/debian/control index f22980fdbc4..c5d98d98f41 100644 --- a/debian/control +++ b/debian/control @@ -56,9 +56,3 @@ Replaces: clickhouse-common-dbg Conflicts: clickhouse-common-dbg Description: debugging symbols for clickhouse-common-static This package contains the debugging symbols for clickhouse-common. - -Package: clickhouse-test -Priority: optional -Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python3, python3-lxml, python3-termcolor, python3-requests, curl, perl, sudo, openssl, netcat-openbsd, telnet, brotli, bsdutils -Description: ClickHouse tests diff --git a/docker/packager/README.md b/docker/packager/README.md index a745f6225fa..223d70916d4 100644 --- a/docker/packager/README.md +++ b/docker/packager/README.md @@ -14,7 +14,6 @@ $ ls -l deb/test_output -rw-r--r-- 1 root root 14940 clickhouse-server_18.14.2+debug_all.deb -rw-r--r-- 1 root root 340206010 clickhouse-server-base_18.14.2+debug_amd64.deb -rw-r--r-- 1 root root 7900 clickhouse-server-common_18.14.2+debug_all.deb --rw-r--r-- 1 root root 2880432 clickhouse-test_18.14.2+debug_all.deb ``` diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile deleted file mode 100644 index 66d1afb309b..00000000000 --- a/docker/test/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:18.04 - -ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/" -ARG version=22.1.1.* - -RUN apt-get update && \ - apt-get install -y apt-transport-https dirmngr && \ - mkdir -p /etc/apt/sources.list.d && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \ - echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \ - apt-get update && \ - env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-test && \ - rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ - apt-get clean - -ENTRYPOINT ["/usr/bin/clickhouse-test"] diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 8202a07f017..3f3240a0b7f 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -11,7 +11,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb; dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -dpkg -i package_folder/clickhouse-test_*.deb + +ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test # install test configs /usr/share/clickhouse-test/config/install.sh diff --git a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile index 562141ba147..dc83e8b8d2e 100644 --- a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile +++ b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile @@ -2,7 +2,7 @@ # 1. build base container # 2. run base conatiner with mounted volumes # 3. commit container as image -FROM ubuntu:18.10 as clickhouse-test-runner-base +FROM ubuntu:20.04 as clickhouse-test-runner-base # A volume where directory with clickhouse packages to be mounted, # for later installing. @@ -11,5 +11,4 @@ VOLUME /packages CMD apt-get update ;\ DEBIAN_FRONTEND=noninteractive \ apt install -y /packages/clickhouse-common-static_*.deb \ - /packages/clickhouse-client_*.deb \ - /packages/clickhouse-test_*.deb + /packages/clickhouse-client_*.deb diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 7c2563f2d86..5fd78502337 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -12,11 +12,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -if [[ -n "$TEST_CASES_FROM_DEB" ]] && [[ "$TEST_CASES_FROM_DEB" -eq 1 ]]; then - dpkg -i package_folder/clickhouse-test_*.deb -else - ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test -fi + +ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test # install test configs /usr/share/clickhouse-test/config/install.sh diff --git a/docker/test/stateless_pytest/Dockerfile b/docker/test/stateless_pytest/Dockerfile index f692f8f39f0..789ee0e9b30 100644 --- a/docker/test/stateless_pytest/Dockerfile +++ b/docker/test/stateless_pytest/Dockerfile @@ -30,5 +30,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \ dpkg -i package_folder/clickhouse-server_*.deb; \ dpkg -i package_folder/clickhouse-client_*.deb; \ - dpkg -i package_folder/clickhouse-test_*.deb; \ python3 -m pytest /usr/share/clickhouse-test/queries -n $(nproc) --reruns=1 --timeout=600 --json=test_output/report.json --html=test_output/report.html --self-contained-html diff --git a/docker/test/stress/README.md b/docker/test/stress/README.md index b1519e7968d..96807b9f9a6 100644 --- a/docker/test/stress/README.md +++ b/docker/test/stress/README.md @@ -5,7 +5,7 @@ This allows to find problems like segmentation fault which cause shutdown of ser Usage: ``` $ ls $HOME/someclickhouse -clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb +clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb $ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output clickhouse/stress-test Selecting previously unselected package clickhouse-common-static. (Reading database ... 14442 files and directories currently installed.) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index e57dbc38ded..de77dec03b9 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -31,13 +31,15 @@ dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -dpkg -i package_folder/clickhouse-test_*.deb function configure() { # install test configs /usr/share/clickhouse-test/config/install.sh + # we mount tests folder from repo to /usr/share + ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test + # avoid too slow startup sudo cat /etc/clickhouse-server/config.d/keeper_port.xml | sed "s|100000|10000|" > /etc/clickhouse-server/config.d/keeper_port.xml.tmp sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md index e9a77b41210..b5b558464ba 100644 --- a/docs/en/development/continuous-integration.md +++ b/docs/en/development/continuous-integration.md @@ -126,7 +126,6 @@ Builds ClickHouse in various configurations for use in further steps. You have t - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - `clickhouse-server_XXX_all.deb` - - `clickhouse-test_XXX_all.deb` - `clickhouse_XXX_amd64.buildinfo` - `clickhouse_XXX_amd64.changes` - `clickhouse`: Main built binary. diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index ea2f3c5196a..2700c747b12 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -21,8 +21,9 @@ from rerun_helper import RerunHelper IMAGE_NAME = 'clickhouse/performance-comparison' -def get_run_command(workspace, result_path, pr_to_test, sha_to_test, additional_env, image): +def get_run_command(workspace, result_path, repo_tests_path, pr_to_test, sha_to_test, additional_env, image): return f"docker run --privileged --volume={workspace}:/workspace --volume={result_path}:/output " \ + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \ f"--cap-add syslog --cap-add sys_admin --cap-add sys_rawio " \ f"-e PR_TO_TEST={pr_to_test} -e SHA_TO_TEST={sha_to_test} {additional_env} " \ f"{image}" @@ -45,6 +46,7 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO) temp_path = os.getenv("TEMP_PATH", os.path.abspath(".")) repo_path = os.getenv("REPO_COPY", os.path.abspath("../../")) + repo_tests_path = os.path.join(repo_path, "tests") ramdrive_path = os.getenv("RAMDRIVE_PATH", os.path.join(temp_path, "ramdrive")) # currently unused, doesn't make tests more stable ramdrive_size = os.getenv("RAMDRIVE_SIZE", '0G') @@ -95,7 +97,7 @@ if __name__ == "__main__": if not os.path.exists(result_path): os.makedirs(result_path) - run_command = get_run_command(result_path, result_path, pr_info.number, pr_info.sha, docker_env, docker_image) + run_command = get_run_command(result_path, result_path, repo_tests_path, pr_info.number, pr_info.sha, docker_env, docker_image) logging.info("Going to run command %s", run_command) run_log_path = os.path.join(temp_path, "runlog.log") with TeePopen(run_command, run_log_path) as process: diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index 7747525df02..1f2ea139a6e 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -32,7 +32,6 @@ class Packages: ("clickhouse-common-static", "amd64"), ("clickhouse-common-static-dbg", "amd64"), ("clickhouse-server", "all"), - ("clickhouse-test", "all"), ) def __init__(self, version: str): diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 1f811b1e25d..bb9cec624ec 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -22,10 +22,11 @@ from rerun_helper import RerunHelper from tee_popen import TeePopen -def get_run_command(build_path, result_folder, server_log_folder, image): +def get_run_command(build_path, result_folder, repo_tests_path, server_log_folder, image): cmd = "docker run --cap-add=SYS_PTRACE -e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' " + \ f"--volume={build_path}:/package_folder " \ f"--volume={result_folder}:/test_output " \ + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \ f"--volume={server_log_folder}:/var/log/clickhouse-server {image}" return cmd @@ -72,6 +73,7 @@ if __name__ == "__main__": stopwatch = Stopwatch() temp_path = TEMP_PATH repo_path = REPO_COPY + repo_tests_path = os.path.join(repo_path, "tests") reports_path = REPORTS_PATH check_name = sys.argv[1] @@ -106,7 +108,7 @@ if __name__ == "__main__": run_log_path = os.path.join(temp_path, "runlog.log") - run_command = get_run_command(packages_path, result_path, server_log_path, docker_image) + run_command = get_run_command(packages_path, result_path, repo_tests_path, server_log_path, docker_image) logging.info("Going to run func tests: %s", run_command) with TeePopen(run_command, run_log_path) as process: diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh index 9f6c2285d93..538d596d263 100644 --- a/utils/release/release_lib.sh +++ b/utils/release/release_lib.sh @@ -243,16 +243,6 @@ function make_rpm { cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec rpm_pack - PACKAGE=clickhouse-test - ARCH=all - TARGET=noarch - deb_unpack - mv ${PACKAGE}-$VERSION_FULL-2.spec ${PACKAGE}-$VERSION_FULL-2.spec_tmp - echo "Requires: python3" >> ${PACKAGE}-$VERSION_FULL-2.spec - #echo "Requires: python3-termcolor" >> ${PACKAGE}-$VERSION-2.spec - cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec - rpm_pack - PACKAGE=clickhouse-common-static ARCH=amd64 TARGET=x86_64 @@ -271,7 +261,7 @@ function make_tgz { VERSION_FULL="${VERSION_STRING}" PACKAGE_DIR=${PACKAGE_DIR=../} - for PACKAGE in clickhouse-server clickhouse-client clickhouse-test clickhouse-common-static clickhouse-common-static-dbg; do + for PACKAGE in clickhouse-server clickhouse-client clickhouse-common-static clickhouse-common-static-dbg; do alien --verbose --scripts --generate --to-tgz ${PACKAGE_DIR}${PACKAGE}_${VERSION_FULL}_*.deb PKGDIR="./${PACKAGE}-${VERSION_FULL}" if [ ! -d "$PKGDIR/install" ]; then