Merge pull request #34606 from qoega/no-clickhouse-test-deb

Remove clickhouse-test deb completely

debian/.gitignore

@@ -10,7 +10,6 @@ clickhouse-common-static/
 clickhouse-server-base/
 clickhouse-server-common/
 clickhouse-server/
-clickhouse-test/
 debhelper-build-stamp
 files
 *.debhelper.log

debian/clickhouse-test.install

@@ -1,2 +0,0 @@
-usr/bin/clickhouse-test
-usr/share/clickhouse-test/*

debian/control

@@ -56,9 +56,3 @@ Replaces: clickhouse-common-dbg
 Conflicts: clickhouse-common-dbg
 Description: debugging symbols for clickhouse-common-static
  This package contains the debugging symbols for clickhouse-common.
-
-Package: clickhouse-test
-Priority: optional
-Architecture: all
-Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python3, python3-lxml, python3-termcolor, python3-requests, curl, perl, sudo, openssl, netcat-openbsd, telnet, brotli, bsdutils
-Description: ClickHouse tests

docker/packager/README.md

@@ -14,7 +14,6 @@ $ ls -l deb/test_output
 -rw-r--r-- 1 root root     14940 clickhouse-server_18.14.2+debug_all.deb
 -rw-r--r-- 1 root root 340206010 clickhouse-server-base_18.14.2+debug_amd64.deb
 -rw-r--r-- 1 root root      7900 clickhouse-server-common_18.14.2+debug_all.deb
--rw-r--r-- 1 root root   2880432 clickhouse-test_18.14.2+debug_all.deb
 
 ```
 

docker/test/Dockerfile

@@ -1,16 +0,0 @@
-FROM ubuntu:18.04
-
-ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/"
-ARG version=22.1.1.*
-
-RUN apt-get update && \
-    apt-get install -y apt-transport-https dirmngr && \
-    mkdir -p /etc/apt/sources.list.d && \
-    apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
-    echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
-    apt-get update && \
-    env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-test && \
-    rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
-    apt-get clean
-
-ENTRYPOINT ["/usr/bin/clickhouse-test"]

docker/test/stateful/run.sh

@@ -11,7 +11,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb;
 dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
 dpkg -i package_folder/clickhouse-server_*.deb
 dpkg -i package_folder/clickhouse-client_*.deb
-dpkg -i package_folder/clickhouse-test_*.deb
+
+ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
 
 # install test configs
 /usr/share/clickhouse-test/config/install.sh

docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile

@@ -2,7 +2,7 @@
 # 1. build base container
 # 2. run base conatiner with mounted volumes
 # 3. commit container as image
-FROM ubuntu:18.10 as clickhouse-test-runner-base
+FROM ubuntu:20.04 as clickhouse-test-runner-base
 
 # A volume where directory with clickhouse packages to be mounted,
 # for later installing.
@@ -11,5 +11,4 @@ VOLUME /packages
 CMD apt-get update ;\
 	DEBIAN_FRONTEND=noninteractive \
 	apt install -y /packages/clickhouse-common-static_*.deb \
-		/packages/clickhouse-client_*.deb \
+		/packages/clickhouse-client_*.deb
-		/packages/clickhouse-test_*.deb

docker/test/stateless/run.sh

@@ -12,11 +12,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb
 dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
 dpkg -i package_folder/clickhouse-server_*.deb
 dpkg -i package_folder/clickhouse-client_*.deb
-if [[ -n "$TEST_CASES_FROM_DEB" ]] && [[ "$TEST_CASES_FROM_DEB" -eq 1 ]]; then
+
-    dpkg -i package_folder/clickhouse-test_*.deb
+ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
-else
-    ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
-fi
 
 # install test configs
 /usr/share/clickhouse-test/config/install.sh

docker/test/stateless_pytest/Dockerfile

@@ -30,5 +30,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
     dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
     dpkg -i package_folder/clickhouse-server_*.deb;  \
     dpkg -i package_folder/clickhouse-client_*.deb; \
-    dpkg -i package_folder/clickhouse-test_*.deb; \
     python3 -m pytest /usr/share/clickhouse-test/queries -n $(nproc) --reruns=1 --timeout=600 --json=test_output/report.json --html=test_output/report.html --self-contained-html

docker/test/stress/README.md

@@ -5,7 +5,7 @@ This allows to find problems like segmentation fault which cause shutdown of ser
 Usage:
 ```
 $ ls $HOME/someclickhouse
-clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb
+clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb
 $ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output clickhouse/stress-test
 Selecting previously unselected package clickhouse-common-static.
 (Reading database ... 14442 files and directories currently installed.)

docker/test/stress/run.sh

@@ -31,13 +31,15 @@ dpkg -i package_folder/clickhouse-common-static_*.deb
 dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
 dpkg -i package_folder/clickhouse-server_*.deb
 dpkg -i package_folder/clickhouse-client_*.deb
-dpkg -i package_folder/clickhouse-test_*.deb
 
 function configure()
 {
     # install test configs
     /usr/share/clickhouse-test/config/install.sh
 
+    # we mount tests folder from repo to /usr/share
+    ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
+
     # avoid too slow startup
     sudo cat /etc/clickhouse-server/config.d/keeper_port.xml | sed "s|<snapshot_distance>100000</snapshot_distance>|<snapshot_distance>10000</snapshot_distance>|" > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
     sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml

docs/en/development/continuous-integration.md

@@ -126,7 +126,6 @@ Builds ClickHouse in various configurations for use in further steps. You have t
   - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
   - `clickhouse-common-staticXXX_amd64.deb`
   - `clickhouse-server_XXX_all.deb`
-  - `clickhouse-test_XXX_all.deb`
   - `clickhouse_XXX_amd64.buildinfo`
   - `clickhouse_XXX_amd64.changes`
   - `clickhouse`: Main built binary.

tests/ci/performance_comparison_check.py

@@ -21,8 +21,9 @@ from rerun_helper import RerunHelper
 
 IMAGE_NAME = 'clickhouse/performance-comparison'
 
-def get_run_command(workspace, result_path, pr_to_test, sha_to_test, additional_env, image):
+def get_run_command(workspace, result_path, repo_tests_path, pr_to_test, sha_to_test, additional_env, image):
     return f"docker run --privileged --volume={workspace}:/workspace --volume={result_path}:/output " \
+        f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \
         f"--cap-add syslog --cap-add sys_admin --cap-add sys_rawio " \
         f"-e PR_TO_TEST={pr_to_test} -e SHA_TO_TEST={sha_to_test} {additional_env} " \
         f"{image}"
@@ -45,6 +46,7 @@ if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
     repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
+    repo_tests_path = os.path.join(repo_path, "tests")
     ramdrive_path = os.getenv("RAMDRIVE_PATH", os.path.join(temp_path, "ramdrive"))
     # currently unused, doesn't make tests more stable
     ramdrive_size = os.getenv("RAMDRIVE_SIZE", '0G')
@@ -95,7 +97,7 @@ if __name__ == "__main__":
     if not os.path.exists(result_path):
         os.makedirs(result_path)
 
-    run_command = get_run_command(result_path, result_path, pr_info.number, pr_info.sha, docker_env, docker_image)
+    run_command = get_run_command(result_path, result_path, repo_tests_path, pr_info.number, pr_info.sha, docker_env, docker_image)
     logging.info("Going to run command %s", run_command)
     run_log_path = os.path.join(temp_path, "runlog.log")
     with TeePopen(run_command, run_log_path) as process:

tests/ci/push_to_artifactory.py

@@ -32,7 +32,6 @@ class Packages:
         ("clickhouse-common-static", "amd64"),
         ("clickhouse-common-static-dbg", "amd64"),
         ("clickhouse-server", "all"),
-        ("clickhouse-test", "all"),
     )
 
     def __init__(self, version: str):

tests/ci/stress_check.py

@@ -22,10 +22,11 @@ from rerun_helper import RerunHelper
 from tee_popen import TeePopen
 
 
-def get_run_command(build_path, result_folder, server_log_folder, image):
+def get_run_command(build_path, result_folder, repo_tests_path, server_log_folder, image):
     cmd = "docker run --cap-add=SYS_PTRACE -e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' " + \
           f"--volume={build_path}:/package_folder "  \
           f"--volume={result_folder}:/test_output " \
+          f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \
           f"--volume={server_log_folder}:/var/log/clickhouse-server {image}"
 
     return cmd
@@ -72,6 +73,7 @@ if __name__ == "__main__":
     stopwatch = Stopwatch()
     temp_path = TEMP_PATH
     repo_path = REPO_COPY
+    repo_tests_path = os.path.join(repo_path, "tests")
     reports_path = REPORTS_PATH
 
     check_name = sys.argv[1]
@@ -106,7 +108,7 @@ if __name__ == "__main__":
 
     run_log_path = os.path.join(temp_path, "runlog.log")
 
-    run_command = get_run_command(packages_path, result_path, server_log_path, docker_image)
+    run_command = get_run_command(packages_path, result_path, repo_tests_path, server_log_path, docker_image)
     logging.info("Going to run func tests: %s", run_command)
 
     with TeePopen(run_command, run_log_path) as process:

utils/release/release_lib.sh

@@ -243,16 +243,6 @@ function make_rpm {
     cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec
     rpm_pack
 
-    PACKAGE=clickhouse-test
-    ARCH=all
-    TARGET=noarch
-    deb_unpack
-    mv ${PACKAGE}-$VERSION_FULL-2.spec ${PACKAGE}-$VERSION_FULL-2.spec_tmp
-    echo "Requires: python3" >> ${PACKAGE}-$VERSION_FULL-2.spec
-    #echo "Requires: python3-termcolor" >> ${PACKAGE}-$VERSION-2.spec
-    cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec
-    rpm_pack
-
     PACKAGE=clickhouse-common-static
     ARCH=amd64
     TARGET=x86_64
@@ -271,7 +261,7 @@ function make_tgz {
     VERSION_FULL="${VERSION_STRING}"
     PACKAGE_DIR=${PACKAGE_DIR=../}
 
-    for PACKAGE in clickhouse-server clickhouse-client clickhouse-test clickhouse-common-static clickhouse-common-static-dbg; do
+    for PACKAGE in clickhouse-server clickhouse-client clickhouse-common-static clickhouse-common-static-dbg; do
         alien --verbose --scripts --generate --to-tgz ${PACKAGE_DIR}${PACKAGE}_${VERSION_FULL}_*.deb
         PKGDIR="./${PACKAGE}-${VERSION_FULL}"
         if [ ! -d "$PKGDIR/install" ]; then