Merge pull request #34606 from qoega/no-clickhouse-test-deb

Remove clickhouse-test deb completely
This commit is contained in:
alesapin 2022-02-17 18:13:44 +03:00 committed by GitHub
commit 8a0153b6d3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 19 additions and 55 deletions

1
debian/.gitignore vendored
View File

@ -10,7 +10,6 @@ clickhouse-common-static/
clickhouse-server-base/
clickhouse-server-common/
clickhouse-server/
clickhouse-test/
debhelper-build-stamp
files
*.debhelper.log

View File

@ -1,2 +0,0 @@
usr/bin/clickhouse-test
usr/share/clickhouse-test/*

6
debian/control vendored
View File

@ -56,9 +56,3 @@ Replaces: clickhouse-common-dbg
Conflicts: clickhouse-common-dbg
Description: debugging symbols for clickhouse-common-static
This package contains the debugging symbols for clickhouse-common.
Package: clickhouse-test
Priority: optional
Architecture: all
Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python3, python3-lxml, python3-termcolor, python3-requests, curl, perl, sudo, openssl, netcat-openbsd, telnet, brotli, bsdutils
Description: ClickHouse tests

View File

@ -14,7 +14,6 @@ $ ls -l deb/test_output
-rw-r--r-- 1 root root 14940 clickhouse-server_18.14.2+debug_all.deb
-rw-r--r-- 1 root root 340206010 clickhouse-server-base_18.14.2+debug_amd64.deb
-rw-r--r-- 1 root root 7900 clickhouse-server-common_18.14.2+debug_all.deb
-rw-r--r-- 1 root root 2880432 clickhouse-test_18.14.2+debug_all.deb
```

View File

@ -1,16 +0,0 @@
FROM ubuntu:18.04
ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/"
ARG version=22.1.1.*
RUN apt-get update && \
apt-get install -y apt-transport-https dirmngr && \
mkdir -p /etc/apt/sources.list.d && \
apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \
echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \
apt-get update && \
env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-test && \
rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
apt-get clean
ENTRYPOINT ["/usr/bin/clickhouse-test"]

View File

@ -11,7 +11,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb;
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
dpkg -i package_folder/clickhouse-server_*.deb
dpkg -i package_folder/clickhouse-client_*.deb
dpkg -i package_folder/clickhouse-test_*.deb
ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
# install test configs
/usr/share/clickhouse-test/config/install.sh

View File

@ -2,7 +2,7 @@
# 1. build base container
# 2. run base conatiner with mounted volumes
# 3. commit container as image
FROM ubuntu:18.10 as clickhouse-test-runner-base
FROM ubuntu:20.04 as clickhouse-test-runner-base
# A volume where directory with clickhouse packages to be mounted,
# for later installing.
@ -11,5 +11,4 @@ VOLUME /packages
CMD apt-get update ;\
DEBIAN_FRONTEND=noninteractive \
apt install -y /packages/clickhouse-common-static_*.deb \
/packages/clickhouse-client_*.deb \
/packages/clickhouse-test_*.deb
/packages/clickhouse-client_*.deb

View File

@ -12,11 +12,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
dpkg -i package_folder/clickhouse-server_*.deb
dpkg -i package_folder/clickhouse-client_*.deb
if [[ -n "$TEST_CASES_FROM_DEB" ]] && [[ "$TEST_CASES_FROM_DEB" -eq 1 ]]; then
dpkg -i package_folder/clickhouse-test_*.deb
else
ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
fi
ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
# install test configs
/usr/share/clickhouse-test/config/install.sh

View File

@ -30,5 +30,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
dpkg -i package_folder/clickhouse-server_*.deb; \
dpkg -i package_folder/clickhouse-client_*.deb; \
dpkg -i package_folder/clickhouse-test_*.deb; \
python3 -m pytest /usr/share/clickhouse-test/queries -n $(nproc) --reruns=1 --timeout=600 --json=test_output/report.json --html=test_output/report.html --self-contained-html

View File

@ -5,7 +5,7 @@ This allows to find problems like segmentation fault which cause shutdown of ser
Usage:
```
$ ls $HOME/someclickhouse
clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb
clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb
$ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output clickhouse/stress-test
Selecting previously unselected package clickhouse-common-static.
(Reading database ... 14442 files and directories currently installed.)

View File

@ -31,13 +31,15 @@ dpkg -i package_folder/clickhouse-common-static_*.deb
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
dpkg -i package_folder/clickhouse-server_*.deb
dpkg -i package_folder/clickhouse-client_*.deb
dpkg -i package_folder/clickhouse-test_*.deb
function configure()
{
# install test configs
/usr/share/clickhouse-test/config/install.sh
# we mount tests folder from repo to /usr/share
ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
# avoid too slow startup
sudo cat /etc/clickhouse-server/config.d/keeper_port.xml | sed "s|<snapshot_distance>100000</snapshot_distance>|<snapshot_distance>10000</snapshot_distance>|" > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml

View File

@ -126,7 +126,6 @@ Builds ClickHouse in various configurations for use in further steps. You have t
- `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
- `clickhouse-common-staticXXX_amd64.deb`
- `clickhouse-server_XXX_all.deb`
- `clickhouse-test_XXX_all.deb`
- `clickhouse_XXX_amd64.buildinfo`
- `clickhouse_XXX_amd64.changes`
- `clickhouse`: Main built binary.

View File

@ -21,8 +21,9 @@ from rerun_helper import RerunHelper
IMAGE_NAME = 'clickhouse/performance-comparison'
def get_run_command(workspace, result_path, pr_to_test, sha_to_test, additional_env, image):
def get_run_command(workspace, result_path, repo_tests_path, pr_to_test, sha_to_test, additional_env, image):
return f"docker run --privileged --volume={workspace}:/workspace --volume={result_path}:/output " \
f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \
f"--cap-add syslog --cap-add sys_admin --cap-add sys_rawio " \
f"-e PR_TO_TEST={pr_to_test} -e SHA_TO_TEST={sha_to_test} {additional_env} " \
f"{image}"
@ -45,6 +46,7 @@ if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
repo_tests_path = os.path.join(repo_path, "tests")
ramdrive_path = os.getenv("RAMDRIVE_PATH", os.path.join(temp_path, "ramdrive"))
# currently unused, doesn't make tests more stable
ramdrive_size = os.getenv("RAMDRIVE_SIZE", '0G')
@ -95,7 +97,7 @@ if __name__ == "__main__":
if not os.path.exists(result_path):
os.makedirs(result_path)
run_command = get_run_command(result_path, result_path, pr_info.number, pr_info.sha, docker_env, docker_image)
run_command = get_run_command(result_path, result_path, repo_tests_path, pr_info.number, pr_info.sha, docker_env, docker_image)
logging.info("Going to run command %s", run_command)
run_log_path = os.path.join(temp_path, "runlog.log")
with TeePopen(run_command, run_log_path) as process:

View File

@ -32,7 +32,6 @@ class Packages:
("clickhouse-common-static", "amd64"),
("clickhouse-common-static-dbg", "amd64"),
("clickhouse-server", "all"),
("clickhouse-test", "all"),
)
def __init__(self, version: str):

View File

@ -22,10 +22,11 @@ from rerun_helper import RerunHelper
from tee_popen import TeePopen
def get_run_command(build_path, result_folder, server_log_folder, image):
def get_run_command(build_path, result_folder, repo_tests_path, server_log_folder, image):
cmd = "docker run --cap-add=SYS_PTRACE -e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' " + \
f"--volume={build_path}:/package_folder " \
f"--volume={result_folder}:/test_output " \
f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \
f"--volume={server_log_folder}:/var/log/clickhouse-server {image}"
return cmd
@ -72,6 +73,7 @@ if __name__ == "__main__":
stopwatch = Stopwatch()
temp_path = TEMP_PATH
repo_path = REPO_COPY
repo_tests_path = os.path.join(repo_path, "tests")
reports_path = REPORTS_PATH
check_name = sys.argv[1]
@ -106,7 +108,7 @@ if __name__ == "__main__":
run_log_path = os.path.join(temp_path, "runlog.log")
run_command = get_run_command(packages_path, result_path, server_log_path, docker_image)
run_command = get_run_command(packages_path, result_path, repo_tests_path, server_log_path, docker_image)
logging.info("Going to run func tests: %s", run_command)
with TeePopen(run_command, run_log_path) as process:

View File

@ -243,16 +243,6 @@ function make_rpm {
cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec
rpm_pack
PACKAGE=clickhouse-test
ARCH=all
TARGET=noarch
deb_unpack
mv ${PACKAGE}-$VERSION_FULL-2.spec ${PACKAGE}-$VERSION_FULL-2.spec_tmp
echo "Requires: python3" >> ${PACKAGE}-$VERSION_FULL-2.spec
#echo "Requires: python3-termcolor" >> ${PACKAGE}-$VERSION-2.spec
cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec
rpm_pack
PACKAGE=clickhouse-common-static
ARCH=amd64
TARGET=x86_64
@ -271,7 +261,7 @@ function make_tgz {
VERSION_FULL="${VERSION_STRING}"
PACKAGE_DIR=${PACKAGE_DIR=../}
for PACKAGE in clickhouse-server clickhouse-client clickhouse-test clickhouse-common-static clickhouse-common-static-dbg; do
for PACKAGE in clickhouse-server clickhouse-client clickhouse-common-static clickhouse-common-static-dbg; do
alien --verbose --scripts --generate --to-tgz ${PACKAGE_DIR}${PACKAGE}_${VERSION_FULL}_*.deb
PKGDIR="./${PACKAGE}-${VERSION_FULL}"
if [ ! -d "$PKGDIR/install" ]; then