ClickHouse/tests/integration/runner

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

438 lines
16 KiB
Plaintext
Raw Normal View History

2020-10-02 16:54:07 +00:00
#!/usr/bin/env python3
2022-06-21 18:53:13 +00:00
# -*- coding: utf-8 -*-
import subprocess
import os
import getpass
2021-07-01 14:41:59 +00:00
import glob
import argparse
import logging
import signal
import subprocess
import sys
2022-06-15 18:08:18 +00:00
import string
import random
def random_str(length=6):
alphabet = string.ascii_lowercase + string.digits
2022-06-21 18:53:13 +00:00
return "".join(random.SystemRandom().choice(alphabet) for _ in range(length))
2022-06-15 18:08:18 +00:00
CUR_FILE_DIR = os.path.dirname(os.path.realpath(__file__))
2020-04-06 18:30:51 +00:00
DEFAULT_CLICKHOUSE_ROOT = os.path.abspath(os.path.join(CUR_FILE_DIR, "../../"))
CURRENT_WORK_DIR = os.getcwd()
VOLUME_NAME = "clickhouse_integration_tests"
CONTAINER_NAME = f"{VOLUME_NAME}_{random_str()}"
2020-06-29 11:19:06 +00:00
CONFIG_DIR_IN_REPO = "programs/server"
INTEGRATION_DIR_IN_REPO = "tests/integration"
2020-10-11 02:19:16 +00:00
SRC_DIR_IN_REPO = "src"
2020-06-29 11:19:06 +00:00
2021-09-08 10:03:54 +00:00
DIND_INTEGRATION_TESTS_IMAGE_NAME = "clickhouse/integration-tests-runner"
2022-06-21 18:53:13 +00:00
def check_args_and_update_paths(args):
2020-06-29 11:19:06 +00:00
if args.clickhouse_root:
2020-07-31 12:08:19 +00:00
if not os.path.isabs(args.clickhouse_root):
2020-06-29 11:19:06 +00:00
CLICKHOUSE_ROOT = os.path.abspath(args.clickhouse_root)
else:
CLICKHOUSE_ROOT = args.clickhouse_root
else:
2022-06-21 18:53:13 +00:00
logging.info(
"ClickHouse root is not set. Will use %s" % (DEFAULT_CLICKHOUSE_ROOT)
)
2020-06-29 11:19:06 +00:00
CLICKHOUSE_ROOT = DEFAULT_CLICKHOUSE_ROOT
if not os.path.isabs(args.binary):
args.binary = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.binary))
2021-03-16 15:35:57 +00:00
if not args.odbc_bridge_binary:
2022-06-21 18:53:13 +00:00
args.odbc_bridge_binary = os.path.join(
os.path.dirname(args.binary), "clickhouse-odbc-bridge"
)
2021-03-16 15:35:57 +00:00
elif not os.path.isabs(args.odbc_bridge_binary):
2022-06-21 18:53:13 +00:00
args.odbc_bridge_binary = os.path.abspath(
os.path.join(CURRENT_WORK_DIR, args.odbc_bridge_binary)
)
2021-03-16 15:35:57 +00:00
if not args.library_bridge_binary:
2022-06-21 18:53:13 +00:00
args.library_bridge_binary = os.path.join(
os.path.dirname(args.binary), "clickhouse-library-bridge"
)
2021-03-16 15:35:57 +00:00
elif not os.path.isabs(args.library_bridge_binary):
2022-06-21 18:53:13 +00:00
args.library_bridge_binary = os.path.abspath(
os.path.join(CURRENT_WORK_DIR, args.library_bridge_binary)
)
2020-06-29 11:19:06 +00:00
if args.base_configs_dir:
if not os.path.isabs(args.base_configs_dir):
2022-06-21 18:53:13 +00:00
args.base_configs_dir = os.path.abspath(
os.path.join(CURRENT_WORK_DIR, args.base_configs_dir)
)
2020-06-29 11:19:06 +00:00
else:
2022-06-21 18:53:13 +00:00
args.base_configs_dir = os.path.abspath(
os.path.join(CLICKHOUSE_ROOT, CONFIG_DIR_IN_REPO)
)
logging.info(
"Base configs dir is not set. Will use %s" % (args.base_configs_dir)
)
2020-06-29 11:19:06 +00:00
if args.cases_dir:
if not os.path.isabs(args.cases_dir):
2022-06-21 18:53:13 +00:00
args.cases_dir = os.path.abspath(
os.path.join(CURRENT_WORK_DIR, args.cases_dir)
)
2020-06-29 11:19:06 +00:00
else:
2022-06-21 18:53:13 +00:00
args.cases_dir = os.path.abspath(
os.path.join(CLICKHOUSE_ROOT, INTEGRATION_DIR_IN_REPO)
2022-06-21 18:53:13 +00:00
)
2021-02-24 15:14:21 +00:00
logging.info("Cases dir is not set. Will use %s" % (args.cases_dir))
2020-10-11 02:19:16 +00:00
if args.src_dir:
if not os.path.isabs(args.src_dir):
args.src_dir = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.src_dir))
else:
args.src_dir = os.path.abspath(os.path.join(CLICKHOUSE_ROOT, SRC_DIR_IN_REPO))
2021-02-24 15:14:21 +00:00
logging.info("src dir is not set. Will use %s" % (args.src_dir))
2020-10-11 02:19:16 +00:00
2022-06-21 18:53:13 +00:00
logging.info(
"base_configs_dir: {}, binary: {}, cases_dir: {} ".format(
args.base_configs_dir, args.binary, args.cases_dir
)
)
2020-06-29 11:19:06 +00:00
2022-06-21 18:53:13 +00:00
for path in [
args.binary,
args.odbc_bridge_binary,
args.library_bridge_binary,
args.base_configs_dir,
args.cases_dir,
CLICKHOUSE_ROOT,
]:
if not os.path.exists(path):
2020-06-29 11:19:06 +00:00
raise Exception("Path {} doesn't exist".format(path))
2021-04-30 09:18:12 +00:00
if args.dockerd_volume:
if not os.path.isabs(args.dockerd_volume):
2022-06-21 18:53:13 +00:00
args.src_dir = os.path.abspath(
os.path.join(CURRENT_WORK_DIR, args.dockerd_volume)
)
if (not os.path.exists(os.path.join(args.base_configs_dir, "config.xml"))) and (
not os.path.exists(os.path.join(args.base_configs_dir, "config.yaml"))
):
raise Exception(
"No config.xml or config.yaml in {}".format(args.base_configs_dir)
)
2020-06-29 11:19:06 +00:00
2022-06-21 18:53:13 +00:00
if (not os.path.exists(os.path.join(args.base_configs_dir, "users.xml"))) and (
not os.path.exists(os.path.join(args.base_configs_dir, "users.yaml"))
):
raise Exception(
"No users.xml or users.yaml in {}".format(args.base_configs_dir)
)
2020-06-29 11:19:06 +00:00
def docker_kill_handler_handler(signum, frame):
2022-06-21 18:53:13 +00:00
subprocess.check_call(
'docker ps --all --quiet --filter name={name} --format="{{{{.ID}}}}"'.format(
2022-06-21 18:53:13 +00:00
name=CONTAINER_NAME
),
shell=True,
)
raise KeyboardInterrupt("Killed by Ctrl+C")
2022-06-21 18:53:13 +00:00
signal.signal(signal.SIGINT, docker_kill_handler_handler)
2020-06-29 11:19:06 +00:00
# Integration tests runner should allow to run tests on several versions of ClickHouse.
# Integration tests should be portable.
# To run integration tests following artfacts should be sufficient:
# - clickhouse binaries (env CLICKHOUSE_TESTS_SERVER_BIN_PATH or --binary arg)
# - clickhouse default configs(config.xml, users.xml) from same version as binary (env CLICKHOUSE_TESTS_BASE_CONFIG_DIR or --base-configs-dir arg)
2021-03-16 15:35:57 +00:00
# - odbc bridge binary (env CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH or --odbc-bridge-binary arg)
# - library bridge binary (env CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH or --library-bridge-binary)
2020-06-29 11:19:06 +00:00
# - tests/integration directory with all test cases and configs (env CLICKHOUSE_TESTS_INTEGRATION_PATH or --cases-dir)
#
# 1) --clickhouse-root is only used to determine other paths on default places
# 2) path of runner script is used to determine paths for trivial case, when we run it from repository
if __name__ == "__main__":
2022-06-21 18:53:13 +00:00
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [ %(process)d ] %(levelname)s : %(message)s (%(filename)s:%(lineno)s, %(funcName)s)",
)
parser = argparse.ArgumentParser(description="ClickHouse integration tests runner")
parser.add_argument(
"--binary",
2022-06-21 18:53:13 +00:00
default=os.environ.get(
"CLICKHOUSE_TESTS_SERVER_BIN_PATH",
os.environ.get("CLICKHOUSE_TESTS_CLIENT_BIN_PATH", "/usr/bin/clickhouse"),
),
help="Path to clickhouse binary. For example /usr/bin/clickhouse",
)
2019-01-30 08:24:16 +00:00
parser.add_argument(
2021-03-16 15:35:57 +00:00
"--odbc-bridge-binary",
default=os.environ.get("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH", ""),
2022-06-21 18:53:13 +00:00
help="Path to clickhouse-odbc-bridge binary. Defaults to clickhouse-odbc-bridge in the same dir as clickhouse.",
)
2019-01-30 08:24:16 +00:00
2021-03-16 15:35:57 +00:00
parser.add_argument(
"--library-bridge-binary",
default=os.environ.get("CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH", ""),
2022-06-21 18:53:13 +00:00
help="Path to clickhouse-library-bridge binary. Defaults to clickhouse-library-bridge in the same dir as clickhouse.",
)
2021-03-16 15:35:57 +00:00
parser.add_argument(
2020-06-29 11:19:06 +00:00
"--base-configs-dir",
default=os.environ.get("CLICKHOUSE_TESTS_BASE_CONFIG_DIR"),
2022-06-21 18:53:13 +00:00
help="Path to clickhouse base configs directory with config.xml/users.xml",
)
2020-06-29 11:19:06 +00:00
parser.add_argument(
"--cases-dir",
default=os.environ.get("CLICKHOUSE_TESTS_INTEGRATION_PATH"),
2022-06-21 18:53:13 +00:00
help="Path to integration tests cases and configs directory. For example tests/integration in repository",
)
2020-10-11 02:19:16 +00:00
parser.add_argument(
"--src-dir",
default=os.environ.get("CLICKHOUSE_SRC_DIR"),
2022-06-21 18:53:13 +00:00
help="Path to the 'src' directory in repository. Used to provide schemas (e.g. *.proto) for some tests when those schemas are located in the 'src' directory",
)
2020-10-11 02:19:16 +00:00
parser.add_argument(
"--clickhouse-root",
2022-06-21 18:53:13 +00:00
help="Path to repository root folder. Used to take configuration from repository default paths.",
)
parser.add_argument(
"--command",
2022-06-21 18:53:13 +00:00
default="",
help="Set it to run some other command in container (for example bash)",
)
2018-11-26 10:23:03 +00:00
parser.add_argument(
"--disable-net-host",
2022-06-21 18:53:13 +00:00
action="store_true",
2018-11-26 10:23:03 +00:00
default=False,
2022-06-21 18:53:13 +00:00
help="Don't use net host in parent docker container",
)
2018-11-26 10:23:03 +00:00
2021-05-24 08:23:04 +00:00
parser.add_argument(
"--network",
2022-06-21 18:53:13 +00:00
help="Set network driver for runnner container (defaults to `host`)",
)
2021-05-24 08:23:04 +00:00
parser.add_argument(
"--docker-image-version",
default="latest",
2022-06-21 18:53:13 +00:00
help="Version of docker image which runner will use to run tests",
)
2020-08-10 13:35:08 +00:00
parser.add_argument(
"--docker-compose-images-tags",
action="append",
2022-06-21 18:53:13 +00:00
help="Set non-default tags for images used in docker compose recipes(yandex/my_container:my_tag)",
)
2021-02-19 14:42:43 +00:00
parser.add_argument(
2022-06-21 18:53:13 +00:00
"-n", "--parallel", action="store", dest="parallel", help="Parallelism"
)
2021-02-19 14:42:43 +00:00
2023-01-04 15:47:50 +00:00
parser.add_argument(
"--no-random", action="store", dest="no_random", help="Disable tests order randomization"
)
parser.add_argument(
"--pre-pull", action="store_true", default=False, dest="pre_pull", help="Pull images for docker_compose before all other actions"
)
2021-03-19 11:44:28 +00:00
parser.add_argument(
2022-06-21 18:53:13 +00:00
"-t",
"--tests_list",
2021-03-19 11:44:28 +00:00
action="store",
2022-06-21 18:53:13 +00:00
nargs="+",
2021-03-19 11:44:28 +00:00
default=[],
dest="tests_list",
2022-06-21 18:53:13 +00:00
help="List of tests to run",
)
2021-03-19 11:44:28 +00:00
parser.add_argument(
2022-06-21 18:53:13 +00:00
"-k",
"--keyword_expression",
action="store",
dest="keyword_expression",
2022-06-21 18:53:13 +00:00
help="pytest keyword expression",
)
2021-04-14 16:04:17 +00:00
parser.add_argument(
"--tmpfs",
2022-06-21 18:53:13 +00:00
action="store_true",
2021-04-14 16:04:17 +00:00
default=False,
dest="tmpfs",
2022-06-21 18:53:13 +00:00
help="Use tmpfs for dockerd files",
)
2021-04-14 16:04:17 +00:00
2021-06-04 15:00:59 +00:00
parser.add_argument(
"--cleanup-containers",
2022-06-21 18:53:13 +00:00
action="store_true",
2021-06-04 15:00:59 +00:00
default=False,
dest="cleanup_containers",
2022-06-21 18:53:13 +00:00
help="Remove all running containers on test session start",
)
2021-06-04 15:00:59 +00:00
2021-04-30 09:18:12 +00:00
parser.add_argument(
"--dockerd-volume-dir",
2022-06-21 18:53:13 +00:00
action="store",
2021-04-30 09:18:12 +00:00
dest="dockerd_volume",
2022-06-21 18:53:13 +00:00
help="Bind volume to this dir to use for dockerd files",
)
2021-04-30 09:18:12 +00:00
2022-06-21 18:53:13 +00:00
parser.add_argument("pytest_args", nargs="*", help="args for pytest command")
args = parser.parse_args()
check_args_and_update_paths(args)
2021-02-19 14:42:43 +00:00
parallel_args = ""
if args.parallel:
parallel_args += "--dist=loadfile"
2021-02-20 14:59:39 +00:00
parallel_args += " -n {}".format(args.parallel)
2021-02-19 14:42:43 +00:00
2022-12-29 14:24:23 +00:00
rand_args = ""
# if not args.no_random:
# rand_args += f"--random-seed={os.getpid()}"
2022-12-29 14:24:23 +00:00
2018-11-26 10:23:03 +00:00
net = ""
2021-05-24 08:23:04 +00:00
if args.network:
net = "--net={}".format(args.network)
elif not args.disable_net_host:
2018-11-26 10:23:03 +00:00
net = "--net=host"
2020-08-10 13:35:08 +00:00
env_tags = ""
2020-08-11 08:36:26 +00:00
if args.docker_compose_images_tags is not None:
2020-08-11 08:35:31 +00:00
for img_tag in args.docker_compose_images_tags:
[image, tag] = img_tag.split(":")
2021-09-08 10:03:54 +00:00
if image == "clickhouse/mysql-golang-client":
2020-08-11 08:35:31 +00:00
env_tags += "-e {}={} ".format("DOCKER_MYSQL_GOLANG_CLIENT_TAG", tag)
2022-01-02 23:56:23 +00:00
elif image == "clickhouse/dotnet-client":
env_tags += "-e {}={} ".format("DOCKER_DOTNET_CLIENT_TAG", tag)
2021-09-08 10:03:54 +00:00
elif image == "clickhouse/mysql-java-client":
2020-08-11 08:35:31 +00:00
env_tags += "-e {}={} ".format("DOCKER_MYSQL_JAVA_CLIENT_TAG", tag)
2021-09-08 10:03:54 +00:00
elif image == "clickhouse/mysql-js-client":
2020-08-11 08:35:31 +00:00
env_tags += "-e {}={} ".format("DOCKER_MYSQL_JS_CLIENT_TAG", tag)
2021-09-08 10:03:54 +00:00
elif image == "clickhouse/mysql-php-client":
2020-08-11 08:35:31 +00:00
env_tags += "-e {}={} ".format("DOCKER_MYSQL_PHP_CLIENT_TAG", tag)
2021-09-08 10:03:54 +00:00
elif image == "clickhouse/postgresql-java-client":
2020-08-11 08:35:31 +00:00
env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag)
2021-09-08 10:03:54 +00:00
elif image == "clickhouse/integration-test":
2020-09-29 08:58:04 +00:00
env_tags += "-e {}={} ".format("DOCKER_BASE_TAG", tag)
elif image == "clickhouse/kerberized-hadoop":
env_tags += "-e {}={} ".format("DOCKER_KERBERIZED_HADOOP_TAG", tag)
2021-09-08 10:03:54 +00:00
elif image == "clickhouse/kerberos-kdc":
2022-01-06 12:48:16 +00:00
env_tags += "-e {}={} ".format("DOCKER_KERBEROS_KDC_TAG", tag)
2020-08-11 08:35:31 +00:00
else:
2021-02-24 15:14:21 +00:00
logging.info("Unknown image %s" % (image))
2020-08-10 13:35:08 +00:00
# create named volume which will be used inside to store images and other docker related files,
# to avoid redownloading it every time
#
# should be removed manually when not needed
2021-04-14 16:04:17 +00:00
dockerd_internal_volume = ""
if args.tmpfs:
dockerd_internal_volume = "--tmpfs /var/lib/docker -e DOCKER_RAMDISK=true"
2021-04-30 09:18:12 +00:00
elif args.dockerd_volume:
2022-06-21 18:53:13 +00:00
dockerd_internal_volume = (
"--mount type=bind,source={},target=/var/lib/docker".format(
args.dockerd_volume
)
)
2021-04-14 16:04:17 +00:00
else:
2021-11-16 08:59:38 +00:00
try:
2022-06-21 18:53:13 +00:00
subprocess.check_call(
f"docker volume create {VOLUME_NAME}_volume", shell=True
)
2021-11-16 08:59:38 +00:00
except Exception as ex:
2022-08-04 18:12:30 +00:00
print("Volume creation failed, probably it already exists, exception", ex)
# TODO: this part cleans out stale volumes produced by container name
# randomizer, we should remove it after Sep 2022
try:
subprocess.check_call(
f"docker volume ls -q | grep '{VOLUME_NAME}_.*_volume' | xargs --no-run-if-empty docker volume rm",
shell=True,
)
except Exception as ex:
print("Probably, some stale volumes still there, just continue:", ex)
# TODO END
dockerd_internal_volume = f"--volume={VOLUME_NAME}_volume:/var/lib/docker"
2021-06-04 15:00:59 +00:00
# If enabled we kill and remove containers before pytest session run.
env_cleanup = ""
if args.cleanup_containers:
env_cleanup = "-e PYTEST_CLEANUP_CONTAINERS=1"
# enable tty mode & interactive for docker if we have real tty
tty = ""
if sys.stdout.isatty() and sys.stdin.isatty():
tty = "-it"
2021-07-01 14:41:59 +00:00
# Remove old logs.
for old_log_path in glob.glob(args.cases_dir + "/pytest*.log"):
os.remove(old_log_path)
2021-04-14 16:04:17 +00:00
if args.keyword_expression:
2022-06-21 18:53:13 +00:00
args.pytest_args += ["-k", args.keyword_expression]
2023-01-04 15:47:50 +00:00
cmd_base = "docker run {net} {tty} --rm --name {name} --privileged \
2021-03-16 15:35:57 +00:00
--volume={odbc_bridge_bin}:/clickhouse-odbc-bridge --volume={bin}:/clickhouse \
2022-04-05 23:40:56 +00:00
--volume={library_bridge_bin}:/clickhouse-library-bridge \
2020-07-28 13:11:29 +00:00
--volume={base_cfg}:/clickhouse-config --volume={cases_dir}:/ClickHouse/tests/integration \
2020-10-11 02:19:16 +00:00
--volume={src_dir}/Server/grpc_protos:/ClickHouse/src/Server/grpc_protos \
--volume=/run:/run/host:ro \
2021-06-02 15:08:16 +00:00
{dockerd_internal_volume} -e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 \
-e XTABLES_LOCKFILE=/run/host/xtables.lock \
2022-07-07 20:19:15 +00:00
-e PYTHONUNBUFFERED=1 \
{env_tags} {env_cleanup} -e PYTEST_OPTS='{parallel} {opts} {tests_list} {rand} -vvv' {img}".format(
2018-11-26 10:23:03 +00:00
net=net,
tty=tty,
bin=args.binary,
2021-03-16 15:35:57 +00:00
odbc_bridge_bin=args.odbc_bridge_binary,
library_bridge_bin=args.library_bridge_binary,
2020-06-29 11:19:06 +00:00
base_cfg=args.base_configs_dir,
cases_dir=args.cases_dir,
2020-10-11 02:19:16 +00:00
src_dir=args.src_dir,
2020-08-10 13:35:08 +00:00
env_tags=env_tags,
2021-06-04 15:00:59 +00:00
env_cleanup=env_cleanup,
2021-02-19 14:42:43 +00:00
parallel=parallel_args,
2022-12-29 14:24:23 +00:00
rand=rand_args,
2022-06-21 18:53:13 +00:00
opts=" ".join(args.pytest_args).replace("'", "\\'"),
tests_list=" ".join(args.tests_list),
2021-04-14 16:04:17 +00:00
dockerd_internal_volume=dockerd_internal_volume,
img=DIND_INTEGRATION_TESTS_IMAGE_NAME + ":" + args.docker_image_version,
name=CONTAINER_NAME,
)
2023-01-04 15:47:50 +00:00
cmd = cmd_base + " " + args.command
cmd_pre_pull = cmd_base + " find /compose -name docker_compose_*.yml -exec docker-compose -f '{}' pull \;"
containers = subprocess.check_output(
f"docker ps --all --quiet --filter name={CONTAINER_NAME} --format={{{{.ID}}}}",
shell=True,
universal_newlines=True,
).splitlines()
if containers:
print(f"Trying to kill containers name={CONTAINER_NAME} ids={containers}")
subprocess.check_call(f"docker kill {' '.join(containers)}", shell=True)
print(f"Containers {containers} killed")
2023-01-04 15:47:50 +00:00
if args.pre_pull:
print(("Running pre pull as: '" + cmd_pre_pull + "'."))
subprocess.check_call(cmd_pre_pull, shell=True)
2020-10-02 16:54:07 +00:00
print(("Running pytest container as: '" + cmd + "'."))
2019-07-21 11:45:01 +00:00
subprocess.check_call(cmd, shell=True)