diff --git a/tests/testflows/aes_encryption/regression.py b/tests/testflows/aes_encryption/regression.py
index e50ac0a3f8b..6c4598dae28 100755
--- a/tests/testflows/aes_encryption/regression.py
+++ b/tests/testflows/aes_encryption/regression.py
@@ -53,7 +53,7 @@ xfails = {
RQ_SRS008_AES_Functions_DifferentModes("1.0")
)
@XFails(xfails)
-def regression(self, local, clickhouse_binary_path):
+def regression(self, local, clickhouse_binary_path, stress=None, parallel=None):
"""ClickHouse AES encryption functions regression module.
"""
nodes = {
diff --git a/tests/testflows/example/regression.py b/tests/testflows/example/regression.py
index 2c0a778d39b..cb58b42ba4c 100755
--- a/tests/testflows/example/regression.py
+++ b/tests/testflows/example/regression.py
@@ -2,7 +2,7 @@
import sys
from testflows.core import *
-append_path(sys.path, "..")
+append_path(sys.path, "..")
from helpers.cluster import Cluster
from helpers.argparser import argparser
@@ -10,13 +10,13 @@ from helpers.argparser import argparser
@TestFeature
@Name("example")
@ArgumentParser(argparser)
-def regression(self, local, clickhouse_binary_path):
+def regression(self, local, clickhouse_binary_path, stress=None, parallel=None):
"""Simple example of how you can use TestFlows to test ClickHouse.
"""
nodes = {
"clickhouse": ("clickhouse1",),
}
-
+
with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster:
self.context.cluster = cluster
diff --git a/tests/testflows/helpers/argparser.py b/tests/testflows/helpers/argparser.py
index 033c15a3bfe..03014becb76 100644
--- a/tests/testflows/helpers/argparser.py
+++ b/tests/testflows/helpers/argparser.py
@@ -1,5 +1,12 @@
import os
+def onoff(v):
+ if v in ["yes", "1", "on"]:
+ return True
+ elif v in ["no", "0", "off"]:
+ return False
+ raise ValueError(f"invalid {v}")
+
def argparser(parser):
"""Default argument parser for regressions.
"""
@@ -10,4 +17,10 @@ def argparser(parser):
parser.add_argument("--clickhouse-binary-path",
type=str, dest="clickhouse_binary_path",
help="path to ClickHouse binary, default: /usr/bin/clickhouse", metavar="path",
- default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse"))
\ No newline at end of file
+ default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse"))
+
+ parser.add_argument("--stress", action="store_true", default=False,
+ help="enable stress testing (might take a long time)")
+
+ parser.add_argument("--parallel", type=onoff, default=True, choices=["yes", "no", "on", "off", 0, 1],
+ help="enable parallelism for tests that support it")
\ No newline at end of file
diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py
old mode 100644
new mode 100755
index 01ebfa8424b..27e06a7c17f
--- a/tests/testflows/helpers/cluster.py
+++ b/tests/testflows/helpers/cluster.py
@@ -7,6 +7,7 @@ import tempfile
from testflows.core import *
from testflows.asserts import error
from testflows.connect import Shell
+from testflows.uexpect import ExpectTimeoutError
class QueryRuntimeException(Exception):
"""Exception during query execution on the server.
@@ -25,7 +26,7 @@ class Node(object):
def repr(self):
return f"Node(name='{self.name}')"
- def restart(self, timeout=120, safe=True):
+ def restart(self, timeout=300, safe=True):
"""Restart node.
"""
with self.cluster.lock:
@@ -42,18 +43,18 @@ class Node(object):
class ClickHouseNode(Node):
"""Node with ClickHouse server.
"""
- def wait_healthy(self, timeout=120):
+ def wait_healthy(self, timeout=300):
with By(f"waiting until container {self.name} is healthy"):
start_time = time.time()
while True:
- if self.query("select 1", no_checks=1, timeout=120, steps=False).exitcode == 0:
+ if self.query("select 1", no_checks=1, timeout=300, steps=False).exitcode == 0:
break
if time.time() - start_time < timeout:
time.sleep(2)
continue
assert False, "container is not healthy"
- def restart(self, timeout=120, safe=True, wait_healthy=True):
+ def restart(self, timeout=300, safe=True, wait_healthy=True):
"""Restart node.
"""
if safe:
@@ -79,32 +80,43 @@ class ClickHouseNode(Node):
def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False,
raise_on_exception=False, step=By, settings=None, *args, **kwargs):
"""Execute and check query.
-
:param sql: sql query
:param message: expected message that should be in the output, default: None
:param exitcode: expected exitcode, default: None
"""
+ settings = list(settings or [])
+
+ if hasattr(current().context, "default_query_settings"):
+ settings += current().context.default_query_settings
+
if len(sql) > 1024:
with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query:
query.write(sql)
query.flush()
command = f"cat \"{query.name}\" | {self.cluster.docker_compose} exec -T {self.name} clickhouse client -n"
- for setting in settings or []:
+ for setting in settings:
name, value = setting
command += f" --{name} \"{value}\""
description = f"""
echo -e \"{sql[:100]}...\" > {query.name}
{command}
"""
- with step("executing command", description=description) if steps else NullStep():
- r = self.cluster.bash(None)(command, *args, **kwargs)
+ with step("executing command", description=description, format_description=False) if steps else NullStep():
+ try:
+ r = self.cluster.bash(None)(command, *args, **kwargs)
+ except ExpectTimeoutError:
+ self.cluster.close_bash(None)
else:
command = f"echo -e \"{sql}\" | clickhouse client -n"
- for setting in settings or []:
+ for setting in settings:
name, value = setting
command += f" --{name} \"{value}\""
- with step("executing command", description=command) if steps else NullStep():
- r = self.cluster.bash(self.name)(command, *args, **kwargs)
+ with step("executing command", description=command, format_description=False) if steps else NullStep():
+ try:
+ r = self.cluster.bash(self.name)(command, *args, **kwargs)
+ except ExpectTimeoutError:
+ self.cluster.close_bash(self.name)
+ raise
if no_checks:
return r
@@ -135,6 +147,7 @@ class Cluster(object):
docker_compose="docker-compose", docker_compose_project_dir=None,
docker_compose_file="docker-compose.yml"):
+ self.terminating = False
self._bash = {}
self.clickhouse_binary_path = clickhouse_binary_path
self.configs_dir = configs_dir
@@ -168,7 +181,7 @@ class Cluster(object):
self.docker_compose += f" --no-ansi --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\""
self.lock = threading.Lock()
- def shell(self, node, timeout=120):
+ def shell(self, node, timeout=300):
"""Returns unique shell terminal to be used.
"""
if node is None:
@@ -181,14 +194,22 @@ class Cluster(object):
shell.timeout = timeout
return shell
- def bash(self, node, timeout=120):
+ def bash(self, node, timeout=300):
"""Returns thread-local bash terminal
to a specific node.
-
:param node: name of the service
"""
+ test = current()
+
+ if self.terminating:
+ if test and (test.cflags & MANDATORY):
+ pass
+ else:
+ raise InterruptedError("terminating")
+
current_thread = threading.current_thread()
- id = f"{current_thread.ident}-{node}"
+ id = f"{current_thread.name}-{node}"
+
with self.lock:
if self._bash.get(id) is None:
if node is None:
@@ -197,9 +218,30 @@ class Cluster(object):
self._bash[id] = Shell(command=[
"/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting"
], name=node).__enter__()
+
self._bash[id].timeout = timeout
+
+ # clean up any stale open shells for threads that have exited
+ active_thread_names = {thread.name for thread in threading.enumerate()}
+
+ for bash_id in list(self._bash.keys()):
+ thread_name, node_name = bash_id.rsplit("-", 1)
+ if thread_name not in active_thread_names:
+ self._bash[bash_id].__exit__(None, None, None)
+ del self._bash[bash_id]
+
return self._bash[id]
+ def close_bash(self, node):
+ current_thread = threading.current_thread()
+ id = f"{current_thread.name}-{node}"
+
+ with self.lock:
+ if self._bash.get(id) is None:
+ return
+ self._bash[id].__exit__(None, None, None)
+ del self._bash[id]
+
def __enter__(self):
with Given("docker-compose cluster"):
self.up()
@@ -211,20 +253,21 @@ class Cluster(object):
self.down()
finally:
with self.lock:
- for shell in list(self._bash.values()):
+ for shell in self._bash.values():
shell.__exit__(type, value, traceback)
def node(self, name):
"""Get object with node bound methods.
-
:param name: name of service name
"""
if name.startswith("clickhouse"):
return ClickHouseNode(self, name)
return Node(self, name)
- def down(self, timeout=120):
+ def down(self, timeout=300):
"""Bring cluster down by executing docker-compose down."""
+ self.terminating = True
+
try:
bash = self.bash(None)
with self.lock:
@@ -236,7 +279,7 @@ class Cluster(object):
else:
self._bash[id] = shell
finally:
- return self.command(None, f"{self.docker_compose} down", timeout=timeout)
+ return self.command(None, f"{self.docker_compose} down --timeout 60", bash=bash, timeout=timeout)
def up(self, timeout=30*60):
if self.local:
@@ -265,7 +308,7 @@ class Cluster(object):
if cmd.exitcode != 0:
continue
with And("executing docker-compose down just in case it is up"):
- cmd = self.command(None, f"{self.docker_compose} down 2>&1 | tee", exitcode=None, timeout=timeout)
+ cmd = self.command(None, f"{self.docker_compose} down --remove-orphans 2>&1 | tee", exitcode=None, timeout=timeout)
if cmd.exitcode != 0:
continue
with And("executing docker-compose up"):
@@ -286,18 +329,22 @@ class Cluster(object):
for name in self.nodes["clickhouse"]:
self.node(name).wait_healthy()
- def command(self, node, command, message=None, exitcode=None, steps=True, *args, **kwargs):
+ def command(self, node, command, message=None, exitcode=None, steps=True, bash=None, *args, **kwargs):
"""Execute and check command.
-
:param node: name of the service
:param command: command
:param message: expected message that should be in the output, default: None
:param exitcode: expected exitcode, default: None
:param steps: don't break command into steps, default: True
"""
- debug(f"command() {node}, {command}")
with By("executing command", description=command, format_description=False) if steps else NullStep():
- r = self.bash(node)(command, *args, **kwargs)
+ if bash is None:
+ bash = self.bash(node)
+ try:
+ r = bash(command, *args, **kwargs)
+ except ExpectTimeoutError:
+ self.close_bash(node)
+ raise
if exitcode is not None:
with Then(f"exitcode should be {exitcode}", format_name=False) if steps else NullStep():
assert r.exitcode == exitcode, error(r.output)
diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py
index 0e9d06cf84a..9cc9aa85f93 100755
--- a/tests/testflows/ldap/regression.py
+++ b/tests/testflows/ldap/regression.py
@@ -9,7 +9,7 @@ from helpers.argparser import argparser
@TestModule
@Name("ldap")
@ArgumentParser(argparser)
-def regression(self, local, clickhouse_binary_path):
+def regression(self, local, clickhouse_binary_path, parallel=None, stress=None):
"""ClickHouse LDAP integration regression module.
"""
args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path}
@@ -18,4 +18,4 @@ def regression(self, local, clickhouse_binary_path):
Feature(test=load("ldap.external_user_directory.regression", "regression"))(**args)
if main():
- regression()
+ regression()
\ No newline at end of file
diff --git a/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml b/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml
index ada8eec5fc9..a7ed0d6e2b4 100644
--- a/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml
+++ b/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml
@@ -58,9 +58,44 @@
9440
1
-
-
-
+
+
+
+
+
+ clickhouse1
+ 9440
+ 1
+
+
+
+
+
+
+ clickhouse1
+ 9000
+
+
+
+
+ clickhouse2
+ 9000
+
+
+
+
+
+
+ clickhouse1
+ 9000
+
+
+ clickhouse2
+ 9000
+
+
+
+
clickhouse2
@@ -73,8 +108,20 @@
9000
-
-
+
+
+
+
+ clickhouse2
+ 9000
+
+
+ clickhouse3
+ 9000
+
+
+
+
clickhouse1
@@ -94,6 +141,22 @@
+
+
+
+ clickhouse1
+ 9000
+
+
+ clickhouse2
+ 9000
+
+
+ clickhouse3
+ 9000
+
+
+
diff --git a/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml b/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml
index ca65ffd5e04..768d2250b79 100644
--- a/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml
+++ b/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml
@@ -3,6 +3,7 @@
/etc/clickhouse-server/ssl/server.crt
/etc/clickhouse-server/ssl/server.key
+ /etc/clickhouse-server/ssl/dhparam.pem
none
true
diff --git a/tests/testflows/rbac/configs/clickhouse/config.xml b/tests/testflows/rbac/configs/clickhouse/config.xml
index 65187edf806..4ec12232539 100644
--- a/tests/testflows/rbac/configs/clickhouse/config.xml
+++ b/tests/testflows/rbac/configs/clickhouse/config.xml
@@ -69,7 +69,7 @@
-
+ 0.0.0.0
/var/lib/clickhouse/access/
+
+
+
+
+ users.xml
+
+
+
+ /var/lib/clickhouse/access/
+
+
+
users.xml
@@ -160,7 +172,7 @@
-
+
@@ -220,7 +232,7 @@
See https://clickhouse.yandex/docs/en/table_engines/replication/
-->
-
+
-
+