thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove subprocess_call from cluster.py

Browse Source
This commit is contained in:
Yatsishin Ilya 2021-06-04 13:14:32 +03:00
parent d715395751
commit e075130112
1 changed files with 15 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
tests/integration/helpers/cluster.py
View File

@ -30,6 +30,7 @@ from kazoo.client import KazooClient
from kazoo.exceptions import KazooException
from minio import Minio
from minio.deleteobjects import DeleteObject
from helpers.test_tools import assert_eq_with_retry
import docker
@ -51,7 +52,7 @@ def _create_env_file(path, variables):
f.write("=".join([var, value]) + "\n")
return path
def run_and_check(args, env=None, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, nothrow=False, detach=False):
def run_and_check(args, env=None, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300, nothrow=False, detach=False):
if detach:
subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env, shell=shell)
return
@ -105,12 +106,6 @@ def subprocess_check_call(args, detach=False, nothrow=False):
#logging.info('run:' + ' '.join(args))
return run_and_check(args, detach=detach, nothrow=nothrow)
def subprocess_call(args):
# Uncomment for debugging..;
# logging.debug('run:', ' ' . join(args))
run_and_check(args)
def get_odbc_bridge_path():
path = os.environ.get('CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH')
if path is None:
@ -378,13 +373,13 @@ class ClickHouseCluster:
def cleanup(self):
# Just in case kill unstopped containers from previous launch
try:
result = subprocess_call(['docker', 'container', 'list', '-a', '-f name={self.project_name}'])
result = run_and_check(['docker', 'container', 'list', '-a', '-f name={self.project_name}'])
if int(result) > 1:
logging.debug("Trying to kill unstopped containers...")
subprocess_call(['docker', 'kill', f'`docker container list -a -f name={self.project_name}`'])
subprocess_call(['docker', 'rm', f'`docker container list -a -f name={self.project_name}`'])
run_and_check(['docker', 'kill', f'`docker container list -a -f name={self.project_name}`'])
run_and_check(['docker', 'rm', f'`docker container list -a -f name={self.project_name}`'])
logging.debug("Unstopped containers killed")
subprocess_call(['docker-compose', 'ps', '--services', '--all'])
run_and_check(['docker-compose', 'ps', '--services', '--all'])
else:
logging.debug(f"No running containers for project: {self.project_name}")
except:
@ -394,7 +389,7 @@ class ClickHouseCluster:
# try:
# logging.debug("Trying to prune unused networks...")
# subprocess_call(['docker', 'network', 'prune', '-f'])
# run_and_check(['docker', 'network', 'prune', '-f'])
# logging.debug("Networks pruned")
# except:
# pass
@ -403,7 +398,7 @@ class ClickHouseCluster:
# try:
# logging.debug("Trying to prune unused images...")
# subprocess_call(['docker', 'image', 'prune', '-f'])
# run_and_check(['docker', 'image', 'prune', '-f'])
# logging.debug("Images pruned")
# except:
# pass
@ -412,7 +407,7 @@ class ClickHouseCluster:
try:
logging.debug("Trying to prune unused volumes...")
subprocess_call(['docker', 'volume', 'prune', '-f'])
run_and_check(['docker', 'volume', 'prune', '-f'])
logging.debug("Volumes pruned")
except:
pass
@ -949,7 +944,7 @@ class ClickHouseCluster:
errors += [str(ex)]
time.sleep(1)
subprocess_call(['docker-compose', 'ps', '--services', '--all'])
run_and_check(['docker-compose', 'ps', '--services', '--all'])
logging.error("Can't connect to MySQL Client:{}".format(errors))
raise Exception("Cannot wait MySQL Client container")
@ -967,7 +962,7 @@ class ClickHouseCluster:
errors += [str(ex)]
time.sleep(0.5)
subprocess_call(['docker-compose', 'ps', '--services', '--all'])
run_and_check(['docker-compose', 'ps', '--services', '--all'])
logging.error("Can't connect to MySQL:{}".format(errors))
raise Exception("Cannot wait MySQL container")
@ -984,7 +979,7 @@ class ClickHouseCluster:
logging.debug("Can't connect to MySQL 8 " + str(ex))
time.sleep(0.5)
subprocess_call(['docker-compose', 'ps', '--services', '--all'])
run_and_check(['docker-compose', 'ps', '--services', '--all'])
raise Exception("Cannot wait MySQL 8 container")
def wait_mysql_cluster_to_start(self, timeout=180):
@ -1004,7 +999,7 @@ class ClickHouseCluster:
errors += [str(ex)]
time.sleep(0.5)
subprocess_call(['docker-compose', 'ps', '--services', '--all'])
run_and_check(['docker-compose', 'ps', '--services', '--all'])
logging.error("Can't connect to MySQL:{}".format(errors))
raise Exception("Cannot wait MySQL container")
@ -1425,7 +1420,7 @@ class ClickHouseCluster:
run_and_check(clickhouse_start_cmd)
logging.debug("ClickHouse instance created")
start_timeout = 180.0 # seconds
start_timeout = 300.0 # seconds
for instance in self.instances.values():
instance.docker_client = self.docker_client
instance.ip_address = self.get_instance_ip(instance.name)
@ -1881,7 +1876,7 @@ class ClickHouseInstance:
return None
return None
def restart_with_latest_version(self, stop_start_wait_sec=120, callback_onstop=None, signal=60):
def restart_with_latest_version(self, stop_start_wait_sec=300, callback_onstop=None, signal=60):
if not self.stay_alive:
raise Exception("Cannot restart not stay alive container")
self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root')
@ -1908,7 +1903,6 @@ class ClickHouseInstance:
"cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"],
user='root')
self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid()))
from helpers.test_tools import assert_eq_with_retry
# wait start
assert_eq_with_retry(self, "select 1", "1", retry_count=retries)