thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove subprocess_call from cluster.py

Browse Source
This commit is contained in:
Yatsishin Ilya 2021-06-04 13:14:32 +03:00
parent d715395751
commit e075130112
1 changed files with 15 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
tests/integration/helpers/cluster.py
View File

@ -30,6 +30,7 @@ from kazoo.client import KazooClient
from kazoo.exceptions import KazooException from kazoo.exceptions import KazooException
from minio import Minio from minio import Minio
from minio.deleteobjects import DeleteObject from minio.deleteobjects import DeleteObject
from helpers.test_tools import assert_eq_with_retry
import docker import docker
@ -51,7 +52,7 @@ def _create_env_file(path, variables):
f.write("=".join([var, value]) + "\n") f.write("=".join([var, value]) + "\n")
return path return path
def run_and_check(args, env=None, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, nothrow=False, detach=False): def run_and_check(args, env=None, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300, nothrow=False, detach=False):
if detach: if detach:
subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env, shell=shell) subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env, shell=shell)
return return
@ -105,12 +106,6 @@ def subprocess_check_call(args, detach=False, nothrow=False):
#logging.info('run:' + ' '.join(args)) #logging.info('run:' + ' '.join(args))
return run_and_check(args, detach=detach, nothrow=nothrow) return run_and_check(args, detach=detach, nothrow=nothrow)
def subprocess_call(args):
# Uncomment for debugging..;
# logging.debug('run:', ' ' . join(args))
run_and_check(args)
def get_odbc_bridge_path(): def get_odbc_bridge_path():
path = os.environ.get('CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH') path = os.environ.get('CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH')
if path is None: if path is None:
@ -378,13 +373,13 @@ class ClickHouseCluster:
def cleanup(self): def cleanup(self):
# Just in case kill unstopped containers from previous launch # Just in case kill unstopped containers from previous launch
try: try:
result = subprocess_call(['docker', 'container', 'list', '-a', '-f name={self.project_name}']) result = run_and_check(['docker', 'container', 'list', '-a', '-f name={self.project_name}'])
if int(result) > 1: if int(result) > 1:
logging.debug("Trying to kill unstopped containers...") logging.debug("Trying to kill unstopped containers...")
subprocess_call(['docker', 'kill', f'`docker container list -a -f name={self.project_name}`']) run_and_check(['docker', 'kill', f'`docker container list -a -f name={self.project_name}`'])
subprocess_call(['docker', 'rm', f'`docker container list -a -f name={self.project_name}`']) run_and_check(['docker', 'rm', f'`docker container list -a -f name={self.project_name}`'])
logging.debug("Unstopped containers killed") logging.debug("Unstopped containers killed")
subprocess_call(['docker-compose', 'ps', '--services', '--all']) run_and_check(['docker-compose', 'ps', '--services', '--all'])
else: else:
logging.debug(f"No running containers for project: {self.project_name}") logging.debug(f"No running containers for project: {self.project_name}")
except: except:
@ -394,7 +389,7 @@ class ClickHouseCluster:
# try: # try:
# logging.debug("Trying to prune unused networks...") # logging.debug("Trying to prune unused networks...")
# subprocess_call(['docker', 'network', 'prune', '-f']) # run_and_check(['docker', 'network', 'prune', '-f'])
# logging.debug("Networks pruned") # logging.debug("Networks pruned")
# except: # except:
# pass # pass
@ -403,7 +398,7 @@ class ClickHouseCluster:
# try: # try:
# logging.debug("Trying to prune unused images...") # logging.debug("Trying to prune unused images...")
# subprocess_call(['docker', 'image', 'prune', '-f']) # run_and_check(['docker', 'image', 'prune', '-f'])
# logging.debug("Images pruned") # logging.debug("Images pruned")
# except: # except:
# pass # pass
@ -412,7 +407,7 @@ class ClickHouseCluster:
try: try:
logging.debug("Trying to prune unused volumes...") logging.debug("Trying to prune unused volumes...")
subprocess_call(['docker', 'volume', 'prune', '-f']) run_and_check(['docker', 'volume', 'prune', '-f'])
logging.debug("Volumes pruned") logging.debug("Volumes pruned")
except: except:
pass pass
@ -949,7 +944,7 @@ class ClickHouseCluster:
errors += [str(ex)] errors += [str(ex)]
time.sleep(1) time.sleep(1)
subprocess_call(['docker-compose', 'ps', '--services', '--all']) run_and_check(['docker-compose', 'ps', '--services', '--all'])
logging.error("Can't connect to MySQL Client:{}".format(errors)) logging.error("Can't connect to MySQL Client:{}".format(errors))
raise Exception("Cannot wait MySQL Client container") raise Exception("Cannot wait MySQL Client container")
@ -967,7 +962,7 @@ class ClickHouseCluster:
errors += [str(ex)] errors += [str(ex)]
time.sleep(0.5) time.sleep(0.5)
subprocess_call(['docker-compose', 'ps', '--services', '--all']) run_and_check(['docker-compose', 'ps', '--services', '--all'])
logging.error("Can't connect to MySQL:{}".format(errors)) logging.error("Can't connect to MySQL:{}".format(errors))
raise Exception("Cannot wait MySQL container") raise Exception("Cannot wait MySQL container")
@ -984,7 +979,7 @@ class ClickHouseCluster:
logging.debug("Can't connect to MySQL 8 " + str(ex)) logging.debug("Can't connect to MySQL 8 " + str(ex))
time.sleep(0.5) time.sleep(0.5)
subprocess_call(['docker-compose', 'ps', '--services', '--all']) run_and_check(['docker-compose', 'ps', '--services', '--all'])
raise Exception("Cannot wait MySQL 8 container") raise Exception("Cannot wait MySQL 8 container")
def wait_mysql_cluster_to_start(self, timeout=180): def wait_mysql_cluster_to_start(self, timeout=180):
@ -1004,7 +999,7 @@ class ClickHouseCluster:
errors += [str(ex)] errors += [str(ex)]
time.sleep(0.5) time.sleep(0.5)
subprocess_call(['docker-compose', 'ps', '--services', '--all']) run_and_check(['docker-compose', 'ps', '--services', '--all'])
logging.error("Can't connect to MySQL:{}".format(errors)) logging.error("Can't connect to MySQL:{}".format(errors))
raise Exception("Cannot wait MySQL container") raise Exception("Cannot wait MySQL container")
@ -1425,7 +1420,7 @@ class ClickHouseCluster:
run_and_check(clickhouse_start_cmd) run_and_check(clickhouse_start_cmd)
logging.debug("ClickHouse instance created") logging.debug("ClickHouse instance created")
start_timeout = 180.0 # seconds start_timeout = 300.0 # seconds
for instance in self.instances.values(): for instance in self.instances.values():
instance.docker_client = self.docker_client instance.docker_client = self.docker_client
instance.ip_address = self.get_instance_ip(instance.name) instance.ip_address = self.get_instance_ip(instance.name)
@ -1881,7 +1876,7 @@ class ClickHouseInstance:
return None return None
return None return None
def restart_with_latest_version(self, stop_start_wait_sec=120, callback_onstop=None, signal=60): def restart_with_latest_version(self, stop_start_wait_sec=300, callback_onstop=None, signal=60):
if not self.stay_alive: if not self.stay_alive:
raise Exception("Cannot restart not stay alive container") raise Exception("Cannot restart not stay alive container")
self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root') self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root')
@ -1908,7 +1903,6 @@ class ClickHouseInstance:
"cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"], "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"],
user='root') user='root')
self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid()))
from helpers.test_tools import assert_eq_with_retry
# wait start # wait start
assert_eq_with_retry(self, "select 1", "1", retry_count=retries) assert_eq_with_retry(self, "select 1", "1", retry_count=retries)