add check for fatal

This commit is contained in:
Yatsishin Ilya 2021-03-03 11:09:44 +03:00
parent de20de8a2f
commit ec3d63bfe4

View File

@ -955,6 +955,7 @@ class ClickHouseCluster:
def shutdown(self, kill=True):
sanitizer_assert_instance = None
fatal_log = None
if self.up_called:
with open(self.docker_logs_path, "w+") as f:
try:
@ -967,6 +968,15 @@ class ClickHouseCluster:
sanitizer_assert_instance = line.split('|')[0].strip()
break
for instance in list(self.instances.values()):
if instance.contains_in_log(SANITIZER_SIGN):
sanitizer_assert_instance = instance.grep_in_log(SANITIZER_SIGN)
logging.ERROR(f"Sanitizer in instance {instance.name} fatal log {fatal_log}")
if instance.contains_in_log("Fatal"):
fatal_log = instance.grep_in_log("Fatal")
logging.ERROR(f"Crash in instance {instance.name} fatal log {fatal_log}")
if kill:
try:
subprocess_check_call(self.base_cmd + ['stop', '--timeout', '20'])
@ -1002,6 +1012,7 @@ class ClickHouseCluster:
raise Exception(
"Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance))
def pause_container(self, instance_name):
subprocess_check_call(self.base_cmd + ['pause', instance_name])
@ -1287,6 +1298,11 @@ class ClickHouseInstance:
["bash", "-c", 'grep "{}" /var/log/clickhouse-server/clickhouse-server.log || true'.format(substring)])
return len(result) > 0
def grep_in_log(self, substring):
result = self.exec_in_container(
["bash", "-c", 'grep "{}" /var/log/clickhouse-server/clickhouse-server.log || true'.format(substring)])
return result
def file_exists(self, path):
return self.exec_in_container(
["bash", "-c", "echo $(if [ -e '{}' ]; then echo 'yes'; else echo 'no'; fi)".format(path)]) == 'yes\n'