fix

docker/test/stress/stress

@@ -88,9 +88,10 @@ if __name__ == "__main__":
         logging.info("Checking if some queries hung")
         cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1")
         res = call(cmd, shell=True, stderr=STDOUT)
-        hung_check_status = "Hung check\t{}\n".format('FAIL' if res else 'OK')
-        open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write(hung_check_status)
+        hung_check_status = "No queries hung\tOK\n"
         if res != 0:
             logging.info("Hung check failed with exit code {}".format(res))
+            hung_check_status = "Hung check failed\tFAIL\n"
+        open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write(hung_check_status)
 
     logging.info("Stress test finished")

tests/clickhouse-test

@@ -212,7 +212,8 @@ def get_stacktraces_from_gdb(server_pid):
     try:
         return subprocess.check_output(cmd, shell=True).decode('utf-8')
     except Exception as ex:
-        return "Error occured while receiving stack traces from gdb: {}".format(str(ex))
+        print("Error occured while receiving stack traces from gdb: {}".format(str(ex)))
+        return None
 
 
 # collect server stacktraces from system.stack_trace table
@@ -224,21 +225,24 @@ def get_stacktraces_from_clickhouse(client):
                "arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace "
                "FROM system.stack_trace format Vertical\"".format(client), shell=True).decode('utf-8')
     except Exception as ex:
-        return "Error occured while receiving stack traces from client: {}".format(str(ex))
+        print("Error occured while receiving stack traces from client: {}".format(str(ex)))
+        return None
 
 
 def get_server_pid(server_tcp_port):
-    cmd = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port)
+    # lsof does not work in stress tests for some reason
+    cmd_lsof = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port)
+    cmd_pidof = "pidof -s clickhouse-server"
+    commands = [cmd_lsof, cmd_pidof]
     output = None
-    try:
-        output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
-        if output:
-            return int(output)
-        else:
-            return None # server dead
-    except Exception as e:
-        print("Cannot get server pid, got {}: {}", output, e)
-        return None
+    for cmd in commands:
+        try:
+            output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+            if output:
+                return int(output)
+        except Exception as e:
+            print("Cannot get server pid with {}, got {}: {}", cmd, output, e)
+    return None # most likely server dead
 
 
 def colored(text, args, color=None, on_color=None, attrs=None):
@@ -796,21 +800,23 @@ def main(args):
 
             clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000')
             server_pid = get_server_pid(clickhouse_tcp_port)
+            bt = None
             if server_pid:
                 print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port))
-
-                # It does not work in Sandbox
-                #print("\nCollecting stacktraces from system.stacktraces table:")
-                #print(get_stacktraces_from_clickhouse(args.client))
-
                 print("\nCollecting stacktraces from all running threads with gdb:")
-                print(get_stacktraces_from_gdb(server_pid))
-            else:
+                bt = get_stacktraces_from_gdb(server_pid)
+            if bt is None:
+                print("\nCollecting stacktraces from system.stacktraces table:")
+                bt = get_stacktraces_from_clickhouse(args.client)
+            if bt is None:
                 print(
                     colored(
                         "\nUnable to locate ClickHouse server process listening at TCP port {}. "
                         "It must have crashed or exited prematurely!".format(clickhouse_tcp_port),
                         args, "red", attrs=["bold"]))
+            else:
+                print(bt)
+
 
             exit_code = 1
         else:

tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh

@@ -14,7 +14,6 @@ for i in $(seq $REPLICAS); do
     $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000"
 done
 
-
 $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10)"
 $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10, 40)"