Limit stress test to one hour

docker/test/stress/stress

@@ -33,13 +33,18 @@ def get_options(i):
     return options
 
 
-def run_func_test(cmd, output_prefix, num_processes, skip_tests_option):
+def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_time_limit):
     skip_list_opt = get_skip_list_cmd(cmd)
+
+    global_time_limit_option = None
+    if global_time_limit:
+        global_time_limit_option = "--global_time_limit={}".format(global_time_limit)
+
     output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)]
     pipes = []
     for i in range(0, len(output_paths)):
         f = open(output_paths[i], 'w')
-        full_command = "{} {} {} {}".format(cmd, skip_list_opt, get_options(i), skip_tests_option)
+        full_command = "{} {} {} {} {}".format(cmd, skip_list_opt, get_options(i), global_time_limit_option, skip_tests_option)
         logging.info("Run func tests '%s'", full_command)
         p = Popen(full_command, shell=True, stdout=f, stderr=f)
         pipes.append(p)
@@ -57,13 +62,14 @@ if __name__ == "__main__":
     parser.add_argument("--perf-test-xml-path", default='/usr/share/clickhouse-test/performance/')
     parser.add_argument("--server-log-folder", default='/var/log/clickhouse-server')
     parser.add_argument("--output-folder")
+    parser.add_argument("--global-time-limit", type=int, default=3600)
     parser.add_argument("--num-parallel", default=cpu_count() // 3);
 
     args = parser.parse_args()
     func_pipes = []
     perf_process = None
     perf_process = run_perf_test(args.perf_test_cmd, args.perf_test_xml_path, args.output_folder)
-    func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests)
+    func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit)
 
     logging.info("Will wait functests to finish")
     while True:

tests/clickhouse-test

@@ -16,7 +16,7 @@ from subprocess import Popen
 from subprocess import PIPE
 from subprocess import CalledProcessError
 from datetime import datetime
-from time import sleep
+from time import time, sleep
 from errno import ESRCH
 try:
     import termcolor
@@ -167,6 +167,7 @@ def colored(text, args, color=None, on_color=None, attrs=None):
 
 SERVER_DIED = False
 exit_code = 0
+stop_time = None
 
 
 # def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total):
@@ -174,6 +175,7 @@ def run_tests_array(all_tests_with_params):
     all_tests, suite, suite_dir, suite_tmp_dir, run_total = all_tests_with_params
     global exit_code
     global SERVER_DIED
+    global stop_time
 
     OP_SQUARE_BRACKET = colored("[", args, attrs=['bold'])
     CL_SQUARE_BRACKET = colored("]", args, attrs=['bold'])
@@ -202,6 +204,10 @@ def run_tests_array(all_tests_with_params):
         if SERVER_DIED:
             break
 
+        if stop_time and time() > stop_time:
+            print("\nStop tests run because global time limit is exceeded.\n")
+            break
+
         case_file = os.path.join(suite_dir, case)
         (name, ext) = os.path.splitext(case)
 
@@ -457,6 +463,7 @@ def collect_build_flags(client):
 
 def main(args):
     global SERVER_DIED
+    global stop_time
     global exit_code
     global server_logs_level
 
@@ -496,6 +503,10 @@ def main(args):
     # Shell scripts could change logging level
     os.environ.setdefault("CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL", server_logs_level)
 
+    # This code is bad as the time is not monotonic
+    if args.global_time_limit:
+        stop_time = time() + args.global_time_limit
+
     if args.zookeeper is None:
         code, out = commands.getstatusoutput(args.extract_from_config + " --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l')
         try:
@@ -716,6 +727,7 @@ if __name__ == '__main__':
     parser.add_argument('--configserver', default= '/etc/clickhouse-server/config.xml', help='Preprocessed server config')
     parser.add_argument('-o', '--output', help='Output xUnit compliant test report directory')
     parser.add_argument('-t', '--timeout', type=int, default=600, help='Timeout for each test case in seconds')
+    parser.add_argument('--global_time_limit', type=int, help='Stop if executing more than specified time (after current test finished)')
     parser.add_argument('test', nargs='*', help='Optional test case name regex')
     parser.add_argument('-d', '--disabled', action='store_true', default=False, help='Also run disabled tests')
     parser.add_argument('--stop', action='store_true', default=None, dest='stop', help='Stop on network errors')