diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 173cb8442d6..a49d99e83b5 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -33,13 +33,18 @@ def get_options(i): return options -def run_func_test(cmd, output_prefix, num_processes, skip_tests_option): +def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_time_limit): skip_list_opt = get_skip_list_cmd(cmd) + + global_time_limit_option = None + if global_time_limit: + global_time_limit_option = "--global_time_limit={}".format(global_time_limit) + output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)] pipes = [] for i in range(0, len(output_paths)): f = open(output_paths[i], 'w') - full_command = "{} {} {} {}".format(cmd, skip_list_opt, get_options(i), skip_tests_option) + full_command = "{} {} {} {} {}".format(cmd, skip_list_opt, get_options(i), global_time_limit_option, skip_tests_option) logging.info("Run func tests '%s'", full_command) p = Popen(full_command, shell=True, stdout=f, stderr=f) pipes.append(p) @@ -57,13 +62,14 @@ if __name__ == "__main__": parser.add_argument("--perf-test-xml-path", default='/usr/share/clickhouse-test/performance/') parser.add_argument("--server-log-folder", default='/var/log/clickhouse-server') parser.add_argument("--output-folder") + parser.add_argument("--global-time-limit", type=int, default=3600) parser.add_argument("--num-parallel", default=cpu_count() // 3); args = parser.parse_args() func_pipes = [] perf_process = None perf_process = run_perf_test(args.perf_test_cmd, args.perf_test_xml_path, args.output_folder) - func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests) + func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit) logging.info("Will wait functests to finish") while True: diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 6f7fa72eea9..becb5247a14 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -16,7 +16,7 @@ from subprocess import Popen from subprocess import PIPE from subprocess import CalledProcessError from datetime import datetime -from time import sleep +from time import time, sleep from errno import ESRCH try: import termcolor @@ -167,6 +167,7 @@ def colored(text, args, color=None, on_color=None, attrs=None): SERVER_DIED = False exit_code = 0 +stop_time = None # def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total): @@ -174,6 +175,7 @@ def run_tests_array(all_tests_with_params): all_tests, suite, suite_dir, suite_tmp_dir, run_total = all_tests_with_params global exit_code global SERVER_DIED + global stop_time OP_SQUARE_BRACKET = colored("[", args, attrs=['bold']) CL_SQUARE_BRACKET = colored("]", args, attrs=['bold']) @@ -202,6 +204,10 @@ def run_tests_array(all_tests_with_params): if SERVER_DIED: break + if stop_time and time() > stop_time: + print("\nStop tests run because global time limit is exceeded.\n") + break + case_file = os.path.join(suite_dir, case) (name, ext) = os.path.splitext(case) @@ -457,6 +463,7 @@ def collect_build_flags(client): def main(args): global SERVER_DIED + global stop_time global exit_code global server_logs_level @@ -496,6 +503,10 @@ def main(args): # Shell scripts could change logging level os.environ.setdefault("CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL", server_logs_level) + # This code is bad as the time is not monotonic + if args.global_time_limit: + stop_time = time() + args.global_time_limit + if args.zookeeper is None: code, out = commands.getstatusoutput(args.extract_from_config + " --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l') try: @@ -716,6 +727,7 @@ if __name__ == '__main__': parser.add_argument('--configserver', default= '/etc/clickhouse-server/config.xml', help='Preprocessed server config') parser.add_argument('-o', '--output', help='Output xUnit compliant test report directory') parser.add_argument('-t', '--timeout', type=int, default=600, help='Timeout for each test case in seconds') + parser.add_argument('--global_time_limit', type=int, help='Stop if executing more than specified time (after current test finished)') parser.add_argument('test', nargs='*', help='Optional test case name regex') parser.add_argument('-d', '--disabled', action='store_true', default=False, help='Also run disabled tests') parser.add_argument('--stop', action='store_true', default=None, dest='stop', help='Stop on network errors')