thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to specify arguments to client in functional tests.

Browse Source
This commit is contained in:
Nikolai Kochetov 2019-10-11 13:30:32 +03:00
parent 9a89ee8eab
commit 48bfcab428
1 changed files with 40 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
dbms/tests/clickhouse-test
View File

@ -45,13 +45,25 @@ def remove_control_characters(s):
s = re.sub(r"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", s)
return s
def run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr_file):
if ext == '.sql':
command = "{0} --send_logs_level={1} --testmode --multiquery < {2} > {3} 2> {4}".format(args.client_with_database, server_logs_level, case_file, stdout_file, stderr_file)
else:
command = "{} > {} 2> {}".format(case_file, stdout_file, stderr_file)
proc = Popen(command, shell = True)
def run_single_test(args, ext, server_logs_level, client_options, case_file, stdout_file, stderr_file):
params = {
'client': args.client_with_database,
'logs_level': server_logs_level,
'options': client_options,
'test': case_file,
'stdout': stdout_file,
'stderr': stderr_file,
}
if ext == '.sql':
pattern = "{client} --send_logs_level={logs_level} --testmode --multiquery < {test} > {stdout} 2> {stderr}"
command = pattern.format(**params)
else:
command = "{test} > {stdout} 2> {stderr}".format(**params)
proc = Popen(command, shell=True)
start_time = datetime.now()
while (datetime.now() - start_time).total_seconds() < args.timeout and proc.poll() is None:
sleep(0.01)
@ -67,15 +79,18 @@ def run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr
return proc, stdout, stderr
def need_retry(stderr):
return any(msg in stderr for msg in MESSAGES_TO_RETRY)
def get_processlist(client_cmd):
try:
return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True)
except:
return "" # server seems dead
def get_stacktraces(server_pid):
cmd = "gdb -q -ex 'set pagination off' -ex 'backtrace' -ex 'thread apply all backtrace' -ex 'detach' -ex 'quit' --pid {} 2>/dev/null".format(server_pid)
try:
@ -83,6 +98,7 @@ def get_stacktraces(server_pid):
except Exception as ex:
return "Error occured while receiving stack traces {}".format(str(ex))
def get_server_pid(server_tcp_port):
cmd = "lsof -i tcp:{port} | grep '*:{port}'".format(port=server_tcp_port)
try:
@ -95,17 +111,19 @@ def get_server_pid(server_tcp_port):
except Exception as ex:
return None
def colored(text, args, color=None, on_color=None, attrs=None):
if termcolor and (sys.stdout.isatty() or args.force_color):
return termcolor.colored(text, color, on_color, attrs)
else:
return text
SERVER_DIED = False
exit_code = 0
#def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total):
# def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total):
def run_tests_array(all_tests_with_params):
all_tests, suite, suite_dir, suite_tmp_dir, run_total = all_tests_with_params
global exit_code
@ -125,6 +143,8 @@ def run_tests_array(all_tests_with_params):
failures = 0
failures_chain = 0
client_options = get_additional_client_options(args)
if len(all_tests):
print("\nRunning {} {} tests.".format(len(all_tests), suite) + "\n")
@ -170,7 +190,7 @@ def run_tests_array(all_tests_with_params):
stdout_file = os.path.join(suite_tmp_dir, name) + '.stdout'
stderr_file = os.path.join(suite_tmp_dir, name) + '.stderr'
proc, stdout, stderr = run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr_file)
proc, stdout, stderr = run_single_test(args, ext, server_logs_level, client_options, case_file, stdout_file, stderr_file)
if proc.returncode is None:
try:
proc.kill()
@ -183,7 +203,7 @@ def run_tests_array(all_tests_with_params):
else:
counter = 1
while proc.returncode != 0 and need_retry(stderr):
proc, stdout, stderr = run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr_file)
proc, stdout, stderr = run_single_test(args, ext, server_logs_level, client_options, case_file, stdout_file, stderr_file)
sleep(2**counter)
counter += 1
if counter > 6:
@ -245,8 +265,10 @@ def run_tests_array(all_tests_with_params):
else:
print(colored("\n{passed_total} tests passed. {skipped_total} tests skipped.".format(passed_total = passed_total, skipped_total = skipped_total), args, "green", attrs=["bold"]))
server_logs_level = "warning"
def main(args):
global SERVER_DIED
global exit_code
@ -435,6 +457,10 @@ def find_binary(name):
return False
def get_additional_client_options(args):
return ' '.join('--' + option for option in args.client_option)
if __name__ == '__main__':
parser=ArgumentParser(description='ClickHouse functional tests')
parser.add_argument('-q', '--queries', help='Path to queries dir')
@ -468,6 +494,7 @@ if __name__ == '__main__':
group=parser.add_mutually_exclusive_group(required=False)
group.add_argument('--shard', action='store_true', default=None, dest='shard', help='Run sharding related tests (required to clickhouse-server listen 127.0.0.2 127.0.0.3)')
group.add_argument('--no-shard', action='store_false', default=None, dest='shard', help='Do not run shard related tests')
group.add_argument('--client-option', nargs='+', help='Specify additional client argument')
args = parser.parse_args()
@ -504,6 +531,9 @@ if __name__ == '__main__':
if os.getenv("CLICKHOUSE_DATABASE"):
args.client += ' --database=' + os.getenv("CLICKHOUSE_DATABASE")
if args.client_option:
os.environ['CLICKHOUSE_CLIENT_OPT'] += ' ' + get_additional_client_options(args)
args.client_with_database = args.client
if not args.database:
def random_str(length=6):
@ -521,6 +551,6 @@ if __name__ == '__main__':
args.extract_from_config = args.binary + ' extract-from-config'
if args.jobs is None:
args.jobs=multiprocessing.cpu_count()
args.jobs = multiprocessing.cpu_count()
main(args)