mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 16:50:48 +00:00
Merge pull request #10902 from ClickHouse/recreate-databases-in-tests
clickhouse-test: recreate database for every test
This commit is contained in:
alexey-milovidov
GitHub
commit
4175167455
@ -51,8 +51,27 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std
|
||||
|
||||
# print(client_options)
|
||||
|
||||
if args.database:
|
||||
database = args.database
|
||||
os.environ.setdefault("CLICKHOUSE_DATABASE", database)
|
||||
|
||||
else:
|
||||
# If --database is not specified, we will create temporary database with unique name
|
||||
# And we will recreate and drop it for each test
|
||||
def random_str(length=6):
|
||||
import random
|
||||
import string
|
||||
alphabet = string.ascii_lowercase + string.digits
|
||||
return ''.join(random.choice(alphabet) for _ in range(length))
|
||||
database = 'test_{suffix}'.format(suffix=random_str())
|
||||
|
||||
clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
clickhouse_proc_create.communicate("CREATE DATABASE " + database)
|
||||
|
||||
os.environ["CLICKHOUSE_DATABASE"] = database
|
||||
|
||||
params = {
|
||||
'client': args.client_with_database,
|
||||
'client': args.client + ' --database=' + database,
|
||||
'logs_level': server_logs_level,
|
||||
'options': client_options,
|
||||
'test': case_file,
|
||||
@ -73,11 +92,15 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std
|
||||
while (datetime.now() - start_time).total_seconds() < args.timeout and proc.poll() is None:
|
||||
sleep(0.01)
|
||||
|
||||
if not args.database:
|
||||
clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
clickhouse_proc_create.communicate("DROP DATABASE " + database)
|
||||
|
||||
total_time = (datetime.now() - start_time).total_seconds()
|
||||
|
||||
# Normalize randomized database names in stdout, stderr files.
|
||||
os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=args.database, file=stdout_file))
|
||||
os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=args.database, file=stderr_file))
|
||||
os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=database, file=stdout_file))
|
||||
os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=database, file=stderr_file))
|
||||
|
||||
stdout = open(stdout_file, 'r').read() if os.path.exists(stdout_file) else ''
|
||||
stdout = unicode(stdout, errors='replace', encoding='utf-8')
|
||||
@ -108,6 +131,7 @@ def get_stacktraces_from_gdb(server_pid):
|
||||
|
||||
|
||||
# collect server stacktraces from system.stack_trace table
|
||||
# it does not work in Sandbox
|
||||
def get_stacktraces_from_clickhouse(client):
|
||||
try:
|
||||
return subprocess.check_output("{} --allow_introspection_functions=1 --query \"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace FROM system.stack_trace format Vertical\"".format(client), shell=True)
|
||||
@ -207,7 +231,7 @@ def run_tests_array(all_tests_with_params):
|
||||
else:
|
||||
|
||||
if args.testname:
|
||||
clickhouse_proc = Popen(shlex.split(args.client_with_database), stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
clickhouse_proc.communicate("SELECT 'Running test {suite}/{case} from pid={pid}';".format(pid = os.getpid(), case = case, suite = suite))
|
||||
|
||||
reference_file = os.path.join(suite_dir, name) + '.reference'
|
||||
@ -330,7 +354,6 @@ def main(args):
|
||||
if args.configclient:
|
||||
os.environ.setdefault("CLICKHOUSE_CONFIG_CLIENT", args.configclient)
|
||||
os.environ.setdefault("CLICKHOUSE_TMP", tmp_dir)
|
||||
os.environ.setdefault("CLICKHOUSE_DATABASE", args.database)
|
||||
|
||||
# Force to print server warnings in stderr
|
||||
# Shell scripts could change logging level
|
||||
@ -353,11 +376,12 @@ def main(args):
|
||||
else:
|
||||
args.shard = False
|
||||
|
||||
clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database)
|
||||
if args.database != "test":
|
||||
if args.database and args.database != "test":
|
||||
clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test")
|
||||
clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database)
|
||||
|
||||
clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test")
|
||||
|
||||
def is_test_from_dir(suite_dir, case):
|
||||
case_file = os.path.join(suite_dir, case)
|
||||
@ -460,7 +484,7 @@ def main(args):
|
||||
total_tests_run += tests_n
|
||||
|
||||
if args.hung_check:
|
||||
processlist = get_processlist(args.client_with_database)
|
||||
processlist = get_processlist(args.client)
|
||||
if processlist:
|
||||
print(colored("\nFound hung queries in processlist:", args, "red", attrs=["bold"]))
|
||||
print(processlist)
|
||||
@ -469,8 +493,11 @@ def main(args):
|
||||
server_pid = get_server_pid(clickhouse_tcp_port)
|
||||
if server_pid:
|
||||
print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port))
|
||||
print("\nCollecting stacktraces from system.stacktraces table:")
|
||||
print(get_stacktraces_from_clickhouse(args.client))
|
||||
|
||||
# It does not work in Sandbox
|
||||
#print("\nCollecting stacktraces from system.stacktraces table:")
|
||||
#print(get_stacktraces_from_clickhouse(args.client))
|
||||
|
||||
print("\nCollecting stacktraces from all running threads with gdb:")
|
||||
print(get_stacktraces_from_gdb(server_pid))
|
||||
else:
|
||||
@ -609,17 +636,6 @@ if __name__ == '__main__':
|
||||
|
||||
os.environ['CLICKHOUSE_URL_PARAMS'] += get_additional_client_options_url(args)
|
||||
|
||||
|
||||
args.client_with_database = args.client
|
||||
if not args.database:
|
||||
def random_str(length=6):
|
||||
import random
|
||||
import string
|
||||
alphabet = string.ascii_lowercase + string.digits
|
||||
return ''.join(random.choice(alphabet) for _ in range(length))
|
||||
args.database = 'test_{suffix}'.format(suffix=random_str())
|
||||
args.client_with_database += ' --database=' + args.database
|
||||
|
||||
if args.extract_from_config is None:
|
||||
if os.access(args.binary + '-extract-from-config', os.X_OK):
|
||||
args.extract_from_config = args.binary + '-extract-from-config'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user