mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 16:50:48 +00:00
Merge pull request #20946 from ClickHouse/stress_test_results
Move some CI-related scripts to github
This commit is contained in:
tavplubix
GitHub
commit
e3141124fd
@ -2,5 +2,6 @@
|
||||
FROM yandex/clickhouse-binary-builder
|
||||
|
||||
COPY run.sh /run.sh
|
||||
COPY process_split_build_smoke_test_result.py /
|
||||
|
||||
CMD /run.sh
|
||||
|
||||
61
docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py
Executable file
61
docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py
Executable file
@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
import csv
|
||||
|
||||
RESULT_LOG_NAME = "run.log"
|
||||
|
||||
def process_result(result_folder):
|
||||
|
||||
status = "success"
|
||||
description = 'Server started and responded'
|
||||
summary = [("Smoke test", "OK")]
|
||||
with open(os.path.join(result_folder, RESULT_LOG_NAME), 'r') as run_log:
|
||||
lines = run_log.read().split('\n')
|
||||
if not lines or lines[0].strip() != 'OK':
|
||||
status = "failure"
|
||||
logging.info("Lines is not ok: %s", str('\n'.join(lines)))
|
||||
summary = [("Smoke test", "FAIL")]
|
||||
description = 'Server failed to respond, see result in logs'
|
||||
|
||||
result_logs = []
|
||||
server_log_path = os.path.join(result_folder, "clickhouse-server.log")
|
||||
stderr_log_path = os.path.join(result_folder, "stderr.log")
|
||||
client_stderr_log_path = os.path.join(result_folder, "clientstderr.log")
|
||||
|
||||
if os.path.exists(server_log_path):
|
||||
result_logs.append(server_log_path)
|
||||
|
||||
if os.path.exists(stderr_log_path):
|
||||
result_logs.append(stderr_log_path)
|
||||
|
||||
if os.path.exists(client_stderr_log_path):
|
||||
result_logs.append(client_stderr_log_path)
|
||||
|
||||
return status, description, summary, result_logs
|
||||
|
||||
|
||||
def write_results(results_file, status_file, results, status):
|
||||
with open(results_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerows(results)
|
||||
with open(status_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerow(status)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of split build smoke test")
|
||||
parser.add_argument("--in-results-dir", default='/test_output/')
|
||||
parser.add_argument("--out-results-file", default='/test_output/test_results.tsv')
|
||||
parser.add_argument("--out-status-file", default='/test_output/check_status.tsv')
|
||||
args = parser.parse_args()
|
||||
|
||||
state, description, test_results, logs = process_result(args.in_results_dir)
|
||||
logging.info("Result parsed")
|
||||
status = (state, description)
|
||||
write_results(args.out_results_file, args.out_status_file, test_results, status)
|
||||
logging.info("Result written")
|
||||
@ -5,16 +5,18 @@ set -x
|
||||
install_and_run_server() {
|
||||
mkdir /unpacked
|
||||
tar -xzf /package_folder/shared_build.tgz -C /unpacked --strip 1
|
||||
LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-server --config /unpacked/config/config.xml >/var/log/clickhouse-server/stderr.log 2>&1 &
|
||||
LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-server --config /unpacked/config/config.xml >/test_output/stderr.log 2>&1 &
|
||||
}
|
||||
|
||||
run_client() {
|
||||
for i in {1..100}; do
|
||||
sleep 1
|
||||
LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-client --query "select 'OK'" 2>/var/log/clickhouse-server/clientstderr.log && break
|
||||
LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-client --query "select 'OK'" > /test_output/run.log 2> /test_output/clientstderr.log && break
|
||||
[[ $i == 100 ]] && echo 'FAIL'
|
||||
done
|
||||
}
|
||||
|
||||
install_and_run_server
|
||||
run_client
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /test_output/clickhouse-server.log
|
||||
/process_split_build_smoke_test_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
|
||||
@ -10,4 +10,5 @@ RUN mkdir /sqlancer && \
|
||||
RUN cd /sqlancer/sqlancer-master && mvn package -DskipTests
|
||||
|
||||
COPY run.sh /
|
||||
COPY process_sqlancer_result.py /
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
74
docker/test/sqlancer/process_sqlancer_result.py
Executable file
74
docker/test/sqlancer/process_sqlancer_result.py
Executable file
@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
import csv
|
||||
|
||||
|
||||
def process_result(result_folder):
|
||||
status = "success"
|
||||
summary = []
|
||||
paths = []
|
||||
tests = ["TLPWhere", "TLPGroupBy", "TLPHaving", "TLPWhereGroupBy", "TLPDistinct", "TLPAggregate"]
|
||||
|
||||
for test in tests:
|
||||
err_path = '{}/{}.err'.format(result_folder, test)
|
||||
out_path = '{}/{}.out'.format(result_folder, test)
|
||||
if not os.path.exists(err_path):
|
||||
logging.info("No output err on path %s", err_path)
|
||||
summary.append((test, "SKIPPED"))
|
||||
elif not os.path.exists(out_path):
|
||||
logging.info("No output log on path %s", out_path)
|
||||
else:
|
||||
paths.append(err_path)
|
||||
paths.append(out_path)
|
||||
with open(err_path, 'r') as f:
|
||||
if 'AssertionError' in f.read():
|
||||
summary.append((test, "FAIL"))
|
||||
else:
|
||||
summary.append((test, "OK"))
|
||||
|
||||
logs_path = '{}/logs.tar.gz'.format(result_folder)
|
||||
if not os.path.exists(logs_path):
|
||||
logging.info("No logs tar on path %s", logs_path)
|
||||
else:
|
||||
paths.append(logs_path)
|
||||
stdout_path = '{}/stdout.log'.format(result_folder)
|
||||
if not os.path.exists(stdout_path):
|
||||
logging.info("No stdout log on path %s", stdout_path)
|
||||
else:
|
||||
paths.append(stdout_path)
|
||||
stderr_path = '{}/stderr.log'.format(result_folder)
|
||||
if not os.path.exists(stderr_path):
|
||||
logging.info("No stderr log on path %s", stderr_path)
|
||||
else:
|
||||
paths.append(stderr_path)
|
||||
|
||||
description = "SQLancer test run. See report"
|
||||
|
||||
return status, description, summary, paths
|
||||
|
||||
|
||||
def write_results(results_file, status_file, results, status):
|
||||
with open(results_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerows(results)
|
||||
with open(status_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerow(status)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of sqlancer test")
|
||||
parser.add_argument("--in-results-dir", default='/test_output/')
|
||||
parser.add_argument("--out-results-file", default='/test_output/test_results.tsv')
|
||||
parser.add_argument("--out-status-file", default='/test_output/check_status.tsv')
|
||||
args = parser.parse_args()
|
||||
|
||||
state, description, test_results, logs = process_result(args.in_results_dir)
|
||||
logging.info("Result parsed")
|
||||
status = (state, description)
|
||||
write_results(args.out_results_file, args.out_status_file, test_results, status)
|
||||
logging.info("Result written")
|
||||
@ -29,4 +29,5 @@ tail -n 1000 /var/log/clickhouse-server/stderr.log > /test_output/stderr.log
|
||||
tail -n 1000 /var/log/clickhouse-server/stdout.log > /test_output/stdout.log
|
||||
tail -n 1000 /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log
|
||||
|
||||
/process_sqlancer_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
ls /test_output
|
||||
|
||||
@ -65,3 +65,11 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
|
||||
fi
|
||||
|
||||
clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
|
||||
./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||:
|
||||
mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
|
||||
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
|
||||
tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
|
||||
fi
|
||||
|
||||
@ -46,4 +46,5 @@ ENV NUM_TRIES=1
|
||||
ENV MAX_RUN_TIME=0
|
||||
|
||||
COPY run.sh /
|
||||
COPY process_functional_tests_result.py /
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
118
docker/test/stateless/process_functional_tests_result.py
Executable file
118
docker/test/stateless/process_functional_tests_result.py
Executable file
@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
import csv
|
||||
|
||||
OK_SIGN = "[ OK "
|
||||
FAIL_SING = "[ FAIL "
|
||||
TIMEOUT_SING = "[ Timeout! "
|
||||
UNKNOWN_SIGN = "[ UNKNOWN "
|
||||
SKIPPED_SIGN = "[ SKIPPED "
|
||||
HUNG_SIGN = "Found hung queries in processlist"
|
||||
|
||||
def process_test_log(log_path):
|
||||
total = 0
|
||||
skipped = 0
|
||||
unknown = 0
|
||||
failed = 0
|
||||
success = 0
|
||||
hung = False
|
||||
test_results = []
|
||||
with open(log_path, 'r') as test_file:
|
||||
for line in test_file:
|
||||
line = line.strip()
|
||||
if HUNG_SIGN in line:
|
||||
hung = True
|
||||
if any(sign in line for sign in (OK_SIGN, FAIL_SING, UNKNOWN_SIGN, SKIPPED_SIGN)):
|
||||
test_name = line.split(' ')[2].split(':')[0]
|
||||
|
||||
test_time = ''
|
||||
try:
|
||||
time_token = line.split(']')[1].strip().split()[0]
|
||||
float(time_token)
|
||||
test_time = time_token
|
||||
except:
|
||||
pass
|
||||
|
||||
total += 1
|
||||
if TIMEOUT_SING in line:
|
||||
failed += 1
|
||||
test_results.append((test_name, "Timeout", test_time))
|
||||
elif FAIL_SING in line:
|
||||
failed += 1
|
||||
test_results.append((test_name, "FAIL", test_time))
|
||||
elif UNKNOWN_SIGN in line:
|
||||
unknown += 1
|
||||
test_results.append((test_name, "FAIL", test_time))
|
||||
elif SKIPPED_SIGN in line:
|
||||
skipped += 1
|
||||
test_results.append((test_name, "SKIPPED", test_time))
|
||||
else:
|
||||
success += int(OK_SIGN in line)
|
||||
test_results.append((test_name, "OK", test_time))
|
||||
return total, skipped, unknown, failed, success, hung, test_results
|
||||
|
||||
def process_result(result_path):
|
||||
test_results = []
|
||||
state = "success"
|
||||
description = ""
|
||||
files = os.listdir(result_path)
|
||||
if files:
|
||||
logging.info("Find files in result folder %s", ','.join(files))
|
||||
result_path = os.path.join(result_path, 'test_result.txt')
|
||||
else:
|
||||
result_path = None
|
||||
description = "No output log"
|
||||
state = "error"
|
||||
|
||||
if result_path and os.path.exists(result_path):
|
||||
total, skipped, unknown, failed, success, hung, test_results = process_test_log(result_path)
|
||||
is_flacky_check = 1 < int(os.environ.get('NUM_TRIES', 1))
|
||||
# If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately)
|
||||
# But it's Ok for "flaky checks" - they can contain just one test for check which is marked as skipped.
|
||||
if failed != 0 or unknown != 0 or (success == 0 and (not is_flacky_check)):
|
||||
state = "failure"
|
||||
|
||||
if hung:
|
||||
description = "Some queries hung, "
|
||||
state = "failure"
|
||||
else:
|
||||
description = ""
|
||||
|
||||
description += "fail: {}, passed: {}".format(failed, success)
|
||||
if skipped != 0:
|
||||
description += ", skipped: {}".format(skipped)
|
||||
if unknown != 0:
|
||||
description += ", unknown: {}".format(unknown)
|
||||
else:
|
||||
state = "failure"
|
||||
description = "Output log doesn't exist"
|
||||
test_results = []
|
||||
|
||||
return state, description, test_results
|
||||
|
||||
|
||||
def write_results(results_file, status_file, results, status):
|
||||
with open(results_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerows(results)
|
||||
with open(status_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerow(status)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of functional tests")
|
||||
parser.add_argument("--in-results-dir", default='/test_output/')
|
||||
parser.add_argument("--out-results-file", default='/test_output/test_results.tsv')
|
||||
parser.add_argument("--out-status-file", default='/test_output/check_status.tsv')
|
||||
args = parser.parse_args()
|
||||
|
||||
state, description, test_results = process_result(args.in_results_dir)
|
||||
logging.info("Result parsed")
|
||||
status = (state, description)
|
||||
write_results(args.out_results_file, args.out_status_file, test_results, status)
|
||||
logging.info("Result written")
|
||||
@ -72,5 +72,12 @@ export -f run_tests
|
||||
|
||||
timeout "$MAX_RUN_TIME" bash -c run_tests ||:
|
||||
|
||||
./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||:
|
||||
mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
|
||||
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
|
||||
tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
|
||||
fi
|
||||
tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||:
|
||||
tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||:
|
||||
|
||||
@ -53,10 +53,14 @@ handle SIGBUS stop print
|
||||
handle SIGABRT stop print
|
||||
continue
|
||||
thread apply all backtrace
|
||||
continue
|
||||
detach
|
||||
quit
|
||||
" > script.gdb
|
||||
|
||||
gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" &
|
||||
# FIXME Hung check may work incorrectly because of attached gdb
|
||||
# 1. False positives are possible
|
||||
# 2. We cannot attach another gdb to get stacktraces if some queries hung
|
||||
gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" >> /test_output/gdb.log &
|
||||
}
|
||||
|
||||
configure
|
||||
@ -78,11 +82,55 @@ clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
|
||||
clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
|
||||
clickhouse-client --query "SHOW TABLES FROM test"
|
||||
|
||||
./stress --hung-check --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" && echo "OK" > /test_output/script_exit_code.txt || echo "FAIL" > /test_output/script_exit_code.txt
|
||||
./stress --hung-check --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" \
|
||||
&& echo -e 'Test script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Test script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
|
||||
stop
|
||||
start
|
||||
|
||||
clickhouse-client --query "SELECT 'Server successfuly started'" > /test_output/alive_check.txt || echo 'Server failed to start' > /test_output/alive_check.txt
|
||||
clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Server failed to start\tFAIL' >> /test_output/test_results.tsv
|
||||
|
||||
[ -f /var/log/clickhouse-server/clickhouse-server.log ] || echo -e "Server log does not exist\tFAIL"
|
||||
[ -f /var/log/clickhouse-server/stderr.log ] || echo -e "Stderr log does not exist\tFAIL"
|
||||
|
||||
# Print Fatal log messages to stdout
|
||||
zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.log
|
||||
|
||||
# Grep logs for sanitizer asserts, crashes and other critical errors
|
||||
|
||||
# Sanitizer asserts
|
||||
zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
zgrep -Fav "ASan doesn't fully support makecontext/swapcontext functions" > /dev/null \
|
||||
&& echo -e 'Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No sanitizer asserts\tOK' >> /test_output/test_results.tsv
|
||||
rm -f /test_output/tmp
|
||||
|
||||
# Logical errors
|
||||
zgrep -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \
|
||||
&& echo -e 'Logical error thrown (see clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No logical errors\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# Crash
|
||||
zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \
|
||||
&& echo -e 'Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Not crashed\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
# It also checks for OOM or crash without stacktrace (printed by watchdog)
|
||||
zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.log > /dev/null \
|
||||
&& echo -e 'Fatal message in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
zgrep -Fa "########################################" /test_output/* > /dev/null \
|
||||
&& echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv
|
||||
|
||||
# Put logs into /test_output/
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz
|
||||
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
|
||||
mv /var/log/clickhouse-server/stderr.log /test_output/
|
||||
|
||||
# Write check result into check_status.tsv
|
||||
clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by (lower(test) like '%hung%') LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv
|
||||
[ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv
|
||||
|
||||
@ -58,6 +58,27 @@ def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_t
|
||||
time.sleep(0.5)
|
||||
return pipes
|
||||
|
||||
def prepare_for_hung_check():
|
||||
# FIXME this function should not exist, but...
|
||||
|
||||
# We attach gdb to clickhouse-server before running tests
|
||||
# to print stacktraces of all crashes even if clickhouse cannot print it for some reason.
|
||||
# However, it obstruct checking for hung queries.
|
||||
logging.info("Will terminate gdb (if any)")
|
||||
call("kill -TERM $(pidof gdb)", shell=True, stderr=STDOUT)
|
||||
|
||||
# Some tests execute SYSTEM STOP MERGES or similar queries.
|
||||
# It may cause some ALTERs to hang.
|
||||
# Possibly we should fix tests and forbid to use such queries without specifying table.
|
||||
call("clickhouse client -q 'SYSTEM START MERGES'", shell=True, stderr=STDOUT)
|
||||
call("clickhouse client -q 'SYSTEM START DISTRIBUTED SENDS'", shell=True, stderr=STDOUT)
|
||||
call("clickhouse client -q 'SYSTEM START TTL MERGES'", shell=True, stderr=STDOUT)
|
||||
call("clickhouse client -q 'SYSTEM START MOVES'", shell=True, stderr=STDOUT)
|
||||
call("clickhouse client -q 'SYSTEM START FETCHES'", shell=True, stderr=STDOUT)
|
||||
call("clickhouse client -q 'SYSTEM START REPLICATED SENDS'", shell=True, stderr=STDOUT)
|
||||
call("clickhouse client -q 'SYSTEM START REPLICATION QUEUES'", shell=True, stderr=STDOUT)
|
||||
|
||||
time.sleep(30)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
@ -88,11 +109,14 @@ if __name__ == "__main__":
|
||||
|
||||
logging.info("All processes finished")
|
||||
if args.hung_check:
|
||||
prepare_for_hung_check()
|
||||
logging.info("Checking if some queries hung")
|
||||
cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1")
|
||||
res = call(cmd, shell=True, stderr=STDOUT)
|
||||
hung_check_status = "No queries hung\tOK\n"
|
||||
if res != 0:
|
||||
logging.info("Hung check failed with exit code {}".format(res))
|
||||
sys.exit(1)
|
||||
hung_check_status = "Hung check failed\tFAIL\n"
|
||||
open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write(hung_check_status)
|
||||
|
||||
logging.info("Stress test finished")
|
||||
|
||||
@ -10,14 +10,6 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
yamllint \
|
||||
&& pip3 install codespell
|
||||
|
||||
|
||||
# For |& syntax
|
||||
SHELL ["bash", "-c"]
|
||||
|
||||
CMD cd /ClickHouse/utils/check-style && \
|
||||
./check-style -n |& tee /test_output/style_output.txt && \
|
||||
./check-typos |& tee /test_output/typos_output.txt && \
|
||||
./check-whitespaces -n |& tee /test_output/whitespaces_output.txt && \
|
||||
./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt && \
|
||||
./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt && \
|
||||
true
|
||||
COPY run.sh /
|
||||
COPY process_style_check_result.py /
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
96
docker/test/style/process_style_check_result.py
Executable file
96
docker/test/style/process_style_check_result.py
Executable file
@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
import csv
|
||||
|
||||
|
||||
def process_result(result_folder):
|
||||
status = "success"
|
||||
description = ""
|
||||
test_results = []
|
||||
|
||||
style_log_path = '{}/style_output.txt'.format(result_folder)
|
||||
if not os.path.exists(style_log_path):
|
||||
logging.info("No style check log on path %s", style_log_path)
|
||||
return "exception", "No style check log", []
|
||||
elif os.stat(style_log_path).st_size != 0:
|
||||
description += "Style check failed. "
|
||||
test_results.append(("Style check", "FAIL"))
|
||||
status = "failure" # Disabled for now
|
||||
else:
|
||||
test_results.append(("Style check", "OK"))
|
||||
|
||||
typos_log_path = '{}/typos_output.txt'.format(result_folder)
|
||||
if not os.path.exists(style_log_path):
|
||||
logging.info("No typos check log on path %s", style_log_path)
|
||||
return "exception", "No typos check log", []
|
||||
elif os.stat(typos_log_path).st_size != 0:
|
||||
description += "Typos check failed. "
|
||||
test_results.append(("Typos check", "FAIL"))
|
||||
status = "failure"
|
||||
else:
|
||||
test_results.append(("Typos check", "OK"))
|
||||
|
||||
whitespaces_log_path = '{}/whitespaces_output.txt'.format(result_folder)
|
||||
if not os.path.exists(style_log_path):
|
||||
logging.info("No whitespaces check log on path %s", style_log_path)
|
||||
return "exception", "No whitespaces check log", []
|
||||
elif os.stat(whitespaces_log_path).st_size != 0:
|
||||
description += "Whitespaces check failed. "
|
||||
test_results.append(("Whitespaces check", "FAIL"))
|
||||
status = "failure"
|
||||
else:
|
||||
test_results.append(("Whitespaces check", "OK"))
|
||||
|
||||
duplicate_log_path = '{}/duplicate_output.txt'.format(result_folder)
|
||||
if not os.path.exists(duplicate_log_path):
|
||||
logging.info("No header duplicates check log on path %s", duplicate_log_path)
|
||||
return "exception", "No header duplicates check log", []
|
||||
elif os.stat(duplicate_log_path).st_size != 0:
|
||||
description += " Header duplicates check failed. "
|
||||
test_results.append(("Header duplicates check", "FAIL"))
|
||||
status = "failure"
|
||||
else:
|
||||
test_results.append(("Header duplicates check", "OK"))
|
||||
|
||||
shellcheck_log_path = '{}/shellcheck_output.txt'.format(result_folder)
|
||||
if not os.path.exists(shellcheck_log_path):
|
||||
logging.info("No shellcheck log on path %s", shellcheck_log_path)
|
||||
return "exception", "No shellcheck log", []
|
||||
elif os.stat(shellcheck_log_path).st_size != 0:
|
||||
description += " Shellcheck check failed. "
|
||||
test_results.append(("Shellcheck ", "FAIL"))
|
||||
status = "failure"
|
||||
else:
|
||||
test_results.append(("Shellcheck", "OK"))
|
||||
|
||||
if not description:
|
||||
description += "Style check success"
|
||||
|
||||
return status, description, test_results
|
||||
|
||||
|
||||
def write_results(results_file, status_file, results, status):
|
||||
with open(results_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerows(results)
|
||||
with open(status_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerow(status)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of style check")
|
||||
parser.add_argument("--in-results-dir", default='/test_output/')
|
||||
parser.add_argument("--out-results-file", default='/test_output/test_results.tsv')
|
||||
parser.add_argument("--out-status-file", default='/test_output/check_status.tsv')
|
||||
args = parser.parse_args()
|
||||
|
||||
state, description, test_results = process_result(args.in_results_dir)
|
||||
logging.info("Result parsed")
|
||||
status = (state, description)
|
||||
write_results(args.out_results_file, args.out_status_file, test_results, status)
|
||||
logging.info("Result written")
|
||||
9
docker/test/style/run.sh
Executable file
9
docker/test/style/run.sh
Executable file
@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv
|
||||
./check-style -n |& tee /test_output/style_output.txt
|
||||
./check-typos |& tee /test_output/typos_output.txt
|
||||
./check-whitespaces -n |& tee /test_output/whitespaces_output.txt
|
||||
./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt
|
||||
./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt
|
||||
/process_style_check_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
@ -61,6 +61,7 @@ RUN set -eux; \
|
||||
|
||||
COPY modprobe.sh /usr/local/bin/modprobe
|
||||
COPY dockerd-entrypoint.sh /usr/local/bin/
|
||||
COPY process_testflows_result.py /usr/local/bin/
|
||||
|
||||
RUN set -x \
|
||||
&& addgroup --system dockremap \
|
||||
@ -72,5 +73,5 @@ RUN set -x \
|
||||
VOLUME /var/lib/docker
|
||||
EXPOSE 2375
|
||||
ENTRYPOINT ["dockerd-entrypoint.sh"]
|
||||
CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json"]
|
||||
CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv"]
|
||||
|
||||
|
||||
67
docker/test/testflows/runner/process_testflows_result.py
Executable file
67
docker/test/testflows/runner/process_testflows_result.py
Executable file
@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
|
||||
|
||||
def process_result(result_folder):
|
||||
json_path = os.path.join(result_folder, "results.json")
|
||||
if not os.path.exists(json_path):
|
||||
return "success", "No testflows in branch", None, []
|
||||
|
||||
test_binary_log = os.path.join(result_folder, "test.log")
|
||||
with open(json_path) as source:
|
||||
results = json.loads(source.read())
|
||||
|
||||
total_tests = 0
|
||||
total_ok = 0
|
||||
total_fail = 0
|
||||
total_other = 0
|
||||
test_results = []
|
||||
for test in results["tests"]:
|
||||
test_name = test['test']['test_name']
|
||||
test_result = test['result']['result_type'].upper()
|
||||
test_time = str(test['result']['message_rtime'])
|
||||
total_tests += 1
|
||||
if test_result == "OK":
|
||||
total_ok += 1
|
||||
elif test_result == "FAIL" or test_result == "ERROR":
|
||||
total_fail += 1
|
||||
else:
|
||||
total_other += 1
|
||||
|
||||
test_results.append((test_name, test_result, test_time))
|
||||
if total_fail != 0:
|
||||
status = "failure"
|
||||
else:
|
||||
status = "success"
|
||||
|
||||
description = "failed: {}, passed: {}, other: {}".format(total_fail, total_ok, total_other)
|
||||
return status, description, test_results, [json_path, test_binary_log]
|
||||
|
||||
|
||||
def write_results(results_file, status_file, results, status):
|
||||
with open(results_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerows(results)
|
||||
with open(status_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerow(status)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of Testflows tests")
|
||||
parser.add_argument("--in-results-dir", default='./')
|
||||
parser.add_argument("--out-results-file", default='./test_results.tsv')
|
||||
parser.add_argument("--out-status-file", default='./check_status.tsv')
|
||||
args = parser.parse_args()
|
||||
|
||||
state, description, test_results, logs = process_result(args.in_results_dir)
|
||||
logging.info("Result parsed")
|
||||
status = (state, description)
|
||||
write_results(args.out_results_file, args.out_status_file, test_results, status)
|
||||
logging.info("Result written")
|
||||
|
||||
@ -5,6 +5,6 @@ ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
RUN apt-get install gdb
|
||||
|
||||
CMD service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test ''; \
|
||||
gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms | tee test_output/test_result.txt
|
||||
|
||||
COPY run.sh /
|
||||
COPY process_unit_tests_result.py /
|
||||
CMD ["/bin/bash", "/run.sh"]
|
||||
|
||||
96
docker/test/unit/process_unit_tests_result.py
Executable file
96
docker/test/unit/process_unit_tests_result.py
Executable file
@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
import csv
|
||||
|
||||
OK_SIGN = 'OK ]'
|
||||
FAILED_SIGN = 'FAILED ]'
|
||||
SEGFAULT = 'Segmentation fault'
|
||||
SIGNAL = 'received signal SIG'
|
||||
PASSED = 'PASSED'
|
||||
|
||||
def get_test_name(line):
|
||||
elements = reversed(line.split(' '))
|
||||
for element in elements:
|
||||
if '(' not in element and ')' not in element:
|
||||
return element
|
||||
raise Exception("No test name in line '{}'".format(line))
|
||||
|
||||
def process_result(result_folder):
|
||||
summary = []
|
||||
total_counter = 0
|
||||
failed_counter = 0
|
||||
result_log_path = '{}/test_result.txt'.format(result_folder)
|
||||
if not os.path.exists(result_log_path):
|
||||
logging.info("No output log on path %s", result_log_path)
|
||||
return "exception", "No output log", []
|
||||
|
||||
status = "success"
|
||||
description = ""
|
||||
passed = False
|
||||
with open(result_log_path, 'r') as test_result:
|
||||
for line in test_result:
|
||||
if OK_SIGN in line:
|
||||
logging.info("Found ok line: '%s'", line)
|
||||
test_name = get_test_name(line.strip())
|
||||
logging.info("Test name: '%s'", test_name)
|
||||
summary.append((test_name, "OK"))
|
||||
total_counter += 1
|
||||
elif FAILED_SIGN in line and 'listed below' not in line and 'ms)' in line:
|
||||
logging.info("Found fail line: '%s'", line)
|
||||
test_name = get_test_name(line.strip())
|
||||
logging.info("Test name: '%s'", test_name)
|
||||
summary.append((test_name, "FAIL"))
|
||||
total_counter += 1
|
||||
failed_counter += 1
|
||||
elif SEGFAULT in line:
|
||||
logging.info("Found segfault line: '%s'", line)
|
||||
status = "failure"
|
||||
description += "Segmentation fault. "
|
||||
break
|
||||
elif SIGNAL in line:
|
||||
logging.info("Received signal line: '%s'", line)
|
||||
status = "failure"
|
||||
description += "Exit on signal. "
|
||||
break
|
||||
elif PASSED in line:
|
||||
logging.info("PASSED record found: '%s'", line)
|
||||
passed = True
|
||||
|
||||
if not passed:
|
||||
status = "failure"
|
||||
description += "PASSED record not found. "
|
||||
|
||||
if failed_counter != 0:
|
||||
status = "failure"
|
||||
|
||||
if not description:
|
||||
description += "fail: {}, passed: {}".format(failed_counter, total_counter - failed_counter)
|
||||
|
||||
return status, description, summary
|
||||
|
||||
|
||||
def write_results(results_file, status_file, results, status):
|
||||
with open(results_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerows(results)
|
||||
with open(status_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerow(status)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of unit tests")
|
||||
parser.add_argument("--in-results-dir", default='/test_output/')
|
||||
parser.add_argument("--out-results-file", default='/test_output/test_results.tsv')
|
||||
parser.add_argument("--out-status-file", default='/test_output/check_status.tsv')
|
||||
args = parser.parse_args()
|
||||
|
||||
state, description, test_results = process_result(args.in_results_dir)
|
||||
logging.info("Result parsed")
|
||||
status = (state, description)
|
||||
write_results(args.out_results_file, args.out_status_file, test_results, status)
|
||||
logging.info("Result written")
|
||||
|
||||
7
docker/test/unit/run.sh
Normal file
7
docker/test/unit/run.sh
Normal file
@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
|
||||
service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test '';
|
||||
gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms | tee test_output/test_result.txt
|
||||
./process_unit_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
@ -216,11 +216,12 @@ def get_processlist(args):
|
||||
|
||||
# collect server stacktraces using gdb
|
||||
def get_stacktraces_from_gdb(server_pid):
|
||||
cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid)
|
||||
try:
|
||||
cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid)
|
||||
return subprocess.check_output(cmd, shell=True).decode('utf-8')
|
||||
except Exception as ex:
|
||||
return "Error occured while receiving stack traces from gdb: {}".format(str(ex))
|
||||
print("Error occured while receiving stack traces from gdb: {}".format(str(ex)))
|
||||
return None
|
||||
|
||||
|
||||
# collect server stacktraces from system.stack_trace table
|
||||
@ -230,21 +231,26 @@ def get_stacktraces_from_clickhouse(client):
|
||||
return subprocess.check_output("{} --allow_introspection_functions=1 --query "
|
||||
"\"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), "
|
||||
"arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace "
|
||||
"FROM system.stack_trace format Vertical\"".format(client), shell=True).decode('utf-8')
|
||||
"FROM system.stack_trace format Vertical\"".format(client), shell=True, stderr=subprocess.STDOUT).decode('utf-8')
|
||||
except Exception as ex:
|
||||
return "Error occured while receiving stack traces from client: {}".format(str(ex))
|
||||
print("Error occured while receiving stack traces from client: {}".format(str(ex)))
|
||||
return None
|
||||
|
||||
|
||||
def get_server_pid(server_tcp_port):
|
||||
cmd = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port)
|
||||
try:
|
||||
output = subprocess.check_output(cmd, shell=True)
|
||||
if output:
|
||||
return int(output)
|
||||
else:
|
||||
return None # server dead
|
||||
except Exception:
|
||||
return None
|
||||
# lsof does not work in stress tests for some reason
|
||||
cmd_lsof = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port)
|
||||
cmd_pidof = "pidof -s clickhouse-server"
|
||||
commands = [cmd_lsof, cmd_pidof]
|
||||
output = None
|
||||
for cmd in commands:
|
||||
try:
|
||||
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
|
||||
if output:
|
||||
return int(output)
|
||||
except Exception as e:
|
||||
print("Cannot get server pid with {}, got {}: {}", cmd, output, e)
|
||||
return None # most likely server dead
|
||||
|
||||
|
||||
def colored(text, args, color=None, on_color=None, attrs=None):
|
||||
@ -806,21 +812,26 @@ def main(args):
|
||||
|
||||
clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000')
|
||||
server_pid = get_server_pid(clickhouse_tcp_port)
|
||||
bt = None
|
||||
if server_pid:
|
||||
print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port))
|
||||
|
||||
# It does not work in Sandbox
|
||||
#print("\nCollecting stacktraces from system.stacktraces table:")
|
||||
#print(get_stacktraces_from_clickhouse(args.client))
|
||||
|
||||
print("\nCollecting stacktraces from all running threads with gdb:")
|
||||
print(get_stacktraces_from_gdb(server_pid))
|
||||
else:
|
||||
bt = get_stacktraces_from_gdb(server_pid)
|
||||
if len(bt) < 1000:
|
||||
print("Got suspiciously small stacktraces: ", bt)
|
||||
bt = None
|
||||
if bt is None:
|
||||
print("\nCollecting stacktraces from system.stacktraces table:")
|
||||
bt = get_stacktraces_from_clickhouse(args.client)
|
||||
if bt is None:
|
||||
print(
|
||||
colored(
|
||||
"\nUnable to locate ClickHouse server process listening at TCP port {}. "
|
||||
"It must have crashed or exited prematurely!".format(clickhouse_tcp_port),
|
||||
args, "red", attrs=["bold"]))
|
||||
else:
|
||||
print(bt)
|
||||
|
||||
|
||||
exit_code = 1
|
||||
else:
|
||||
|
||||
524
tests/integration/ci-runner.py
Executable file
524
tests/integration/ci-runner.py
Executable file
@ -0,0 +1,524 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
import os
|
||||
import time
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
import random
|
||||
import json
|
||||
import csv
|
||||
|
||||
|
||||
MAX_RETRY = 2
|
||||
SLEEP_BETWEEN_RETRIES = 5
|
||||
CLICKHOUSE_BINARY_PATH = "/usr/bin/clickhouse"
|
||||
CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "/usr/bin/clickhouse-odbc-bridge"
|
||||
|
||||
TRIES_COUNT = 10
|
||||
MAX_TIME_SECONDS = 3600
|
||||
|
||||
|
||||
def get_tests_to_run(pr_info):
|
||||
result = set([])
|
||||
changed_files = pr_info['changed_files']
|
||||
|
||||
if changed_files is None:
|
||||
return []
|
||||
|
||||
for fpath in changed_files:
|
||||
if 'tests/integration/test_' in fpath:
|
||||
logging.info('File %s changed and seems like integration test', fpath)
|
||||
result.add(fpath.split('/')[2])
|
||||
return list(result)
|
||||
|
||||
|
||||
def filter_existing_tests(tests_to_run, repo_path):
|
||||
result = []
|
||||
for relative_test_path in tests_to_run:
|
||||
if os.path.exists(os.path.join(repo_path, 'tests/integration', relative_test_path)):
|
||||
result.append(relative_test_path)
|
||||
else:
|
||||
logging.info("Skipping test %s, seems like it was removed", relative_test_path)
|
||||
return result
|
||||
|
||||
|
||||
def _get_deselect_option(tests):
|
||||
return ' '.join(['--deselect {}'.format(t) for t in tests])
|
||||
|
||||
|
||||
def parse_test_results_output(fname):
|
||||
read = False
|
||||
description_output = []
|
||||
with open(fname, 'r') as out:
|
||||
for line in out:
|
||||
if read and line.strip() and not line.startswith('=='):
|
||||
description_output.append(line.strip())
|
||||
if 'short test summary info' in line:
|
||||
read = True
|
||||
return description_output
|
||||
|
||||
|
||||
def get_counters(output):
|
||||
counters = {
|
||||
"ERROR": set([]),
|
||||
"PASSED": set([]),
|
||||
"FAILED": set([]),
|
||||
}
|
||||
|
||||
for line in output:
|
||||
if '.py' in line:
|
||||
line_arr = line.strip().split(' ')
|
||||
state = line_arr[0]
|
||||
test_name = ' '.join(line_arr[1:])
|
||||
if ' - ' in test_name:
|
||||
test_name = test_name[:test_name.find(' - ')]
|
||||
if state in counters:
|
||||
counters[state].add(test_name)
|
||||
else:
|
||||
logging.info("Strange line %s", line)
|
||||
else:
|
||||
logging.info("Strange line %s")
|
||||
return {k: list(v) for k, v in counters.items()}
|
||||
|
||||
|
||||
def parse_test_times(fname):
|
||||
read = False
|
||||
description_output = []
|
||||
with open(fname, 'r') as out:
|
||||
for line in out:
|
||||
if read and '==' in line:
|
||||
break
|
||||
if read and line.strip():
|
||||
description_output.append(line.strip())
|
||||
if 'slowest durations' in line:
|
||||
read = True
|
||||
return description_output
|
||||
|
||||
|
||||
def get_test_times(output):
|
||||
result = defaultdict(float)
|
||||
for line in output:
|
||||
if '.py' in line:
|
||||
line_arr = line.strip().split(' ')
|
||||
test_time = line_arr[0]
|
||||
test_name = ' '.join([elem for elem in line_arr[2:] if elem])
|
||||
if test_name not in result:
|
||||
result[test_name] = 0.0
|
||||
result[test_name] += float(test_time[:-1])
|
||||
return result
|
||||
|
||||
|
||||
def clear_ip_tables_and_restart_daemons():
|
||||
logging.info("Dump iptables after run %s", subprocess.check_output("iptables -L", shell=True))
|
||||
try:
|
||||
logging.info("Killing all alive docker containers")
|
||||
subprocess.check_output("docker kill $(docker ps -q)", shell=True)
|
||||
except subprocess.CalledProcessError as err:
|
||||
logging.info("docker kill excepted: " + str(err))
|
||||
|
||||
try:
|
||||
logging.info("Removing all docker containers")
|
||||
subprocess.check_output("docker rm $(docker ps -a -q) --force", shell=True)
|
||||
except subprocess.CalledProcessError as err:
|
||||
logging.info("docker rm excepted: " + str(err))
|
||||
|
||||
try:
|
||||
logging.info("Stopping docker daemon")
|
||||
subprocess.check_output("service docker stop", shell=True)
|
||||
except subprocess.CalledProcessError as err:
|
||||
logging.info("docker stop excepted: " + str(err))
|
||||
|
||||
try:
|
||||
for i in range(200):
|
||||
try:
|
||||
logging.info("Restarting docker %s", i)
|
||||
subprocess.check_output("service docker start", shell=True)
|
||||
subprocess.check_output("docker ps", shell=True)
|
||||
break
|
||||
except subprocess.CalledProcessError as err:
|
||||
time.sleep(0.5)
|
||||
logging.info("Waiting docker to start, current %s", str(err))
|
||||
else:
|
||||
raise Exception("Docker daemon doesn't responding")
|
||||
except subprocess.CalledProcessError as err:
|
||||
logging.info("Can't reload docker: " + str(err))
|
||||
|
||||
try:
|
||||
for i in xrange(1000):
|
||||
# when rules will be empty, it will raise exception
|
||||
subprocess.check_call("iptables -D DOCKER-USER 1", shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL
|
||||
except:
|
||||
logging.info("All iptables rules cleared")
|
||||
|
||||
|
||||
class ClickhouseIntegrationTestsRunner:
|
||||
|
||||
def __init__(self, result_path, params):
|
||||
self.result_path = result_path
|
||||
self.params = params
|
||||
|
||||
self.image_versions = self.params['docker_images_with_versions']
|
||||
self.shuffle_groups = self.params['shuffle_test_groups']
|
||||
self.flaky_check = 'flaky check' in self.params['context_name']
|
||||
|
||||
def path(self):
|
||||
return self.result_path
|
||||
|
||||
def base_path(self):
|
||||
return os.path.join(str(self.result_path), '../')
|
||||
|
||||
def should_skip_tests(self):
|
||||
return []
|
||||
|
||||
def get_image_with_version(self, name):
|
||||
if name in self.image_versions:
|
||||
return name + ":" + self.image_versions[name]
|
||||
logging.warn("Cannot find image %s in params list %s", name, self.image_versions)
|
||||
if ':' not in name:
|
||||
return name + ":latest"
|
||||
return name
|
||||
|
||||
def get_single_image_version(self):
|
||||
name = self.get_images_names()[0]
|
||||
if name in self.image_versions:
|
||||
return self.image_versions[name]
|
||||
logging.warn("Cannot find image %s in params list %s", name, self.image_versions)
|
||||
return 'latest'
|
||||
|
||||
def shuffle_test_groups(self):
|
||||
return self.shuffle_groups != 0
|
||||
|
||||
@staticmethod
|
||||
def get_images_names():
|
||||
return ["yandex/clickhouse-integration-tests-runner", "yandex/clickhouse-mysql-golang-client",
|
||||
"yandex/clickhouse-mysql-java-client", "yandex/clickhouse-mysql-js-client",
|
||||
"yandex/clickhouse-mysql-php-client", "yandex/clickhouse-postgresql-java-client",
|
||||
"yandex/clickhouse-integration-test", "yandex/clickhouse-kerberos-kdc",
|
||||
"yandex/clickhouse-integration-helper", ]
|
||||
|
||||
|
||||
def _can_run_with(self, path, opt):
|
||||
with open(path, 'r') as script:
|
||||
for line in script:
|
||||
if opt in line:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _install_clickhouse(self, debs_path):
|
||||
for package in ('clickhouse-common-static_', 'clickhouse-server_', 'clickhouse-client', 'clickhouse-common-static-dbg_'): # order matters
|
||||
logging.info("Installing package %s", package)
|
||||
for f in os.listdir(debs_path):
|
||||
if package in f:
|
||||
full_path = os.path.join(debs_path, f)
|
||||
logging.info("Package found in %s", full_path)
|
||||
log_name = "install_" + f + ".log"
|
||||
log_path = os.path.join(str(self.path()), log_name)
|
||||
with open(log_path, 'w') as log:
|
||||
cmd = "dpkg -i {}".format(full_path)
|
||||
logging.info("Executing installation cmd %s", cmd)
|
||||
retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait()
|
||||
if retcode == 0:
|
||||
logging.info("Instsallation of %s successfull", full_path)
|
||||
else:
|
||||
raise Exception("Installation of %s failed", full_path)
|
||||
break
|
||||
else:
|
||||
raise Exception("Package with {} not found".format(package))
|
||||
logging.info("Unstripping binary")
|
||||
# logging.info("Unstring %s", subprocess.check_output("eu-unstrip /usr/bin/clickhouse {}".format(CLICKHOUSE_BINARY_PATH), shell=True))
|
||||
|
||||
logging.info("All packages installed")
|
||||
os.chmod(CLICKHOUSE_BINARY_PATH, 0o777)
|
||||
os.chmod(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, 0o777)
|
||||
result_path_bin = os.path.join(str(self.base_path()), "clickhouse")
|
||||
result_path_bridge = os.path.join(str(self.base_path()), "clickhouse-odbc-bridge")
|
||||
shutil.copy(CLICKHOUSE_BINARY_PATH, result_path_bin)
|
||||
shutil.copy(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, result_path_bridge)
|
||||
return None, None
|
||||
|
||||
def _compress_logs(self, path, result_path):
|
||||
subprocess.check_call("tar czf {} -C {} .".format(result_path, path), shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL
|
||||
|
||||
def _get_all_tests(self, repo_path):
|
||||
image_cmd = self._get_runner_image_cmd(repo_path)
|
||||
cmd = "cd {}/tests/integration && ./runner {} ' --setup-plan' | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' > all_tests.txt".format(repo_path, image_cmd)
|
||||
logging.info("Getting all tests with cmd '%s'", cmd)
|
||||
subprocess.check_call(cmd, shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL
|
||||
|
||||
all_tests_file_path = "{}/tests/integration/all_tests.txt".format(repo_path)
|
||||
if not os.path.isfile(all_tests_file_path) or os.path.getsize(all_tests_file_path) == 0:
|
||||
raise Exception("There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(all_tests_file_path))
|
||||
|
||||
all_tests = []
|
||||
with open(all_tests_file_path, "r") as all_tests_file:
|
||||
for line in all_tests_file:
|
||||
all_tests.append(line.strip())
|
||||
return list(sorted(all_tests))
|
||||
|
||||
def group_test_by_file(self, tests):
|
||||
result = {}
|
||||
for test in tests:
|
||||
test_file = test.split('::')[0]
|
||||
if test_file not in result:
|
||||
result[test_file] = []
|
||||
result[test_file].append(test)
|
||||
return result
|
||||
|
||||
def _update_counters(self, main_counters, current_counters):
|
||||
for test in current_counters["PASSED"]:
|
||||
if test not in main_counters["PASSED"]:
|
||||
if test in main_counters["FAILED"]:
|
||||
main_counters["FAILED"].remove(test)
|
||||
if test in main_counters["ERROR"]:
|
||||
main_counters["ERROR"].remove(test)
|
||||
main_counters["PASSED"].append(test)
|
||||
|
||||
for state in ("ERROR", "FAILED"):
|
||||
for test in current_counters[state]:
|
||||
if test in main_counters["PASSED"]:
|
||||
continue
|
||||
if test not in main_counters[state]:
|
||||
main_counters[state].append(test)
|
||||
|
||||
def _get_runner_image_cmd(self, repo_path):
|
||||
image_cmd = ''
|
||||
if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-image-version'):
|
||||
for img in self.get_images_names():
|
||||
if img == "yandex/clickhouse-integration-tests-runner":
|
||||
runner_version = self.get_single_image_version()
|
||||
logging.info("Can run with custom docker image version %s", runner_version)
|
||||
image_cmd += ' --docker-image-version={} '.format(runner_version)
|
||||
else:
|
||||
if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-compose-images-tags'):
|
||||
image_cmd += '--docker-compose-images-tags={} '.format(self.get_image_with_version(img))
|
||||
else:
|
||||
image_cmd = ''
|
||||
logging.info("Cannot run with custom docker image version :(")
|
||||
return image_cmd
|
||||
|
||||
def run_test_group(self, repo_path, test_group, tests_in_group, num_tries):
|
||||
image_cmd = self._get_runner_image_cmd(repo_path)
|
||||
counters = {
|
||||
"ERROR": [],
|
||||
"PASSED": [],
|
||||
"FAILED": [],
|
||||
}
|
||||
tests_times = defaultdict(float)
|
||||
test_group_str = test_group.replace('/', '_').replace('.', '_')
|
||||
|
||||
for i in range(num_tries):
|
||||
logging.info("Running test group %s for the %s retry", test_group, i)
|
||||
clear_ip_tables_and_restart_daemons()
|
||||
|
||||
output_path = os.path.join(str(self.path()), "test_output_" + test_group_str + "_" + str(i) + ".log")
|
||||
log_name = "integration_run_" + test_group_str + "_" + str(i) + ".txt"
|
||||
log_path = os.path.join(str(self.path()), log_name)
|
||||
logging.info("Will wait output inside %s", output_path)
|
||||
|
||||
test_names = set([])
|
||||
for test_name in tests_in_group:
|
||||
if test_name not in counters["PASSED"]:
|
||||
if '[' in test_name:
|
||||
test_names.add(test_name[:test_name.find('[')])
|
||||
else:
|
||||
test_names.add(test_name)
|
||||
|
||||
test_cmd = ' '.join([test for test in sorted(test_names)])
|
||||
cmd = "cd {}/tests/integration && ./runner {} '-ss {} -rfEp --color=no --durations=0 {}' | tee {}".format(
|
||||
repo_path, image_cmd, test_cmd, _get_deselect_option(self.should_skip_tests()), output_path)
|
||||
|
||||
with open(log_path, 'w') as log:
|
||||
logging.info("Executing cmd: %s", cmd)
|
||||
retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait()
|
||||
if retcode == 0:
|
||||
logging.info("Run %s group successfully", test_group)
|
||||
else:
|
||||
logging.info("Some tests failed")
|
||||
|
||||
if os.path.exists(output_path):
|
||||
lines = parse_test_results_output(output_path)
|
||||
new_counters = get_counters(lines)
|
||||
times_lines = parse_test_times(output_path)
|
||||
new_tests_times = get_test_times(times_lines)
|
||||
self._update_counters(counters, new_counters)
|
||||
for test_name, test_time in new_tests_times.items():
|
||||
tests_times[test_name] = test_time
|
||||
os.remove(output_path)
|
||||
if len(counters["PASSED"]) == len(tests_in_group):
|
||||
logging.info("All tests from group %s passed", test_group)
|
||||
break
|
||||
if len(counters["PASSED"]) >= 0 and len(counters["FAILED"]) == 0 and len(counters["ERROR"]) == 0:
|
||||
logging.info("Seems like all tests passed but some of them are skipped or deselected. Ignoring them and finishing group.")
|
||||
break
|
||||
else:
|
||||
for test in tests_in_group:
|
||||
if test not in counters["PASSED"] and test not in counters["ERROR"] and test not in counters["FAILED"]:
|
||||
counters["ERROR"].append(test)
|
||||
|
||||
return counters, tests_times, log_name, log_path
|
||||
|
||||
def run_flaky_check(self, repo_path, build_path):
|
||||
pr_info = self.params['pr_info']
|
||||
|
||||
# pytest swears, if we require to run some tests which was renamed or deleted
|
||||
tests_to_run = filter_existing_tests(get_tests_to_run(pr_info), repo_path)
|
||||
if not tests_to_run:
|
||||
logging.info("No tests to run found")
|
||||
return 'success', 'Nothing to run', [('Nothing to run', 'OK')], ''
|
||||
|
||||
self._install_clickhouse(build_path)
|
||||
logging.info("Found '%s' tests to run", ' '.join(tests_to_run))
|
||||
result_state = "success"
|
||||
description_prefix = "No flaky tests: "
|
||||
start = time.time()
|
||||
logging.info("Starting check with retries")
|
||||
final_retry = 0
|
||||
log_paths = []
|
||||
for i in range(TRIES_COUNT):
|
||||
final_retry += 1
|
||||
logging.info("Running tests for the %s time", i)
|
||||
counters, tests_times, log_name, log_path = self.run_test_group(repo_path, "flaky", tests_to_run, 1)
|
||||
log_paths.append(log_path)
|
||||
if counters["FAILED"]:
|
||||
logging.info("Found failed tests: %s", ' '.join(counters["FAILED"]))
|
||||
description_prefix = "Flaky tests found: "
|
||||
result_state = "failure"
|
||||
break
|
||||
if counters["ERROR"]:
|
||||
description_prefix = "Flaky tests found: "
|
||||
logging.info("Found error tests: %s", ' '.join(counters["ERROR"]))
|
||||
result_state = "error"
|
||||
break
|
||||
logging.info("Try is OK, all tests passed, going to clear env")
|
||||
clear_ip_tables_and_restart_daemons()
|
||||
logging.info("And going to sleep for some time")
|
||||
if time.time() - start > MAX_TIME_SECONDS:
|
||||
logging.info("Timeout reached, going to finish flaky check")
|
||||
break
|
||||
time.sleep(5)
|
||||
|
||||
logging.info("Finally all tests done, going to compress test dir")
|
||||
test_logs = os.path.join(str(self.path()), "./test_dir.tar")
|
||||
self._compress_logs("{}/tests/integration".format(repo_path), test_logs)
|
||||
logging.info("Compression finished")
|
||||
|
||||
test_result = []
|
||||
for state in ("ERROR", "FAILED", "PASSED"):
|
||||
if state == "PASSED":
|
||||
text_state = "OK"
|
||||
elif state == "FAILED":
|
||||
text_state = "FAIL"
|
||||
else:
|
||||
text_state = state
|
||||
test_result += [(c + ' (✕' + str(final_retry) + ')', text_state, str(tests_times[c])) for c in counters[state]]
|
||||
status_text = description_prefix + ', '.join([str(n).lower().replace('failed', 'fail') + ': ' + str(len(c)) for n, c in counters.items()])
|
||||
|
||||
return result_state, status_text, test_result, [test_logs] + log_paths
|
||||
|
||||
def run_impl(self, repo_path, build_path):
|
||||
if self.flaky_check:
|
||||
return self.run_flaky_check(repo_path, build_path)
|
||||
|
||||
self._install_clickhouse(build_path)
|
||||
logging.info("Dump iptables before run %s", subprocess.check_output("iptables -L", shell=True))
|
||||
all_tests = self._get_all_tests(repo_path)
|
||||
logging.info("Found %s tests first 3 %s", len(all_tests), ' '.join(all_tests[:3]))
|
||||
grouped_tests = self.group_test_by_file(all_tests)
|
||||
logging.info("Found %s tests groups", len(grouped_tests))
|
||||
|
||||
counters = {
|
||||
"ERROR": [],
|
||||
"PASSED": [],
|
||||
"FAILED": [],
|
||||
}
|
||||
tests_times = defaultdict(float)
|
||||
|
||||
logs = []
|
||||
items_to_run = list(grouped_tests.items())
|
||||
|
||||
logging.info("Total test groups %s", len(items_to_run))
|
||||
if self.shuffle_test_groups():
|
||||
logging.info("Shuffling test groups")
|
||||
random.shuffle(items_to_run)
|
||||
|
||||
for group, tests in items_to_run:
|
||||
logging.info("Running test group %s countaining %s tests", group, len(tests))
|
||||
group_counters, group_test_times, log_name, log_path = self.run_test_group(repo_path, group, tests, MAX_RETRY)
|
||||
total_tests = 0
|
||||
for counter, value in group_counters.items():
|
||||
logging.info("Tests from group %s stats, %s count %s", group, counter, len(value))
|
||||
counters[counter] += value
|
||||
logging.info("Totally have %s with status %s", len(counters[counter]), counter)
|
||||
total_tests += len(counters[counter])
|
||||
logging.info("Totally finished tests %s/%s", total_tests, len(all_tests))
|
||||
|
||||
for test_name, test_time in group_test_times.items():
|
||||
tests_times[test_name] = test_time
|
||||
logs.append(log_path)
|
||||
if len(counters["FAILED"]) + len(counters["ERROR"]) >= 20:
|
||||
logging.info("Collected more than 20 failed/error tests, stopping")
|
||||
break
|
||||
|
||||
logging.info("Finally all tests done, going to compress test dir")
|
||||
test_logs = os.path.join(str(self.path()), "./test_dir.tar")
|
||||
self._compress_logs("{}/tests/integration".format(repo_path), test_logs)
|
||||
logging.info("Compression finished")
|
||||
|
||||
if counters["FAILED"] or counters["ERROR"]:
|
||||
logging.info("Overall status failure, because we have tests in FAILED or ERROR state")
|
||||
result_state = "failure"
|
||||
else:
|
||||
logging.info("Overall success!")
|
||||
result_state = "success"
|
||||
|
||||
test_result = []
|
||||
for state in ("ERROR", "FAILED", "PASSED"):
|
||||
if state == "PASSED":
|
||||
text_state = "OK"
|
||||
elif state == "FAILED":
|
||||
text_state = "FAIL"
|
||||
else:
|
||||
text_state = state
|
||||
test_result += [(c, text_state, str(tests_times[c])) for c in counters[state]]
|
||||
|
||||
status_text = "fail: {}, passed: {}, error: {}".format(len(counters['FAILED']), len(counters['PASSED']), len(counters['ERROR']))
|
||||
|
||||
if not counters or sum(len(counter) for counter in counters.values()) == 0:
|
||||
status_text = "No tests found for some reason! It's a bug"
|
||||
result_state = "failure"
|
||||
|
||||
if '(memory)' in self.params['context_name']:
|
||||
result_state = "success"
|
||||
|
||||
return result_state, status_text, test_result, [test_logs] + logs
|
||||
|
||||
def write_results(results_file, status_file, results, status):
|
||||
with open(results_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerows(results)
|
||||
with open(status_file, 'w') as f:
|
||||
out = csv.writer(f, delimiter='\t')
|
||||
out.writerow(status)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
|
||||
repo_path = os.environ.get("CLICKHOUSE_TESTS_REPO_PATH")
|
||||
build_path = os.environ.get("CLICKHOUSE_TESTS_BUILD_PATH")
|
||||
result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH")
|
||||
params_path = os.environ.get("CLICKHOUSE_TESTS_JSON_PARAMS_PATH")
|
||||
|
||||
params = json.loads(open(params_path, 'r').read())
|
||||
runner = ClickhouseIntegrationTestsRunner(result_path, params)
|
||||
|
||||
logging.info("Running tests")
|
||||
state, description, test_results, logs = runner.run_impl(repo_path, build_path)
|
||||
logging.info("Tests finished")
|
||||
|
||||
status = (state, description)
|
||||
out_results_file = os.path.join(str(runner.path()), "test_results.tsv")
|
||||
out_status_file = os.path.join(str(runner.path()), "check_status.tsv")
|
||||
write_results(out_results_file, out_status_file, test_results, status)
|
||||
logging.info("Result written")
|
||||
@ -49,7 +49,6 @@ def fill_nodes(nodes, shard):
|
||||
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
node_1_1 = cluster.add_instance('node_1_1', with_zookeeper=True, main_configs=['configs/remote_servers.xml'])
|
||||
node_1_2 = cluster.add_instance('node_1_2', with_zookeeper=True, main_configs=['configs/remote_servers.xml'])
|
||||
node_1_3 = cluster.add_instance('node_1_3', with_zookeeper=True, main_configs=['configs/remote_servers.xml'])
|
||||
|
||||
@ -6,6 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "drop user if exists u_00600"
|
||||
${CLICKHOUSE_CLIENT} -q "create user u_00600 settings max_execution_time=60, readonly=1"
|
||||
|
||||
function wait_for_query_to_start()
|
||||
{
|
||||
@ -22,7 +24,7 @@ $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query_id=hello&replace_running_query=1" -d
|
||||
# Wait for it to be replaced
|
||||
wait
|
||||
|
||||
${CLICKHOUSE_CLIENT_BINARY} --user=readonly --query_id=42 --query='SELECT 2, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' &
|
||||
${CLICKHOUSE_CLIENT_BINARY} --user=u_00600 --query_id=42 --query='SELECT 2, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' &
|
||||
wait_for_query_to_start '42'
|
||||
|
||||
# Trying to run another query with the same query_id
|
||||
@ -39,3 +41,4 @@ wait_for_query_to_start '42'
|
||||
${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --replace_running_query_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null
|
||||
wait
|
||||
${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --query='SELECT 44'
|
||||
${CLICKHOUSE_CLIENT} -q "drop user u_00600"
|
||||
|
||||
@ -278,6 +278,7 @@
|
||||
"00534_functions_bad_arguments4",
|
||||
"00534_functions_bad_arguments9",
|
||||
"00564_temporary_table_management",
|
||||
"00600_replace_running_query",
|
||||
"00626_replace_partition_from_table_zookeeper",
|
||||
"00652_replicated_mutations_zookeeper",
|
||||
"00687_top_and_offset",
|
||||
@ -522,6 +523,7 @@
|
||||
"00575_illegal_column_exception_when_drop_depen_column",
|
||||
"00599_create_view_with_subquery",
|
||||
"00604_show_create_database",
|
||||
"00600_replace_running_query",
|
||||
"00612_http_max_query_size",
|
||||
"00619_union_highlite",
|
||||
"00620_optimize_on_nonleader_replica_zookeeper",
|
||||
|
||||
@ -120,3 +120,11 @@ if __name__ == "__main__":
|
||||
print(("Running testflows container as: '" + cmd + "'."))
|
||||
# testflows return non zero error code on failed tests
|
||||
subprocess.call(cmd, shell=True)
|
||||
|
||||
result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH", None)
|
||||
if result_path is not None:
|
||||
move_from = os.path.join(args.clickhouse_root, 'tests/testflows')
|
||||
status = os.path.join(move_from, 'check_status.tsv')
|
||||
results = os.path.join(move_from, 'test_results.tsv')
|
||||
subprocess.call("mv {} {}".format(status, result_path), shell=True)
|
||||
subprocess.call("mv {} {}".format(results, result_path), shell=True)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user