mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Merge master
This commit is contained in:
alesapin
commit
87fc69f37b
@ -1,3 +1,7 @@
|
||||
option (ENABLE_BROTLI "Enable brotli" ON)
|
||||
|
||||
if (ENABLE_BROTLI)
|
||||
|
||||
option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${NOT_UNBUNDLED})
|
||||
|
||||
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h")
|
||||
@ -27,4 +31,6 @@ elseif (NOT MISSING_INTERNAL_BROTLI_LIBRARY)
|
||||
set (USE_BROTLI 1)
|
||||
endif ()
|
||||
|
||||
endif()
|
||||
|
||||
message (STATUS "Using brotli=${USE_BROTLI}: ${BROTLI_INCLUDE_DIR} : ${BROTLI_LIBRARY}")
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
option (ENABLE_PARQUET "Enable parquet" ON)
|
||||
|
||||
if (ENABLE_PARQUET)
|
||||
|
||||
if (NOT OS_FREEBSD) # Freebsd: ../contrib/arrow/cpp/src/arrow/util/bit-util.h:27:10: fatal error: endian.h: No such file or directory
|
||||
option(USE_INTERNAL_PARQUET_LIBRARY "Set to FALSE to use system parquet library instead of bundled" ${NOT_UNBUNDLED})
|
||||
endif()
|
||||
@ -61,6 +65,8 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
endif()
|
||||
|
||||
if(USE_PARQUET)
|
||||
message(STATUS "Using Parquet: ${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ; ${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ; ${THRIFT_LIBRARY}")
|
||||
else()
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
option (ENABLE_PROTOBUF "Enable protobuf" ON)
|
||||
|
||||
if (ENABLE_PROTOBUF)
|
||||
|
||||
option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ${NOT_UNBUNDLED})
|
||||
|
||||
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt")
|
||||
@ -98,4 +102,6 @@ if(OS_FREEBSD AND SANITIZE STREQUAL "address")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
endif()
|
||||
|
||||
message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY}")
|
||||
|
||||
@ -48,7 +48,6 @@ def remove_control_characters(s):
|
||||
s = re.sub(ur"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", s)
|
||||
return s
|
||||
|
||||
|
||||
def run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr_file):
|
||||
if ext == '.sql':
|
||||
command = "{0} --send_logs_level={1} --testmode --multiquery < {2} > {3} 2> {4}".format(args.client, server_logs_level, case_file, stdout_file, stderr_file)
|
||||
@ -70,6 +69,31 @@ def run_single_test(args, ext, server_logs_level, case_file, stdout_file, stderr
|
||||
def need_retry(stderr):
|
||||
return any(msg in stderr for msg in MESSAGES_TO_RETRY)
|
||||
|
||||
def get_processlist(client_cmd):
|
||||
try:
|
||||
return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True)
|
||||
except:
|
||||
return "" # server seems dead
|
||||
|
||||
def get_stacktraces(server_pid):
|
||||
cmd = "gdb -q -ex 'set pagination off' -ex 'backtrace' -ex 'thread apply all backtrace' -ex 'detach' -ex 'quit' --pid {} 2>/dev/null".format(server_pid)
|
||||
try:
|
||||
return subprocess.check_output(cmd, shell=True)
|
||||
except Exception as ex:
|
||||
return "Error occured while receiving stack traces {}".format(str(ex))
|
||||
|
||||
def get_server_pid(server_tcp_port):
|
||||
cmd = "lsof -i tcp:{port} | grep '*:{port}'".format(port=server_tcp_port)
|
||||
try:
|
||||
output = subprocess.check_output(cmd, shell=True)
|
||||
if output:
|
||||
columns = output.strip().split(' ')
|
||||
return int(columns[1])
|
||||
else:
|
||||
return None # server dead
|
||||
except Exception as ex:
|
||||
return None
|
||||
|
||||
def main(args):
|
||||
|
||||
SERVER_DIED = False
|
||||
@ -341,12 +365,28 @@ def main(args):
|
||||
|
||||
failures_total = failures_total + failures
|
||||
|
||||
exit_code = 0
|
||||
if failures_total > 0:
|
||||
print(colored("\nHaving {failures_total} errors! {passed_total} tests passed. {skipped_total} tests skipped.".format(passed_total = passed_total, skipped_total = skipped_total, failures_total = failures_total), "red", attrs=["bold"]))
|
||||
sys.exit(1)
|
||||
exit_code = 1
|
||||
else:
|
||||
print(colored("\n{passed_total} tests passed. {skipped_total} tests skipped.".format(passed_total = passed_total, skipped_total = skipped_total), "green", attrs=["bold"]))
|
||||
sys.exit(0)
|
||||
|
||||
if args.hung_check:
|
||||
processlist = get_processlist(args.client)
|
||||
if processlist:
|
||||
server_pid = get_server_pid(os.getenv("CLICKHOUSE_PORT_TCP", '9000'))
|
||||
print(colored("\nFound hung queries in processlist:", "red", attrs=["bold"]))
|
||||
print(processlist)
|
||||
if server_pid:
|
||||
print("\nStacktraces of all threads:")
|
||||
print(get_stacktraces(server_pid))
|
||||
exit_code = 1
|
||||
else:
|
||||
print(colored("\nNo queries hung.", "green", attrs=["bold"]))
|
||||
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
def find_binary(name):
|
||||
paths = os.environ.get("PATH").split(':')
|
||||
@ -358,31 +398,32 @@ def find_binary(name):
|
||||
return os.access(os.path.join('/usr/bin', name), os.X_OK)
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser(description = 'ClickHouse functional tests')
|
||||
parser.add_argument('-q', '--queries', help = 'Path to queries dir')
|
||||
parser.add_argument('--tmp', help = 'Path to tmp dir')
|
||||
parser.add_argument('-b', '--binary', default = 'clickhouse', help = 'Main clickhouse binary')
|
||||
parser.add_argument('-c', '--client', help = 'Client program')
|
||||
parser.add_argument('--extract_from_config', help = 'extract-from-config program')
|
||||
parser.add_argument('--configclient', help = 'Client config (if you use not default ports)')
|
||||
parser.add_argument('--configserver', default= '/etc/clickhouse-server/config.xml', help = 'Preprocessed server config')
|
||||
parser.add_argument('-o', '--output', help = 'Output xUnit compliant test report directory')
|
||||
parser.add_argument('-t', '--timeout', type = int, default = 600, help = 'Timeout for each test case in seconds')
|
||||
parser.add_argument('test', nargs = '?', help = 'Optional test case name regex')
|
||||
parser.add_argument('-d', '--disabled', action = 'store_true', default = False, help = 'Also run disabled tests')
|
||||
parser.add_argument('--stop', action = 'store_true', default = None, dest = 'stop', help = 'Stop on network errors')
|
||||
parser.add_argument('--order', default = 'desc', help = 'Run order (asc, desc, random)')
|
||||
parser.add_argument('--testname', action = 'store_true', default = None, dest = 'testname', help = 'Make query with test name before test run')
|
||||
parser=ArgumentParser(description='ClickHouse functional tests')
|
||||
parser.add_argument('-q', '--queries', help='Path to queries dir')
|
||||
parser.add_argument('--tmp', help='Path to tmp dir')
|
||||
parser.add_argument('-b', '--binary', default='clickhouse', help='Main clickhouse binary')
|
||||
parser.add_argument('-c', '--client', help='Client program')
|
||||
parser.add_argument('--extract_from_config', help='extract-from-config program')
|
||||
parser.add_argument('--configclient', help='Client config (if you use not default ports)')
|
||||
parser.add_argument('--configserver', default= '/etc/clickhouse-server/config.xml', help='Preprocessed server config')
|
||||
parser.add_argument('-o', '--output', help='Output xUnit compliant test report directory')
|
||||
parser.add_argument('-t', '--timeout', type=int, default=600, help='Timeout for each test case in seconds')
|
||||
parser.add_argument('test', nargs='?', help='Optional test case name regex')
|
||||
parser.add_argument('-d', '--disabled', action='store_true', default=False, help='Also run disabled tests')
|
||||
parser.add_argument('--stop', action='store_true', default=None, dest='stop', help='Stop on network errors')
|
||||
parser.add_argument('--order', default='desc', help='Run order (asc, desc, random)')
|
||||
parser.add_argument('--testname', action='store_true', default=None, dest='testname', help='Make query with test name before test run')
|
||||
parser.add_argument('--hung-check', action='store_true', default=False)
|
||||
|
||||
parser.add_argument('--no-stateless', action = 'store_true', help = 'Disable all stateless tests')
|
||||
parser.add_argument('--skip', nargs='+', help = "Skip these tests")
|
||||
parser.add_argument('--no-long', action = 'store_false', dest = 'no_long', help = 'Do not run long tests')
|
||||
group = parser.add_mutually_exclusive_group(required = False)
|
||||
group.add_argument('--zookeeper', action = 'store_true', default = None, dest = 'zookeeper', help = 'Run zookeeper related tests')
|
||||
group.add_argument('--no-zookeeper', action = 'store_false', default = None, dest = 'zookeeper', help = 'Do not run zookeeper related tests')
|
||||
group = parser.add_mutually_exclusive_group(required = False)
|
||||
group.add_argument('--shard', action = 'store_true', default = None, dest = 'shard', help = 'Run sharding related tests (required to clickhouse-server listen 127.0.0.2 127.0.0.3)')
|
||||
group.add_argument('--no-shard', action = 'store_false', default = None, dest = 'shard', help = 'Do not run shard related tests')
|
||||
parser.add_argument('--no-stateless', action='store_true', help='Disable all stateless tests')
|
||||
parser.add_argument('--skip', nargs='+', help="Skip these tests")
|
||||
parser.add_argument('--no-long', action='store_false', dest='no_long', help='Do not run long tests')
|
||||
group=parser.add_mutually_exclusive_group(required=False)
|
||||
group.add_argument('--zookeeper', action='store_true', default=None, dest='zookeeper', help='Run zookeeper related tests')
|
||||
group.add_argument('--no-zookeeper', action='store_false', default=None, dest='zookeeper', help='Do not run zookeeper related tests')
|
||||
group=parser.add_mutually_exclusive_group(required=False)
|
||||
group.add_argument('--shard', action='store_true', default=None, dest='shard', help='Run sharding related tests (required to clickhouse-server listen 127.0.0.2 127.0.0.3)')
|
||||
group.add_argument('--no-shard', action='store_false', default=None, dest='shard', help='Do not run shard related tests')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@ -20,7 +20,9 @@ RUN apt-get update -y \
|
||||
netcat-openbsd \
|
||||
telnet \
|
||||
moreutils \
|
||||
brotli
|
||||
brotli \
|
||||
gdb \
|
||||
lsof
|
||||
|
||||
|
||||
ENV TZ=Europe/Moscow
|
||||
@ -42,4 +44,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
|
||||
echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \
|
||||
echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \
|
||||
service zookeeper start; sleep 5; \
|
||||
service clickhouse-server start && sleep 5 && clickhouse-test --shard --zookeeper $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
service clickhouse-server start && sleep 5 && clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
|
||||
Loading…
Reference in New Issue
Block a user