mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge pull request #18372 from nikitamikhaylov/fix-arcadia-fifth
Return clickhouse-test-server for Arcadia needs
This commit is contained in:
Nikita Mikhaylov
GitHub
commit
9d3908d093
3
tests/clickhouse-client.xml
Normal file
3
tests/clickhouse-client.xml
Normal file
@ -0,0 +1,3 @@
|
||||
<yandex>
|
||||
<insert_format_max_block_size>100000</insert_format_max_block_size>
|
||||
</yandex>
|
||||
166
tests/clickhouse-test-server
Executable file
166
tests/clickhouse-test-server
Executable file
@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -x
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)
|
||||
DATA_DIR=${DATA_DIR:=`mktemp -d /tmp/clickhouse.test..XXXXX`}
|
||||
DATA_DIR_PATTERN=${DATA_DIR_PATTERN:=/tmp/clickhouse} # path from config file, will be replaced to temporary
|
||||
LOG_DIR=${LOG_DIR:=$DATA_DIR/log}
|
||||
export CLICKHOUSE_BINARY_NAME=${CLICKHOUSE_BINARY_NAME:="clickhouse"}
|
||||
( [ -x "$ROOT_DIR/programs/${CLICKHOUSE_BINARY_NAME}-server" ] || [ -x "$ROOT_DIR/programs/${CLICKHOUSE_BINARY_NAME}" ] ) && BUILD_DIR=${BUILD_DIR:=$ROOT_DIR} # Build without separate build dir
|
||||
[ -d "$ROOT_DIR/build${BUILD_TYPE}" ] && BUILD_DIR=${BUILD_DIR:=$ROOT_DIR/build${BUILD_TYPE}}
|
||||
BUILD_DIR=${BUILD_DIR:=$ROOT_DIR}
|
||||
[ -x ${CLICKHOUSE_BINARY_NAME}-server" ] && [ -x ${CLICKHOUSE_BINARY_NAME}-client" ] && BIN_DIR= # Allow run in /usr/bin
|
||||
( [ -x "$BUILD_DIR/programs/${CLICKHOUSE_BINARY_NAME}" ] || [ -x "$BUILD_DIR/programs/${CLICKHOUSE_BINARY_NAME}-server" ] ) && BIN_DIR=${BIN_DIR:=$BUILD_DIR/programs/}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-server" ] && CLICKHOUSE_SERVER=${CLICKHOUSE_SERVER:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-server}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}" ] && CLICKHOUSE_SERVER=${CLICKHOUSE_SERVER:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME} server}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-client" ] && CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-client}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}" ] && CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME} client}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-extract-from-config" ] && CLICKHOUSE_EXTRACT=${CLICKHOUSE_EXTRACT:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME}-extract-from-config}
|
||||
[ -x "$BIN_DIR/${CLICKHOUSE_BINARY_NAME}" ] && CLICKHOUSE_EXTRACT=${CLICKHOUSE_EXTRACT:=$BIN_DIR/${CLICKHOUSE_BINARY_NAME} extract-from-config}
|
||||
|
||||
[ -f "$CUR_DIR/server-test.xml" ] && CONFIG_DIR=${CONFIG_DIR=$CUR_DIR}/
|
||||
CONFIG_CLIENT_DIR=${CONFIG_CLIENT_DIR=$CONFIG_DIR}
|
||||
CONFIG_SERVER_DIR=${CONFIG_SERVER_DIR=$CONFIG_DIR}
|
||||
[ ! -f "${CONFIG_CLIENT_DIR}client-test.xml" ] && CONFIG_CLIENT_DIR=${CONFIG_CLIENT_DIR:=/etc/clickhouse-client/}
|
||||
[ ! -f "${CONFIG_SERVER_DIR}server-test.xml" ] && CONFIG_SERVER_DIR=${CONFIG_SERVER_DIR:=/etc/clickhouse-server/}
|
||||
export CLICKHOUSE_CONFIG_CLIENT=${CLICKHOUSE_CONFIG_CLIENT:=${CONFIG_CLIENT_DIR}client-test.xml}
|
||||
export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:=${CONFIG_SERVER_DIR}server-test.xml}
|
||||
CLICKHOUSE_CONFIG_USERS=${CONFIG_SERVER_DIR}users.xml
|
||||
[ ! -f "$CLICKHOUSE_CONFIG_USERS" ] && CLICKHOUSE_CONFIG_USERS=$CUR_DIR/../programs/server/users.xml
|
||||
CLICKHOUSE_CONFIG_USERS_D=${CONFIG_SERVER_DIR}users.d
|
||||
[ ! -d "$CLICKHOUSE_CONFIG_USERS_D" ] && CLICKHOUSE_CONFIG_USERS_D=$CUR_DIR/../programs/server/users.d
|
||||
[ -x "$CUR_DIR/clickhouse-test" ] && TEST_DIR=${TEST_DIR=$CUR_DIR/}
|
||||
[ -d "$CUR_DIR/queries" ] && QUERIES_DIR=${QUERIES_DIR=$CUR_DIR/queries}
|
||||
[ ! -d "$QUERIES_DIR" ] && [ -d "/usr/local/share/clickhouse-test/queries" ] && QUERIES_DIR=${QUERIES_DIR=/usr/local/share/clickhouse-test/queries}
|
||||
[ ! -d "$QUERIES_DIR" ] && [ -d "/usr/share/clickhouse-test/queries" ] && QUERIES_DIR=${QUERIES_DIR=/usr/share/clickhouse-test/queries}
|
||||
|
||||
TEST_PORT_RANDOM=${TEST_PORT_RANDOM=1}
|
||||
if [ "${TEST_PORT_RANDOM}" ]; then
|
||||
CLICKHOUSE_PORT_BASE=${CLICKHOUSE_PORT_BASE:=$(( ( RANDOM % 50000 ) + 10000 ))}
|
||||
CLICKHOUSE_PORT_TCP=${CLICKHOUSE_PORT_TCP:=$(($CLICKHOUSE_PORT_BASE + 1))}
|
||||
CLICKHOUSE_PORT_HTTP=${CLICKHOUSE_PORT_HTTP:=$(($CLICKHOUSE_PORT_BASE + 2))}
|
||||
CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:=$(($CLICKHOUSE_PORT_BASE + 3))}
|
||||
CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=$(($CLICKHOUSE_PORT_BASE + 4))}
|
||||
CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=$(($CLICKHOUSE_PORT_BASE + 5))}
|
||||
CLICKHOUSE_PORT_ODBC_BRIDGE=${CLICKHOUSE_ODBC_BRIDGE:=$(($CLICKHOUSE_PORT_BASE + 6))}
|
||||
fi
|
||||
|
||||
rm -rf $DATA_DIR || true
|
||||
mkdir -p $LOG_DIR $DATA_DIR/etc || true
|
||||
|
||||
if [ "$DATA_DIR_PATTERN" != "$DATA_DIR" ]; then
|
||||
cat $CLICKHOUSE_CONFIG | sed -e s!$DATA_DIR_PATTERN!$DATA_DIR! > $DATA_DIR/etc/server-config.xml
|
||||
export CLICKHOUSE_CONFIG=$DATA_DIR/etc/server-config.xml
|
||||
cp $CLICKHOUSE_CONFIG_USERS $DATA_DIR/etc
|
||||
cp -R -L $CLICKHOUSE_CONFIG_USERS_D $DATA_DIR/etc
|
||||
cat ${CONFIG_SERVER_DIR}/ints_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/ints_dictionary.xml
|
||||
cat ${CONFIG_SERVER_DIR}/strings_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/strings_dictionary.xml
|
||||
cat ${CONFIG_SERVER_DIR}/decimals_dictionary.xml | sed -e s!9000!$CLICKHOUSE_PORT_TCP! > $DATA_DIR/etc/decimals_dictionary.xml
|
||||
fi
|
||||
|
||||
CLICKHOUSE_EXTRACT_CONFIG=${CLICKHOUSE_EXTRACT_CONFIG:="${CLICKHOUSE_EXTRACT} --config=$CLICKHOUSE_CONFIG"}
|
||||
CLICKHOUSE_LOG=${CLICKHOUSE_LOG:=${LOG_DIR}clickhouse-server.log}
|
||||
export CLICKHOUSE_PORT_TCP=${CLICKHOUSE_PORT_TCP:=`$CLICKHOUSE_EXTRACT_CONFIG --key=tcp_port || echo 9000`}
|
||||
export CLICKHOUSE_PORT_HTTP=${CLICKHOUSE_PORT_HTTP:=`$CLICKHOUSE_EXTRACT_CONFIG --key=http_port || echo 8123`}
|
||||
export CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:=`$CLICKHOUSE_EXTRACT_CONFIG --key=interserver_http_port || echo 9009`}
|
||||
export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=`$CLICKHOUSE_EXTRACT_CONFIG --key=tcp_port_secure`}
|
||||
export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=`$CLICKHOUSE_EXTRACT_CONFIG --key=https_port`}
|
||||
export CLICKHOUSE_ODBC_BRIDGE=${CLICKHOUSE_ODBC_BRIDGE:=`$CLICKHOUSE_EXTRACT_CONFIG --key=odbc_bridge.port || echo 9018`}
|
||||
|
||||
DHPARAM=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.dhParamsFile`
|
||||
PRIVATEKEY=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.privateKeyFile`
|
||||
CERT=`$CLICKHOUSE_EXTRACT_CONFIG --key=openSSL.server.certificateFile`
|
||||
# Do not generate in case broken extract-config
|
||||
[ -n "$DHPARAM" ] && openssl dhparam -out $DHPARAM 256
|
||||
[ -n "$PRIVATEKEY" ] && [ -n "$CERT" ] && openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout $PRIVATEKEY -out $CERT
|
||||
|
||||
if [ "$TEST_GDB" ] || [ "$GDB" ]; then
|
||||
echo -e "run \nset pagination off \nset logging file $LOG_DIR/server.gdb.log \nset logging on \nbacktrace \nthread apply all backtrace \nbacktrace \ndetach \nquit " > $DATA_DIR/gdb.cmd
|
||||
GDB=${GDB:="gdb -x $DATA_DIR/gdb.cmd --args "}
|
||||
fi
|
||||
|
||||
# Start a local clickhouse server which will be used to run tests
|
||||
|
||||
# TODO: fix change shard ports:
|
||||
# --remote_servers.test_shard_localhost_secure.shard.replica.port=$CLICKHOUSE_PORT_TCP_SECURE \
|
||||
# --remote_servers.test_shard_localhost.shard.replica.port=$CLICKHOUSE_PORT_TCP \
|
||||
|
||||
VERSION=`$CLICKHOUSE_CLIENT --version-clean`
|
||||
# If run from compile dir - use in-place compile binary and headers
|
||||
[ -n "$BIN_DIR" ] && INTERNAL_COMPILER_PARAMS="--compiler_executable_root=${INTERNAL_COMPILER_BIN_ROOT:=$BUILD_DIR/programs/} --compiler_headers=$BUILD_DIR/programs/clang/headers/$VERSION/ --compiler_headers_root=$BUILD_DIR/programs/clang/headers/$VERSION/"
|
||||
|
||||
$GDB $CLICKHOUSE_SERVER --config-file=$CLICKHOUSE_CONFIG --log=$CLICKHOUSE_LOG $TEST_SERVER_PARAMS -- \
|
||||
--http_port=$CLICKHOUSE_PORT_HTTP \
|
||||
--tcp_port=$CLICKHOUSE_PORT_TCP \
|
||||
--https_port=$CLICKHOUSE_PORT_HTTPS \
|
||||
--tcp_port_secure=$CLICKHOUSE_PORT_TCP_SECURE \
|
||||
--interserver_http_port=$CLICKHOUSE_PORT_INTERSERVER \
|
||||
--odbc_bridge.port=$CLICKHOUSE_ODBC_BRIDGE \
|
||||
$INTERNAL_COMPILER_PARAMS \
|
||||
$TEST_SERVER_CONFIG_PARAMS \
|
||||
2>&1 > $LOG_DIR/server.stdout.log &
|
||||
CH_PID=$!
|
||||
sleep ${TEST_SERVER_STARTUP_WAIT:=5}
|
||||
|
||||
if [ "$GDB" ]; then
|
||||
# Long symbols read
|
||||
sleep ${TEST_GDB_SLEEP:=60}
|
||||
fi
|
||||
|
||||
tail -n50 $LOG_DIR/*.log || true
|
||||
|
||||
# Define needed stuff to kill test clickhouse server after tests completion
|
||||
function finish {
|
||||
kill $CH_PID || true
|
||||
wait
|
||||
tail -n 50 $LOG_DIR/*.log || true
|
||||
if [ "$GDB" ]; then
|
||||
cat $LOG_DIR/server.gdb.log || true
|
||||
fi
|
||||
rm -rf $DATA_DIR
|
||||
}
|
||||
trap finish EXIT SIGINT SIGQUIT SIGTERM
|
||||
|
||||
# Do tests
|
||||
if [ -n "$*" ]; then
|
||||
$*
|
||||
else
|
||||
TEST_RUN=${TEST_RUN=1}
|
||||
TEST_DICT=${TEST_DICT=1}
|
||||
CLICKHOUSE_CLIENT_QUERY="${CLICKHOUSE_CLIENT} --config ${CLICKHOUSE_CONFIG_CLIENT} --port $CLICKHOUSE_PORT_TCP -m -n -q"
|
||||
$CLICKHOUSE_CLIENT_QUERY 'SELECT * from system.build_options; SELECT * FROM system.clusters;'
|
||||
CLICKHOUSE_TEST="env ${TEST_DIR}clickhouse-test --force-color --binary ${BIN_DIR}${CLICKHOUSE_BINARY_NAME} --configclient $CLICKHOUSE_CONFIG_CLIENT --configserver $CLICKHOUSE_CONFIG --tmp $DATA_DIR/tmp --queries $QUERIES_DIR $TEST_OPT0 $TEST_OPT"
|
||||
if [ "${TEST_RUN_STRESS}" ]; then
|
||||
# Running test in parallel will fail some results (tests can create/fill/drop same tables)
|
||||
TEST_NPROC=${TEST_NPROC:=$(( `nproc || sysctl -n hw.ncpu || echo 2` * 2))}
|
||||
for i in `seq 1 ${TEST_NPROC}`; do
|
||||
$CLICKHOUSE_TEST --order=random --testname --tmp=$DATA_DIR/tmp/tmp${i} &
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "${TEST_RUN_PARALLEL}" ]; then
|
||||
# Running test in parallel will fail some results (tests can create/fill/drop same tables)
|
||||
TEST_NPROC=${TEST_NPROC:=$(( `nproc || sysctl -n hw.ncpu || echo 2` * 2))}
|
||||
for i in `seq 1 ${TEST_NPROC}`; do
|
||||
$CLICKHOUSE_TEST --testname --tmp=$DATA_DIR/tmp/tmp${i} --database=test${i} --parallel=${i}/${TEST_NPROC} &
|
||||
done
|
||||
for job in `jobs -p`; do
|
||||
#echo wait $job
|
||||
wait $job || let "FAIL+=1"
|
||||
done
|
||||
|
||||
#echo $FAIL
|
||||
if [ "$FAIL" != "0" ]; then
|
||||
return $FAIL
|
||||
fi
|
||||
else
|
||||
( [ "$TEST_RUN" ] && $CLICKHOUSE_TEST ) || ${TEST_TRUE:=false}
|
||||
fi
|
||||
|
||||
$CLICKHOUSE_CLIENT_QUERY "SELECT event, value FROM system.events; SELECT metric, value FROM system.metrics; SELECT metric, value FROM system.asynchronous_metrics;"
|
||||
$CLICKHOUSE_CLIENT_QUERY "SELECT 'Still alive'"
|
||||
fi
|
||||
Loading…
Reference in New Issue
Block a user