diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index ac9dcde1f36..3969d93e3c0 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -341,7 +341,7 @@ def main(args): if result_is_different: diff = Popen(['diff', '--unified', reference_file, stdout_file], stdout = PIPE).communicate()[0] diff = unicode(diff, errors='replace', encoding='utf-8') - cat = Popen(['cat', '-A'], stdin=PIPE, stdout=PIPE).communicate(input=diff)[0] + cat = Popen(['cat', '-A'], stdin=PIPE, stdout=PIPE).communicate(input=diff.encode(encoding='utf-8', errors='replace'))[0] failure = et.Element("failure", attrib = {"message": "result differs with reference"}) report_testcase.append(failure) @@ -367,12 +367,13 @@ def main(args): print(colored("Break tests execution", "red")) raise e except: - (exc_type, exc_value) = sys.exc_info()[:2] + import traceback + exc_type, exc_value, tb = sys.exc_info() error = et.Element("error", attrib = {"type": exc_type.__name__, "message": str(exc_value)}) report_testcase.append(error) failures += 1 - print("{0} - Test internal error: {1}\n{2}".format(MSG_FAIL, exc_type.__name__, exc_value)) + print("{0} - Test internal error: {1}\n{2}\n{3}".format(MSG_FAIL, exc_type.__name__, exc_value, "\n".join(traceback.format_tb(tb, 10)))) finally: dump_report(args.output, suite, name, report_testcase) diff --git a/dbms/tests/queries/0_stateless/00354_host_command_line_option.sh b/dbms/tests/queries/0_stateless/00354_host_command_line_option.sh index c158e929ec0..a9a510dc45f 100755 --- a/dbms/tests/queries/0_stateless/00354_host_command_line_option.sh +++ b/dbms/tests/queries/0_stateless/00354_host_command_line_option.sh @@ -3,6 +3,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -$CLICKHOUSE_CLIENT --host=localhost --query="SELECT 1"; -$CLICKHOUSE_CLIENT --host localhost --query "SELECT 1"; -$CLICKHOUSE_CLIENT -hlocalhost -q"SELECT 1"; +clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT 1"; +clickhouse_client_removed_host_parameter --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +clickhouse_client_removed_host_parameter -h"${CLICKHOUSE_HOST}" -q"SELECT 1"; diff --git a/dbms/tests/queries/0_stateless/00368_format_option_collision.sh b/dbms/tests/queries/0_stateless/00368_format_option_collision.sh index a9c1178b495..6d0a355d78a 100755 --- a/dbms/tests/queries/0_stateless/00368_format_option_collision.sh +++ b/dbms/tests/queries/0_stateless/00368_format_option_collision.sh @@ -3,4 +3,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -$CLICKHOUSE_CLIENT --host=localhost --query="SELECT * FROM ext" --format=Vertical --external --file=- --structure="s String" --name=ext --format=JSONEachRow <<< '{"s":"Hello"}' +clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT * FROM ext" --format=Vertical --external --file=- --structure="s String" --name=ext --format=JSONEachRow <<< '{"s":"Hello"}' diff --git a/dbms/tests/queries/0_stateless/00634_logging_shard.sh b/dbms/tests/queries/0_stateless/00634_logging_shard.sh index 3ea0df81bab..7df6e768a8b 100755 --- a/dbms/tests/queries/0_stateless/00634_logging_shard.sh +++ b/dbms/tests/queries/0_stateless/00634_logging_shard.sh @@ -9,6 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) cur_name=$(basename "${BASH_SOURCE[0]}") server_logs_file="${CLICKHOUSE_TMP}/${cur_name}_server.logs" + server_logs="--server_logs_file=$server_logs_file" rm -f "$server_logs_file" diff --git a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index 56a0514e90d..47e9aa07476 100755 --- a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -9,6 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) cur_name=$(basename "${BASH_SOURCE[0]}") server_logs_file=${CLICKHOUSE_TMP}/$cur_name"_server.logs" + server_logs="--server_logs_file=$server_logs_file" rm -f "$server_logs_file" diff --git a/dbms/tests/queries/0_stateless/00646_url_engine.python b/dbms/tests/queries/0_stateless/00646_url_engine.python index fed6b2a4d84..960048dbb8f 100644 --- a/dbms/tests/queries/0_stateless/00646_url_engine.python +++ b/dbms/tests/queries/0_stateless/00646_url_engine.python @@ -5,17 +5,31 @@ import sys import tempfile import threading import os, urllib +import subprocess from io import StringIO from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', '127.0.0.1') +CLICKHOUSE_PORT_HTTP = os.environ.get('CLICKHOUSE_PORT_HTTP', '8123') -SERVER_ADDRESS = ('127.0.0.1', 51234) -SERVER_ADDRESS_STR = 'http://' + ':'.join(str(s) for s in SERVER_ADDRESS) + "/" +##################################################################################### +# This test starts an HTTP server and serves data to clickhouse url-engine based table. +# In order for it to work ip+port of http server (given below) should be +# accessible from clickhouse server. +##################################################################################### + +# IP-address of this host accessible from outside world. +HTTP_SERVER_HOST = subprocess.check_output(['hostname', '-i']).decode('utf-8').strip() +HTTP_SERVER_PORT = int(os.environ.get('CLICKHOUSE_TEST_HOST_EXPOSED_PORT', 51234)) + +# IP address and port of the HTTP server started from this script. +HTTP_SERVER_ADDRESS = (HTTP_SERVER_HOST, HTTP_SERVER_PORT) +HTTP_SERVER_URL_STR = 'http://' + ':'.join(str(s) for s in HTTP_SERVER_ADDRESS) + "/" CSV_DATA = os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names())) - def get_ch_answer(query): - return urllib.urlopen(os.environ.get('CLICKHOUSE_URL', 'http://localhost:' + os.environ.get('CLICKHOUSE_PORT_HTTP', '8123')), data=query).read() + url = os.environ.get('CLICKHOUSE_URL', 'http://{host}:{port}'.format(host=CLICKHOUSE_HOST, port=CLICKHOUSE_PORT_HTTP)) + return urllib.urlopen(url, data=query).read() def check_answers(query, answer): ch_answer = get_ch_answer(query) @@ -75,7 +89,7 @@ class CSVHTTPServer(BaseHTTPRequestHandler): return def start_server(requests_amount): - httpd = HTTPServer(SERVER_ADDRESS, CSVHTTPServer) + httpd = HTTPServer(HTTP_SERVER_ADDRESS, CSVHTTPServer) def real_func(): for i in xrange(requests_amount): @@ -96,12 +110,12 @@ def test_select(table_name="", schema="str String,numuint UInt32,numint Int32,do if table_name: get_ch_answer("drop table if exists {}".format(table_name)) - get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, SERVER_ADDRESS_STR)) + get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, HTTP_SERVER_URL_STR)) for i in xrange(len(requests)): tbl = table_name if not tbl: - tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema) + tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema) check_answers(requests[i].format(tbl=tbl), answers[i]) if table_name: @@ -113,19 +127,19 @@ def test_insert(table_name="", schema="str String,numuint UInt32,numint Int32,do if table_name: get_ch_answer("drop table if exists {}".format(table_name)) - get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, SERVER_ADDRESS_STR)) + get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, HTTP_SERVER_URL_STR)) for req in requests_insert: tbl = table_name if not tbl: - tbl = "table function url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema) + tbl = "table function url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema) get_ch_answer(req.format(tbl=tbl)) for i in xrange(len(requests_select)): tbl = table_name if not tbl: - tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema) + tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema) check_answers(requests_select[i].format(tbl=tbl), answers[i]) if table_name: diff --git a/dbms/tests/queries/0_stateless/00837_minmax_index.sh b/dbms/tests/queries/0_stateless/00837_minmax_index.sh index d38f7bbabfd..67686475970 100755 --- a/dbms/tests/queries/0_stateless/00837_minmax_index.sh +++ b/dbms/tests/queries/0_stateless/00837_minmax_index.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +CLICKHOUSE_CLIENT_OPT="--allow_experimental_data_skipping_indices=1" + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh diff --git a/dbms/tests/queries/0_stateless/00838_unique_index.sh b/dbms/tests/queries/0_stateless/00838_unique_index.sh index f6bea4f083a..c580eb21a9c 100755 --- a/dbms/tests/queries/0_stateless/00838_unique_index.sh +++ b/dbms/tests/queries/0_stateless/00838_unique_index.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +CLICKHOUSE_CLIENT_OPT="--allow_experimental_data_skipping_indices=1" + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh diff --git a/dbms/tests/queries/shell_config.sh b/dbms/tests/queries/shell_config.sh index f086143db65..a972690e0cf 100644 --- a/dbms/tests/queries/shell_config.sh +++ b/dbms/tests/queries/shell_config.sh @@ -39,3 +39,10 @@ export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"} export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} --max-time 10"} export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."} mkdir -p ${CLICKHOUSE_TMP} + +function clickhouse_client_removed_host_parameter() +{ + # removing only `--host=value` and `--host value` (removing '-hvalue' feels to dangerous) with python regex. + # bash regex magic is arcane, but version dependant and weak; sed or awk are not really portable. + $(echo "$CLICKHOUSE_CLIENT" | python -c "import sys, re; print re.sub('--host(\s+|=)[^\s]+', '', sys.stdin.read())") "$@" +} diff --git a/docker/server/local.Dockerfile b/docker/server/local.Dockerfile index 33d7e11f118..390f840513e 100644 --- a/docker/server/local.Dockerfile +++ b/docker/server/local.Dockerfile @@ -1,18 +1,19 @@ -FROM ubuntu:18.04 +# Since right now we can't set volumes to the docker during build, we split building container in stages: +# 1. build base container +# 2. run base conatiner with mounted volumes +# 3. commit container as image +# 4. build final container atop that image +# Middle steps are performed by the bash script. +FROM ubuntu:18.04 as clickhouse-server-base ARG gosu_ver=1.10 -ARG CLICKHOUSE_PACKAGES_DIR -COPY ${CLICKHOUSE_PACKAGES_DIR}/clickhouse-*.deb /packages/ +VOLUME /packages/ -# installing via apt to simulate real-world scenario, where user installs deb package and all it's dependecies automatically. +# update to allow installing dependencies of clickhouse automatically RUN apt update; \ DEBIAN_FRONTEND=noninteractive \ - apt install -y \ - /packages/clickhouse-common-static_*.deb \ - /packages/clickhouse-server_*.deb \ - locales ;\ - rm -rf /packages + apt install -y locales; ADD https://github.com/tianon/gosu/releases/download/${gosu_ver}/gosu-amd64 /bin/gosu @@ -21,10 +22,18 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 +# installing via apt to simulate real-world scenario, where user installs deb package and all it's dependecies automatically. +CMD DEBIAN_FRONTEND=noninteractive \ + apt install -y \ + /packages/clickhouse-common-static_*.deb \ + /packages/clickhouse-server_*.deb ; + +FROM clickhouse-server-base:postinstall as clickhouse-server + RUN mkdir /docker-entrypoint-initdb.d -COPY server/docker_related_config.xml /etc/clickhouse-server/config.d/ -COPY server/entrypoint.sh /entrypoint.sh +COPY docker_related_config.xml /etc/clickhouse-server/config.d/ +COPY entrypoint.sh /entrypoint.sh RUN chmod +x \ /entrypoint.sh \ diff --git a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile index 8aecb7119cc..562141ba147 100644 --- a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile +++ b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile @@ -1,12 +1,15 @@ -FROM ubuntu:18.10 +# Since right now we can't set volumes to the docker during build, we split building container in stages: +# 1. build base container +# 2. run base conatiner with mounted volumes +# 3. commit container as image +FROM ubuntu:18.10 as clickhouse-test-runner-base -ARG CLICKHOUSE_PACKAGES_DIR -COPY ${CLICKHOUSE_PACKAGES_DIR}/clickhouse-*.deb /packages/ +# A volume where directory with clickhouse packages to be mounted, +# for later installing. +VOLUME /packages -RUN apt-get update ;\ +CMD apt-get update ;\ DEBIAN_FRONTEND=noninteractive \ apt install -y /packages/clickhouse-common-static_*.deb \ /packages/clickhouse-client_*.deb \ - /packages/clickhouse-test_*.deb \ - wait-for-it; \ - rm -rf /packages + /packages/clickhouse-test_*.deb diff --git a/docker/test/test_runner.sh b/docker/test/test_runner.sh index afa5c95720b..6e6d4537603 100755 --- a/docker/test/test_runner.sh +++ b/docker/test/test_runner.sh @@ -1,31 +1,84 @@ #!/bin/sh -set -e +set -e -x -# Run tests in docker -# OR -# Build containers from deb packages, copying the tests from the source directory +trap 'rc=$?; echo EXITED WITH: $rc; exit $rc' EXIT + +# CLI option to prevent rebuilding images, just re-run tests with images leftover from previuos time +readonly NO_REBUILD_FLAG="--no-rebuild" readonly CLICKHOUSE_DOCKER_DIR="$(realpath ${1})" -readonly CLICKHOUSE_PACKAGES_DIR="${2}" +readonly CLICKHOUSE_PACKAGES_ARG="${2}" CLICKHOUSE_SERVER_IMAGE="${3}" -# Build test runner image -docker build \ - -f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \ - -t clickhouse-statelest-test-runner:local \ - --build-arg CLICKHOUSE_PACKAGES_DIR="${CLICKHOUSE_PACKAGES_DIR}" \ - "${CLICKHOUSE_DOCKER_DIR}" +if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then + readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild +fi + + +# In order to allow packages directory to be anywhere, and to reduce amoun of context sent to the docker daemon, +# all images are built in multiple stages: +# 1. build base image, install dependencies +# 2. run image with volume mounted, install what needed from those volumes +# 3. tag container as image +# 4. [optional] build another image atop of tagged. + +# TODO: optionally mount most recent clickhouse-test and queries directory from local machine + +if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \ + --target clickhouse-test-runner-base \ + -t clickhouse-test-runner-base:preinstall \ + "${CLICKHOUSE_DOCKER_DIR}/test/stateless" + + docker rm -f clickhouse-test-runner-installing-packages || true + docker run \ + -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ + --name clickhouse-test-runner-installing-packages \ + clickhouse-test-runner-base:preinstall + docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local + docker rm -f clickhouse-test-runner-installing-packages || true +fi + +# # Create a bind-volume to the clickhouse-test script file +# docker volume create --driver local --opt type=none --opt device=/home/enmk/proj/ClickHouse_master/dbms/tests/clickhouse-test --opt o=bind clickhouse-test-script-volume +# docker volume create --driver local --opt type=none --opt device=/home/enmk/proj/ClickHouse_master/dbms/tests/queries --opt o=bind clickhouse-test-queries-dir-volume # Build server image (optional) from local packages if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then - CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse_server:local" + CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local" - docker build \ - -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ - -t "${CLICKHOUSE_SERVER_IMAGE}" \ - --build-arg CLICKHOUSE_PACKAGES_DIR=${CLICKHOUSE_PACKAGES_DIR} \ - "${CLICKHOUSE_DOCKER_DIR}" + if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ + --target clickhouse-server-base \ + -t clickhouse-server-base:preinstall \ + "${CLICKHOUSE_DOCKER_DIR}/server" + + docker rm -f clickhouse_server_base_installing_server || true + docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ + --name clickhouse_server_base_installing_server \ + clickhouse-server-base:preinstall + docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall + + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ + --target clickhouse-server \ + -t "${CLICKHOUSE_SERVER_IMAGE}" \ + "${CLICKHOUSE_DOCKER_DIR}/server" + fi fi -CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" run test-runner \ No newline at end of file +docker rm -f test-runner || true +docker-compose down +CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \ + docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ + create \ + --build --force-recreate + +CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \ + docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ + run \ + --name test-runner \ + test-runner \ No newline at end of file diff --git a/docker/test/test_runner_docker_compose.yaml b/docker/test/test_runner_docker_compose.yaml index 281442f26a4..ba2e525b3a5 100644 --- a/docker/test/test_runner_docker_compose.yaml +++ b/docker/test/test_runner_docker_compose.yaml @@ -4,13 +4,13 @@ services: clickhouse-server: image: ${CLICKHOUSE_SERVER_IMAGE} expose: - - "8123" - - "9000" - - "9009" + - "8123" # HTTP + - "9000" # TCP + - "9009" # HTTP-interserver restart: "no" test-runner: - image: yandex/clickhouse-statelest-test-runner:local + image: clickhouse-statelest-test-runner:local restart: "no" depends_on: @@ -18,13 +18,17 @@ services: environment: # these are used by clickhouse-test to point clickhouse-client to the right server - CLICKHOUSE_HOST=clickhouse-server - - CLICKHOUSE_PORT=8123 + - CLICKHOUSE_PORT=9009 + - CLICKHOUSE_TEST_HOST_EXPOSED_PORT=51234 + expose: + # port for any test to serve data to clickhouse-server on rare occasion (like URL-engine tables in 00646), + # should match value of CLICKHOUSE_TEST_HOST_EXPOSED_PORT above + - "51234" - entrypoint: - - wait-for-it - - clickhouse-server:8123 - - -- - - clickhouse-test - # - -c - # - `which clickhouse-client` - - ${CLICKHOUSE_TEST_ARGS} + # NOTE: Dev-mode: mount newest versions of the queries and clickhouse-test script into container. + # volumes: + # - /home/enmk/proj/ClickHouse_master/dbms/tests/queries:/usr/share/clickhouse-test/queries:ro + # - /home/enmk/proj/ClickHouse_master/dbms/tests/clickhouse-test:/usr/bin/clickhouse-test:ro + + # String-form instead of list-form to allow multiple arguments in "${CLICKHOUSE_TEST_ARGS}" + entrypoint: "clickhouse-test ${CLICKHOUSE_TEST_ARGS}"