Fixed test failures when running clickhouse-server on different host

Fixed test in docker: writing to read-only filesystem Multi-stage builds for test-runner and server to allow putting packages directory anywhere Fixed more tests
2024-11-21 15:12:02 +00:00 · 2019-02-21 12:21:20 +03:00 · 2019-02-21 12:21:20 +03:00 · 9b5950f403
commit 9b5950f403
parent 14b8e64992
13 changed files with 163 additions and 66 deletions
--- a/dbms/tests/clickhouse-test
+++ b/dbms/tests/clickhouse-test
@ -341,7 +341,7 @@ def main(args):
                                        if result_is_different:
                                            diff = Popen(['diff', '--unified', reference_file, stdout_file], stdout = PIPE).communicate()[0]
                                            diff = unicode(diff, errors='replace', encoding='utf-8')
-                                            cat = Popen(['cat', '-A'], stdin=PIPE, stdout=PIPE).communicate(input=diff)[0]
+                                            cat = Popen(['cat', '-A'], stdin=PIPE, stdout=PIPE).communicate(input=diff.encode(encoding='utf-8', errors='replace'))[0]

                                            failure = et.Element("failure", attrib = {"message": "result differs with reference"})
                                            report_testcase.append(failure)
@ -367,12 +367,13 @@ def main(args):
                        print(colored("Break tests execution", "red"))
                        raise e
                    except:
-                        (exc_type, exc_value) = sys.exc_info()[:2]
+                        import traceback
+                        exc_type, exc_value, tb = sys.exc_info()
                        error = et.Element("error", attrib = {"type": exc_type.__name__, "message": str(exc_value)})
                        report_testcase.append(error)

                        failures += 1
-                        print("{0} - Test internal error: {1}\n{2}".format(MSG_FAIL, exc_type.__name__, exc_value))
+                        print("{0} - Test internal error: {1}\n{2}\n{3}".format(MSG_FAIL, exc_type.__name__, exc_value, "\n".join(traceback.format_tb(tb, 10))))
                    finally:
                        dump_report(args.output, suite, name, report_testcase)

--- a/dbms/tests/queries/0_stateless/00354_host_command_line_option.sh
+++ b/dbms/tests/queries/0_stateless/00354_host_command_line_option.sh
@ -3,6 +3,6 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh

-$CLICKHOUSE_CLIENT --host=localhost --query="SELECT 1";
-$CLICKHOUSE_CLIENT --host localhost --query "SELECT 1";
-$CLICKHOUSE_CLIENT -hlocalhost -q"SELECT 1";
+clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT 1";
+clickhouse_client_removed_host_parameter --host "${CLICKHOUSE_HOST}" --query "SELECT 1";
+clickhouse_client_removed_host_parameter -h"${CLICKHOUSE_HOST}" -q"SELECT 1";
--- a/dbms/tests/queries/0_stateless/00368_format_option_collision.sh
+++ b/dbms/tests/queries/0_stateless/00368_format_option_collision.sh
@ -3,4 +3,4 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh

-$CLICKHOUSE_CLIENT --host=localhost --query="SELECT * FROM ext" --format=Vertical --external --file=- --structure="s String" --name=ext --format=JSONEachRow <<< '{"s":"Hello"}'
+clickhouse_client_removed_host_parameter --host="${CLICKHOUSE_HOST}" --query="SELECT * FROM ext" --format=Vertical --external --file=- --structure="s String" --name=ext --format=JSONEachRow <<< '{"s":"Hello"}'
--- a/dbms/tests/queries/0_stateless/00634_logging_shard.sh
+++ b/dbms/tests/queries/0_stateless/00634_logging_shard.sh
@ -9,6 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

 cur_name=$(basename "${BASH_SOURCE[0]}")
 server_logs_file="${CLICKHOUSE_TMP}/${cur_name}_server.logs"
+
 server_logs="--server_logs_file=$server_logs_file"
 rm -f "$server_logs_file"

--- a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh
+++ b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh
@ -9,6 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

 cur_name=$(basename "${BASH_SOURCE[0]}")
 server_logs_file=${CLICKHOUSE_TMP}/$cur_name"_server.logs"
+
 server_logs="--server_logs_file=$server_logs_file"
 rm -f "$server_logs_file"

--- a/dbms/tests/queries/0_stateless/00646_url_engine.python
+++ b/dbms/tests/queries/0_stateless/00646_url_engine.python
@ -5,17 +5,31 @@ import sys
 import tempfile
 import threading
 import os, urllib
+import subprocess
 from io import StringIO
 from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer

+CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', '127.0.0.1')
+CLICKHOUSE_PORT_HTTP = os.environ.get('CLICKHOUSE_PORT_HTTP', '8123')

-SERVER_ADDRESS = ('127.0.0.1', 51234)
-SERVER_ADDRESS_STR = 'http://' + ':'.join(str(s) for s in SERVER_ADDRESS) + "/"
+#####################################################################################
+# This test starts an HTTP server and serves data to clickhouse url-engine based table.
+# In order for it to work ip+port of http server (given below) should be
+# accessible from clickhouse server.
+#####################################################################################
+
+# IP-address of this host accessible from outside world.
+HTTP_SERVER_HOST = subprocess.check_output(['hostname', '-i']).decode('utf-8').strip()
+HTTP_SERVER_PORT = int(os.environ.get('CLICKHOUSE_TEST_HOST_EXPOSED_PORT', 51234))
+
+# IP address and port of the HTTP server started from this script.
+HTTP_SERVER_ADDRESS = (HTTP_SERVER_HOST, HTTP_SERVER_PORT)
+HTTP_SERVER_URL_STR = 'http://' + ':'.join(str(s) for s in HTTP_SERVER_ADDRESS) + "/"
 CSV_DATA = os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))

-
 def get_ch_answer(query):
-    return urllib.urlopen(os.environ.get('CLICKHOUSE_URL', 'http://localhost:' + os.environ.get('CLICKHOUSE_PORT_HTTP', '8123')), data=query).read()
+    url = os.environ.get('CLICKHOUSE_URL', 'http://{host}:{port}'.format(host=CLICKHOUSE_HOST, port=CLICKHOUSE_PORT_HTTP))
+    return urllib.urlopen(url, data=query).read()

 def check_answers(query, answer):
    ch_answer = get_ch_answer(query)
@ -75,7 +89,7 @@ class CSVHTTPServer(BaseHTTPRequestHandler):
        return

 def start_server(requests_amount):
-    httpd = HTTPServer(SERVER_ADDRESS, CSVHTTPServer)
+    httpd = HTTPServer(HTTP_SERVER_ADDRESS, CSVHTTPServer)

    def real_func():
        for i in xrange(requests_amount):
@ -96,12 +110,12 @@ def test_select(table_name="", schema="str String,numuint UInt32,numint Int32,do

    if table_name:
        get_ch_answer("drop table if exists {}".format(table_name))
-        get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, SERVER_ADDRESS_STR))
+        get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, HTTP_SERVER_URL_STR))

    for i in xrange(len(requests)):
        tbl = table_name
        if not tbl:
-            tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema)
+            tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema)
        check_answers(requests[i].format(tbl=tbl), answers[i])

    if table_name:
@ -113,19 +127,19 @@ def test_insert(table_name="", schema="str String,numuint UInt32,numint Int32,do

    if table_name:
        get_ch_answer("drop table if exists {}".format(table_name))
-        get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, SERVER_ADDRESS_STR))
+        get_ch_answer("create table {} ({}) engine=URL('{}', 'CSV')".format(table_name, schema, HTTP_SERVER_URL_STR))

    for req in requests_insert:
        tbl = table_name
        if not tbl:
-            tbl = "table function url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema)
+            tbl = "table function url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema)
        get_ch_answer(req.format(tbl=tbl))


    for i in xrange(len(requests_select)):
        tbl = table_name
        if not tbl:
-            tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=SERVER_ADDRESS_STR, schema=schema)
+            tbl = "url('{addr}', 'CSV', '{schema}')".format(addr=HTTP_SERVER_URL_STR, schema=schema)
        check_answers(requests_select[i].format(tbl=tbl), answers[i])

    if table_name:
--- a/dbms/tests/queries/0_stateless/00837_minmax_index.sh
+++ b/dbms/tests/queries/0_stateless/00837_minmax_index.sh
@ -1,5 +1,7 @@
 #!/usr/bin/env bash

+CLICKHOUSE_CLIENT_OPT="--allow_experimental_data_skipping_indices=1"
+
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh

--- a/dbms/tests/queries/0_stateless/00838_unique_index.sh
+++ b/dbms/tests/queries/0_stateless/00838_unique_index.sh
@ -1,5 +1,7 @@
 #!/usr/bin/env bash

+CLICKHOUSE_CLIENT_OPT="--allow_experimental_data_skipping_indices=1"
+
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh

--- a/dbms/tests/queries/shell_config.sh
+++ b/dbms/tests/queries/shell_config.sh
@ -39,3 +39,10 @@ export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"}
 export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} --max-time 10"}
 export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."}
 mkdir -p ${CLICKHOUSE_TMP}
+
+function clickhouse_client_removed_host_parameter()
+{
+	# removing only `--host=value` and `--host value` (removing '-hvalue' feels to dangerous) with python regex.
+	# bash regex magic is arcane, but version dependant and weak; sed or awk are not really portable.
+	$(echo "$CLICKHOUSE_CLIENT"  | python -c "import sys, re; print re.sub('--host(\s+|=)[^\s]+', '', sys.stdin.read())") "$@"
+}
--- a/docker/server/local.Dockerfile
+++ b/docker/server/local.Dockerfile
@ -1,18 +1,19 @@
-FROM ubuntu:18.04
+# Since right now we can't set volumes to the docker during build, we split building container in stages:
+# 1. build base container
+# 2. run base conatiner with mounted volumes
+# 3. commit container as image
+# 4. build final container atop that image
+# Middle steps are performed by the bash script.

+FROM ubuntu:18.04 as clickhouse-server-base
 ARG gosu_ver=1.10
-ARG CLICKHOUSE_PACKAGES_DIR

-COPY ${CLICKHOUSE_PACKAGES_DIR}/clickhouse-*.deb /packages/
+VOLUME /packages/

-# installing via apt to simulate real-world scenario, where user installs deb package and all it's dependecies automatically.
+# update to allow installing dependencies of clickhouse automatically
 RUN apt update; \
    DEBIAN_FRONTEND=noninteractive \
-    apt install -y \
-        /packages/clickhouse-common-static_*.deb \
-        /packages/clickhouse-server_*.deb \
-        locales ;\
-    rm -rf /packages
+    apt install -y locales;

 ADD https://github.com/tianon/gosu/releases/download/${gosu_ver}/gosu-amd64 /bin/gosu

@ -21,10 +22,18 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8

+# installing via apt to simulate real-world scenario, where user installs deb package and all it's dependecies automatically.
+CMD DEBIAN_FRONTEND=noninteractive \
+    apt install -y \
+        /packages/clickhouse-common-static_*.deb \
+        /packages/clickhouse-server_*.deb ;
+
+FROM clickhouse-server-base:postinstall as clickhouse-server
+
 RUN mkdir /docker-entrypoint-initdb.d

-COPY server/docker_related_config.xml /etc/clickhouse-server/config.d/
-COPY server/entrypoint.sh /entrypoint.sh
+COPY docker_related_config.xml /etc/clickhouse-server/config.d/
+COPY entrypoint.sh /entrypoint.sh

 RUN chmod +x \
    /entrypoint.sh \
--- a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile
+++ b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile
@ -1,12 +1,15 @@
-FROM ubuntu:18.10
+# Since right now we can't set volumes to the docker during build, we split building container in stages:
+# 1. build base container
+# 2. run base conatiner with mounted volumes
+# 3. commit container as image
+FROM ubuntu:18.10 as clickhouse-test-runner-base

-ARG CLICKHOUSE_PACKAGES_DIR
-COPY ${CLICKHOUSE_PACKAGES_DIR}/clickhouse-*.deb /packages/
+# A volume where directory with clickhouse packages to be mounted,
+# for later installing.
+VOLUME /packages

-RUN apt-get update ;\
+CMD apt-get update ;\
 	DEBIAN_FRONTEND=noninteractive \
 	apt install -y /packages/clickhouse-common-static_*.deb \
 		/packages/clickhouse-client_*.deb \
-		/packages/clickhouse-test_*.deb \
-		wait-for-it; \
-	rm -rf /packages
+		/packages/clickhouse-test_*.deb
--- a/docker/test/test_runner.sh
+++ b/docker/test/test_runner.sh
@ -1,31 +1,84 @@
 #!/bin/sh

-set -e
+set -e -x

-# Run tests in docker
-# OR
-# Build containers from deb packages, copying the tests from the source directory 
+trap 'rc=$?; echo EXITED WITH: $rc; exit $rc' EXIT
+
+# CLI option to prevent rebuilding images, just re-run tests with images leftover from previuos time
+readonly NO_REBUILD_FLAG="--no-rebuild"

 readonly CLICKHOUSE_DOCKER_DIR="$(realpath ${1})"
-readonly CLICKHOUSE_PACKAGES_DIR="${2}"
+readonly CLICKHOUSE_PACKAGES_ARG="${2}"
 CLICKHOUSE_SERVER_IMAGE="${3}"

-# Build test runner image
-docker build \
-	-f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
-	-t clickhouse-statelest-test-runner:local \
-	--build-arg CLICKHOUSE_PACKAGES_DIR="${CLICKHOUSE_PACKAGES_DIR}" \
-	"${CLICKHOUSE_DOCKER_DIR}"
+if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
+	readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild
+fi
+
+
+# In order to allow packages directory to be anywhere, and to reduce amoun of context sent to the docker daemon,
+# all images are built in multiple stages:
+# 1. build base image, install dependencies
+# 2. run image with volume mounted, install what needed from those volumes
+# 3. tag container as image
+# 4. [optional] build another image atop of tagged.
+
+# TODO: optionally mount most recent clickhouse-test and queries directory from local machine
+
+if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
+	docker build \
+		-f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
+		--target clickhouse-test-runner-base \
+		-t clickhouse-test-runner-base:preinstall \
+		"${CLICKHOUSE_DOCKER_DIR}/test/stateless"
+
+	docker rm -f clickhouse-test-runner-installing-packages || true
+	docker run \
+		-v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
+		--name clickhouse-test-runner-installing-packages \
+		clickhouse-test-runner-base:preinstall
+	docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local
+	docker rm -f clickhouse-test-runner-installing-packages || true
+fi
+
+# # Create a bind-volume to the clickhouse-test script file
+# docker volume create --driver local --opt type=none --opt device=/home/enmk/proj/ClickHouse_master/dbms/tests/clickhouse-test --opt o=bind clickhouse-test-script-volume
+# docker volume create --driver local --opt type=none --opt device=/home/enmk/proj/ClickHouse_master/dbms/tests/queries --opt o=bind clickhouse-test-queries-dir-volume

 # Build server image (optional) from local packages
 if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then
-	CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse_server:local"
+	CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"

-	docker build \
-		-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
-		-t "${CLICKHOUSE_SERVER_IMAGE}" \
-		--build-arg CLICKHOUSE_PACKAGES_DIR=${CLICKHOUSE_PACKAGES_DIR} \
-		"${CLICKHOUSE_DOCKER_DIR}"
+	if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
+		docker build \
+			-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
+			--target clickhouse-server-base \
+			-t clickhouse-server-base:preinstall \
+			"${CLICKHOUSE_DOCKER_DIR}/server"
+
+		docker rm -f clickhouse_server_base_installing_server || true
+		docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
+			--name clickhouse_server_base_installing_server \
+			clickhouse-server-base:preinstall
+		docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall
+
+		docker build \
+			-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
+			--target clickhouse-server \
+			-t "${CLICKHOUSE_SERVER_IMAGE}" \
+			"${CLICKHOUSE_DOCKER_DIR}/server"
+	fi
 fi

-CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" run test-runner
+docker rm -f test-runner || true
+docker-compose down
+CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
+	docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
+	create \
+	--build --force-recreate
+
+CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
+	docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
+	run \
+	--name test-runner \
+	test-runner
--- a/docker/test/test_runner_docker_compose.yaml
+++ b/docker/test/test_runner_docker_compose.yaml
@ -4,13 +4,13 @@ services:
  clickhouse-server:
    image: ${CLICKHOUSE_SERVER_IMAGE}
    expose:
-      - "8123"
-      - "9000"
-      - "9009"
+      - "8123" # HTTP
+      - "9000" # TCP
+      - "9009" # HTTP-interserver
    restart: "no"

  test-runner:
-    image: yandex/clickhouse-statelest-test-runner:local
+    image: clickhouse-statelest-test-runner:local

    restart: "no"
    depends_on:
@ -18,13 +18,17 @@ services:
    environment:
      # these are used by clickhouse-test to point clickhouse-client to the right server
      - CLICKHOUSE_HOST=clickhouse-server
-      - CLICKHOUSE_PORT=8123
+      - CLICKHOUSE_PORT=9009
+      - CLICKHOUSE_TEST_HOST_EXPOSED_PORT=51234
+    expose:
+      # port for any test to serve data to clickhouse-server on rare occasion (like URL-engine tables in 00646),
+      # should match value of CLICKHOUSE_TEST_HOST_EXPOSED_PORT above
+      - "51234"

-    entrypoint:
-      - wait-for-it
-      - clickhouse-server:8123
-      - -- 
-      - clickhouse-test
-      # - -c
-      # - `which clickhouse-client`
-      - ${CLICKHOUSE_TEST_ARGS}
+    # NOTE: Dev-mode: mount newest versions of the queries and clickhouse-test script into container.
+    # volumes:
+    #   - /home/enmk/proj/ClickHouse_master/dbms/tests/queries:/usr/share/clickhouse-test/queries:ro
+    #   - /home/enmk/proj/ClickHouse_master/dbms/tests/clickhouse-test:/usr/bin/clickhouse-test:ro
+
+    # String-form instead of list-form to allow multiple arguments in "${CLICKHOUSE_TEST_ARGS}"
+    entrypoint: "clickhouse-test ${CLICKHOUSE_TEST_ARGS}"