Preparation

This commit is contained in:
alesapin 2021-09-06 14:33:39 +03:00
parent 8d67a34f93
commit 0eac4529e0
50 changed files with 107 additions and 77 deletions

View File

@ -1,12 +1,14 @@
{
"docker/packager/deb": {
"name": "yandex/clickhouse-deb-builder",
"symlink_name": "clickhouse/deb-builder",
"dependent": [
"docker/packager/unbundled"
]
},
"docker/packager/binary": {
"name": "yandex/clickhouse-binary-builder",
"symlink_name": "clickhouse/binary-builder",
"dependent": [
"docker/test/split_build_smoke_test",
"docker/test/pvs",
@ -15,26 +17,32 @@
},
"docker/packager/unbundled": {
"name": "yandex/clickhouse-unbundled-builder",
"symlink_name": "clickhouse/unbundled-builder",
"dependent": []
},
"docker/test/compatibility/centos": {
"name": "yandex/clickhouse-test-old-centos",
"symlink_name": "clickhouse/test-old-centos",
"dependent": []
},
"docker/test/compatibility/ubuntu": {
"name": "yandex/clickhouse-test-old-ubuntu",
"symlink_name": "clickhouse/test-old-ubuntu",
"dependent": []
},
"docker/test/integration/base": {
"name": "yandex/clickhouse-integration-test",
"symlink_name": "clickhouse/integration-test",
"dependent": []
},
"docker/test/fuzzer": {
"name": "yandex/clickhouse-fuzzer",
"symlink_name": "clickhouse/fuzzer",
"dependent": []
},
"docker/test/performance-comparison": {
"name": "yandex/clickhouse-performance-comparison",
"symlink_name": "clickhouse/performance-comparison",
"dependent": []
},
"docker/test/pvs": {
@ -43,96 +51,113 @@
},
"docker/test/stateless": {
"name": "yandex/clickhouse-stateless-test",
"symlink_name": "clickhouse/stateless-test",
"dependent": [
"docker/test/stateful",
"docker/test/coverage",
"docker/test/unit"
]
},
"docker/test/stateless_pytest": {
"name": "yandex/clickhouse-stateless-pytest",
"dependent": []
},
"docker/test/stateful": {
"name": "yandex/clickhouse-stateful-test",
"symlink_name": "clickhouse/stateful-test",
"dependent": [
"docker/test/stress"
]
},
"docker/test/coverage": {
"name": "yandex/clickhouse-test-coverage",
"symlink_name": "clickhouse/test-coverage",
"dependent": []
},
"docker/test/unit": {
"name": "yandex/clickhouse-unit-test",
"symlink_name": "clickhouse/unit-test",
"dependent": []
},
"docker/test/stress": {
"name": "yandex/clickhouse-stress-test",
"symlink_name": "clickhouse/stress-test",
"dependent": []
},
"docker/test/split_build_smoke_test": {
"name": "yandex/clickhouse-split-build-smoke-test",
"symlink_name": "clickhouse/split-build-smoke-test",
"dependent": []
},
"docker/test/codebrowser": {
"name": "yandex/clickhouse-codebrowser",
"symlink_name": "clickhouse/codebrowser",
"dependent": []
},
"docker/test/integration/runner": {
"name": "yandex/clickhouse-integration-tests-runner",
"symlink_name": "clickhouse/integration-tests-runner",
"dependent": []
},
"docker/test/testflows/runner": {
"name": "yandex/clickhouse-testflows-runner",
"symlink_name": "clickhouse/testflows-runner",
"dependent": []
},
"docker/test/fasttest": {
"name": "yandex/clickhouse-fasttest",
"symlink_name": "clickhouse/fasttest",
"dependent": []
},
"docker/test/style": {
"name": "yandex/clickhouse-style-test",
"symlink_name": "clickhouse/style-test",
"dependent": []
},
"docker/test/integration/s3_proxy": {
"name": "yandex/clickhouse-s3-proxy",
"symlink_name": "clickhouse/s3-proxy",
"dependent": []
},
"docker/test/integration/resolver": {
"name": "yandex/clickhouse-python-bottle",
"symlink_name": "clickhouse/python-bottle",
"dependent": []
},
"docker/test/integration/helper_container": {
"name": "yandex/clickhouse-integration-helper",
"symlink_name": "clickhouse/integration-helper",
"dependent": []
},
"docker/test/integration/mysql_golang_client": {
"name": "yandex/clickhouse-mysql-golang-client",
"symlink_name": "clickhouse/mysql-golang-client",
"dependent": []
},
"docker/test/integration/mysql_java_client": {
"name": "yandex/clickhouse-mysql-java-client",
"symlink_name": "clickhouse/mysql-golang-client",
"dependent": []
},
"docker/test/integration/mysql_js_client": {
"name": "yandex/clickhouse-mysql-js-client",
"symlink_name": "clickhouse/mysql-js-client",
"dependent": []
},
"docker/test/integration/mysql_php_client": {
"name": "yandex/clickhouse-mysql-php-client",
"symlink_name": "clickhouse/mysql-php-client",
"dependent": []
},
"docker/test/integration/postgresql_java_client": {
"name": "yandex/clickhouse-postgresql-java-client",
"symlink_name": "clickhouse/postgresql-java-client",
"dependent": []
},
"docker/test/integration/kerberos_kdc": {
"name": "yandex/clickhouse-kerberos-kdc",
"symlink_name": "clickhouse/kerberos-kdc",
"dependent": []
},
"docker/test/base": {
"name": "yandex/clickhouse-test-base",
"symlink_name": "clickhouse/test-base",
"dependent": [
"docker/test/stateless",
"docker/test/stateless_unbundled",
@ -144,25 +169,30 @@
},
"docker/packager/unbundled": {
"name": "yandex/clickhouse-unbundled-builder",
"symlink_name": "clickhouse/unbundled-builder",
"dependent": [
"docker/test/stateless_unbundled"
]
},
"docker/test/stateless_unbundled": {
"name": "yandex/clickhouse-stateless-unbundled-test",
"symlink_name": "clickhouse/stateless-unbundled-test",
"dependent": [
]
},
"docker/test/integration/kerberized_hadoop": {
"name": "yandex/clickhouse-kerberized-hadoop",
"symlink_name": "clickhouse/kerberized-hadoop",
"dependent": []
},
"docker/test/sqlancer": {
"name": "yandex/clickhouse-sqlancer-test",
"symlink_name": "clickhouse/sqlancer-test",
"dependent": []
},
"docker/test/keeper-jepsen": {
"name": "yandex/clickhouse-keeper-jepsen-test",
"symlink_name": "clickhouse/keeper-jepsen-test",
"dependent": []
}
}

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-binary-builder .
# docker build -t clickhouse/binary-builder .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-deb-builder .
# docker build -t clickhouse/deb-builder .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11

View File

@ -9,9 +9,9 @@ import sys
SCRIPT_PATH = os.path.realpath(__file__)
IMAGE_MAP = {
"deb": "yandex/clickhouse-deb-builder",
"binary": "yandex/clickhouse-binary-builder",
"unbundled": "yandex/clickhouse-unbundled-builder"
"deb": "clickhouse/deb-builder",
"binary": "clickhouse/binary-builder",
"unbundled": "clickhouse/unbundled-builder"
}
def check_image_exists_locally(image_name):

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-unbundled-builder .
FROM yandex/clickhouse-deb-builder
# docker build -t clickhouse/unbundled-builder .
FROM clickhouse/deb-builder
RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
&& wget -nv -O /tmp/arrow-keyring.deb "https://apache.jfrog.io/artifactory/arrow/ubuntu/apache-arrow-apt-source-latest-${CODENAME}.deb" \

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-test-base .
# docker build -t clickhouse/test-base .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11

View File

@ -1,6 +1,6 @@
# docker build --network=host -t yandex/clickhouse-codebrowser .
# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output yandex/clickhouse-codebrowser
FROM yandex/clickhouse-binary-builder
# docker build --network=host -t clickhouse/codebrowser .
# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser
FROM clickhouse/binary-builder
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-test-old-centos .
# docker build -t clickhouse/test-old-centos .
FROM centos:5
CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-test-old-ubuntu .
# docker build -t clickhouse/test-old-ubuntu .
FROM ubuntu:12.04
CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-test-coverage .
FROM yandex/clickhouse-stateless-test
# docker build -t clickhouse/test-coverage .
FROM clickhouse/stateless-test
RUN apt-get update -y \
&& env DEBIAN_FRONTEND=noninteractive \

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-fasttest .
# docker build -t clickhouse/fasttest .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-fuzzer .
FROM yandex/clickhouse-test-base
# docker build -t clickhouse/fuzzer .
FROM clickhouse/test-base
ENV LANG=C.UTF-8
ENV TZ=Europe/Moscow
@ -36,5 +36,5 @@ CMD set -o pipefail \
&& cd /workspace \
&& /run-fuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log
# docker run --network=host --volume <workspace>:/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-fuzzer
# docker run --network=host --volume <workspace>:/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/fuzzer

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-integration-test .
FROM yandex/clickhouse-test-base
# docker build -t clickhouse/integration-test .
FROM clickhouse/test-base
SHELL ["/bin/bash", "-c"]

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-integration-helper .
# docker build -t clickhouse/integration-helper .
# Helper docker container to run iptables without sudo
FROM alpine

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-kerberized-hadoop .
# docker build -t clickhouse/kerberized-hadoop .
FROM sequenceiq/hadoop-docker:2.7.0
RUN sed -i -e 's/^\#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-kerberos-kdc .
# docker build -t clickhouse/kerberos-kdc .
FROM centos:6.6
# old OS to make is faster and smaller

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-mysql-golang-client .
# docker build -t clickhouse/mysql-golang-client .
# MySQL golang client docker container
FROM golang:1.12.2

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-mysql-java-client .
# docker build -t clickhouse/mysql-java-client .
# MySQL Java client docker container
FROM ubuntu:18.04

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-mysql-js-client .
# docker build -t clickhouse/mysql-js-client .
# MySQL JavaScript client docker container
FROM node:8

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-mysql-php-client .
# docker build -t clickhouse/mysql-php-client .
# MySQL PHP client docker container
FROM php:7.3-cli

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-postgresql-java-client .
# docker build -t clickhouse/postgresql-java-client .
# PostgreSQL Java client docker container
FROM ubuntu:18.04

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-python-bottle .
# docker build -t clickhouse/python-bottle .
# Helper docker container to run python bottle apps
FROM python:3

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-integration-tests-runner .
# docker build -t clickhouse/integration-tests-runner .
FROM ubuntu:20.04
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list

View File

@ -1,7 +1,7 @@
version: '2.3'
services:
bridge1:
image: yandex/clickhouse-jdbc-bridge
image: clickhouse/jdbc-bridge
command: |
/bin/bash -c 'cat << EOF > config/datasources/self.json
{

View File

@ -1,7 +1,7 @@
version: '2.3'
services:
zoo1:
image: ${image:-yandex/clickhouse-integration-test}
image: ${image:-clickhouse/integration-test}
restart: always
user: ${user:-}
volumes:
@ -31,7 +31,7 @@ services:
- inet6
- rotate
zoo2:
image: ${image:-yandex/clickhouse-integration-test}
image: ${image:-clickhouse/integration-test}
restart: always
user: ${user:-}
volumes:
@ -61,7 +61,7 @@ services:
- inet6
- rotate
zoo3:
image: ${image:-yandex/clickhouse-integration-test}
image: ${image:-clickhouse/integration-test}
restart: always
user: ${user:-}
volumes:

View File

@ -4,7 +4,7 @@ services:
kerberizedhdfs1:
cap_add:
- DAC_READ_SEARCH
image: yandex/clickhouse-kerberized-hadoop:16621
image: clickhouse/kerberized-hadoop:16621
hostname: kerberizedhdfs1
restart: always
volumes:
@ -22,7 +22,7 @@ services:
entrypoint: /etc/bootstrap.sh -d
hdfskerberos:
image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}
image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}
hostname: hdfskerberos
volumes:
- ${KERBERIZED_HDFS_DIR}/secrets:/tmp/keytab

View File

@ -50,7 +50,7 @@ services:
- label:disable
kafka_kerberos:
image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}
image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest}
hostname: kafka_kerberos
volumes:
- ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab

View File

@ -19,14 +19,14 @@ services:
# HTTP proxies for Minio.
proxy1:
image: yandex/clickhouse-s3-proxy
image: clickhouse/s3-proxy
expose:
- "8080" # Redirect proxy port
- "80" # Reverse proxy port
- "443" # Reverse proxy port (secure)
proxy2:
image: yandex/clickhouse-s3-proxy
image: clickhouse/s3-proxy
expose:
- "8080"
- "80"
@ -34,7 +34,7 @@ services:
# Empty container to run proxy resolver.
resolver:
image: yandex/clickhouse-python-bottle
image: clickhouse/python-bottle
expose:
- "8080"
tty: true

View File

@ -1,6 +1,6 @@
version: '2.3'
services:
golang1:
image: yandex/clickhouse-mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest}
image: clickhouse/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest}
# to keep container running
command: sleep infinity

View File

@ -1,6 +1,6 @@
version: '2.3'
services:
java1:
image: yandex/clickhouse-mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest}
image: clickhouse/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest}
# to keep container running
command: sleep infinity

View File

@ -1,6 +1,6 @@
version: '2.3'
services:
mysqljs1:
image: yandex/clickhouse-mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest}
image: clickhouse/mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest}
# to keep container running
command: sleep infinity

View File

@ -1,6 +1,6 @@
version: '2.3'
services:
php1:
image: yandex/clickhouse-mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest}
image: clickhouse/mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest}
# to keep container running
command: sleep infinity

View File

@ -1,6 +1,6 @@
version: '2.2'
services:
java:
image: yandex/clickhouse-postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest}
image: clickhouse/postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest}
# to keep container running
command: sleep infinity

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-s3-proxy .
# docker build -t clickhouse/s3-proxy .
FROM nginx:alpine
COPY run.sh /run.sh

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-keeper-jepsen-test .
FROM yandex/clickhouse-test-base
# docker build -t clickhouse/keeper-jepsen-test .
FROM clickhouse/test-base
ENV DEBIAN_FRONTEND=noninteractive
ENV CLOJURE_VERSION=1.10.3.814

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-performance-comparison .
# docker build -t clickhouse/performance-comparison .
FROM ubuntu:18.04
ENV LANG=C.UTF-8
@ -54,4 +54,4 @@ COPY * /
# it gives '/bin/sh: 1: [bash,: not found' otherwise.
CMD ["bash", "-c", "node=$((RANDOM % $(numactl --hardware | sed -n 's/^.*available:\\(.*\\)nodes.*$/\\1/p'))); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"]
# docker run --network=host --volume <workspace>:/workspace --volume=<output>:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison
# docker run --network=host --volume <workspace>:/workspace --volume=<output>:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/performance-comparison

View File

@ -116,7 +116,7 @@ pull requests (0 for master) manually.
docker run --network=host --volume=$(pwd)/workspace:/workspace --volume=$(pwd)/output:/output
[-e REF_PR={} -e REF_SHA={}]
-e PR_TO_TEST={} -e SHA_TO_TEST={}
yandex/clickhouse-performance-comparison
clickhouse/performance-comparison
```
Then see the `report.html` in the `output` directory.

View File

@ -1,6 +1,6 @@
# docker build -t yandex/clickhouse-pvs-test .
# docker build -t clickhouse/pvs-test .
FROM yandex/clickhouse-binary-builder
FROM clickhouse/binary-builder
RUN apt-get update --yes \
&& apt-get install \

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-split-build-smoke-test .
FROM yandex/clickhouse-binary-builder
# docker build -t clickhouse/split-build-smoke-test .
FROM clickhouse/binary-builder
COPY run.sh /run.sh
COPY process_split_build_smoke_test_result.py /

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-sqlancer-test .
# docker build -t clickhouse/sqlancer-test .
FROM ubuntu:20.04
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-stateful-test .
FROM yandex/clickhouse-stateless-test
# docker build -t clickhouse/stateful-test .
FROM clickhouse/stateless-test
RUN apt-get update -y \
&& env DEBIAN_FRONTEND=noninteractive \

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-stateless-test .
FROM yandex/clickhouse-test-base
# docker build -t clickhouse/stateless-test .
FROM clickhouse/test-base
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-stateless-pytest .
FROM yandex/clickhouse-test-base
# docker build -t clickhouse/stateless-pytest .
FROM clickhouse/test-base
RUN apt-get update -y && \
apt-get install -y --no-install-recommends \

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-stateless-unbundled-test .
FROM yandex/clickhouse-test-base
# docker build -t clickhouse/stateless-unbundled-test .
FROM clickhouse/test-base
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-stress-test .
FROM yandex/clickhouse-stateful-test
# docker build -t clickhouse/stress-test .
FROM clickhouse/stateful-test
RUN apt-get update -y \
&& env DEBIAN_FRONTEND=noninteractive \

View File

@ -6,7 +6,7 @@ Usage:
```
$ ls $HOME/someclickhouse
clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb
$ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output yandex/clickhouse-stress-test
$ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output clickhouse/stress-test
Selecting previously unselected package clickhouse-common-static.
(Reading database ... 14442 files and directories currently installed.)
...

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-style-test .
# docker build -t clickhouse/style-test .
FROM ubuntu:20.04
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list

View File

@ -49,7 +49,7 @@ fi
# Build server image (optional) from local packages
if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then
CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"
CLICKHOUSE_SERVER_IMAGE="clickhouse/server:local"
if [ "${CLICKHOUSE_PACKAGES_ARG}" != "${NO_REBUILD_FLAG}" ]; then
docker build --network=host \

View File

@ -1,4 +1,4 @@
# docker build -t yandex/clickhouse-testflows-runner .
# docker build -t clickhouse/testflows-runner .
FROM ubuntu:20.04
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list

View File

@ -1,5 +1,5 @@
# docker build -t yandex/clickhouse-unit-test .
FROM yandex/clickhouse-stateless-test
# docker build -t clickhouse/unit-test .
FROM clickhouse/stateless-test
RUN apt-get install gdb