diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/README.md b/dbms/tests/integration_drafts/001_miltiple_disks/README.md new file mode 100644 index 00000000000..7b722001a05 --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/README.md @@ -0,0 +1,83 @@ +## Test stand for multiple disks feature + +Currently for namual tests, can be easily scripted to be the part of inntergration tests. + +To run you need to have docker & docker-compose. + +``` +(Check makefile) +make run +make ch1_shell + > clickhouse-client + +make logs # Ctrl+C +make cleup +``` + +### basic + +* allows to configure multiple disks & folumes & shemas +* clickhouse check that all disks are write-accessible +* clickhouse can create a table with provided schema + +### one volume-one disk custom schema + +* clickhouse puts data to correct folder when schema is used +* clickhouse can do merges / detach / attach / freeze on that folder + +### one volume-multiple disks schema (JBOD scenario) + +* clickhouse uses round-robin to place new parts +* clickhouse can do merges / detach / attach / freeze on that folder + +### two volumes-one disk per volume (fast expensive / slow cheap storage) + +* clickhouse uses round-robin to place new parts +* clickhouse can do merges / detach / attach / freeze on that folder +* clickhouse put parts to different volumes depending on part size + +### use 'default' schema for tables created without schema provided. + + +# ReplicatedMergeTree + +.... + +For all above: +clickhouse respect free space limitation setting. +ClickHouse writes important disk-related information to logs. + +## Queries + +``` +CREATE TABLE schema_default (id UInt64) Engine=MergeTree() ORDER BY (id); + + INSERT INTO schema_default SELECT * FROM numbers(1); +CREATE TABLE schema_default_explicit (id UInt64) Engine=MergeTree() ORDER BY (id) SETTINGS storage_schema_name='default'; +CREATE TABLE schema_default_disk_with_external (id UInt64) Engine=MergeTree() ORDER BY (id) SETTINGS storage_schema_name='default_disk_with_external'; +CREATE TABLE schema_jbod_with_external (id UInt64) Engine=MergeTree() ORDER BY (id) SETTINGS storage_schema_name='jbod_with_external'; + +CREATE TABLE replicated_schema_default (id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}', '{replica}') ORDER BY (id); +CREATE TABLE replicated_schema_default_explicit (id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}', '{replica}') ORDER BY (id) SETTINGS storage_schema_name='default'; +CREATE TABLE replicated_schema_default_disk_with_external (id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}', '{replica}') ORDER BY (id) SETTINGS storage_schema_name='default_disk_with_external'; +CREATE TABLE replicated_schema_jbod_with_external (id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}', '{replica}') ORDER BY (id) SETTINGS storage_schema_name='jbod_with_external'; +``` + + +## Extra acceptance criterias + +* hardlinks problems. Thouse stetements should be able to work properly (or give a proper feedback) on multidisk scenarios + * ALTER TABLE ... UPDATE + * ALTER TABLE ... TABLE + * ALTER TABLE ... MODIFY COLUMN ... + * ALTER TABLE ... CLEAR COLUMN + * ALTER TABLE ... REPLACE PARTITION ... +* Maintainance - system tables show proper values: + * system.parts + * system.tables + * system.part_log (target disk?) +* New system table + * system.volumes + * system.disks + * system.schemas +* chown / create needed disk folders in docker diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/config/ch1.metrika.xml b/dbms/tests/integration_drafts/001_miltiple_disks/config/ch1.metrika.xml new file mode 100644 index 00000000000..58de1e78d44 --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/config/ch1.metrika.xml @@ -0,0 +1,7 @@ + + + test_cluster + 0 + ch2 + + \ No newline at end of file diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/config/ch2.metrika.xml b/dbms/tests/integration_drafts/001_miltiple_disks/config/ch2.metrika.xml new file mode 100644 index 00000000000..58de1e78d44 --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/config/ch2.metrika.xml @@ -0,0 +1,7 @@ + + + test_cluster + 0 + ch2 + + \ No newline at end of file diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/cluster.xml b/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/cluster.xml new file mode 100644 index 00000000000..ecd1c37d054 --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/cluster.xml @@ -0,0 +1,16 @@ + + + + + + ch1 + 9000 + + + ch2 + 9000 + + + + + \ No newline at end of file diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/storage_config.xml b/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/storage_config.xml new file mode 100644 index 00000000000..001aad5cbd6 --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/storage_config.xml @@ -0,0 +1,63 @@ + + + + + + 0 + + + /jbod1/ + 1000000 + + + /jbod2/ + 1000000 + + + /external/ + 1000000 + + + + + + + + jbod1 + 1000000 + + + + + + + + + default + 1000000 + + + external + 10000000 + + + + + + + jbod1 + jbod2 + 10000000 + + + + external + + + + + + + + + \ No newline at end of file diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/zookeeper.xml b/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/zookeeper.xml new file mode 100644 index 00000000000..28aff86fcfd --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/config/conf.d/zookeeper.xml @@ -0,0 +1,8 @@ + + + + zookeeper + 2181 + + + \ No newline at end of file diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/docker-compose.yml b/dbms/tests/integration_drafts/001_miltiple_disks/docker-compose.yml new file mode 100644 index 00000000000..857aa3e6901 --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/docker-compose.yml @@ -0,0 +1,52 @@ +version: '3' + +services: + zookeeper: + image: zookeeper + networks: + - 001_miltiple_disks + + ch1: + image: yandex/clickhouse-server + hostname: ch1 + depends_on: + - "zookeeper" + networks: + - 001_miltiple_disks + tmpfs: + - /jbod1:size=40M + - /jbod2:size=40M + - /external:size=200M + volumes: + - ./config/conf.d:/etc/clickhouse-server/conf.d + - ./config/ch1.metrika.xml:/etc/metrika.xml + # TODO: hacky, but "cheap" - don't need to rebuild docker image. + # for CI & tests automation should be done another way, not relying on build path + - ../../../../build/dbms/programs/clickhouse:/usr/bin/clickhouse + - ./test_setup:/test_setup + entrypoint: ['/test_setup/entrypoint.sh'] + + + ch2: + image: yandex/clickhouse-server + hostname: ch2 + depends_on: + - "zookeeper" + networks: + - 001_miltiple_disks + tmpfs: + - /jbod1:size=40M + - /jbod2:size=40M + - /external:size=200M + volumes: + - ./config/conf.d:/etc/clickhouse-server/conf.d + - ./config/ch2.metrika.xml:/etc/metrika.xml + # TODO: hacky, but "cheap" - don't need to rebuild docker image. + # for CI & tests automation should be done another way, not relying on build path + - ../../../../build/dbms/programs/clickhouse:/usr/bin/clickhouse + - ./test_setup:/test_setup + entrypoint: ['/test_setup/entrypoint.sh'] + + +networks: + 001_miltiple_disks: diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/test_setup/entrypoint.sh b/dbms/tests/integration_drafts/001_miltiple_disks/test_setup/entrypoint.sh new file mode 100755 index 00000000000..d1bb7b60759 --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/test_setup/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ -f /etc/clickhouse-server/conf.d/container_maintanence_mode.flag ]; then + echo "Starting container in maintanence mode. It will sleep unless you shutdown it" + sleep infinity +else + chown -R clickhouse:clickhouse /jbod1 /jbod2 /external + /test_setup/wait-for-it.sh zookeeper:2181 --timeout=0 --strict -- /entrypoint.sh +fi \ No newline at end of file diff --git a/dbms/tests/integration_drafts/001_miltiple_disks/test_setup/wait-for-it.sh b/dbms/tests/integration_drafts/001_miltiple_disks/test_setup/wait-for-it.sh new file mode 100755 index 00000000000..873a1af0507 --- /dev/null +++ b/dbms/tests/integration_drafts/001_miltiple_disks/test_setup/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +# The MIT License (MIT) +# Copyright (c) 2016 Giles Hall +# See https://github.com/vishnubob/wait-for-it/blob/master/LICENSE + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi