From ebc04d05fbb935e92c290a0b0763e8cdae74c340 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 21 Mar 2019 16:10:47 +0100 Subject: [PATCH 1/5] Add way to launch container as custom user --- docker/server/README.md | 7 +++++ docker/server/entrypoint.sh | 55 ++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/docker/server/README.md b/docker/server/README.md index ffcc4bc37a4..4ceea65a934 100644 --- a/docker/server/README.md +++ b/docker/server/README.md @@ -33,6 +33,13 @@ ClickHouse configuration represented with a file "config.xml" ([documentation](h $ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml yandex/clickhouse-server ``` +### Start server as custom user +``` +# $(pwd)/data/clickhouse should exist and be owned by current user +$ docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/data/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" yandex/clickhouse-server +``` +When you use the image with mounting local directories inside you probably would like to not mess your directory tree with files owner and permissions. Then you could use `--user` argument. In this case, you should mount every necessary directory (`/var/lib/clickhouse` and `/var/log/clickhouse-server`) inside the container. Otherwise, image will complain and not start. + ## How to extend this image If you would like to do additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb` it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service. diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index f22942d6dfd..78d591cfd50 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -2,8 +2,15 @@ # set some vars CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" -USER="$(id -u clickhouse)" -GROUP="$(id -g clickhouse)" +if [ x"$UID" == x0 ]; then + USER="$(id -u clickhouse)" + GROUP="$(id -g clickhouse)" + gosu="gosu $USER:$GROUP" +else + USER="$(id -u)" + GROUP="$(id -g)" + gosu="" +fi # port is needed to check if clickhouse-server is ready for connections HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" @@ -18,28 +25,32 @@ ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFI ERROR_LOG_DIR="$(dirname $ERROR_LOG_PATH || true)" FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=format_schema_path || true)" -# ensure directories exist -mkdir -p \ - "$DATA_DIR" \ - "$ERROR_LOG_DIR" \ - "$LOG_DIR" \ - "$TMP_DIR" \ - "$USER_PATH" \ - "$FORMAT_SCHEMA_PATH" +for dir in "$DATA_DIR" \ + "$ERROR_LOG_DIR" \ + "$LOG_DIR" \ + "$TMP_DIR" \ + "$USER_PATH" \ + "$FORMAT_SCHEMA_PATH" +do + # ensure directories exist + if ! mkdir -p "$dir"; then + echo "Couldn't create necessary directory: $dir" + exit 1 + fi + + if [ x"$UID" == x0 ] && [ "$CLICKHOUSE_DO_NOT_CHOWN" != "1" ]; then + # ensure proper directories permissions + chown -R "$USER:$GROUP" "$dir" + elif [ "$(stat -c %u "$dir")" != "$USER" ]; then + echo "Necessary directory '$dir' isn't owned by user with id '$USER'" + exit 1 + fi +done + -if [ "$CLICKHOUSE_DO_NOT_CHOWN" != "1" ]; then - # ensure proper directories permissions - chown -R $USER:$GROUP \ - "$DATA_DIR" \ - "$ERROR_LOG_DIR" \ - "$LOG_DIR" \ - "$TMP_DIR" \ - "$USER_PATH" \ - "$FORMAT_SCHEMA_PATH" -fi if [ -n "$(ls /docker-entrypoint-initdb.d/)" ]; then - gosu clickhouse /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG & + $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG & pid="$!" # check if clickhouse is ready to accept connections @@ -77,7 +88,7 @@ fi # if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then - exec gosu clickhouse /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@" + exec $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@" fi # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image From fc5f5969ded4101435beabdbfc3c07f5903d5935 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 22 Mar 2019 13:29:19 +0100 Subject: [PATCH 2/5] Use argument for gosu version instead of hardcode --- docker/server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 5fdf50750ee..71cf28082d5 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update \ /tmp/* \ && apt-get clean -ADD https://github.com/tianon/gosu/releases/download/1.10/gosu-amd64 /bin/gosu +ADD https://github.com/tianon/gosu/releases/download/$gosu_ver/gosu-amd64 /bin/gosu RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 From 0cc90cf2db312939d1a207e20bd26b7a4e32e794 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 22 Mar 2019 19:15:44 +0300 Subject: [PATCH 3/5] rename insert_sample_with_metadata setting --- dbms/src/Core/Settings.h | 2 +- dbms/src/Storages/ColumnsDescription.cpp | 2 +- .../0_stateless/00760_insert_json_with_defaults.sql | 2 +- .../0_stateless/00851_http_insert_json_defaults.sh | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index fa76ea02631..3f71e8af4c0 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -127,7 +127,6 @@ struct Settings M(SettingUInt64, max_concurrent_queries_for_user, 0, "The maximum number of concurrent requests per user.") \ \ M(SettingBool, insert_deduplicate, true, "For INSERT queries in the replicated table, specifies that deduplication of insertings blocks should be preformed") \ - M(SettingBool, insert_sample_with_metadata, false, "For INSERT queries, specifies that the server need to send metadata about column defaults to the client. This will be used to calculate default expressions.") \ \ M(SettingUInt64, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled.") \ M(SettingMilliseconds, insert_quorum_timeout, 600000, "") \ @@ -153,6 +152,7 @@ struct Settings \ M(SettingBool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow and TSKV formats).") \ M(SettingBool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).") \ + M(SettingBool, input_format_defaults_for_omitted_fields, false, "For input data calculate default expressions for omitted fields (it works for JSONEachRow format).") \ \ M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.") \ \ diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index b1e5bb1f7f9..96630616900 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -427,7 +427,7 @@ ColumnsDescription ColumnsDescription::parse(const String & str) const ColumnsDescription * ColumnsDescription::loadFromContext(const Context & context, const String & db, const String & table) { - if (context.getSettingsRef().insert_sample_with_metadata) + if (context.getSettingsRef().input_format_defaults_for_omitted_fields) { if (context.isTableExist(db, table)) { diff --git a/dbms/tests/queries/0_stateless/00760_insert_json_with_defaults.sql b/dbms/tests/queries/0_stateless/00760_insert_json_with_defaults.sql index cca17cbb9a9..28d5cd6499a 100644 --- a/dbms/tests/queries/0_stateless/00760_insert_json_with_defaults.sql +++ b/dbms/tests/queries/0_stateless/00760_insert_json_with_defaults.sql @@ -1,4 +1,4 @@ -SET insert_sample_with_metadata=1; +SET input_format_defaults_for_omitted_fields=1; CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.defaults; diff --git a/dbms/tests/queries/0_stateless/00851_http_insert_json_defaults.sh b/dbms/tests/queries/0_stateless/00851_http_insert_json_defaults.sh index a927d8c4c2a..82cb38ae45f 100755 --- a/dbms/tests/queries/0_stateless/00851_http_insert_json_defaults.sh +++ b/dbms/tests/queries/0_stateless/00851_http_insert_json_defaults.sh @@ -6,10 +6,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.defaults" $CLICKHOUSE_CLIENT --query="CREATE TABLE test.defaults (x UInt32, y UInt32, a DEFAULT x + y, b Float32 DEFAULT log(1 + x + y), c UInt32 DEFAULT 42, e MATERIALIZED x + y, f ALIAS x + y) ENGINE = Memory" -echo -ne '{"x":1, "y":1}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT%20INTO%20test.defaults%20FORMAT%20JSONEachRow%20SETTINGS%20insert_sample_with_metadata=1" --data-binary @- -echo -ne '{"x":2, "y":2, "c":2}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+insert_sample_with_metadata=1" --data-binary @- -echo -ne '{"x":3, "y":3, "a":3, "b":3, "c":3}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+insert_sample_with_metadata=1" --data-binary @- -echo -ne '{"x":4} {"y":5, "c":5} {"a":6, "b":6, "c":6}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+insert_sample_with_metadata=1" --data-binary @- +echo -ne '{"x":1, "y":1}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT%20INTO%20test.defaults%20FORMAT%20JSONEachRow%20SETTINGS%20input_format_defaults_for_omitted_fields=1" --data-binary @- +echo -ne '{"x":2, "y":2, "c":2}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+input_format_defaults_for_omitted_fields=1" --data-binary @- +echo -ne '{"x":3, "y":3, "a":3, "b":3, "c":3}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+input_format_defaults_for_omitted_fields=1" --data-binary @- +echo -ne '{"x":4} {"y":5, "c":5} {"a":6, "b":6, "c":6}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+input_format_defaults_for_omitted_fields=1" --data-binary @- $CLICKHOUSE_CLIENT --query="SELECT * FROM test.defaults ORDER BY x, y FORMAT JSONEachRow" $CLICKHOUSE_CLIENT --query="DROP TABLE test.defaults" From 2311bbd95ce6cefd4381b14ebf0facba038da412 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 22 Mar 2019 20:24:35 +0300 Subject: [PATCH 4/5] Restore recheck parts in ReplicatedMergeTreeAlterThread in case of error. --- dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp | 2 -- dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index ba3bc77af6b..1b1c43e1cd1 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -33,8 +33,6 @@ ReplicatedMergeTreeAlterThread::ReplicatedMergeTreeAlterThread(StorageReplicated void ReplicatedMergeTreeAlterThread::run() { - bool force_recheck_parts = true; - try { /** We have a description of columns in ZooKeeper, common for all replicas (Example: /clickhouse/tables/02-06/visits/columns), diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h index bb93d06bf6a..d5b8f6c2e67 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h @@ -36,6 +36,7 @@ private: String log_name; Logger * log; BackgroundSchedulePool::TaskHolder task; + bool force_recheck_parts = true; }; } From 56f7d21e8429e6fa0f25e96de62fe1a44adfa727 Mon Sep 17 00:00:00 2001 From: proller Date: Sat, 23 Mar 2019 00:42:24 +0300 Subject: [PATCH 5/5] Fix contrib/ssl --- contrib/ssl | 1 + 1 file changed, 1 insertion(+) create mode 160000 contrib/ssl diff --git a/contrib/ssl b/contrib/ssl new file mode 160000 index 00000000000..ba8de796195 --- /dev/null +++ b/contrib/ssl @@ -0,0 +1 @@ +Subproject commit ba8de796195ff9d8bb0249ce289b83226b848b77