Merge branch 'master' of github.com:yandex/ClickHouse

contrib/ssl

@@ -0,0 +1 @@
+Subproject commit ba8de796195ff9d8bb0249ce289b83226b848b77

dbms/src/Core/Settings.h

@@ -127,7 +127,6 @@ struct Settings
     M(SettingUInt64, max_concurrent_queries_for_user, 0, "The maximum number of concurrent requests per user.") \
     \
     M(SettingBool, insert_deduplicate, true, "For INSERT queries in the replicated table, specifies that deduplication of insertings blocks should be preformed") \
-    M(SettingBool, insert_sample_with_metadata, false, "For INSERT queries, specifies that the server need to send metadata about column defaults to the client. This will be used to calculate default expressions.") \
     \
     M(SettingUInt64, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled.") \
     M(SettingMilliseconds, insert_quorum_timeout, 600000, "") \
@@ -153,6 +152,7 @@ struct Settings
     \
     M(SettingBool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow and TSKV formats).") \
     M(SettingBool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).") \
+    M(SettingBool, input_format_defaults_for_omitted_fields, false, "For input data calculate default expressions for omitted fields (it works for JSONEachRow format).") \
     \
     M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.") \
     \

dbms/src/Storages/ColumnsDescription.cpp

@@ -427,7 +427,7 @@ ColumnsDescription ColumnsDescription::parse(const String & str)
 
 const ColumnsDescription * ColumnsDescription::loadFromContext(const Context & context, const String & db, const String & table)
 {
-    if (context.getSettingsRef().insert_sample_with_metadata)
+    if (context.getSettingsRef().input_format_defaults_for_omitted_fields)
     {
         if (context.isTableExist(db, table))
         {

dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp

@@ -33,8 +33,6 @@ ReplicatedMergeTreeAlterThread::ReplicatedMergeTreeAlterThread(StorageReplicated
 
 void ReplicatedMergeTreeAlterThread::run()
 {
-    bool force_recheck_parts = true;
-
     try
     {
         /** We have a description of columns in ZooKeeper, common for all replicas (Example: /clickhouse/tables/02-06/visits/columns),

dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h

@@ -36,6 +36,7 @@ private:
     String log_name;
     Logger * log;
     BackgroundSchedulePool::TaskHolder task;
+    bool force_recheck_parts = true;
 };
 
 }

dbms/tests/queries/0_stateless/00760_insert_json_with_defaults.sql

@@ -1,4 +1,4 @@
-SET insert_sample_with_metadata=1;
+SET input_format_defaults_for_omitted_fields=1;
 
 CREATE DATABASE IF NOT EXISTS test;
 DROP TABLE IF EXISTS test.defaults;

dbms/tests/queries/0_stateless/00851_http_insert_json_defaults.sh

@@ -6,10 +6,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test.defaults"
 $CLICKHOUSE_CLIENT --query="CREATE TABLE test.defaults (x UInt32, y UInt32, a DEFAULT x + y, b Float32 DEFAULT log(1 + x + y), c UInt32 DEFAULT 42, e MATERIALIZED x + y, f ALIAS x + y) ENGINE = Memory"
 
-echo -ne '{"x":1, "y":1}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT%20INTO%20test.defaults%20FORMAT%20JSONEachRow%20SETTINGS%20insert_sample_with_metadata=1" --data-binary @-
-echo -ne '{"x":2, "y":2, "c":2}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+insert_sample_with_metadata=1" --data-binary @-
-echo -ne '{"x":3, "y":3, "a":3, "b":3, "c":3}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+insert_sample_with_metadata=1" --data-binary @-
-echo -ne '{"x":4} {"y":5, "c":5} {"a":6, "b":6, "c":6}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+insert_sample_with_metadata=1" --data-binary @-
+echo -ne '{"x":1, "y":1}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT%20INTO%20test.defaults%20FORMAT%20JSONEachRow%20SETTINGS%20input_format_defaults_for_omitted_fields=1" --data-binary @-
+echo -ne '{"x":2, "y":2, "c":2}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+input_format_defaults_for_omitted_fields=1" --data-binary @-
+echo -ne '{"x":3, "y":3, "a":3, "b":3, "c":3}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+input_format_defaults_for_omitted_fields=1" --data-binary @-
+echo -ne '{"x":4} {"y":5, "c":5} {"a":6, "b":6, "c":6}\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=INSERT+INTO+test.defaults+FORMAT+JSONEachRow+SETTINGS+input_format_defaults_for_omitted_fields=1" --data-binary @-
 
 $CLICKHOUSE_CLIENT --query="SELECT * FROM test.defaults ORDER BY x, y FORMAT JSONEachRow"
 $CLICKHOUSE_CLIENT --query="DROP TABLE test.defaults"

docker/server/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update \
         /tmp/* \
     && apt-get clean
 
-ADD https://github.com/tianon/gosu/releases/download/1.10/gosu-amd64 /bin/gosu
+ADD https://github.com/tianon/gosu/releases/download/$gosu_ver/gosu-amd64 /bin/gosu
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8

docker/server/README.md

@@ -33,6 +33,13 @@ ClickHouse configuration represented with a file "config.xml" ([documentation](h
 $ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml yandex/clickhouse-server
 ```
 
+### Start server as custom user
+```
+# $(pwd)/data/clickhouse should exist and be owned by current user
+$ docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/data/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" yandex/clickhouse-server
+```
+When you use the image with mounting local directories inside you probably would like to not mess your directory tree with files owner and permissions. Then you could use `--user` argument. In this case, you should mount every necessary directory (`/var/lib/clickhouse` and `/var/log/clickhouse-server`) inside the container. Otherwise, image will complain and not start.
+
 ## How to extend this image
 
 If you would like to do additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb` it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service.

docker/server/entrypoint.sh

@@ -2,8 +2,15 @@
 
 # set some vars
 CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}"
-USER="$(id -u clickhouse)"
-GROUP="$(id -g clickhouse)"
+if [ x"$UID" == x0 ]; then
+    USER="$(id -u clickhouse)"
+    GROUP="$(id -g clickhouse)"
+    gosu="gosu $USER:$GROUP"
+else
+    USER="$(id -u)"
+    GROUP="$(id -g)"
+    gosu=""
+fi
 
 # port is needed to check if clickhouse-server is ready for connections
 HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)"
@@ -18,28 +25,32 @@ ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFI
 ERROR_LOG_DIR="$(dirname $ERROR_LOG_PATH || true)"
 FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=format_schema_path || true)"
 
-# ensure directories exist
-mkdir -p \
-    "$DATA_DIR" \
-    "$ERROR_LOG_DIR" \
-    "$LOG_DIR" \
-    "$TMP_DIR" \
-    "$USER_PATH" \
-    "$FORMAT_SCHEMA_PATH"
+for dir in "$DATA_DIR" \
+  "$ERROR_LOG_DIR" \
+  "$LOG_DIR" \
+  "$TMP_DIR" \
+  "$USER_PATH" \
+  "$FORMAT_SCHEMA_PATH"
+do
+    # ensure directories exist
+    if ! mkdir -p "$dir"; then
+        echo "Couldn't create necessary directory: $dir"
+        exit 1
+    fi
+
+    if [ x"$UID" == x0 ] && [ "$CLICKHOUSE_DO_NOT_CHOWN" != "1" ]; then
+        # ensure proper directories permissions
+        chown -R "$USER:$GROUP" "$dir"
+    elif [ "$(stat -c %u "$dir")" != "$USER" ]; then
+        echo "Necessary directory '$dir' isn't owned by user with id '$USER'"
+        exit 1
+    fi
+done
+
 
-if [ "$CLICKHOUSE_DO_NOT_CHOWN" != "1" ]; then
-    # ensure proper directories permissions
-    chown -R $USER:$GROUP \
-        "$DATA_DIR" \
-        "$ERROR_LOG_DIR" \
-        "$LOG_DIR" \
-        "$TMP_DIR" \
-        "$USER_PATH" \
-        "$FORMAT_SCHEMA_PATH"
-fi
 
 if [ -n "$(ls /docker-entrypoint-initdb.d/)" ]; then
-    gosu clickhouse /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG &
+    $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG &
     pid="$!"
 
     # check if clickhouse is ready to accept connections
@@ -77,7 +88,7 @@ fi
 
 # if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments
 if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
-    exec gosu clickhouse /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@"
+    exec $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@"
 fi
 
 # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image