From f7e93531f840eb1a044c40851290df20e0765f57 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Aug 2020 22:53:49 +0300 Subject: [PATCH 1/8] Simplify init script --- debian/clickhouse-server.init | 69 ++--------------------------------- 1 file changed, 3 insertions(+), 66 deletions(-) diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index b82c70bd6e0..978feb10337 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -160,82 +160,19 @@ initdb() start() { - [ -x $CLICKHOUSE_BINDIR/$PROGRAM ] || exit 0 - local EXIT_STATUS - EXIT_STATUS=0 - - echo -n "Start $PROGRAM service: " - - if is_running; then - echo -n "already running " - EXIT_STATUS=1 - else - ulimit -n 262144 - mkdir -p $CLICKHOUSE_PIDDIR - chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR - initdb - if ! is_running; then - # Lock should not be held while running child process, so we release the lock. Note: obviously, there is race condition. - # But clickhouse-server has protection from simultaneous runs with same data directory. - su -s $SHELL ${CLICKHOUSE_USER} -c "$FLOCK -u 9; $CLICKHOUSE_PROGRAM_ENV exec -a \"$PROGRAM\" \"$CLICKHOUSE_BINDIR/$PROGRAM\" --daemon --pid-file=\"$CLICKHOUSE_PIDFILE\" --config-file=\"$CLICKHOUSE_CONFIG\"" - EXIT_STATUS=$? - if [ $EXIT_STATUS -ne 0 ]; then - return $EXIT_STATUS - fi - fi - fi - - if [ $EXIT_STATUS -eq 0 ]; then - attempts=0 - while ! is_running && [ $attempts -le ${CLICKHOUSE_START_TIMEOUT:=10} ]; do - attempts=$(($attempts + 1)) - sleep 1 - done - if is_running; then - echo "DONE" - else - echo "UNKNOWN" - fi - else - echo "FAILED" - fi - - return $EXIT_STATUS + ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" } stop() { - #local EXIT_STATUS - EXIT_STATUS=0 - - if [ -f $CLICKHOUSE_PIDFILE ]; then - - echo -n "Stop $PROGRAM service: " - - kill -TERM $(cat "$CLICKHOUSE_PIDFILE") - - if ! wait_for_done ${CLICKHOUSE_STOP_TIMEOUT}; then - EXIT_STATUS=2 - echo "TIMEOUT" - else - echo "DONE" - fi - - fi - return $EXIT_STATUS + ${CLICKHOUSE_GENERIC_PROGRAM} stop --pid-path "${CLICKHOUSE_PIDDIR}" } restart() { - check_config - if stop; then - if start; then - return 0 - fi - fi - return 1 + ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" } From a4a612843160840b73e4ab5c546ee11a69e52eca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Aug 2020 23:45:24 +0300 Subject: [PATCH 2/8] Trigger CI --- src/Dictionaries/FileDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 18893a99f4e..402fb876c09 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -32,7 +32,7 @@ FileDictionarySource::FileDictionarySource( { const String user_files_path = context.getUserFilesPath(); if (!startsWith(filepath, user_files_path)) - throw Exception("File path " + filepath + " is not inside " + user_files_path, ErrorCodes::PATH_ACCESS_DENIED); + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", filepath, user_files_path); } } From fc1b112e2b3a4cf03af7206c58831fec582b37d2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 27 Aug 2020 21:50:13 +0300 Subject: [PATCH 3/8] Update clickhouse-server.init --- debian/clickhouse-server.init | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 978feb10337..4764bca768b 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -160,7 +160,7 @@ initdb() start() { - ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" + ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" } @@ -172,7 +172,7 @@ stop() restart() { - ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" + ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" } From bce8166420a289feebd14adc8fc3fa831d47da45 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 27 Aug 2020 21:51:19 +0300 Subject: [PATCH 4/8] Update FileDictionarySource.cpp --- src/Dictionaries/FileDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 402fb876c09..82aea4cbb98 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -60,7 +60,7 @@ BlockInputStreamPtr FileDictionarySource::loadAll() std::string FileDictionarySource::toString() const { - return "File: " + filepath + ' ' + format; + return fmt::format("File: {}, {}", filepath, format); } From 422dc1d83faeb1787066ee71b25c805217dbe6df Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 19 Oct 2020 14:08:42 +0300 Subject: [PATCH 5/8] Missed change --- debian/clickhouse-server.postinst | 102 +----------------------------- 1 file changed, 1 insertion(+), 101 deletions(-) diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst index 6e031ae8f44..61ace2874ae 100644 --- a/debian/clickhouse-server.postinst +++ b/debian/clickhouse-server.postinst @@ -41,105 +41,5 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then fi fi - # Make sure the administrative user exists - if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then - if [ -n "$not_deb_os" ]; then - useradd -r -s /bin/false --home-dir /nonexistent ${CLICKHOUSE_USER} > /dev/null - else - adduser --system --disabled-login --no-create-home --home /nonexistent \ - --shell /bin/false --group --gecos "ClickHouse server" ${CLICKHOUSE_USER} > /dev/null - fi - fi - - # if the user was created manually, make sure the group is there as well - if ! getent group ${CLICKHOUSE_GROUP} > /dev/null; then - groupadd -r ${CLICKHOUSE_GROUP} > /dev/null - fi - - # make sure user is in the correct group - if ! id -Gn ${CLICKHOUSE_USER} | grep -qw ${CLICKHOUSE_USER}; then - usermod -a -G ${CLICKHOUSE_GROUP} ${CLICKHOUSE_USER} > /dev/null - fi - - # check validity of user and group - if [ "$(id -u ${CLICKHOUSE_USER})" -eq 0 ]; then - echo "The ${CLICKHOUSE_USER} system user must not have uid 0 (root). -Please fix this and reinstall this package." >&2 - exit 1 - fi - - if [ "$(id -g ${CLICKHOUSE_GROUP})" -eq 0 ]; then - echo "The ${CLICKHOUSE_USER} system user must not have root as primary group. -Please fix this and reinstall this package." >&2 - exit 1 - fi - - if [ -x "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG" ] && [ -f "$CLICKHOUSE_CONFIG" ]; then - if [ -z "$SHELL" ]; then - SHELL="/bin/sh" - fi - CLICKHOUSE_DATADIR_FROM_CONFIG=$(su -s $SHELL ${CLICKHOUSE_USER} -c "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path") ||: - echo "Path to data directory in ${CLICKHOUSE_CONFIG}: ${CLICKHOUSE_DATADIR_FROM_CONFIG}" - fi - CLICKHOUSE_DATADIR_FROM_CONFIG=${CLICKHOUSE_DATADIR_FROM_CONFIG:=$CLICKHOUSE_DATADIR} - - if [ ! -d ${CLICKHOUSE_DATADIR_FROM_CONFIG} ]; then - mkdir -p ${CLICKHOUSE_DATADIR_FROM_CONFIG} - chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR_FROM_CONFIG} - chmod 700 ${CLICKHOUSE_DATADIR_FROM_CONFIG} - fi - - if [ -d ${CLICKHOUSE_CONFDIR} ]; then - mkdir -p ${CLICKHOUSE_CONFDIR}/users.d - mkdir -p ${CLICKHOUSE_CONFDIR}/config.d - rm -fv ${CLICKHOUSE_CONFDIR}/*-preprocessed.xml ||: - fi - - [ -e ${CLICKHOUSE_CONFDIR}/preprocessed ] || ln -s ${CLICKHOUSE_DATADIR_FROM_CONFIG}/preprocessed_configs ${CLICKHOUSE_CONFDIR}/preprocessed ||: - - if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then - mkdir -p ${CLICKHOUSE_LOGDIR} - chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR} - # Allow everyone to read logs, root and clickhouse to read-write - chmod 775 ${CLICKHOUSE_LOGDIR} - fi - - # Set net_admin capabilities to support introspection of "taskstats" performance metrics from the kernel - # and ipc_lock capabilities to allow mlock of clickhouse binary. - - # 1. Check that "setcap" tool exists. - # 2. Check that an arbitrary program with installed capabilities can run. - # 3. Set the capabilities. - - # The second is important for Docker and systemd-nspawn. - # When the container has no capabilities, - # but the executable file inside the container has capabilities, - # then attempt to run this file will end up with a cryptic "Operation not permitted" message. - - TMPFILE=/tmp/test_setcap.sh - - command -v setcap >/dev/null \ - && echo > $TMPFILE && chmod a+x $TMPFILE && $TMPFILE && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" $TMPFILE && $TMPFILE && rm $TMPFILE \ - && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" "${CLICKHOUSE_BINDIR}/${CLICKHOUSE_GENERIC_PROGRAM}" \ - || echo "Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary. This is optional. Taskstats accounting will be disabled. To enable taskstats accounting you may add the required capability later manually." - - # Clean old dynamic compilation results - if [ -d "${CLICKHOUSE_DATADIR_FROM_CONFIG}/build" ]; then - rm -f ${CLICKHOUSE_DATADIR_FROM_CONFIG}/build/*.cpp ${CLICKHOUSE_DATADIR_FROM_CONFIG}/build/*.so ||: - fi - - if [ -f /usr/share/debconf/confmodule ]; then - db_get clickhouse-server/default-password - defaultpassword="$RET" - if [ -n "$defaultpassword" ]; then - echo "$defaultpassword" > ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml - chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml - chmod 600 ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml - fi - - # everything went well, so now let's reset the password - db_set clickhouse-server/default-password "" - # ... done with debconf here - db_stop - fi + ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "{CLICKHOUSE_DATADIR}" fi From dc728e2febf8f6bca4699d08e207addd328cd7bf Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 19 Oct 2020 14:20:27 +0300 Subject: [PATCH 6/8] Missed char --- debian/clickhouse-server.postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst index 61ace2874ae..a663fe59ce9 100644 --- a/debian/clickhouse-server.postinst +++ b/debian/clickhouse-server.postinst @@ -41,5 +41,5 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then fi fi - ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "{CLICKHOUSE_DATADIR}" + ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}" fi From dc27ad9d53140c0b81362d7b9721c3927d10cfe7 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 19 Oct 2020 15:34:34 +0300 Subject: [PATCH 7/8] Add piddir --- debian/clickhouse-server.postinst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst index a663fe59ce9..dc876f45954 100644 --- a/debian/clickhouse-server.postinst +++ b/debian/clickhouse-server.postinst @@ -2,6 +2,7 @@ set -e # set -x +PROGRAM=clickhouse-server CLICKHOUSE_USER=${CLICKHOUSE_USER:=clickhouse} CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:=${CLICKHOUSE_USER}} # Please note that we don't support paths with whitespaces. This is rather ignorant. @@ -12,6 +13,7 @@ CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin} CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse} EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml +CLICKHOUSE_PIDDIR=/var/run/$PROGRAM [ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule [ -f /etc/default/clickhouse ] && . /etc/default/clickhouse From 104019bd87cdbf2edbe6a855b743fbba6fefb99a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 20 Oct 2020 21:09:15 +0300 Subject: [PATCH 8/8] Set capabilities with caution --- programs/install/Install.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index ae0c22c8fcc..8290118089c 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -548,11 +548,27 @@ int mainEntryClickHouseInstall(int argc, char ** argv) users_config_file.string(), users_d.string()); } - /// Set capabilities for the binary. + /** Set capabilities for the binary. + * + * 1. Check that "setcap" tool exists. + * 2. Check that an arbitrary program with installed capabilities can run. + * 3. Set the capabilities. + * + * The second is important for Docker and systemd-nspawn. + * When the container has no capabilities, + * but the executable file inside the container has capabilities, + * then attempt to run this file will end up with a cryptic "Operation not permitted" message. + */ #if defined(__linux__) fmt::print("Setting capabilities for clickhouse binary. This is optional.\n"); - std::string command = fmt::format("command -v setcap && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {}", main_bin_path.string()); + std::string command = fmt::format("command -v setcap >/dev/null" + " && echo > {0} && chmod a+x {0} && {0} && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {0} && {0} && rm {0}" + " && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {1}" + " || echo \"Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary." + " This is optional. Taskstats accounting will be disabled." + " To enable taskstats accounting you may add the required capability later manually.\"", + "/tmp/test_setcap.sh", main_bin_path.string()); fmt::print(" {}\n", command); executeScript(command); #endif