Merge branch 'master' of github.com:yandex/ClickHouse

2024-11-22 15:42:02 +00:00 · 2017-01-31 22:55:43 +03:00 · 2017-01-31 22:55:43 +03:00 · e2669c88cd
commit e2669c88cd
parent d08dc730bd c51801d776
19 changed files with 204 additions and 224 deletions
--- a/cmake/create_init_script.cmake
+++ b/cmake/create_init_script.cmake
@ -1,38 +0,0 @@
-# В этом файле описаны действия для добавления init.d скрипта. Пока в разработке.
-# Данный файл нужно включать в CMakeLists.txt в каталоге каждого конкретного демона.
-# Пример использования:
-# include (${ClickHouse_SOURCE_DIR}/create_init_script.cmake)
-# create_init_script (divider Divider)
-# Будет создан init.d скрипт с названием divider для демона (бинарника) Divider
-
-macro (create_init_script daemonname)
-	set (filename ${daemonname})
-	# Опционально принимаем filename вторым аргументом.
-	set (extra_args ${ARGN})
-	list (LENGTH extra_args num_extra_args)
-	if (${num_extra_args} GREATER 0)
-		list (GET extra_args 0 optional_arg)
-		set (filename ${optional_arg})
-	endif ()
-	set (tmp_file_name ${filename}.init)
-
-	set (SED_INPLACE_SUFFIX "")
-	if (APPLE OR CMAKE_SYSTEM MATCHES "FreeBSD")
-		set (SED_INPLACE_SUFFIX "''")
-	endif ()
-	add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
-			COMMAND sed -e 's,[@]DAEMON[@],${daemonname},g' < ${PROJECT_SOURCE_DIR}/tools/init.d/template > ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
-			COMMAND sed -i ${SED_INPLACE_SUFFIX} 's,[@]CRONFILE[@],${filename},g' ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
-			COMMAND chmod a+x ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
-			COMMENT "Building ${daemonname}"
-			)
-	add_custom_target (${daemonname}-init.target DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name})
-	install (
-			FILES ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
-			DESTINATION /etc/init.d
-			RENAME ${filename}
-			PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ
-			COMPONENT ${daemonname}
-			)
-	add_dependencies (${daemonname} ${daemonname}-init.target)
-endmacro (create_init_script)
--- a/dbms/src/Client/Client.cpp
+++ b/dbms/src/Client/Client.cpp
@ -60,28 +60,7 @@
 #include <DB/Common/NetException.h>

 #include <common/config_common.h>
-
-/// Different line editing libraries can be used depending on the environment.
-#ifdef USE_READLINE
-	#include <readline/readline.h>
-	#include <readline/history.h>
-#elif USE_LIBEDIT
-	#include <editline/readline.h>
-	#include <editline/history.h>
-#else
-	char * readline(const char * prompt)
-	{
-		std::string s;
-		std::cout << prompt;
-		std::getline(std::cin, s);
-
-		if (!std::cin.good())
-			return nullptr;
-		return strdup(s.data());
-	}
-	#define add_history(...) do {} while (0);
-	#define rl_bind_key(...) do {} while (0);
-#endif
+#include <common/readline_use.h>


 /// http://en.wikipedia.org/wiki/ANSI_escape_code
--- a/dbms/src/Server/CMakeLists.txt
+++ b/dbms/src/Server/CMakeLists.txt
@ -31,14 +31,3 @@ INSTALL(
 	FILES config.xml users.xml
 	DESTINATION  ${CLICKHOUSE_ETC_DIR}/clickhouse-server
 	COMPONENT clickhouse)
-
-if (NOT CMAKE_SYSTEM MATCHES "FreeBSD")
-	INSTALL(
-		FILES metrika.conf
-		DESTINATION /etc/security/limits.d
-		COMPONENT clickhouse)
-
-	# Adding init.d support
-	include (${ClickHouse_SOURCE_DIR}/cmake/create_init_script.cmake)
-	create_init_script (clickhouse-server clickhouse-server)
-endif ()
--- a/dbms/src/Server/config.xml
+++ b/dbms/src/Server/config.xml
@ -51,10 +51,10 @@


 	<!-- Path to data directory, with trailing slash. -->
-	<path>/opt/clickhouse/</path>
+	<path>/var/lib/clickhouse/</path>

 	<!-- Path to temporary data for processing hard queries. -->
-	<tmp_path>/opt/clickhouse/tmp/</tmp_path>
+	<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>

 	<!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
 	<users_config>users.xml</users_config>
--- a/dbms/src/Server/metrika.conf
+++ b/dbms/src/Server/metrika.conf
@ -1,2 +0,0 @@
-metrika	soft	nofile	262144
-metrika	hard	nofile	262144
--- a/dbms/src/Storages/tests/active_parts.py
+++ b/dbms/src/Storages/tests/active_parts.py
@ -2,7 +2,7 @@
 # coding=UTF-8

 # Выводит список активных кусков - кусков, не покрытых никаким другим куском.
-# Использование: `ls /opt/clickhouse/data/merge/visits | active_parts.py`
+# Usage: `ls /var/lib/clickhouse/data/merge/visits | active_parts.py`

 import sys
 import re
--- a/dbms/src/Storages/tests/broken_increment_test.sh
+++ b/dbms/src/Storages/tests/broken_increment_test.sh
@ -11,20 +11,20 @@ echo 'Creating table'
 echo 'CREATE TABLE increment.a (d Date, v UInt64) ENGINE=MergeTree(d, tuple(v), 8192)' | clickhouse-client || exit 3
 echo 'Inserting'
 echo "2014-01-01	42" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated" || exit 4
-ls /opt/clickhouse/data/increment/a/
-cat /opt/clickhouse/data/increment/a/increment.txt
-rm /opt/clickhouse/data/increment/a/increment.txt
+ls /var/lib/clickhouse/data/increment/a/
+cat /var/lib/clickhouse/data/increment/a/increment.txt
+rm /var/lib/clickhouse/data/increment/a/increment.txt
 echo 'Inserting without increment.txt'
 echo "2014-01-01	41" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated"
-ls /opt/clickhouse/data/increment/a/
-cat /opt/clickhouse/data/increment/a/increment.txt
-sudo /etc/init.d/clickhouse-server stop
-sudo /etc/init.d/clickhouse-server start
+ls /var/lib/clickhouse/data/increment/a/
+cat /var/lib/clickhouse/data/increment/a/increment.txt
+sudo service clickhouse-server stop
+sudo service clickhouse-server start
 sleep 10s
-ls /opt/clickhouse/data/increment/a/
-cat /opt/clickhouse/data/increment/a/increment.txt
+ls /var/lib/clickhouse/data/increment/a/
+cat /var/lib/clickhouse/data/increment/a/increment.txt
 echo 'Inserting after restart without increment.txt'
 echo "2014-01-01	43" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated"
-ls /opt/clickhouse/data/increment/a/
-cat /opt/clickhouse/data/increment/a/increment.txt
+ls /var/lib/clickhouse/data/increment/a/
+cat /var/lib/clickhouse/data/increment/a/increment.txt
 echo "SELECT * FROM increment.a" | clickhouse-client
--- a/dbms/src/Storages/tests/collapsing_merge_order_test.sh
+++ b/dbms/src/Storages/tests/collapsing_merge_order_test.sh
@ -15,11 +15,11 @@ echo "('2014-01-01', 'key1', -1, 'val2')" | clickhouse-client --query="INSERT IN

 sudo /etc/init.d/clickhouse-server stop || exit 10

-sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p0/20140101_20140101_1_1_0,m0/} || exit 12
-sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p1/20140101_20140101_1_1_0,m0/20140101_20140101_2_2_0} || exit 13
-sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p1/20140101_20140101_1_1_0,m1/20140101_20140101_2_2_0} || exit 14
-sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p2/20140101_20140101_1_1_0,m1/20140101_20140101_3_3_0} || exit 15
-rm /opt/clickhouse/data/collapsing_test/m{0,1}/increment.txt || exit 29
+sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p0/20140101_20140101_1_1_0,m0/} || exit 12
+sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p1/20140101_20140101_1_1_0,m0/20140101_20140101_2_2_0} || exit 13
+sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p1/20140101_20140101_1_1_0,m1/20140101_20140101_2_2_0} || exit 14
+sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p2/20140101_20140101_1_1_0,m1/20140101_20140101_3_3_0} || exit 15
+rm /var/lib/clickhouse/data/collapsing_test/m{0,1}/increment.txt || exit 29

 sudo /etc/init.d/clickhouse-server start || exit 16

@ -29,9 +29,9 @@ echo "OPTIMIZE TABLE collapsing_test.m1" | clickhouse-client || exit 18

 sudo /etc/init.d/clickhouse-server stop || exit 19

-sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p0/20140101_20140101_1_1_0,m1/} || exit 20
-sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p2/20140101_20140101_1_1_0,m0/20140101_20140101_3_3_0} || exit 21
-rm /opt/clickhouse/data/collapsing_test/m{0,1}/increment.txt || exit 29
+sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p0/20140101_20140101_1_1_0,m1/} || exit 20
+sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p2/20140101_20140101_1_1_0,m0/20140101_20140101_3_3_0} || exit 21
+rm /var/lib/clickhouse/data/collapsing_test/m{0,1}/increment.txt || exit 29

 sudo /etc/init.d/clickhouse-server start || exit 22

@ -39,7 +39,7 @@ sleep 10s
 echo "OPTIMIZE TABLE collapsing_test.m0" | clickhouse-client || exit 23
 echo "OPTIMIZE TABLE collapsing_test.m1" | clickhouse-client || exit 23

-ls /opt/clickhouse/data/collapsing_test/m{0,1}
+ls /var/lib/clickhouse/data/collapsing_test/m{0,1}

 echo "SELECT * FROM collapsing_test.m0" | clickhouse-client || exit 24
 echo
@ -51,7 +51,7 @@ echo "('2014-01-01', 'key2', 1, 'val')" | clickhouse-client --query="INSERT INTO
 echo "OPTIMIZE TABLE collapsing_test.m0" | clickhouse-client || exit 30
 echo "OPTIMIZE TABLE collapsing_test.m1" | clickhouse-client || exit 31

-ls /opt/clickhouse/data/collapsing_test/m{0,1}
+ls /var/lib/clickhouse/data/collapsing_test/m{0,1}

 echo "SELECT * FROM collapsing_test.m0" | clickhouse-client | tee /tmp/t1 || exit 24
 echo
--- a/dbms/src/Storages/tests/mergetest.sh
+++ b/dbms/src/Storages/tests/mergetest.sh
@ -3,7 +3,7 @@
 # См. таску CONV-8849.
 # Симулируем ситуацию, когда половина одного файла с засечками не успела записаться на диск

-path='/opt/clickhouse/data/mergetest/a/'
+path='/var/lib/clickhouse/data/mergetest/a/'

 echo 'Creating table'
 echo 'CREATE DATABASE IF NOT EXISTS mergetest' | clickhouse-client || exit 1
--- a/debian/clickhouse-server-base.install
+++ b/debian/clickhouse-server-base.install
@ -1,5 +1,6 @@
 /usr/bin/clickhouse
 /usr/bin/clickhouse-server
+/etc/init.d/clickhouse-server
 /etc/cron.d/clickhouse-server
-/etc/security/limits.d/metrika.conf
 /usr/share/clickhouse/*
+/etc/security/limits.d/clickhouse.conf
--- a/debian/clickhouse-server-base.postinst
+++ b/debian/clickhouse-server-base.postinst
@ -1,6 +1,14 @@
 #!/bin/sh
 set -e

+CLICKHOUSE_USER=clickhouse
+CLICKHOUSE_GROUP=${CLICKHOUSE_USER}
+CLICKHOUSE_DATADIR=/var/lib/clickhouse
+CLICKHOUSE_DATADIR_OLD=/opt/clickhouse  # remove after 2017-06-01
+CLICKHOUSE_LOGDIR=/var/log/clickhouse
+CLICKHOUSE_SERVER_ETCDIR=/etc/clickhouse-server
+
+
 if [ "$1" = configure ]; then

 	if [ -x "/etc/init.d/clickhouse-server" ]; then
@ -8,48 +16,62 @@ if [ "$1" = configure ]; then
 	fi

 	# Make sure the administrative user exists
-	if ! getent passwd metrika > /dev/null; then
+	if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then
 		adduser --system --disabled-login --no-create-home --home /nonexistent \
-			--shell /bin/false --group --gecos "Clickhouse server" metrika > /dev/null
+			--shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null
 	fi

 	# if the user was created manually, make sure the group is there as well
-	if ! getent group metrika > /dev/null; then
-		addgroup --system metrika > /dev/null
+	if ! getent group ${CLICKHOUSE_GROUP} > /dev/null; then
+		addgroup --system ${CLICKHOUSE_GROUP} > /dev/null
 	fi

 	# make sure user is in the correct group
-	if ! id -Gn metrika | grep -qw metrika; then
-		adduser metrika metrika > /dev/null
+	if ! id -Gn ${CLICKHOUSE_USER} | grep -qw ${CLICKHOUSE_USER}; then
+		adduser ${CLICKHOUSE_USER} ${CLICKHOUSE_GROUP} > /dev/null
 	fi

 	# check validity of user and group
-	if [ "`id -u metrika`" -eq 0 ]; then
-		echo "The metrika system user must not have uid 0 (root).
+	if [ "`id -u ${CLICKHOUSE_USER}`" -eq 0 ]; then
+		echo "The ${CLICKHOUSE_USER} system user must not have uid 0 (root).
 Please fix this and reinstall this package." >&2
 		exit 1
 	fi

-	if [ "`id -g metrika`" -eq 0 ]; then
-		echo "The metrika system user must not have root as primary group.
+	if [ "`id -g ${CLICKHOUSE_GROUP}`" -eq 0 ]; then
+		echo "The ${CLICKHOUSE_USER} system user must not have root as primary group.
 Please fix this and reinstall this package." >&2
 		exit 1
 	fi

-	if [ ! -d "/opt/clickhouse" ]; then
+	if [ ! -d ${CLICKHOUSE_DATADIR} ]; then
+
+		# only for compatibility for old /opt/clickhouse, remove after 2017-06-01
+		if [ -d ${CLICKHOUSE_DATADIR_OLD} ]; then
+			ln -s ${CLICKHOUSE_DATADIR_OLD} ${CLICKHOUSE_DATADIR}
+		else
+			# DONT remove after 2017-06-01 :
+			mkdir -p ${CLICKHOUSE_DATADIR}
+		fi
+
 		# ensure home directory ownership
-		mkdir -p /opt/clickhouse
-		#su -s /bin/sh metrika -c "test -O /opt/clickhouse && test -G /opt/clickhouse" || \
-		chown metrika:metrika /opt/clickhouse
-		chmod 0700 /opt/clickhouse
+		#su -s /bin/sh clickhouse -c "test -O /var/lib/clickhouse && test -G /var/lib/clickhouse" || \
+		chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR}
+		chmod 0700 ${CLICKHOUSE_DATADIR}
 	fi

+	if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then
+		mkdir -p ${CLICKHOUSE_LOGDIR}
+		chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}
+		chmod 1770 ${CLICKHOUSE_LOGDIR}
+	fi
+
+	# -R only for compatibility for old metrika user, remove -R after 2017-06-01
+	su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_SERVER_ETCDIR}" || chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_SERVER_ETCDIR}
+
 	# Clean old dynamic compilation results
-	if [ -d "/opt/clickhouse/build" ]; then
-		rm -f /opt/clickhouse/build/*.cpp /opt/clickhouse/build/*.so ||:
+	if [ -d "${CLICKHOUSE_DATADIR}/build" ]; then
+		rm -f ${CLICKHOUSE_DATADIR}/build/*.cpp ${CLICKHOUSE_DATADIR}/build/*.so ||:
 	fi

 fi
-
-
-exit 0
--- a/debian/rules
+++ b/debian/rules
@ -84,16 +84,6 @@ install: build
 	# создаем дефолтный cron, если нет крон-файла созданного пользователем
 	# cron_name имеет вид daemonname
 	# Для ClickHouse не должно быть лишних зависимостей.
-	for I in debian/tmp/etc/init.d/*; \
-	do \
-		cron_name=`basename $$I`; \
-		if [ ! -d debian/tmp/etc/cron.d ]; then \
-			mkdir -p debian/tmp/etc/cron.d; \
-		fi; \
-		if [ ! -f debian/tmp/etc/cron.d/$$cron_name ]; then \
-			echo "#*/10 * * * * root /etc/init.d/$$cron_name condstart 1>/dev/null 2>&1" > debian/tmp/etc/cron.d/$$cron_name; \
-		fi; \
-	done; \
 	# Making necessary *.install files
 	for DAEMON_PKG in ${DAEMONS}; do \
 		if [ ! -e "debian/$$DAEMON_PKG.install" ]; then \
@ -112,7 +102,7 @@ install: build
 		if [ ! -e "debian/$$DAEMON_PKG.postinst" ]; then \
 			echo "# automatically created" > debian/$$DAEMON_PKG.postinst; \
 			echo "mkdir -p /etc/$$DAEMON_PKG/conf.d" >> debian/$$DAEMON_PKG.postinst; \
-			echo "chown -R metrika: /etc/$$DAEMON_PKG" >> debian/$$DAEMON_PKG.postinst; \
+			echo "chown -R clickhouse: /etc/$$DAEMON_PKG" >> debian/$$DAEMON_PKG.postinst; \
 			if [ -e "debian/tmp/etc/init.d/$$DAEMON_PKG" ]; then \
 				if echo $$DAEMON_PKG | grep server > /dev/null; then\
 					echo "update-rc.d $$DAEMON_PKG defaults > /dev/null || exit \$$?" >> debian/$$DAEMON_PKG.postinst; \
@ -123,19 +113,17 @@ install: build
 			\
 		else \
 			echo >> debian/$$DAEMON_PKG.postinst; \
-			echo "mkdir -p /etc/$$DAEMON_PKG/conf.d; chown -R metrika: /etc/$$DAEMON_PKG" >> debian/$$DAEMON_PKG.postinst; \
+			echo "mkdir -p /etc/$$DAEMON_PKG/conf.d; chown -R clickhouse: /etc/$$DAEMON_PKG" >> debian/$$DAEMON_PKG.postinst; \
 		fi;  \
 	done


-	# В случае сборки clickhouse-server, добавим в пакет бинарник clang-а, ld и заголовочные файлы - для динамической компиляции.
-	if [ -e debian/tmp/etc/init.d/clickhouse-server ]; then \
-		mkdir -p debian/tmp/usr/share/clickhouse/bin debian/tmp/usr/share/clickhouse/headers; \
-		debian/copy_clang_binaries.sh debian/tmp/usr/share/clickhouse/bin/; \
-		./copy_headers.sh . debian/tmp/usr/share/clickhouse/headers; \
-	fi;
+	# In case building clickhouse-server, adding to package binary of clang, ld and header files - for dynamic compilation.
+	mkdir -p debian/tmp/usr/share/clickhouse/bin debian/tmp/usr/share/clickhouse/headers
+	debian/copy_clang_binaries.sh debian/tmp/usr/share/clickhouse/bin/
+	./copy_headers.sh . debian/tmp/usr/share/clickhouse/headers

-	# Создаём документацию
+	# Making docs
 	cp LICENSE debian/copyright

 	for DAEMON_PKG in ${DAEMONS}; do \
@ -148,6 +136,13 @@ install: build
 	ln -s clickhouse-server.docs debian/clickhouse-server-common.docs
 	ln -s clickhouse-server.docs debian/clickhouse-server-metrika.docs

+	mkdir -p debian/tmp/etc/security/limits.d
+	cp tools/etc/security/limits.d/clickhouse.conf debian/tmp/etc/security/limits.d
+	mkdir -p debian/tmp/etc/init.d
+	cp tools/etc/init.d/clickhouse-server debian/tmp/etc/init.d
+	mkdir -p debian/tmp/etc/cron.d
+	cp tools/etc/cron.d/clickhouse-server debian/tmp/etc/cron.d
+
 # Build architecture-independent files here.
 binary-indep: build install
 # We have nothing to do by default.
--- a/libs/libcommon/cmake/version.cmake
+++ b/libs/libcommon/cmake/version.cmake
@ -1,6 +1,6 @@
 #This strings autochanged from release_lib.sh :
-set(VERSION_DESCRIBE v1.1.54140-testing)
-set(VERSION_REVISION 54140)
+set(VERSION_DESCRIBE v1.1.54141-testing)
+set(VERSION_REVISION 54141)
 #===end of autochange

 set(VERSION_MAJOR 1)
--- a/libs/libcommon/include/common/readline_use.h
+++ b/libs/libcommon/include/common/readline_use.h
@ -0,0 +1,28 @@
+#pragma once
+
+#include <common/config_common.h>
+
+/// Different line editing libraries can be used depending on the environment.
+#if USE_READLINE
+	#include <readline/readline.h>
+	#include <readline/history.h>
+#elif USE_LIBEDIT
+	#include <editline/readline.h>
+	#include <editline/history.h>
+#else
+	#include <string>
+	#include <cstring>
+	#include <iostream>
+	inline char * readline(const char * prompt)
+	{
+		std::string s;
+		std::cout << prompt;
+		std::getline(std::cin, s);
+
+		if (!std::cin.good())
+			return nullptr;
+		return strdup(s.data());
+	}
+	#define add_history(...) do {} while (0);
+	#define rl_bind_key(...) do {} while (0);
+#endif
--- a/libs/libcommon/src/get_revision_lib.sh
+++ b/libs/libcommon/src/get_revision_lib.sh
@ -1,14 +1,5 @@
-# Filter non-release tags
-function tag_filter {
-	grep -E "^v1\.1\.[0-9]{5}-testing$"
-}
-
 # Get last revision number
 function get_revision {
 	BASEDIR=`dirname "$0"`
-	CURDIR=`pwd`
-	cd ${BASEDIR}
-	git rev-parse --git-dir >/dev/null 2>/dev/null || cd ${CURDIR}
-	git tag | tag_filter | tail -1 | sed 's/^v1\.1\.\(.*\)-testing$/\1/'
-	cd ${CURDIR}
+	grep "set(VERSION_REVISION" ${BASEDIR}/libs/libcommon/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\))$/\1/'
 }
--- a/libs/libzkutil/src/tests/zkutil_test.cpp
+++ b/libs/libzkutil/src/tests/zkutil_test.cpp
@ -1,11 +1,10 @@
 #include <zkutil/ZooKeeper.h>
 #include <zkutil/KeeperException.h>
 #include <iostream>
-#include <readline/readline.h>
-#include <readline/history.h>
 #include <sstream>
 #include <Poco/ConsoleChannel.h>
 #include <common/logger_useful.h>
+#include <common/readline_use.h>
 #include <DB/IO/ReadHelpers.h>
 #include <DB/IO/ReadBufferFromString.h>

--- a/tools/etc/cron.d/clickhouse-server
+++ b/tools/etc/cron.d/clickhouse-server
@ -0,0 +1 @@
+#*/10 * * * * root /etc/init.d/clickhouse-server condstart 1>/dev/null 2>&1
--- a/tools/etc/init.d/clickhouse-server
+++ b/tools/etc/init.d/clickhouse-server
@ -1,31 +1,37 @@
-#!/bin/bash
+#!/bin/sh
 ### BEGIN INIT INFO
-# Provides:          @DAEMON@
+# Provides:          clickhouse-server
 # Default-Start:     2 3 4 5
 # Default-Stop:      0 1 6
 # Required-Start:
 # Required-Stop:
-# Short-Description: Yandex daemon
+# Short-Description: Yandex clickhouse-server daemon
 ### END INIT INFO

-USER=metrika
-GROUP=metrika
+
+CLICKHOUSE_USER=clickhouse
+CLICKHOUSE_GROUP=${CLICKHOUSE_USER}
 SHELL=/bin/bash
-PROGRAM=@DAEMON@
+PROGRAM=clickhouse-server
 SYSCONFDIR=/etc/$PROGRAM
-LOGDIR=/var/log/$PROGRAM
+CLICKHOUSE_LOGDIR=/var/log/$CLICKHOUSE_USER
+CLICKHOUSE_LOGDIR_USER=root
+CLICKHOUSE_DATADIR=/var/lib/$CLICKHOUSE_USER
+CLICKHOUSE_DATADIR_OLD=/opt/clickhouse
 LOCALSTATEDIR=/var/lock
 BINDIR=/usr/bin
-CRONFILE=/etc/cron.d/@CRONFILE@
-CNFFILE=$SYSCONFDIR/config.xml
+CLICKHOUSE_CRONFILE=/etc/cron.d/clickhouse-server
+CLICKHOUSE_CONFIG=$SYSCONFDIR/config.xml
 LOCKFILE=$LOCALSTATEDIR/$PROGRAM
 RETVAL=0
-
-DEFAULT_NUMBER_OF_PROCESSES=1
-NUMBER_OF_PROCESSES=$DEFAULT_NUMBER_OF_PROCESSES
-
 LOG_FILE=""

+CLICKHOUSE_PIDDIR=/var/run/$PROGRAM
+CLICKHOUSE_PIDFILE="$CLICKHOUSE_PIDDIR/$PROGRAM.pid"
+
+# Override defaults from optional config file
+test -f /etc/default/clickhouse && . /etc/default/clickhouse
+
 # On x86_64, check for required instruction set.
 if uname -mpi | grep -q 'x86_64'; then
 	if ! grep -q 'sse4_2' /proc/cpuinfo; then
@ -42,8 +48,8 @@ if uname -mpi | grep -q 'x86_64'; then
 			# gcc -xc -Os -static -nostdlib - <<< 'void _start() { __asm__("pcmpgtq %%xmm0, %%xmm1; mov $0x3c, %%rax; xor %%rdi, %%rdi; syscall":::"memory"); }' && strip -R .note.gnu.build-id -R .comment -R .eh_frame -s ./a.out && gzip -c -9 ./a.out | base64 -w0; echo

 			if ! (echo -n 'H4sICAwAW1cCA2Eub3V0AKt39XFjYmRkgAEmBjsGEI+H0QHMd4CKGyCUAMUsGJiBJDNQNUiYlQEZOKDQclB9cnD9CmCSBYqJBRxQOvBpSQobGfqIAWn8FuYnPI4fsAGyPQz/87MeZtArziguKSpJTGLQK0mtKGGgGHADMSgoYH6AhTMPNHyE0NQzYuEzYzEXFr6CBPQDANAsXKTwAQAA' | base64 -d | gzip -d > /tmp/clickhouse_test_sse42 && chmod a+x /tmp/clickhouse_test_sse42 && /tmp/clickhouse_test_sse42); then
-				echo 'SSE 4.2 instruction set is not supported'
-				exit 3
+				echo 'Warning! SSE 4.2 instruction set is not supported'
+				#exit 3
 			fi
 		fi
 	fi
@ -51,15 +57,11 @@ fi

 # С помощью xmlstarlet пытаемся взять некоторые параметры из конфига
 if command -v xmlstarlet >/dev/null 2>&1; then
-	NUMBER_OF_PROCESSES=$(xmlstarlet sel -t -v "/yandex/number_of_processes" $CNFFILE || echo $DEFAULT_NUMBER_OF_PROCESSES)
-	LOG_FILE=$(xmlstarlet sel -t -v "/yandex/logger/log" $CNFFILE || echo $LOG_FILE)
+	LOG_FILE=$(xmlstarlet sel -t -v "/yandex/logger/log" $CLICKHOUSE_CONFIG || echo $LOG_FILE)
 fi

-PIDDIR=/var/run/$PROGRAM
-PIDFILE_PREFIX=$PIDDIR/$PROGRAM
-PIDFILE_RE="$PIDFILE_PREFIX[0-9]*.pid"

-SUPPORTED_COMMANDS="{start|stop|status|restart|forcestop|forcerestart|reload|condstart|condstop|condrestart|condreload}"
+SUPPORTED_COMMANDS="{start|stop|status|restart|forcestop|forcerestart|reload|condstart|condstop|condrestart|condreload|initdb}"
 is_supported_command()
 {
 	echo $SUPPORTED_COMMANDS | grep -E "(\{|\|)$1(\||})" &> /dev/null
@ -67,66 +69,49 @@ is_supported_command()

 generate_program_name()
 {
-	if [ $NUMBER_OF_PROCESSES -eq 1 ]; then
-		echo $PROGRAM
-	else
-		echo $PROGRAM$1
-	fi
+	echo $PROGRAM
 }

 generate_pid_name()
 {
-	if [ $NUMBER_OF_PROCESSES -gt 1 ]; then
-		echo $PIDFILE_PREFIX$1.pid
-	else
-		echo $PIDFILE_PREFIX.pid
-	fi
+	echo $CLICKHOUSE_PIDFILE
 }

 specific_log_file_for_each_process()
 {
-	# Не будем менять имя лог файла, при одном процессе
-	# Не выставляем, если имя лог файла пустое
-	if [ $NUMBER_OF_PROCESSES -gt 1 ] && [ "$LOG_FILE" != "" ]; then
-		log_file=$(echo $LOG_FILE | sed "s/\.log/$1.log/")
-		echo "--log-file=\"$log_file\""
-	fi
-
 	echo ""
 }

 find_pid_files()
 {
-	[[ -e $PIDDIR ]] && find $PIDDIR -regex "$PIDFILE_RE"
+	[ -e $CLICKHOUSE_PIDDIR ] && find $CLICKHOUSE_PIDFILE
 }

 is_running()
 {
 	pidfile=$1

-	[ -r "$pidfile" ] && pgrep -s $(cat "$pidfile") 1> /dev/null 2> /dev/null
+	[ -r $pidfile ] && pgrep -s $(cat "$pidfile") 1> /dev/null 2> /dev/null
 }

 running_processes()
 {
 	pidfiles=$(find_pid_files)
 	running=0
-	for pidfile in $pidfiles; do
-		if is_running $pidfile; then
-			running=$(($running + 1))
-		fi
-	done
+	if is_running $CLICKHOUSE_PIDFILE; then
+		running=$(($running + 1))
+	fi
 	echo $running
 }

 any_runs()
 {
-	if [[ $(running_processes) -gt  0 ]]; then return 0; else return 1; fi
+	if [ $(running_processes) -gt 0 ]; then return 0; else return 1; fi
 }

 all_runs()
 {
-	[[ $(running_processes) -eq $NUMBER_OF_PROCESSES ]]
+	[ $(running_processes) -eq 1 ]
 }

 wait4done()
@ -136,6 +121,39 @@ wait4done()
 	done
 }

+
+initdb()
+{
+	if ! getent group ${CLICKHOUSE_USER} >/dev/null; then
+		echo "Can't chown to non-existing user ${CLICKHOUSE_USER}"
+		return
+	fi
+	if ! getent passwd ${CLICKHOUSE_GROUP} >/dev/null; then
+		echo "Can't chown to non-existing group ${CLICKHOUSE_GROUP}"
+		return
+	fi
+
+	if ! `su -s $SHELL ${CLICKHOUSE_USER} -c "test -r ${CLICKHOUSE_CONFIG}"`; then
+		echo "Warning! clickhouse config [${CLICKHOUSE_CONFIG}] not readable by user [${CLICKHOUSE_USER}]"
+	fi
+
+	if ! `su -s $SHELL ${CLICKHOUSE_USER} -c "test -O ${CLICKHOUSE_DATADIR} && test -G ${CLICKHOUSE_DATADIR}"`; then
+		echo "Changing owner of [${CLICKHOUSE_DATADIR}] to [${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}]"
+		chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR}
+	fi
+
+	if ! `su -s $SHELL ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_LOGDIR}"`; then
+		echo "Changing owner of [${CLICKHOUSE_LOGDIR}] to [${CLICKHOUSE_LOGDIR_USER}:${CLICKHOUSE_GROUP}]"
+		chown -R ${CLICKHOUSE_LOGDIR_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}
+	fi
+
+	#Temporary fix for old metrika user, remove after 2017-06-01
+	if [ ! -z ${CLICKHOUSE_DATADIR_OLD} ] && [ -d ${CLICKHOUSE_DATADIR_OLD} ] && [ ! `su -s $SHELL ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_LOGDIR}"` ]; then
+		echo "Changing owner of old [${CLICKHOUSE_DATADIR_OLD}] to [${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}]"
+		chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR_OLD}
+	fi
+}
+
 start()
 {
 	[ -x $BINDIR/$PROGRAM ] || exit 0
@ -149,26 +167,21 @@ start()
 		echo -n "already running "
 		EXIT_STATUS=1
 	else
-		mkdir -p $LOGDIR
-		mkdir -p $PIDDIR
-		chown -R $USER:$GROUP $LOGDIR
-		chown -R $USER:$GROUP $PIDDIR
-		chown -R $USER:$GROUP $SYSCONFDIR
-
-		for i in $(seq 1 $NUMBER_OF_PROCESSES); do
-			if ! is_running $(generate_pid_name $i); then
-				rm -f $(generate_pid_name $i)
+		mkdir -p $CLICKHOUSE_PIDDIR
+		chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR
+		initdb
+			if ! is_running $CLICKHOUSE_PIDFILE; then
+				rm -f $CLICKHOUSE_PIDFILE
 				# чтобы лок не удерживался в течении времени жизни дочернего процесса, освободим лок
-				su -l $USER -s $SHELL -c "flock -u 9; exec -a $(generate_program_name $i) \"$BINDIR/$PROGRAM\" --daemon --pid-file=\"$(generate_pid_name $i)\" --config-file=\"$CNFFILE\" $(specific_log_file_for_each_process $i)"
+				su -l $CLICKHOUSE_USER -s $SHELL -c "flock -u 9; exec -a $(generate_program_name $i) \"$BINDIR/$PROGRAM\" --daemon --pid-file=\"$CLICKHOUSE_PIDFILE\" --config-file=\"$CLICKHOUSE_CONFIG\" $(specific_log_file_for_each_process $i)"
 				EXIT_STATUS=$?
-				if [[ $EXIT_STATUS -ne 0 ]]; then
+				if [ $EXIT_STATUS -ne 0 ]; then
 					break
 				fi
 			fi
-		done
 	fi

-	if [[ $EXIT_STATUS -eq 0 ]]; then
+	if [ $EXIT_STATUS -eq 0 ]; then
 		echo "DONE"
 	else
 		echo "FAILED"
@ -184,9 +197,7 @@ stop()

 	echo -n "Stop $PROGRAM service: "

-	for pid_file in $(find_pid_files); do
-		kill -TERM `cat "$pid_file"`
-	done
+	kill -TERM `cat "$CLICKHOUSE_PIDFILE"`

 	wait4done

@ -207,9 +218,7 @@ forcestop()

 	echo -n "Stop $PROGRAM service: "

-	for pid_file in $(find_pid_files); do
-		kill -9 `cat "$pid_file"`
-	done
+	kill -9 `cat "$CLICKHOUSE_PIDFILE"`

 	wait4done

@ -225,17 +234,18 @@ forcerestart()

 enable_cron()
 {
-	sed -i 's/^#*//' "$CRONFILE"
+	[ ! -z $CLICKHOUSE_CRONFILE ] && sed -i 's/^#*//' "$CLICKHOUSE_CRONFILE"
 }

 disable_cron()
 {
-	sed -i 's/^#*/#/' "$CRONFILE"
+	[ ! -z $CLICKHOUSE_CRONFILE ] && sed -i 's/^#*/#/' "$CLICKHOUSE_CRONFILE"
 }

 is_cron_disabled()
 {
-	[[ `grep -E "^#.*" $CRONFILE` == `cat $CRONFILE` ]];
+	[ -z $CLICKHOUSE_CRONFILE ] && return 0
+	[[ `grep -E "^#.*" $CLICKHOUSE_CRONFILE` == `cat $CLICKHOUSE_CRONFILE` ]];
 }

 main()
@ -273,6 +283,9 @@ main()
 	condreload)
 		any_runs && restart
 		;;
+	initdb)
+		initdb
+		;;
 	esac

 	exit $EXIT_STATUS
@ -280,13 +293,13 @@ main()

 status()
 {
-	if [[ $(running_processes) -eq $NUMBER_OF_PROCESSES ]]; then
+	if [ $(running_processes) -eq 1 ]; then
 		echo "$PROGRAM service is running"
 	else
 		if is_cron_disabled; then
 			echo "$PROGRAM service is stopped";
 		else
-			echo "$PROGRAM: $(($NUMBER_OF_PROCESSES - $(running_processes))) of $NUMBER_OF_PROCESSES processes unexpectedly terminated"
+			echo "$PROGRAM: $((1 - $(running_processes))) of 1 processes unexpectedly terminated"
 		fi
 	fi
 }
@ -297,7 +310,7 @@ if ! is_supported_command "$1"; then
 	exit 2
 fi

-if [[ "$1" == "status" ]]; then
+if "$1" == "status"; then
 	status
 	exit 0
 fi
--- a/tools/etc/security/limits.d/clickhouse.conf
+++ b/tools/etc/security/limits.d/clickhouse.conf
@ -0,0 +1,2 @@
+clickhouse	soft	nofile	262144
+clickhouse	hard	nofile	262144
				`@ -0,0 +1 @@`
				`#/10 * * * root /etc/init.d/clickhouse-server condstart 1>/dev/null 2>&1`