thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:yandex/ClickHouse

Browse Source
This commit is contained in:
Alexey Milovidov 2017-01-31 22:55:43 +03:00
commit e2669c88cd
19 changed files with 204 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
cmake/create_init_script.cmake
View File

@ -1,38 +0,0 @@
# В этом файле описаны действия для добавления init.d скрипта. Пока в разработке.
# Данный файл нужно включать в CMakeLists.txt в каталоге каждого конкретного демона.
# Пример использования:
# include (${ClickHouse_SOURCE_DIR}/create_init_script.cmake)
# create_init_script (divider Divider)
# Будет создан init.d скрипт с названием divider для демона (бинарника) Divider
macro (create_init_script daemonname)
set (filename ${daemonname})
# Опционально принимаем filename вторым аргументом.
set (extra_args ${ARGN})
list (LENGTH extra_args num_extra_args)
if (${num_extra_args} GREATER 0)
list (GET extra_args 0 optional_arg)
set (filename ${optional_arg})
endif ()
set (tmp_file_name ${filename}.init)
set (SED_INPLACE_SUFFIX "")
if (APPLE OR CMAKE_SYSTEM MATCHES "FreeBSD")
set (SED_INPLACE_SUFFIX "''")
endif ()
add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
COMMAND sed -e 's,[@]DAEMON[@],${daemonname},g' < ${PROJECT_SOURCE_DIR}/tools/init.d/template > ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
COMMAND sed -i ${SED_INPLACE_SUFFIX} 's,[@]CRONFILE[@],${filename},g' ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
COMMAND chmod a+x ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
COMMENT "Building ${daemonname}"
)
add_custom_target (${daemonname}-init.target DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name})
install (
FILES ${CMAKE_CURRENT_BINARY_DIR}/${tmp_file_name}
DESTINATION /etc/init.d
RENAME ${filename}
PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ
COMPONENT ${daemonname}
)
add_dependencies (${daemonname} ${daemonname}-init.target)
endmacro (create_init_script)

23
dbms/src/Client/Client.cpp
View File

@ -60,28 +60,7 @@
#include <DB/Common/NetException.h> #include <DB/Common/NetException.h>
#include <common/config_common.h> #include <common/config_common.h>
#include <common/readline_use.h>
/// Different line editing libraries can be used depending on the environment.
#ifdef USE_READLINE
#include <readline/readline.h>
#include <readline/history.h>
#elif USE_LIBEDIT
#include <editline/readline.h>
#include <editline/history.h>
#else
char * readline(const char * prompt)
{
std::string s;
std::cout << prompt;
std::getline(std::cin, s);
if (!std::cin.good())
return nullptr;
return strdup(s.data());
}
#define add_history(...) do {} while (0);
#define rl_bind_key(...) do {} while (0);
#endif
/// http://en.wikipedia.org/wiki/ANSI_escape_code /// http://en.wikipedia.org/wiki/ANSI_escape_code

11
dbms/src/Server/CMakeLists.txt
View File

@ -31,14 +31,3 @@ INSTALL(
FILES config.xml users.xml FILES config.xml users.xml
DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server
COMPONENT clickhouse) COMPONENT clickhouse)
if (NOT CMAKE_SYSTEM MATCHES "FreeBSD")
INSTALL(
FILES metrika.conf
DESTINATION /etc/security/limits.d
COMPONENT clickhouse)
# Adding init.d support
include (${ClickHouse_SOURCE_DIR}/cmake/create_init_script.cmake)
create_init_script (clickhouse-server clickhouse-server)
endif ()

4
dbms/src/Server/config.xml
View File

@ -51,10 +51,10 @@
<!-- Path to data directory, with trailing slash. --> <!-- Path to data directory, with trailing slash. -->
<path>/opt/clickhouse/</path> <path>/var/lib/clickhouse/</path>
<!-- Path to temporary data for processing hard queries. --> <!-- Path to temporary data for processing hard queries. -->
<tmp_path>/opt/clickhouse/tmp/</tmp_path> <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<!-- Path to configuration file with users, access rights, profiles of settings, quotas. --> <!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
<users_config>users.xml</users_config> <users_config>users.xml</users_config>

2
dbms/src/Server/metrika.conf
View File

@ -1,2 +0,0 @@
metrika soft nofile 262144
metrika hard nofile 262144

2
dbms/src/Storages/tests/active_parts.py
View File

@ -2,7 +2,7 @@
# coding=UTF-8 # coding=UTF-8
# Выводит список активных кусков - кусков, не покрытых никаким другим куском. # Выводит список активных кусков - кусков, не покрытых никаким другим куском.
# Использование: `ls /opt/clickhouse/data/merge/visits | active_parts.py` # Usage: `ls /var/lib/clickhouse/data/merge/visits | active_parts.py`
import sys import sys
import re import re

22
dbms/src/Storages/tests/broken_increment_test.sh
View File

@ -11,20 +11,20 @@ echo 'Creating table'
echo 'CREATE TABLE increment.a (d Date, v UInt64) ENGINE=MergeTree(d, tuple(v), 8192)' | clickhouse-client || exit 3 echo 'CREATE TABLE increment.a (d Date, v UInt64) ENGINE=MergeTree(d, tuple(v), 8192)' | clickhouse-client || exit 3
echo 'Inserting' echo 'Inserting'
echo "2014-01-01 42" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated" || exit 4 echo "2014-01-01 42" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated" || exit 4
ls /opt/clickhouse/data/increment/a/ ls /var/lib/clickhouse/data/increment/a/
cat /opt/clickhouse/data/increment/a/increment.txt cat /var/lib/clickhouse/data/increment/a/increment.txt
rm /opt/clickhouse/data/increment/a/increment.txt rm /var/lib/clickhouse/data/increment/a/increment.txt
echo 'Inserting without increment.txt' echo 'Inserting without increment.txt'
echo "2014-01-01 41" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated" echo "2014-01-01 41" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated"
ls /opt/clickhouse/data/increment/a/ ls /var/lib/clickhouse/data/increment/a/
cat /opt/clickhouse/data/increment/a/increment.txt cat /var/lib/clickhouse/data/increment/a/increment.txt
sudo /etc/init.d/clickhouse-server stop sudo service clickhouse-server stop
sudo /etc/init.d/clickhouse-server start sudo service clickhouse-server start
sleep 10s sleep 10s
ls /opt/clickhouse/data/increment/a/ ls /var/lib/clickhouse/data/increment/a/
cat /opt/clickhouse/data/increment/a/increment.txt cat /var/lib/clickhouse/data/increment/a/increment.txt
echo 'Inserting after restart without increment.txt' echo 'Inserting after restart without increment.txt'
echo "2014-01-01 43" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated" echo "2014-01-01 43" | clickhouse-client --query="INSERT INTO increment.a FORMAT TabSeparated"
ls /opt/clickhouse/data/increment/a/ ls /var/lib/clickhouse/data/increment/a/
cat /opt/clickhouse/data/increment/a/increment.txt cat /var/lib/clickhouse/data/increment/a/increment.txt
echo "SELECT * FROM increment.a" | clickhouse-client echo "SELECT * FROM increment.a" | clickhouse-client

20
dbms/src/Storages/tests/collapsing_merge_order_test.sh
View File

@ -15,11 +15,11 @@ echo "('2014-01-01', 'key1', -1, 'val2')" | clickhouse-client --query="INSERT IN
sudo /etc/init.d/clickhouse-server stop || exit 10 sudo /etc/init.d/clickhouse-server stop || exit 10
sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p0/20140101_20140101_1_1_0,m0/} || exit 12 sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p0/20140101_20140101_1_1_0,m0/} || exit 12
sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p1/20140101_20140101_1_1_0,m0/20140101_20140101_2_2_0} || exit 13 sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p1/20140101_20140101_1_1_0,m0/20140101_20140101_2_2_0} || exit 13
sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p1/20140101_20140101_1_1_0,m1/20140101_20140101_2_2_0} || exit 14 sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p1/20140101_20140101_1_1_0,m1/20140101_20140101_2_2_0} || exit 14
sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p2/20140101_20140101_1_1_0,m1/20140101_20140101_3_3_0} || exit 15 sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p2/20140101_20140101_1_1_0,m1/20140101_20140101_3_3_0} || exit 15
rm /opt/clickhouse/data/collapsing_test/m{0,1}/increment.txt || exit 29 rm /var/lib/clickhouse/data/collapsing_test/m{0,1}/increment.txt || exit 29
sudo /etc/init.d/clickhouse-server start || exit 16 sudo /etc/init.d/clickhouse-server start || exit 16
@ -29,9 +29,9 @@ echo "OPTIMIZE TABLE collapsing_test.m1" | clickhouse-client || exit 18
sudo /etc/init.d/clickhouse-server stop || exit 19 sudo /etc/init.d/clickhouse-server stop || exit 19
sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p0/20140101_20140101_1_1_0,m1/} || exit 20 sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p0/20140101_20140101_1_1_0,m1/} || exit 20
sudo -u metrika cp -r /opt/clickhouse/data/collapsing_test/{p2/20140101_20140101_1_1_0,m0/20140101_20140101_3_3_0} || exit 21 sudo -u clickhouse cp -r /var/lib/clickhouse/data/collapsing_test/{p2/20140101_20140101_1_1_0,m0/20140101_20140101_3_3_0} || exit 21
rm /opt/clickhouse/data/collapsing_test/m{0,1}/increment.txt || exit 29 rm /var/lib/clickhouse/data/collapsing_test/m{0,1}/increment.txt || exit 29
sudo /etc/init.d/clickhouse-server start || exit 22 sudo /etc/init.d/clickhouse-server start || exit 22
@ -39,7 +39,7 @@ sleep 10s
echo "OPTIMIZE TABLE collapsing_test.m0" | clickhouse-client || exit 23 echo "OPTIMIZE TABLE collapsing_test.m0" | clickhouse-client || exit 23
echo "OPTIMIZE TABLE collapsing_test.m1" | clickhouse-client || exit 23 echo "OPTIMIZE TABLE collapsing_test.m1" | clickhouse-client || exit 23
ls /opt/clickhouse/data/collapsing_test/m{0,1} ls /var/lib/clickhouse/data/collapsing_test/m{0,1}
echo "SELECT * FROM collapsing_test.m0" | clickhouse-client || exit 24 echo "SELECT * FROM collapsing_test.m0" | clickhouse-client || exit 24
echo echo
@ -51,7 +51,7 @@ echo "('2014-01-01', 'key2', 1, 'val')" | clickhouse-client --query="INSERT INTO
echo "OPTIMIZE TABLE collapsing_test.m0" | clickhouse-client || exit 30 echo "OPTIMIZE TABLE collapsing_test.m0" | clickhouse-client || exit 30
echo "OPTIMIZE TABLE collapsing_test.m1" | clickhouse-client || exit 31 echo "OPTIMIZE TABLE collapsing_test.m1" | clickhouse-client || exit 31
ls /opt/clickhouse/data/collapsing_test/m{0,1} ls /var/lib/clickhouse/data/collapsing_test/m{0,1}
echo "SELECT * FROM collapsing_test.m0" | clickhouse-client | tee /tmp/t1 || exit 24 echo "SELECT * FROM collapsing_test.m0" | clickhouse-client | tee /tmp/t1 || exit 24
echo echo

2
dbms/src/Storages/tests/mergetest.sh
View File

@ -3,7 +3,7 @@
# См. таску CONV-8849. # См. таску CONV-8849.
# Симулируем ситуацию, когда половина одного файла с засечками не успела записаться на диск # Симулируем ситуацию, когда половина одного файла с засечками не успела записаться на диск
path='/opt/clickhouse/data/mergetest/a/' path='/var/lib/clickhouse/data/mergetest/a/'
echo 'Creating table' echo 'Creating table'
echo 'CREATE DATABASE IF NOT EXISTS mergetest' | clickhouse-client || exit 1 echo 'CREATE DATABASE IF NOT EXISTS mergetest' | clickhouse-client || exit 1

3
debian/clickhouse-server-base.install vendored
View File

@ -1,5 +1,6 @@
/usr/bin/clickhouse /usr/bin/clickhouse
/usr/bin/clickhouse-server /usr/bin/clickhouse-server
/etc/init.d/clickhouse-server
/etc/cron.d/clickhouse-server /etc/cron.d/clickhouse-server
/etc/security/limits.d/metrika.conf
/usr/share/clickhouse/* /usr/share/clickhouse/*
/etc/security/limits.d/clickhouse.conf

62
debian/clickhouse-server-base.postinst vendored
View File

@ -1,6 +1,14 @@
#!/bin/sh #!/bin/sh
set -e set -e
CLICKHOUSE_USER=clickhouse
CLICKHOUSE_GROUP=${CLICKHOUSE_USER}
CLICKHOUSE_DATADIR=/var/lib/clickhouse
CLICKHOUSE_DATADIR_OLD=/opt/clickhouse # remove after 2017-06-01
CLICKHOUSE_LOGDIR=/var/log/clickhouse
CLICKHOUSE_SERVER_ETCDIR=/etc/clickhouse-server
if [ "$1" = configure ]; then if [ "$1" = configure ]; then
if [ -x "/etc/init.d/clickhouse-server" ]; then if [ -x "/etc/init.d/clickhouse-server" ]; then
@ -8,48 +16,62 @@ if [ "$1" = configure ]; then
fi fi
# Make sure the administrative user exists # Make sure the administrative user exists
if ! getent passwd metrika > /dev/null; then if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then
adduser --system --disabled-login --no-create-home --home /nonexistent \ adduser --system --disabled-login --no-create-home --home /nonexistent \
--shell /bin/false --group --gecos "Clickhouse server" metrika > /dev/null --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null
fi fi
# if the user was created manually, make sure the group is there as well # if the user was created manually, make sure the group is there as well
if ! getent group metrika > /dev/null; then if ! getent group ${CLICKHOUSE_GROUP} > /dev/null; then
addgroup --system metrika > /dev/null addgroup --system ${CLICKHOUSE_GROUP} > /dev/null
fi fi
# make sure user is in the correct group # make sure user is in the correct group
if ! id -Gn metrika | grep -qw metrika; then if ! id -Gn ${CLICKHOUSE_USER} | grep -qw ${CLICKHOUSE_USER}; then
adduser metrika metrika > /dev/null adduser ${CLICKHOUSE_USER} ${CLICKHOUSE_GROUP} > /dev/null
fi fi
# check validity of user and group # check validity of user and group
if [ "`id -u metrika`" -eq 0 ]; then if [ "`id -u ${CLICKHOUSE_USER}`" -eq 0 ]; then
echo "The metrika system user must not have uid 0 (root). echo "The ${CLICKHOUSE_USER} system user must not have uid 0 (root).
Please fix this and reinstall this package." >&2 Please fix this and reinstall this package." >&2
exit 1 exit 1
fi fi
if [ "`id -g metrika`" -eq 0 ]; then if [ "`id -g ${CLICKHOUSE_GROUP}`" -eq 0 ]; then
echo "The metrika system user must not have root as primary group. echo "The ${CLICKHOUSE_USER} system user must not have root as primary group.
Please fix this and reinstall this package." >&2 Please fix this and reinstall this package." >&2
exit 1 exit 1
fi fi
if [ ! -d "/opt/clickhouse" ]; then if [ ! -d ${CLICKHOUSE_DATADIR} ]; then
# only for compatibility for old /opt/clickhouse, remove after 2017-06-01
if [ -d ${CLICKHOUSE_DATADIR_OLD} ]; then
ln -s ${CLICKHOUSE_DATADIR_OLD} ${CLICKHOUSE_DATADIR}
else
# DONT remove after 2017-06-01 :
mkdir -p ${CLICKHOUSE_DATADIR}
fi
# ensure home directory ownership # ensure home directory ownership
mkdir -p /opt/clickhouse #su -s /bin/sh clickhouse -c "test -O /var/lib/clickhouse && test -G /var/lib/clickhouse" || \
#su -s /bin/sh metrika -c "test -O /opt/clickhouse && test -G /opt/clickhouse" || \ chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR}
chown metrika:metrika /opt/clickhouse chmod 0700 ${CLICKHOUSE_DATADIR}
chmod 0700 /opt/clickhouse
fi fi
if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then
mkdir -p ${CLICKHOUSE_LOGDIR}
chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}
chmod 1770 ${CLICKHOUSE_LOGDIR}
fi
# -R only for compatibility for old metrika user, remove -R after 2017-06-01
su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_SERVER_ETCDIR}" || chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_SERVER_ETCDIR}
# Clean old dynamic compilation results # Clean old dynamic compilation results
if [ -d "/opt/clickhouse/build" ]; then if [ -d "${CLICKHOUSE_DATADIR}/build" ]; then
rm -f /opt/clickhouse/build/*.cpp /opt/clickhouse/build/*.so ||: rm -f ${CLICKHOUSE_DATADIR}/build/*.cpp ${CLICKHOUSE_DATADIR}/build/*.so ||:
fi fi
fi fi
exit 0

33
debian/rules vendored
View File

@ -84,16 +84,6 @@ install: build
# создаем дефолтный cron, если нет крон-файла созданного пользователем # создаем дефолтный cron, если нет крон-файла созданного пользователем
# cron_name имеет вид daemonname # cron_name имеет вид daemonname
# Для ClickHouse не должно быть лишних зависимостей. # Для ClickHouse не должно быть лишних зависимостей.
for I in debian/tmp/etc/init.d/*; \
do \
cron_name=`basename $$I`; \
if [ ! -d debian/tmp/etc/cron.d ]; then \
mkdir -p debian/tmp/etc/cron.d; \
fi; \
if [ ! -f debian/tmp/etc/cron.d/$$cron_name ]; then \
echo "#*/10 * * * * root /etc/init.d/$$cron_name condstart 1>/dev/null 2>&1" > debian/tmp/etc/cron.d/$$cron_name; \
fi; \
done; \
# Making necessary *.install files # Making necessary *.install files
for DAEMON_PKG in ${DAEMONS}; do \ for DAEMON_PKG in ${DAEMONS}; do \
if [ ! -e "debian/$$DAEMON_PKG.install" ]; then \ if [ ! -e "debian/$$DAEMON_PKG.install" ]; then \
@ -112,7 +102,7 @@ install: build
if [ ! -e "debian/$$DAEMON_PKG.postinst" ]; then \ if [ ! -e "debian/$$DAEMON_PKG.postinst" ]; then \
echo "# automatically created" > debian/$$DAEMON_PKG.postinst; \ echo "# automatically created" > debian/$$DAEMON_PKG.postinst; \
echo "mkdir -p /etc/$$DAEMON_PKG/conf.d" >> debian/$$DAEMON_PKG.postinst; \ echo "mkdir -p /etc/$$DAEMON_PKG/conf.d" >> debian/$$DAEMON_PKG.postinst; \
echo "chown -R metrika: /etc/$$DAEMON_PKG" >> debian/$$DAEMON_PKG.postinst; \ echo "chown -R clickhouse: /etc/$$DAEMON_PKG" >> debian/$$DAEMON_PKG.postinst; \
if [ -e "debian/tmp/etc/init.d/$$DAEMON_PKG" ]; then \ if [ -e "debian/tmp/etc/init.d/$$DAEMON_PKG" ]; then \
if echo $$DAEMON_PKG | grep server > /dev/null; then\ if echo $$DAEMON_PKG | grep server > /dev/null; then\
echo "update-rc.d $$DAEMON_PKG defaults > /dev/null || exit \$$?" >> debian/$$DAEMON_PKG.postinst; \ echo "update-rc.d $$DAEMON_PKG defaults > /dev/null || exit \$$?" >> debian/$$DAEMON_PKG.postinst; \
@ -123,19 +113,17 @@ install: build
\ \
else \ else \
echo >> debian/$$DAEMON_PKG.postinst; \ echo >> debian/$$DAEMON_PKG.postinst; \
echo "mkdir -p /etc/$$DAEMON_PKG/conf.d; chown -R metrika: /etc/$$DAEMON_PKG" >> debian/$$DAEMON_PKG.postinst; \ echo "mkdir -p /etc/$$DAEMON_PKG/conf.d; chown -R clickhouse: /etc/$$DAEMON_PKG" >> debian/$$DAEMON_PKG.postinst; \
fi; \ fi; \
done done
# В случае сборки clickhouse-server, добавим в пакет бинарник clang-а, ld и заголовочные файлы - для динамической компиляции. # In case building clickhouse-server, adding to package binary of clang, ld and header files - for dynamic compilation.
if [ -e debian/tmp/etc/init.d/clickhouse-server ]; then \ mkdir -p debian/tmp/usr/share/clickhouse/bin debian/tmp/usr/share/clickhouse/headers
mkdir -p debian/tmp/usr/share/clickhouse/bin debian/tmp/usr/share/clickhouse/headers; \ debian/copy_clang_binaries.sh debian/tmp/usr/share/clickhouse/bin/
debian/copy_clang_binaries.sh debian/tmp/usr/share/clickhouse/bin/; \ ./copy_headers.sh . debian/tmp/usr/share/clickhouse/headers
./copy_headers.sh . debian/tmp/usr/share/clickhouse/headers; \
fi;
# Создаём документацию # Making docs
cp LICENSE debian/copyright cp LICENSE debian/copyright
for DAEMON_PKG in ${DAEMONS}; do \ for DAEMON_PKG in ${DAEMONS}; do \
@ -148,6 +136,13 @@ install: build
ln -s clickhouse-server.docs debian/clickhouse-server-common.docs ln -s clickhouse-server.docs debian/clickhouse-server-common.docs
ln -s clickhouse-server.docs debian/clickhouse-server-metrika.docs ln -s clickhouse-server.docs debian/clickhouse-server-metrika.docs
mkdir -p debian/tmp/etc/security/limits.d
cp tools/etc/security/limits.d/clickhouse.conf debian/tmp/etc/security/limits.d
mkdir -p debian/tmp/etc/init.d
cp tools/etc/init.d/clickhouse-server debian/tmp/etc/init.d
mkdir -p debian/tmp/etc/cron.d
cp tools/etc/cron.d/clickhouse-server debian/tmp/etc/cron.d
# Build architecture-independent files here. # Build architecture-independent files here.
binary-indep: build install binary-indep: build install
# We have nothing to do by default. # We have nothing to do by default.

4
libs/libcommon/cmake/version.cmake
View File

@ -1,6 +1,6 @@
#This strings autochanged from release_lib.sh : #This strings autochanged from release_lib.sh :
set(VERSION_DESCRIBE v1.1.54140-testing) set(VERSION_DESCRIBE v1.1.54141-testing)
set(VERSION_REVISION 54140) set(VERSION_REVISION 54141)
#===end of autochange #===end of autochange
set(VERSION_MAJOR 1) set(VERSION_MAJOR 1)

28
libs/libcommon/include/common/readline_use.h Normal file
View File

@ -0,0 +1,28 @@
#pragma once
#include <common/config_common.h>
/// Different line editing libraries can be used depending on the environment.
#if USE_READLINE
#include <readline/readline.h>
#include <readline/history.h>
#elif USE_LIBEDIT
#include <editline/readline.h>
#include <editline/history.h>
#else
#include <string>
#include <cstring>
#include <iostream>
inline char * readline(const char * prompt)
{
std::string s;
std::cout << prompt;
std::getline(std::cin, s);
if (!std::cin.good())
return nullptr;
return strdup(s.data());
}
#define add_history(...) do {} while (0);
#define rl_bind_key(...) do {} while (0);
#endif

11
libs/libcommon/src/get_revision_lib.sh
View File

@ -1,14 +1,5 @@
# Filter non-release tags
function tag_filter {
grep -E "^v1\.1\.[0-9]{5}-testing$"
}
# Get last revision number # Get last revision number
function get_revision { function get_revision {
BASEDIR=`dirname "$0"` BASEDIR=`dirname "$0"`
CURDIR=`pwd` grep "set(VERSION_REVISION" ${BASEDIR}/libs/libcommon/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\))$/\1/'
cd ${BASEDIR}
git rev-parse --git-dir >/dev/null 2>/dev/null || cd ${CURDIR}
git tag | tag_filter | tail -1 | sed 's/^v1\.1\.\(.*\)-testing$/\1/'
cd ${CURDIR}
} }

3
libs/libzkutil/src/tests/zkutil_test.cpp
View File

@ -1,11 +1,10 @@
#include <zkutil/ZooKeeper.h> #include <zkutil/ZooKeeper.h>
#include <zkutil/KeeperException.h> #include <zkutil/KeeperException.h>
#include <iostream> #include <iostream>
#include <readline/readline.h>
#include <readline/history.h>
#include <sstream> #include <sstream>
#include <Poco/ConsoleChannel.h> #include <Poco/ConsoleChannel.h>
#include <common/logger_useful.h> #include <common/logger_useful.h>
#include <common/readline_use.h>
#include <DB/IO/ReadHelpers.h> #include <DB/IO/ReadHelpers.h>
#include <DB/IO/ReadBufferFromString.h> #include <DB/IO/ReadBufferFromString.h>

1
tools/etc/cron.d/clickhouse-server Normal file
View File

@ -0,0 +1 @@
#*/10 * * * * root /etc/init.d/clickhouse-server condstart 1>/dev/null 2>&1

151
tools/init.d/template → tools/etc/init.d/clickhouse-server
View File

@ -1,31 +1,37 @@
#!/bin/bash #!/bin/sh
### BEGIN INIT INFO ### BEGIN INIT INFO
# Provides: @DAEMON@ # Provides: clickhouse-server
# Default-Start: 2 3 4 5 # Default-Start: 2 3 4 5
# Default-Stop: 0 1 6 # Default-Stop: 0 1 6
# Required-Start: # Required-Start:
# Required-Stop: # Required-Stop:
# Short-Description: Yandex daemon # Short-Description: Yandex clickhouse-server daemon
### END INIT INFO ### END INIT INFO
USER=metrika
GROUP=metrika CLICKHOUSE_USER=clickhouse
CLICKHOUSE_GROUP=${CLICKHOUSE_USER}
SHELL=/bin/bash SHELL=/bin/bash
PROGRAM=@DAEMON@ PROGRAM=clickhouse-server
SYSCONFDIR=/etc/$PROGRAM SYSCONFDIR=/etc/$PROGRAM
LOGDIR=/var/log/$PROGRAM CLICKHOUSE_LOGDIR=/var/log/$CLICKHOUSE_USER
CLICKHOUSE_LOGDIR_USER=root
CLICKHOUSE_DATADIR=/var/lib/$CLICKHOUSE_USER
CLICKHOUSE_DATADIR_OLD=/opt/clickhouse
LOCALSTATEDIR=/var/lock LOCALSTATEDIR=/var/lock
BINDIR=/usr/bin BINDIR=/usr/bin
CRONFILE=/etc/cron.d/@CRONFILE@ CLICKHOUSE_CRONFILE=/etc/cron.d/clickhouse-server
CNFFILE=$SYSCONFDIR/config.xml CLICKHOUSE_CONFIG=$SYSCONFDIR/config.xml
LOCKFILE=$LOCALSTATEDIR/$PROGRAM LOCKFILE=$LOCALSTATEDIR/$PROGRAM
RETVAL=0 RETVAL=0
DEFAULT_NUMBER_OF_PROCESSES=1
NUMBER_OF_PROCESSES=$DEFAULT_NUMBER_OF_PROCESSES
LOG_FILE="" LOG_FILE=""
CLICKHOUSE_PIDDIR=/var/run/$PROGRAM
CLICKHOUSE_PIDFILE="$CLICKHOUSE_PIDDIR/$PROGRAM.pid"
# Override defaults from optional config file
test -f /etc/default/clickhouse && . /etc/default/clickhouse
# On x86_64, check for required instruction set. # On x86_64, check for required instruction set.
if uname -mpi | grep -q 'x86_64'; then if uname -mpi | grep -q 'x86_64'; then
if ! grep -q 'sse4_2' /proc/cpuinfo; then if ! grep -q 'sse4_2' /proc/cpuinfo; then
@ -42,8 +48,8 @@ if uname -mpi | grep -q 'x86_64'; then
# gcc -xc -Os -static -nostdlib - <<< 'void _start() { __asm__("pcmpgtq %%xmm0, %%xmm1; mov $0x3c, %%rax; xor %%rdi, %%rdi; syscall":::"memory"); }' && strip -R .note.gnu.build-id -R .comment -R .eh_frame -s ./a.out && gzip -c -9 ./a.out | base64 -w0; echo # gcc -xc -Os -static -nostdlib - <<< 'void _start() { __asm__("pcmpgtq %%xmm0, %%xmm1; mov $0x3c, %%rax; xor %%rdi, %%rdi; syscall":::"memory"); }' && strip -R .note.gnu.build-id -R .comment -R .eh_frame -s ./a.out && gzip -c -9 ./a.out | base64 -w0; echo
if ! (echo -n 'H4sICAwAW1cCA2Eub3V0AKt39XFjYmRkgAEmBjsGEI+H0QHMd4CKGyCUAMUsGJiBJDNQNUiYlQEZOKDQclB9cnD9CmCSBYqJBRxQOvBpSQobGfqIAWn8FuYnPI4fsAGyPQz/87MeZtArziguKSpJTGLQK0mtKGGgGHADMSgoYH6AhTMPNHyE0NQzYuEzYzEXFr6CBPQDANAsXKTwAQAA' | base64 -d | gzip -d > /tmp/clickhouse_test_sse42 && chmod a+x /tmp/clickhouse_test_sse42 && /tmp/clickhouse_test_sse42); then if ! (echo -n 'H4sICAwAW1cCA2Eub3V0AKt39XFjYmRkgAEmBjsGEI+H0QHMd4CKGyCUAMUsGJiBJDNQNUiYlQEZOKDQclB9cnD9CmCSBYqJBRxQOvBpSQobGfqIAWn8FuYnPI4fsAGyPQz/87MeZtArziguKSpJTGLQK0mtKGGgGHADMSgoYH6AhTMPNHyE0NQzYuEzYzEXFr6CBPQDANAsXKTwAQAA' | base64 -d | gzip -d > /tmp/clickhouse_test_sse42 && chmod a+x /tmp/clickhouse_test_sse42 && /tmp/clickhouse_test_sse42); then
echo 'SSE 4.2 instruction set is not supported' echo 'Warning! SSE 4.2 instruction set is not supported'
exit 3 #exit 3
fi fi
fi fi
fi fi
@ -51,15 +57,11 @@ fi
# С помощью xmlstarlet пытаемся взять некоторые параметры из конфига # С помощью xmlstarlet пытаемся взять некоторые параметры из конфига
if command -v xmlstarlet >/dev/null 2>&1; then if command -v xmlstarlet >/dev/null 2>&1; then
NUMBER_OF_PROCESSES=$(xmlstarlet sel -t -v "/yandex/number_of_processes" $CNFFILE || echo $DEFAULT_NUMBER_OF_PROCESSES) LOG_FILE=$(xmlstarlet sel -t -v "/yandex/logger/log" $CLICKHOUSE_CONFIG || echo $LOG_FILE)
LOG_FILE=$(xmlstarlet sel -t -v "/yandex/logger/log" $CNFFILE || echo $LOG_FILE)
fi fi
PIDDIR=/var/run/$PROGRAM
PIDFILE_PREFIX=$PIDDIR/$PROGRAM
PIDFILE_RE="$PIDFILE_PREFIX[0-9]*.pid"
SUPPORTED_COMMANDS="{start|stop|status|restart|forcestop|forcerestart|reload|condstart|condstop|condrestart|condreload}" SUPPORTED_COMMANDS="{start|stop|status|restart|forcestop|forcerestart|reload|condstart|condstop|condrestart|condreload|initdb}"
is_supported_command() is_supported_command()
{ {
echo $SUPPORTED_COMMANDS | grep -E "(\{|\|)$1(\||})" &> /dev/null echo $SUPPORTED_COMMANDS | grep -E "(\{|\|)$1(\||})" &> /dev/null
@ -67,66 +69,49 @@ is_supported_command()
generate_program_name() generate_program_name()
{ {
if [ $NUMBER_OF_PROCESSES -eq 1 ]; then
echo $PROGRAM echo $PROGRAM
else
echo $PROGRAM$1
fi
} }
generate_pid_name() generate_pid_name()
{ {
if [ $NUMBER_OF_PROCESSES -gt 1 ]; then echo $CLICKHOUSE_PIDFILE
echo $PIDFILE_PREFIX$1.pid
else
echo $PIDFILE_PREFIX.pid
fi
} }
specific_log_file_for_each_process() specific_log_file_for_each_process()
{ {
# Не будем менять имя лог файла, при одном процессе
# Не выставляем, если имя лог файла пустое
if [ $NUMBER_OF_PROCESSES -gt 1 ] && [ "$LOG_FILE" != "" ]; then
log_file=$(echo $LOG_FILE | sed "s/\.log/$1.log/")
echo "--log-file=\"$log_file\""
fi
echo "" echo ""
} }
find_pid_files() find_pid_files()
{ {
[[ -e $PIDDIR ]] && find $PIDDIR -regex "$PIDFILE_RE" [ -e $CLICKHOUSE_PIDDIR ] && find $CLICKHOUSE_PIDFILE
} }
is_running() is_running()
{ {
pidfile=$1 pidfile=$1
[ -r "$pidfile" ] && pgrep -s $(cat "$pidfile") 1> /dev/null 2> /dev/null [ -r $pidfile ] && pgrep -s $(cat "$pidfile") 1> /dev/null 2> /dev/null
} }
running_processes() running_processes()
{ {
pidfiles=$(find_pid_files) pidfiles=$(find_pid_files)
running=0 running=0
for pidfile in $pidfiles; do if is_running $CLICKHOUSE_PIDFILE; then
if is_running $pidfile; then
running=$(($running + 1)) running=$(($running + 1))
fi fi
done
echo $running echo $running
} }
any_runs() any_runs()
{ {
if [[ $(running_processes) -gt 0 ]]; then return 0; else return 1; fi if [ $(running_processes) -gt 0 ]; then return 0; else return 1; fi
} }
all_runs() all_runs()
{ {
[[ $(running_processes) -eq $NUMBER_OF_PROCESSES ]] [ $(running_processes) -eq 1 ]
} }
wait4done() wait4done()
@ -136,6 +121,39 @@ wait4done()
done done
} }
initdb()
{
if ! getent group ${CLICKHOUSE_USER} >/dev/null; then
echo "Can't chown to non-existing user ${CLICKHOUSE_USER}"
return
fi
if ! getent passwd ${CLICKHOUSE_GROUP} >/dev/null; then
echo "Can't chown to non-existing group ${CLICKHOUSE_GROUP}"
return
fi
if ! `su -s $SHELL ${CLICKHOUSE_USER} -c "test -r ${CLICKHOUSE_CONFIG}"`; then
echo "Warning! clickhouse config [${CLICKHOUSE_CONFIG}] not readable by user [${CLICKHOUSE_USER}]"
fi
if ! `su -s $SHELL ${CLICKHOUSE_USER} -c "test -O ${CLICKHOUSE_DATADIR} && test -G ${CLICKHOUSE_DATADIR}"`; then
echo "Changing owner of [${CLICKHOUSE_DATADIR}] to [${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}]"
chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR}
fi
if ! `su -s $SHELL ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_LOGDIR}"`; then
echo "Changing owner of [${CLICKHOUSE_LOGDIR}] to [${CLICKHOUSE_LOGDIR_USER}:${CLICKHOUSE_GROUP}]"
chown -R ${CLICKHOUSE_LOGDIR_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}
fi
#Temporary fix for old metrika user, remove after 2017-06-01
if [ ! -z ${CLICKHOUSE_DATADIR_OLD} ] && [ -d ${CLICKHOUSE_DATADIR_OLD} ] && [ ! `su -s $SHELL ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_LOGDIR}"` ]; then
echo "Changing owner of old [${CLICKHOUSE_DATADIR_OLD}] to [${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}]"
chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR_OLD}
fi
}
start() start()
{ {
[ -x $BINDIR/$PROGRAM ] || exit 0 [ -x $BINDIR/$PROGRAM ] || exit 0
@ -149,26 +167,21 @@ start()
echo -n "already running " echo -n "already running "
EXIT_STATUS=1 EXIT_STATUS=1
else else
mkdir -p $LOGDIR mkdir -p $CLICKHOUSE_PIDDIR
mkdir -p $PIDDIR chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR
chown -R $USER:$GROUP $LOGDIR initdb
chown -R $USER:$GROUP $PIDDIR if ! is_running $CLICKHOUSE_PIDFILE; then
chown -R $USER:$GROUP $SYSCONFDIR rm -f $CLICKHOUSE_PIDFILE
for i in $(seq 1 $NUMBER_OF_PROCESSES); do
if ! is_running $(generate_pid_name $i); then
rm -f $(generate_pid_name $i)
# чтобы лок не удерживался в течении времени жизни дочернего процесса, освободим лок # чтобы лок не удерживался в течении времени жизни дочернего процесса, освободим лок
su -l $USER -s $SHELL -c "flock -u 9; exec -a $(generate_program_name $i) \"$BINDIR/$PROGRAM\" --daemon --pid-file=\"$(generate_pid_name $i)\" --config-file=\"$CNFFILE\" $(specific_log_file_for_each_process $i)" su -l $CLICKHOUSE_USER -s $SHELL -c "flock -u 9; exec -a $(generate_program_name $i) \"$BINDIR/$PROGRAM\" --daemon --pid-file=\"$CLICKHOUSE_PIDFILE\" --config-file=\"$CLICKHOUSE_CONFIG\" $(specific_log_file_for_each_process $i)"
EXIT_STATUS=$? EXIT_STATUS=$?
if [[ $EXIT_STATUS -ne 0 ]]; then if [ $EXIT_STATUS -ne 0 ]; then
break break
fi fi
fi fi
done
fi fi
if [[ $EXIT_STATUS -eq 0 ]]; then if [ $EXIT_STATUS -eq 0 ]; then
echo "DONE" echo "DONE"
else else
echo "FAILED" echo "FAILED"
@ -184,9 +197,7 @@ stop()
echo -n "Stop $PROGRAM service: " echo -n "Stop $PROGRAM service: "
for pid_file in $(find_pid_files); do kill -TERM `cat "$CLICKHOUSE_PIDFILE"`
kill -TERM `cat "$pid_file"`
done
wait4done wait4done
@ -207,9 +218,7 @@ forcestop()
echo -n "Stop $PROGRAM service: " echo -n "Stop $PROGRAM service: "
for pid_file in $(find_pid_files); do kill -9 `cat "$CLICKHOUSE_PIDFILE"`
kill -9 `cat "$pid_file"`
done
wait4done wait4done
@ -225,17 +234,18 @@ forcerestart()
enable_cron() enable_cron()
{ {
sed -i 's/^#*//' "$CRONFILE" [ ! -z $CLICKHOUSE_CRONFILE ] && sed -i 's/^#*//' "$CLICKHOUSE_CRONFILE"
} }
disable_cron() disable_cron()
{ {
sed -i 's/^#*/#/' "$CRONFILE" [ ! -z $CLICKHOUSE_CRONFILE ] && sed -i 's/^#*/#/' "$CLICKHOUSE_CRONFILE"
} }
is_cron_disabled() is_cron_disabled()
{ {
[[ `grep -E "^#.*" $CRONFILE` == `cat $CRONFILE` ]]; [ -z $CLICKHOUSE_CRONFILE ] && return 0
[[ `grep -E "^#.*" $CLICKHOUSE_CRONFILE` == `cat $CLICKHOUSE_CRONFILE` ]];
} }
main() main()
@ -273,6 +283,9 @@ main()
condreload) condreload)
any_runs && restart any_runs && restart
;; ;;
initdb)
initdb
;;
esac esac
exit $EXIT_STATUS exit $EXIT_STATUS
@ -280,13 +293,13 @@ main()
status() status()
{ {
if [[ $(running_processes) -eq $NUMBER_OF_PROCESSES ]]; then if [ $(running_processes) -eq 1 ]; then
echo "$PROGRAM service is running" echo "$PROGRAM service is running"
else else
if is_cron_disabled; then if is_cron_disabled; then
echo "$PROGRAM service is stopped"; echo "$PROGRAM service is stopped";
else else
echo "$PROGRAM: $(($NUMBER_OF_PROCESSES - $(running_processes))) of $NUMBER_OF_PROCESSES processes unexpectedly terminated" echo "$PROGRAM: $((1 - $(running_processes))) of 1 processes unexpectedly terminated"
fi fi
fi fi
} }
@ -297,7 +310,7 @@ if ! is_supported_command "$1"; then
exit 2 exit 2
fi fi
if [[ "$1" == "status" ]]; then if "$1" == "status"; then
status status
exit 0 exit 0
fi fi

2
tools/etc/security/limits.d/clickhouse.conf Normal file
View File

@ -0,0 +1,2 @@
clickhouse soft nofile 262144
clickhouse hard nofile 262144