diff --git a/CHANGELOG.md b/CHANGELOG.md index ef7ff17d2ee..b9c71f69f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,7 +79,7 @@ This is a bugfix release. The following bugs were fixed: ## Major changes: -* Improved security: all server files are created with 0640 permissions. +* Improved security: all server files are created with 0640 permissions (can be changed via config parameter). * Improved error messages for queries with invalid syntax. * Significantly reduced memory consumption and improved performance when merging large sections of MergeTree data. * Significantly increased the performance of data merges for the ReplacingMergeTree engine. diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index 97dce55a89e..f48d5e43a4c 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -77,7 +77,7 @@ * Добавлен параметр max_size для агрегатной функции `groupArray(max_size)(column)`, и оптимизирована её производительность ## Основные изменения: -* Улучшение безопасности: все файлы сервера создаются с правами 0640 +* Улучшение безопасности: все файлы сервера создаются с правами 0640 (можно поменять, через параметр в конфиге). * Улучшены сообщения об ошибках в случае синтаксически неверных запросов * Значительно уменьшен расход оперативной памяти и улучшена производительность слияний больших MergeTree-кусков данных * Значительно увеличена производительность слияний данных для движка ReplacingMergeTree diff --git a/dbms/src/Server/PerformanceTest.cpp b/dbms/src/Server/PerformanceTest.cpp index 26e779bfac4..c975898eb37 100644 --- a/dbms/src/Server/PerformanceTest.cpp +++ b/dbms/src/Server/PerformanceTest.cpp @@ -931,7 +931,7 @@ private: else throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name); - times_to_run = test_config->getUInt("times_to_run"); + times_to_run = test_config->getUInt("times_to_run", 1); stop_conditions_by_run.clear(); TestStopConditions stop_conditions_template; diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index f39186487ac..a1574e00e00 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -22,10 +22,10 @@ StorageSystemTables::StorageSystemTables(const std::string & name_) : name(name_), columns { - {"database", std::make_shared()}, - {"name", std::make_shared()}, - {"engine", std::make_shared()}, - {"metadata_modification_time", std::make_shared()} + {"database", std::make_shared()}, + {"name", std::make_shared()}, + {"engine", std::make_shared()}, + {"metadata_modification_time", std::make_shared()} } { } diff --git a/dbms/tests/queries/0_stateless/00507_nullable.reference b/dbms/tests/queries/0_stateless/00507_nullable.reference deleted file mode 100644 index 1c42f9991e2..00000000000 --- a/dbms/tests/queries/0_stateless/00507_nullable.reference +++ /dev/null @@ -1,2 +0,0 @@ -1970-01-02 2 0 FAILED 2 -still alive diff --git a/dbms/tests/queries/0_stateless/00507_nullable.sql b/dbms/tests/queries/0_stateless/00507_nullable.sql deleted file mode 100644 index 46e7aef5614..00000000000 --- a/dbms/tests/queries/0_stateless/00507_nullable.sql +++ /dev/null @@ -1,41 +0,0 @@ -CREATE DATABASE IF NOT EXISTS test; -DROP TABLE IF EXISTS test.runs; -DROP TABLE IF EXISTS test.tests; - -CREATE TABLE test.runs -( - date Date, - id UInt64, - t_id UInt64, - status Enum8('OK' = 1, 'FAILED' = 2, 'SKIPPED' = 3, 'DISCOVERED' = 4), - run_id UInt64 DEFAULT id -) ENGINE = MergeTree(date, (t_id, id), 8192); - -CREATE TABLE test.tests -( - date Date, - id UInt64, - path Nullable(String), - suite_id Nullable(String) -) ENGINE = MergeTree(date, id, 8192); - -INSERT INTO test.tests (date, id) VALUES (1,1); -INSERT INTO test.runs (date, id) VALUES (1,1); -INSERT INTO test.runs (date, id, status) VALUES (1,2, 'FAILED'); -INSERT INTO test.tests (date, id, path) VALUES (1,2 ,'rtline1'); - -SELECT * -FROM test.runs AS r -WHERE (r.status = 'FAILED') AND ( -( - SELECT path - FROM test.tests AS t - WHERE t.id = r.id - LIMIT 1 -) LIKE 'rtline%') -LIMIT 1; - -SELECT 'still alive'; - -DROP TABLE test.runs; -DROP TABLE test.tests; diff --git a/debian/clickhouse-server-base.postinst b/debian/clickhouse-server-base.postinst index 3dc07fcb384..476dccfd271 100644 --- a/debian/clickhouse-server-base.postinst +++ b/debian/clickhouse-server-base.postinst @@ -4,81 +4,67 @@ set -e CLICKHOUSE_USER=clickhouse CLICKHOUSE_GROUP=${CLICKHOUSE_USER} CLICKHOUSE_DATADIR=/var/lib/clickhouse -CLICKHOUSE_DATADIR_OLD=/opt/clickhouse # remove after 2017-06-01 CLICKHOUSE_LOGDIR=/var/log/clickhouse-server -CLICKHOUSE_SERVER_ETCDIR=/etc/clickhouse-server if [ "$1" = configure ]; then - if [ -x "/etc/init.d/clickhouse-server" ]; then - update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $? - fi + if [ -x "/etc/init.d/clickhouse-server" ]; then + update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $? + fi - # Make sure the administrative user exists - if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then - adduser --system --disabled-login --no-create-home --home /nonexistent \ - --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null - fi + # Make sure the administrative user exists + if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then + adduser --system --disabled-login --no-create-home --home /nonexistent \ + --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null + fi - # if the user was created manually, make sure the group is there as well - if ! getent group ${CLICKHOUSE_GROUP} > /dev/null; then - addgroup --system ${CLICKHOUSE_GROUP} > /dev/null - fi + # if the user was created manually, make sure the group is there as well + if ! getent group ${CLICKHOUSE_GROUP} > /dev/null; then + addgroup --system ${CLICKHOUSE_GROUP} > /dev/null + fi - # make sure user is in the correct group - if ! id -Gn ${CLICKHOUSE_USER} | grep -qw ${CLICKHOUSE_USER}; then - adduser ${CLICKHOUSE_USER} ${CLICKHOUSE_GROUP} > /dev/null - fi + # make sure user is in the correct group + if ! id -Gn ${CLICKHOUSE_USER} | grep -qw ${CLICKHOUSE_USER}; then + adduser ${CLICKHOUSE_USER} ${CLICKHOUSE_GROUP} > /dev/null + fi - # check validity of user and group - if [ "`id -u ${CLICKHOUSE_USER}`" -eq 0 ]; then - echo "The ${CLICKHOUSE_USER} system user must not have uid 0 (root). + # check validity of user and group + if [ "`id -u ${CLICKHOUSE_USER}`" -eq 0 ]; then + echo "The ${CLICKHOUSE_USER} system user must not have uid 0 (root). Please fix this and reinstall this package." >&2 - exit 1 - fi + exit 1 + fi - if [ "`id -g ${CLICKHOUSE_GROUP}`" -eq 0 ]; then - echo "The ${CLICKHOUSE_USER} system user must not have root as primary group. + if [ "`id -g ${CLICKHOUSE_GROUP}`" -eq 0 ]; then + echo "The ${CLICKHOUSE_USER} system user must not have root as primary group. Please fix this and reinstall this package." >&2 - exit 1 - fi + exit 1 + fi - if [ ! -d ${CLICKHOUSE_DATADIR} ]; then - # only for compatibility for old /opt/clickhouse, remove after 2017-06-01 - if [ -d ${CLICKHOUSE_DATADIR_OLD} ]; then - ln -s ${CLICKHOUSE_DATADIR_OLD} ${CLICKHOUSE_DATADIR} - else - # DONT remove after 2017-06-01 : - mkdir -p ${CLICKHOUSE_DATADIR} - chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR} - chmod 700 ${CLICKHOUSE_DATADIR} - fi - fi + if [ ! -d ${CLICKHOUSE_DATADIR} ]; then + mkdir -p ${CLICKHOUSE_DATADIR} + chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR} + chmod 700 ${CLICKHOUSE_DATADIR} + fi - if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then - mkdir -p ${CLICKHOUSE_LOGDIR} - chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR} - # Allow everyone to read logs, root and clickhouse to read-write - chmod 775 ${CLICKHOUSE_LOGDIR} - fi + if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then + mkdir -p ${CLICKHOUSE_LOGDIR} + chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR} + # Allow everyone to read logs, root and clickhouse to read-write + chmod 775 ${CLICKHOUSE_LOGDIR} + fi - if [ -d ${CLICKHOUSE_LOGDIR} ]; then - # only for compatibility for old metrika user, remove string after 2017-06-01 : - su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_LOGDIR}" || chown -R root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}; chmod -R ug+rw ${CLICKHOUSE_LOGDIR} - fi + if [ -d ${CLICKHOUSE_LOGDIR} ]; then + # only for compatibility for old metrika user, remove string after 2017-06-01 + su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_LOGDIR}" || chown -R root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}; chmod -R ug+rw ${CLICKHOUSE_LOGDIR} + fi - if [ -d ${CLICKHOUSE_SERVER_ETCDIR} ]; then - # -R only for compatibility for old metrika user, remove -R after 2017-06-01 - su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_SERVER_ETCDIR}" || chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_SERVER_ETCDIR} - fi - - - # Clean old dynamic compilation results - if [ -d "${CLICKHOUSE_DATADIR}/build" ]; then - rm -f ${CLICKHOUSE_DATADIR}/build/*.cpp ${CLICKHOUSE_DATADIR}/build/*.so ||: - fi + # Clean old dynamic compilation results + if [ -d "${CLICKHOUSE_DATADIR}/build" ]; then + rm -f ${CLICKHOUSE_DATADIR}/build/*.cpp ${CLICKHOUSE_DATADIR}/build/*.so ||: + fi fi diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 9e51bf417c2..3261825b74b 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -95,6 +95,10 @@ check_config() initdb() { + if [ -d ${SYSCONFDIR} ]; then + su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${SYSCONFDIR}" || chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${SYSCONFDIR} + fi + if [ -x "$BINDIR/$GENERIC_PROGRAM" ]; then CLICKHOUSE_DATADIR_FROM_CONFIG=$(su -s $SHELL ${CLICKHOUSE_USER} -c "$BINDIR/$GENERIC_PROGRAM extract-from-config --config-file=\"$CLICKHOUSE_CONFIG\" --key=path") if [ "(" "$?" -ne "0" ")" -o "(" -z "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ")" ]; then @@ -138,12 +142,6 @@ initdb() echo "Changing owner of [${CLICKHOUSE_LOGDIR}] to [${CLICKHOUSE_LOGDIR_USER}:${CLICKHOUSE_GROUP}]" chown ${CLICKHOUSE_LOGDIR_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR} fi - - # Temporary fix for old metrika user, remove after 2017-06-01 - if [ ! -z ${CLICKHOUSE_DATADIR_OLD} ] && [ -d ${CLICKHOUSE_DATADIR_OLD} ] && ! $(su -s $SHELL ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_DATADIR_OLD}") ; then - echo "Changing owner of old [${CLICKHOUSE_DATADIR_OLD}] to [${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}]" - chown -RL ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR_OLD} - fi } diff --git a/docs/en/interfaces/third-party_client_libraries.rst b/docs/en/interfaces/third-party_client_libraries.rst index 0ee86db23f7..cf87beb66f2 100644 --- a/docs/en/interfaces/third-party_client_libraries.rst +++ b/docs/en/interfaces/third-party_client_libraries.rst @@ -7,6 +7,7 @@ There exist third-party client libraries for ClickHouse: - `infi.clickhouse_orm `_ - `sqlalchemy-clickhouse `_ - `clickhouse-driver `_ + - `clickhouse-client `_ * PHP - `clickhouse-php-client `_ - `PhpClickHouseClient `_ diff --git a/docs/en/operations/access_rights.rst b/docs/en/operations/access_rights.rst index a2e0c8da2cb..020bb64f5f5 100644 --- a/docs/en/operations/access_rights.rst +++ b/docs/en/operations/access_rights.rst @@ -40,7 +40,9 @@ Users are recorded in the ``users`` section. Let's look at part of the ``users.x Here we can see that two users are declared: ``default`` and ``web``. We added the ``web`` user ourselves. -The ``default`` user is chosen in cases when the username is not passed, so this user must be present in the config file. The ``default`` user is also used for distributed query processing - the system accesses remote servers under this username. So the ``default`` user must have an empty password and must not have substantial restrictions or quotas - otherwise, distributed queries will fail. +The ``default`` user is chosen in cases when the username is not passed. The ``default`` user can also be used for distributed query processing - the system accesses remote servers using this username if no ``user`` and ``password`` were configured for that server inside cluster configuration (see also section about "Distributed" table engine). + +For connection to the server inside cluster you should use the user without any substantial restrictions or quotas - otherwise, distributed queries will fail. The password is specified in plain text directly in the config. In this regard, you should not consider these passwords as providing security against potential malicious attacks. Rather, they are necessary for protection from Yandex employees. diff --git a/docs/ru/interfaces/third-party_client_libraries.rst b/docs/ru/interfaces/third-party_client_libraries.rst index 7460e2300c0..d35207965e0 100644 --- a/docs/ru/interfaces/third-party_client_libraries.rst +++ b/docs/ru/interfaces/third-party_client_libraries.rst @@ -7,6 +7,7 @@ - `infi.clickhouse_orm `_ - `sqlalchemy-clickhouse `_ - `clickhouse-driver `_ + - `clickhouse-client `_ * PHP - `clickhouse-php-client `_ - `PhpClickHouseClient `_ diff --git a/docs/ru/operations/access_rights.rst b/docs/ru/operations/access_rights.rst index 4078b357252..83bb19c7808 100644 --- a/docs/ru/operations/access_rights.rst +++ b/docs/ru/operations/access_rights.rst @@ -60,7 +60,9 @@ Здесь видно объявление двух пользователей - ``default`` и ``web``. Пользователя ``web`` мы добавили самостоятельно. -Пользователь ``default`` выбирается в случаях, когда имя пользователя не передаётся, поэтому такой пользователь должен присутствовать в конфигурационном файле обязательно. Также пользователь ``default`` используется при распределённой обработки запроса - система ходит на удалённые серверы под ним. Поэтому, у пользователя ``default`` должен быть пустой пароль и не должно быть выставлено существенных ограничений или квот - иначе распределённые запросы сломаются. +Пользователь ``default`` выбирается в случаях, когда имя пользователя не передаётся. Также пользователь ``default`` может использоваться при распределённой обработке запроса - если в конфигурации кластера для сервера не указаны ``user`` и ``password``. (см. раздел о движке "Distributed"). + +Пользователь, который используется для обмена информацией между серверами, объединенными в кластер, не должен иметь существенных ограничений или квот - иначе распределённые запросы сломаются. Пароль указывается либо в открытом виде (не рекомендуется), либо в виде SHA-256. Хэш не содержит соль. В связи с этим, не следует рассматривать такие пароли, как защиту от потенциального злоумышленника. Скорее, они нужны для защиты от сотрудников. diff --git a/utils/report/clickhouse_report.sh b/utils/report/clickhouse_report.sh index 369529b50e0..253b8812066 100755 --- a/utils/report/clickhouse_report.sh +++ b/utils/report/clickhouse_report.sh @@ -18,3 +18,5 @@ top -bn1 tail -n200 /var/log/clickhouse-server/clickhouse-server.err.log tail -n200 /var/log/clickhouse-server/clickhouse-server.log tail -n100 /var/log/clickhouse-server/stderr +cat /etc/lsb-release +uname -a