Do not write preprocessed configs to /etc/ (#2443)

CHANGELOG.draft.md

@@ -1 +1,4 @@
 * Настройка `enable_optimize_predicate_expression` выключена по-умолчанию.
+
+### Улучшения:
+* Файлы *-preprocessed.xml записываются в директорию с данными (/var/lib/clickhouse/preprocessed_configs). Для /etc/clickhouse-server больше не нужен +w для пользователя clickhouse. Для удобства создан симлинк /var/lib/clickhouse/preprocessed_configs -> /etc/clickhouse-server/preprocessed

dbms/programs/extract-from-config/ExtractFromConfig.cpp

@@ -26,18 +26,18 @@ static void setupLogging(const std::string & log_level)
 static std::string extractFromConfig(
         const std::string & config_path, const std::string & key, bool process_zk_includes, bool try_get = false)
 {
-    ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
+    DB::ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
     bool has_zk_includes;
-    XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes);
+    DB::XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes);
     if (has_zk_includes && process_zk_includes)
     {
-        ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml));
+        DB::ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml));
         zkutil::ZooKeeperPtr zookeeper = std::make_shared<zkutil::ZooKeeper>(
                 *bootstrap_configuration, "zookeeper");
         zkutil::ZooKeeperNodeCache zk_node_cache([&] { return zookeeper; });
         config_xml = processor.processConfig(&has_zk_includes, &zk_node_cache);
     }
-    ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
+    DB::ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
     // do not throw exception if not found
     if (try_get)
         return configuration->getString(key, "");

dbms/programs/local/LocalServer.cpp

@@ -115,9 +115,11 @@ try
     /// Load config files if exists
     if (config().has("config-file") || Poco::File("config.xml").exists())
     {
-        ConfigProcessor config_processor(config().getString("config-file", "config.xml"), false, true);
+        const auto config_path = config().getString("config-file", "config.xml");
+        ConfigProcessor config_processor(config_path, false, true);
+        config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
         auto loaded_config = config_processor.loadConfig();
-        config_processor.savePreprocessedConfig(loaded_config);
+        config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", DBMS_DEFAULT_PATH));
         config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
     }
 
@@ -348,7 +350,7 @@ void LocalServer::setupUsers()
         const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
         ConfigProcessor config_processor(users_config_path);
         const auto loaded_config = config_processor.loadConfig();
-        config_processor.savePreprocessedConfig(loaded_config);
+        config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
         users_config = loaded_config.configuration;
     }
     else

dbms/programs/server/Server.cpp

@@ -96,7 +96,7 @@ void Server::initialize(Poco::Util::Application & self)
 
 std::string Server::getDefaultCorePath() const
 {
-    return getCanonicalPath(config().getString("path")) + "cores";
+    return getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)) + "cores";
 }
 
 int Server::main(const std::vector<std::string> & /*args*/)
@@ -129,7 +129,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
         ConfigProcessor config_processor(config_path);
         loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
             main_config_zk_node_cache, /* fallback_to_preprocessed = */ true);
-        config_processor.savePreprocessedConfig(loaded_config);
+        config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
         config().removeConfiguration(old_configuration.get());
         config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
     }
@@ -160,7 +160,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     }
 #endif
 
-    std::string path = getCanonicalPath(config().getString("path"));
+    std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH));
     std::string default_database = config().getString("default_database", "default");
 
     global_context->setPath(path);
@@ -301,6 +301,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
     auto main_config_reloader = std::make_unique<ConfigReloader>(config_path,
         include_from_path,
+        config().getString("path", ""),
         std::move(main_config_zk_node_cache),
         [&](ConfigurationPtr config)
         {
@@ -322,6 +323,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     }
     auto users_config_reloader = std::make_unique<ConfigReloader>(users_config_path,
         include_from_path,
+        config().getString("path", ""),
         zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }),
         [&](ConfigurationPtr config) { global_context->setUsersConfig(config); },
         /* already_loaded = */ false);

dbms/src/Client/Connection.cpp

@@ -27,7 +27,6 @@
 #include <Poco/Net/SecureStreamSocket.h>
 #endif
 
-
 namespace CurrentMetrics
 {
     extern const Metric SendExternalTables;

dbms/src/Common/Config/ConfigProcessor.cpp

@@ -20,6 +20,11 @@
 
 using namespace Poco::XML;
 
+namespace DB
+{
+
+/// For cutting prerpocessed path to this base
+std::string main_config_path;
 
 /// Extracts from a string the first encountered number consisting of at least two digits.
 static std::string numberFromHost(const std::string & s)
@@ -40,13 +45,6 @@ static std::string numberFromHost(const std::string & s)
     return "";
 }
 
-static std::string preprocessedConfigPath(const std::string & path)
-{
-    Poco::Path preprocessed_path(path);
-    preprocessed_path.setBaseName(preprocessed_path.getBaseName() + PREPROCESSED_SUFFIX);
-    return preprocessed_path.toString();
-}
-
 bool ConfigProcessor::isPreprocessedFile(const std::string & path)
 {
     return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX);
@@ -59,7 +57,6 @@ ConfigProcessor::ConfigProcessor(
     bool log_to_console,
     const Substitutions & substitutions_)
     : path(path_)
-    , preprocessed_path(preprocessedConfigPath(path))
     , throw_on_bad_incl(throw_on_bad_incl_)
     , substitutions(substitutions_)
     /// We need larger name pool to allow to support vast amount of users in users.xml files for ClickHouse.
@@ -522,7 +519,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes
 
     ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
 
-    return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml};
+    return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml, path};
 }
 
 ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
@@ -556,11 +553,44 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
 
     ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
 
-    return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml};
+    return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml, path};
 }
 
-void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
+void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir)
 {
+    if (preprocessed_path.empty())
+    {
+        auto new_path = loaded_config.config_path;
+        if (new_path.substr(0, main_config_path.size()) == main_config_path)
+            new_path.replace(0, main_config_path.size(), "");
+        std::replace(new_path.begin(), new_path.end(), '/', '_');
+
+        if (preprocessed_dir.empty())
+        {
+            if (!loaded_config.configuration->has("path"))
+            {
+                // Will use current directory
+                auto parent_path = Poco::Path(loaded_config.config_path).makeParent();
+                preprocessed_dir = parent_path.toString();
+                Poco::Path poco_new_path(new_path);
+                poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX);
+                new_path = poco_new_path.toString();
+            }
+            else
+            {
+                preprocessed_dir = loaded_config.configuration->getString("path") + "/preprocessed_configs/";
+            }
+        }
+        else
+        {
+            preprocessed_dir += "/preprocessed_configs/";
+        }
+
+        preprocessed_path = preprocessed_dir + new_path;
+        auto path = Poco::Path(preprocessed_path).makeParent();
+        if (!path.toString().empty())
+            Poco::File(path).createDirectories();
+    }
     try
     {
         DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml);
@@ -570,3 +600,10 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
         LOG_WARNING(log, "Couldn't save preprocessed config to " << preprocessed_path << ": " << e.displayText());
     }
 }
+
+void ConfigProcessor::setConfigPath(const std::string & config_path)
+{
+    main_config_path = config_path;
+}
+
+}

dbms/src/Common/Config/ConfigProcessor.h

@@ -24,6 +24,9 @@ namespace zkutil
     class ZooKeeperNodeCache;
 }
 
+namespace DB
+{
+
 using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
 using XMLDocumentPtr = Poco::AutoPtr<Poco::XML::Document>;
 
@@ -72,6 +75,7 @@ public:
         bool has_zk_includes;
         bool loaded_from_preprocessed;
         XMLDocumentPtr preprocessed_xml;
+        std::string config_path;
     };
 
     /// If allow_zk_includes is true, expect that the configuration XML can contain from_zk nodes.
@@ -85,7 +89,12 @@ public:
         zkutil::ZooKeeperNodeCache & zk_node_cache,
         bool fallback_to_preprocessed = false);
 
-    void savePreprocessedConfig(const LoadedConfig & loaded_config);
+    /// Save preprocessed config to specified directory.
+    /// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/
+    void savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir);
+
+    /// Set path of main config.xml . It will be cutted from all configs placed to preprocessed_configs/
+    void setConfigPath(const std::string & config_path);
 
 public:
     using Files = std::vector<std::string>;
@@ -99,7 +108,7 @@ public:
 
 private:
     const std::string path;
-    const std::string preprocessed_path;
+    std::string preprocessed_path;
 
     bool throw_on_bad_incl;
 
@@ -127,3 +136,5 @@ private:
             zkutil::ZooKeeperNodeCache * zk_node_cache,
             std::unordered_set<std::string> & contributing_zk_paths);
 };
+
+}

dbms/src/Common/Config/ConfigReloader.cpp

@@ -15,10 +15,12 @@ constexpr decltype(ConfigReloader::reload_interval) ConfigReloader::reload_inter
 ConfigReloader::ConfigReloader(
         const std::string & path_,
         const std::string & include_from_path_,
+        const std::string & preprocessed_dir_,
         zkutil::ZooKeeperNodeCache && zk_node_cache_,
         Updater && updater_,
         bool already_loaded)
     : path(path_), include_from_path(include_from_path_)
+    , preprocessed_dir(preprocessed_dir_)
     , zk_node_cache(std::move(zk_node_cache_))
     , updater(std::move(updater_))
 {
@@ -98,7 +100,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac
             tryLogCurrentException(log, "Error loading config from `" + path + "'");
             return;
         }
-        config_processor.savePreprocessedConfig(loaded_config);
+        config_processor.savePreprocessedConfig(loaded_config, preprocessed_dir);
 
         /** We should remember last modification time if and only if config was sucessfully loaded
          * Otherwise a race condition could occur during config files update:

dbms/src/Common/Config/ConfigReloader.h

@@ -33,6 +33,7 @@ public:
     ConfigReloader(
             const std::string & path,
             const std::string & include_from_path,
+            const std::string & preprocessed_dir,
             zkutil::ZooKeeperNodeCache && zk_node_cache,
             Updater && updater,
             bool already_loaded);
@@ -70,6 +71,7 @@ private:
 
     std::string path;
     std::string include_from_path;
+    std::string preprocessed_dir;
     FilesChangesTracker files;
     zkutil::ZooKeeperNodeCache zk_node_cache;
 

dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp

@@ -23,7 +23,7 @@ int main(int argc, char ** argv)
             return 3;
         }
 
-        ConfigProcessor processor(argv[1], false, true);
+        DB::ConfigProcessor processor(argv[1], false, true);
         auto config = processor.loadConfig().configuration;
         zkutil::ZooKeeper zk(*config, "zookeeper");
         zkutil::EventPtr watch = std::make_shared<Poco::Event>();

dbms/src/Core/Defines.h

@@ -66,6 +66,8 @@
 /// the number is unmotivated
 #define DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT 15
 
+#define DBMS_DEFAULT_PATH "/var/lib/clickhouse/"
+
 // more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
 
 #if defined(_MSC_VER)

dbms/src/Interpreters/ExternalLoader.cpp

@@ -1,4 +1,5 @@
-#include <Interpreters/ExternalLoader.h>
+#include "ExternalLoader.h"
+#include <Core/Defines.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/MemoryTracker.h>
 #include <Common/Exception.h>
@@ -42,12 +43,12 @@ void ExternalLoader::reloadPeriodically()
 }
 
 
-ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config,
+ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config_main,
                                const ExternalLoaderUpdateSettings & update_settings,
                                const ExternalLoaderConfigSettings & config_settings,
                                std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
                                Logger * log, const std::string & loadable_object_name)
-        : config(config)
+        : config_main(config_main)
         , update_settings(update_settings)
         , config_settings(config_settings)
         , config_repository(std::move(config_repository))
@@ -214,7 +215,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
 
 void ExternalLoader::reloadFromConfigFiles(const bool throw_on_error, const bool force_reload, const std::string & only_dictionary)
 {
-    const auto config_paths = config_repository->list(config, config_settings.path_setting_name);
+    const auto config_paths = config_repository->list(config_main, config_settings.path_setting_name);
 
     for (const auto & config_path : config_paths)
     {
@@ -262,7 +263,7 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const
         const auto last_modified = config_repository->getLastModificationTime(config_path);
         if (force_reload || last_modified > config_last_modified)
         {
-            auto loaded_config = config_repository->load(config_path);
+            auto loaded_config = config_repository->load(config_path, config_main.getString("path", DBMS_DEFAULT_PATH));
 
             loadable_objects_defined_in_config[config_path].clear();
 

dbms/src/Interpreters/ExternalLoader.h

@@ -91,7 +91,7 @@ public:
     using ObjectsMap = std::unordered_map<std::string, LoadableInfo>;
 
     /// Objects will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
-    ExternalLoader(const Configuration & config,
+    ExternalLoader(const Configuration & config_main,
                    const ExternalLoaderUpdateSettings & update_settings,
                    const ExternalLoaderConfigSettings & config_settings,
                    std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
@@ -151,7 +151,7 @@ private:
 
     pcg64 rnd_engine{randomSeed()};
 
-    const Configuration & config;
+    const Configuration & config_main;
     const ExternalLoaderUpdateSettings & update_settings;
     const ExternalLoaderConfigSettings & config_settings;
 

dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp

@@ -61,11 +61,11 @@ Poco::Timestamp ExternalLoaderConfigRepository::getLastModificationTime(
 }
 
 Poco::AutoPtr<Poco::Util::AbstractConfiguration> ExternalLoaderConfigRepository::load(
-    const std::string & config_file) const
+    const std::string & config_file, const std::string & preprocessed_dir) const
 {
     ConfigProcessor config_processor{config_file};
     ConfigProcessor::LoadedConfig preprocessed = config_processor.loadConfig();
-    config_processor.savePreprocessedConfig(preprocessed);
+    config_processor.savePreprocessedConfig(preprocessed, preprocessed_dir);
     return preprocessed.configuration;
 }
 

dbms/src/Interpreters/ExternalLoaderConfigRepository.h

@@ -19,7 +19,7 @@ public:
 
     Poco::Timestamp getLastModificationTime(const std::string & config_file) const override;
 
-    Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file) const override;
+    Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file, const std::string & preprocessed_dir = "") const override;
 };
 
 }

dbms/src/Interpreters/IExternalLoaderConfigRepository.h

@@ -23,7 +23,7 @@ public:
 
     virtual Poco::Timestamp getLastModificationTime(const std::string & config_file) const = 0;
 
-    virtual Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file) const = 0;
+    virtual Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file, const std::string & preprocessed_dir = "") const = 0;
 
     virtual ~IExternalLoaderConfigRepository() {}
 };

dbms/src/Interpreters/tests/users.cpp

@@ -189,11 +189,11 @@ void runOneTest(const TestDescriptor & test_descriptor)
     const auto path_name = createTmpPath("users.xml");
     createFile(path_name, test_descriptor.config_content);
 
-    ConfigurationPtr config;
+    DB::ConfigurationPtr config;
 
     try
     {
-        config = ConfigProcessor(path_name).loadConfig().configuration;
+        config = DB::ConfigProcessor(path_name).loadConfig().configuration;
     }
     catch (const Poco::Exception & ex)
     {

debian/clickhouse-client.postinst

@@ -4,6 +4,3 @@ set -e
 CLICKHOUSE_USER=${CLICKHOUSE_USER=clickhouse}
 
 mkdir -p /etc/clickhouse-client/conf.d
-
-# user created by clickhouse-server package
-chown -R ${CLICKHOUSE_USER} /etc/clickhouse-client || true

debian/clickhouse-server.cron.d

@@ -1 +1 @@
-#*/10 * * * * root (which service > /dev/null 2>&1 && (service clickhouse-server condstart || true)) || /etc/init.d/clickhouse-server condstart > /dev/null 2>&1
+#*/10 * * * * root (which service > /dev/null 2>&1 && (service clickhouse-server condstart ||:)) || /etc/init.d/clickhouse-server condstart > /dev/null 2>&1

debian/clickhouse-server.init

@@ -100,10 +100,6 @@ check_config()
 
 initdb()
 {
-    if [ -d ${SYSCONFDIR} ]; then
-        su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${SYSCONFDIR}" || chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${SYSCONFDIR}
-    fi
-
     if [ -x "$BINDIR/$EXTRACT_FROM_CONFIG" ]; then
         CLICKHOUSE_DATADIR_FROM_CONFIG=$(su -s $SHELL ${CLICKHOUSE_USER} -c "$BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path")
         if [ "(" "$?" -ne "0" ")" -o "(" -z "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ")" ]; then
@@ -128,7 +124,7 @@ initdb()
     fi
 
     if ! $(su -s $SHELL ${CLICKHOUSE_USER} -c "test -O \"${CLICKHOUSE_DATADIR_FROM_CONFIG}\" && test -G \"${CLICKHOUSE_DATADIR_FROM_CONFIG}\""); then
-        if [ $(dirname "${CLICKHOUSE_DATADIR_FROM_CONFIG}") == "/" ]; then
+        if [ $(dirname "${CLICKHOUSE_DATADIR_FROM_CONFIG}") = "/" ]; then
             echo "Directory ${CLICKHOUSE_DATADIR_FROM_CONFIG} seems too dangerous to chown."
         else
             if [ ! -e "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ]; then

debian/clickhouse-server.postinst

@@ -9,7 +9,7 @@ CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR=/var/log/clickhouse-server}
 CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR=/usr/bin}
 CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM=clickhouse}
 
-OS=${OS=`lsb_release -is 2>/dev/null || uname -s || true`}
+OS=${OS=`lsb_release -is 2>/dev/null || uname -s ||:`}
 
 test -f /etc/default/clickhouse && . /etc/default/clickhouse
 
@@ -68,9 +68,6 @@ Please fix this and reinstall this package." >&2
         exit 1
     fi
 
-    if [ -d ${CLICKHOUSE_CONFDIR} ]; then
-        su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_CONFDIR}" || chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_CONFDIR}
-    fi
 
     if [ ! -d ${CLICKHOUSE_DATADIR} ]; then
         mkdir -p ${CLICKHOUSE_DATADIR}
@@ -78,6 +75,12 @@ Please fix this and reinstall this package." >&2
         chmod 700 ${CLICKHOUSE_DATADIR}
     fi
 
+    if [ -d ${CLICKHOUSE_CONFDIR} ]; then
+        rm -v ${CLICKHOUSE_CONFDIR}/*-preprocessed.xml ||:
+    fi
+
+    ln -s ${CLICKHOUSE_DATADIR}/preprocessed_configs ${CLICKHOUSE_CONFDIR}/preprocessed ||:
+
     if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then
         mkdir -p ${CLICKHOUSE_LOGDIR}
         chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}

debian/clickhouse-server.preinst

@@ -2,5 +2,5 @@
 
 if [ "$1" = "upgrade" ]; then
     # Return etc/cron.d/clickhouse-server to original state
-    service clickhouse-server disable_cron || true
+    service clickhouse-server disable_cron ||:
 fi

debian/clickhouse-server.prerm

@@ -2,5 +2,5 @@
 
 if [ "$1" = "upgrade" ] || [ "$1" = "remove" ]; then
     # Return etc/cron.d/clickhouse-server to original state
-    service clickhouse-server disable_cron || true
+    service clickhouse-server disable_cron ||:
 fi

debian/clickhouse-server.service

@@ -8,8 +8,6 @@ Group=clickhouse
 PermissionsStartOnly=true
 Restart=always
 RestartSec=30
-ExecStartPre=-/usr/bin/chown clickhouse:clickhouse -R /etc/clickhouse-server
-ExecStartPre=-/bin/chown clickhouse:clickhouse -R /etc/clickhouse-server
 ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml
 LimitCORE=infinity
 LimitNOFILE=500000

debian/pbuilder-hooks/A00ccache

@@ -7,9 +7,9 @@ echo "CCACHEDIR=$CCACHEDIR CCACHE_DIR=$CCACHE_DIR SET_CCACHEDIR=$SET_CCACHEDIR"
 [ -z "$CCACHE_DIR" ] && export CCACHE_DIR=${CCACHEDIR:=${SET_CCACHEDIR=/var/cache/pbuilder/ccache}}
 
 if [ -n "$CCACHE_DIR" ]; then
-    mkdir -p $CCACHE_DIR $DISTCC_DIR || true
+    mkdir -p $CCACHE_DIR $DISTCC_DIR ||:
-    chown -R $BUILDUSERID:$BUILDUSERID $CCACHE_DIR $DISTCC_DIR || true
+    chown -R $BUILDUSERID:$BUILDUSERID $CCACHE_DIR $DISTCC_DIR ||:
-    chmod -R a+rwx $CCACHE_DIR $DISTCC_DIR || true
+    chmod -R a+rwx $CCACHE_DIR $DISTCC_DIR ||:
 fi
 
 df -h

debian/pbuilder-hooks/B90test-server

@@ -9,13 +9,13 @@ TEST_PORT_RANDOM=${TEST_PORT_RANDOM=1}
 
 if [ "${PACKAGE_INSTALL}" ]; then
     for PKG in $(ls /tmp/buildd/*.deb | sed -e's,.*/,,;s,_.*,,' ); do
-        apt-get install -y --force-yes "$PKG" || true
+        apt-get install -y --force-yes "$PKG" ||:
-        apt-get remove -y "$PKG" || true
+        apt-get remove -y "$PKG" ||:
     done
 
-    dpkg --auto-deconfigure -i /tmp/buildd/*.deb || true
+    dpkg --auto-deconfigure -i /tmp/buildd/*.deb ||:
-    apt install -y -f --allow-downgrades || true
+    apt install -y -f --allow-downgrades ||:
-    dpkg -l | grep clickhouse || true
+    dpkg -l | grep clickhouse ||:
 
     # Some test references uses specific timezone
     ln -fs /usr/share/zoneinfo/Europe/Moscow /etc/localtime
@@ -49,20 +49,20 @@ if [ "${TEST_CONNECT}" ]; then
         echo "<yandex><tcp_port>${CLICKHOUSE_PORT_TCP}</tcp_port><tcp_port_secure>${CLICKHOUSE_PORT_TCP_SECURE}</tcp_port_secure>${CLICKHOUSE_SSL_CONFIG}</yandex>" > /etc/clickhouse-client/config.xml
         openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
         openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
-        chmod a+r /etc/clickhouse-server/* /etc/clickhouse-client/*
+        chmod a+r /etc/clickhouse-server/* /etc/clickhouse-client/* ||:
         CLIENT_ADD+="--secure --port ${CLICKHOUSE_PORT_TCP_SECURE}"
     else
         CLIENT_ADD+="--port ${CLICKHOUSE_PORT_TCP}"
     fi
 
     # For debug
-    # tail -n +1 -- /etc/clickhouse-server/*.xml /etc/clickhouse-server/config.d/*.xml || true
+    # tail -n +1 -- /etc/clickhouse-server/*.xml /etc/clickhouse-server/config.d/*.xml ||:
 
     function finish {
         service clickhouse-server stop
-        tail -n 100 /var/log/clickhouse-server/*.log || true
+        tail -n 100 /var/log/clickhouse-server/*.log ||:
         sleep 1
-        killall -9 clickhouse-server || true
+        killall -9 clickhouse-server ||:
     }
     trap finish EXIT SIGINT SIGQUIT SIGTERM
 
@@ -70,7 +70,7 @@ if [ "${TEST_CONNECT}" ]; then
     sleep ${TEST_SERVER_STARTUP_WAIT:=5}
 
     # TODO: remove me or make only on error:
-    tail -n100 /var/log/clickhouse-server/*.log || true
+    tail -n100 /var/log/clickhouse-server/*.log ||:
 
     clickhouse-client --port $CLICKHOUSE_PORT_TCP -q "SELECT * from system.build_options;"
     clickhouse-client ${CLIENT_ADD} -q "SELECT toDateTime(1);"

debian/pbuilder-hooks/C99kill-make

@@ -2,4 +2,4 @@
 
 # Try stop parallel build after timeout
 
-killall make gcc gcc-7 g++-7 gcc-8 g++-8 clang clang-5.0 clang++-5.0 clang-6.0 clang++-6.0 clang-7 clang++-7 || true
+killall make gcc gcc-7 g++-7 gcc-8 g++-8 clang clang-5.0 clang++-5.0 clang-6.0 clang++-6.0 clang-7 clang++-7 ||:

libs/libdaemon/include/daemon/BaseDaemon.h

@@ -224,7 +224,7 @@ protected:
     std::atomic_size_t sigint_signals_counter{0};
 
     std::string config_path;
-    ConfigProcessor::LoadedConfig loaded_config;
+    DB::ConfigProcessor::LoadedConfig loaded_config;
     Poco::Util::AbstractConfiguration * last_configuration = nullptr;
 
 private:

libs/libdaemon/src/BaseDaemon.cpp

@@ -586,7 +586,10 @@ void BaseDaemon::reloadConfiguration()
       * (It's convenient to log in console when you start server without any command line parameters.)
       */
     config_path = config().getString("config-file", "config.xml");
-    loaded_config = ConfigProcessor(config_path, false, true).loadConfig(/* allow_zk_includes = */ true);
+    DB::ConfigProcessor config_processor(config_path, false, true);
+    config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
+    loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
+
     if (last_configuration != nullptr)
         config().removeConfiguration(last_configuration);
     last_configuration = loaded_config.configuration.duplicate();
@@ -895,7 +898,7 @@ void BaseDaemon::initialize(Application & self)
         umask(umask_num);
     }
 
-    ConfigProcessor(config_path).savePreprocessedConfig(loaded_config);
+    DB::ConfigProcessor(config_path).savePreprocessedConfig(loaded_config, "");
 
     /// Write core dump on crash.
     {

utils/config-processor/config-processor.cpp

@@ -11,8 +11,8 @@ int main(int argc, char ** argv)
             return 3;
         }
 
-        ConfigProcessor processor(argv[1], false, true);
+        DB::ConfigProcessor processor(argv[1], false, true);
-        XMLDocumentPtr document = processor.processConfig();
+        DB::XMLDocumentPtr document = processor.processConfig();
         Poco::XML::DOMWriter().writeNode(std::cout, document);
     }
     catch (Poco::Exception & e)