diff --git a/CHANGELOG.draft.md b/CHANGELOG.draft.md index 50a7665dc63..0674a4193fe 100644 --- a/CHANGELOG.draft.md +++ b/CHANGELOG.draft.md @@ -1 +1,4 @@ * Настройка `enable_optimize_predicate_expression` выключена по-умолчанию. + +### Улучшения: +* Файлы *-preprocessed.xml записываются в директорию с данными (/var/lib/clickhouse/preprocessed_configs). Для /etc/clickhouse-server больше не нужен +w для пользователя clickhouse. Для удобства создан симлинк /var/lib/clickhouse/preprocessed_configs -> /etc/clickhouse-server/preprocessed diff --git a/dbms/programs/extract-from-config/ExtractFromConfig.cpp b/dbms/programs/extract-from-config/ExtractFromConfig.cpp index 7f0122256dd..af9550e4547 100644 --- a/dbms/programs/extract-from-config/ExtractFromConfig.cpp +++ b/dbms/programs/extract-from-config/ExtractFromConfig.cpp @@ -26,18 +26,18 @@ static void setupLogging(const std::string & log_level) static std::string extractFromConfig( const std::string & config_path, const std::string & key, bool process_zk_includes, bool try_get = false) { - ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false); + DB::ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false); bool has_zk_includes; - XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes); + DB::XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes); if (has_zk_includes && process_zk_includes) { - ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml)); + DB::ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml)); zkutil::ZooKeeperPtr zookeeper = std::make_shared( *bootstrap_configuration, "zookeeper"); zkutil::ZooKeeperNodeCache zk_node_cache([&] { return zookeeper; }); config_xml = processor.processConfig(&has_zk_includes, &zk_node_cache); } - ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml)); + DB::ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml)); // do not throw exception if not found if (try_get) return configuration->getString(key, ""); diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index 606ce6b5b37..0dab224c7f1 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -115,9 +115,11 @@ try /// Load config files if exists if (config().has("config-file") || Poco::File("config.xml").exists()) { - ConfigProcessor config_processor(config().getString("config-file", "config.xml"), false, true); + const auto config_path = config().getString("config-file", "config.xml"); + ConfigProcessor config_processor(config_path, false, true); + config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); auto loaded_config = config_processor.loadConfig(); - config_processor.savePreprocessedConfig(loaded_config); + config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", DBMS_DEFAULT_PATH)); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } @@ -348,7 +350,7 @@ void LocalServer::setupUsers() const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml")); ConfigProcessor config_processor(users_config_path); const auto loaded_config = config_processor.loadConfig(); - config_processor.savePreprocessedConfig(loaded_config); + config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH)); users_config = loaded_config.configuration; } else diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 03d9a8268c0..8d3bff9ebfb 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -96,7 +96,7 @@ void Server::initialize(Poco::Util::Application & self) std::string Server::getDefaultCorePath() const { - return getCanonicalPath(config().getString("path")) + "cores"; + return getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)) + "cores"; } int Server::main(const std::vector & /*args*/) @@ -129,7 +129,7 @@ int Server::main(const std::vector & /*args*/) ConfigProcessor config_processor(config_path); loaded_config = config_processor.loadConfigWithZooKeeperIncludes( main_config_zk_node_cache, /* fallback_to_preprocessed = */ true); - config_processor.savePreprocessedConfig(loaded_config); + config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH)); config().removeConfiguration(old_configuration.get()); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } @@ -160,7 +160,7 @@ int Server::main(const std::vector & /*args*/) } #endif - std::string path = getCanonicalPath(config().getString("path")); + std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); std::string default_database = config().getString("default_database", "default"); global_context->setPath(path); @@ -301,6 +301,7 @@ int Server::main(const std::vector & /*args*/) std::string include_from_path = config().getString("include_from", "/etc/metrika.xml"); auto main_config_reloader = std::make_unique(config_path, include_from_path, + config().getString("path", ""), std::move(main_config_zk_node_cache), [&](ConfigurationPtr config) { @@ -322,6 +323,7 @@ int Server::main(const std::vector & /*args*/) } auto users_config_reloader = std::make_unique(users_config_path, include_from_path, + config().getString("path", ""), zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }), [&](ConfigurationPtr config) { global_context->setUsersConfig(config); }, /* already_loaded = */ false); diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 60fd070064b..2c704e0aa54 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -27,7 +27,6 @@ #include #endif - namespace CurrentMetrics { extern const Metric SendExternalTables; diff --git a/dbms/src/Common/Config/ConfigProcessor.cpp b/dbms/src/Common/Config/ConfigProcessor.cpp index cc7660f9641..fa8f3867e73 100644 --- a/dbms/src/Common/Config/ConfigProcessor.cpp +++ b/dbms/src/Common/Config/ConfigProcessor.cpp @@ -20,6 +20,11 @@ using namespace Poco::XML; +namespace DB +{ + +/// For cutting prerpocessed path to this base +std::string main_config_path; /// Extracts from a string the first encountered number consisting of at least two digits. static std::string numberFromHost(const std::string & s) @@ -40,13 +45,6 @@ static std::string numberFromHost(const std::string & s) return ""; } -static std::string preprocessedConfigPath(const std::string & path) -{ - Poco::Path preprocessed_path(path); - preprocessed_path.setBaseName(preprocessed_path.getBaseName() + PREPROCESSED_SUFFIX); - return preprocessed_path.toString(); -} - bool ConfigProcessor::isPreprocessedFile(const std::string & path) { return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX); @@ -59,7 +57,6 @@ ConfigProcessor::ConfigProcessor( bool log_to_console, const Substitutions & substitutions_) : path(path_) - , preprocessed_path(preprocessedConfigPath(path)) , throw_on_bad_incl(throw_on_bad_incl_) , substitutions(substitutions_) /// We need larger name pool to allow to support vast amount of users in users.xml files for ClickHouse. @@ -522,7 +519,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml)); - return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml}; + return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml, path}; } ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( @@ -556,11 +553,44 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes( ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml)); - return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml}; + return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml, path}; } -void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config) +void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir) { + if (preprocessed_path.empty()) + { + auto new_path = loaded_config.config_path; + if (new_path.substr(0, main_config_path.size()) == main_config_path) + new_path.replace(0, main_config_path.size(), ""); + std::replace(new_path.begin(), new_path.end(), '/', '_'); + + if (preprocessed_dir.empty()) + { + if (!loaded_config.configuration->has("path")) + { + // Will use current directory + auto parent_path = Poco::Path(loaded_config.config_path).makeParent(); + preprocessed_dir = parent_path.toString(); + Poco::Path poco_new_path(new_path); + poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX); + new_path = poco_new_path.toString(); + } + else + { + preprocessed_dir = loaded_config.configuration->getString("path") + "/preprocessed_configs/"; + } + } + else + { + preprocessed_dir += "/preprocessed_configs/"; + } + + preprocessed_path = preprocessed_dir + new_path; + auto path = Poco::Path(preprocessed_path).makeParent(); + if (!path.toString().empty()) + Poco::File(path).createDirectories(); + } try { DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml); @@ -570,3 +600,10 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config) LOG_WARNING(log, "Couldn't save preprocessed config to " << preprocessed_path << ": " << e.displayText()); } } + +void ConfigProcessor::setConfigPath(const std::string & config_path) +{ + main_config_path = config_path; +} + +} diff --git a/dbms/src/Common/Config/ConfigProcessor.h b/dbms/src/Common/Config/ConfigProcessor.h index 8663ecb682f..af227c554fa 100644 --- a/dbms/src/Common/Config/ConfigProcessor.h +++ b/dbms/src/Common/Config/ConfigProcessor.h @@ -24,6 +24,9 @@ namespace zkutil class ZooKeeperNodeCache; } +namespace DB +{ + using ConfigurationPtr = Poco::AutoPtr; using XMLDocumentPtr = Poco::AutoPtr; @@ -72,6 +75,7 @@ public: bool has_zk_includes; bool loaded_from_preprocessed; XMLDocumentPtr preprocessed_xml; + std::string config_path; }; /// If allow_zk_includes is true, expect that the configuration XML can contain from_zk nodes. @@ -85,7 +89,12 @@ public: zkutil::ZooKeeperNodeCache & zk_node_cache, bool fallback_to_preprocessed = false); - void savePreprocessedConfig(const LoadedConfig & loaded_config); + /// Save preprocessed config to specified directory. + /// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/ + void savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir); + + /// Set path of main config.xml . It will be cutted from all configs placed to preprocessed_configs/ + void setConfigPath(const std::string & config_path); public: using Files = std::vector; @@ -99,7 +108,7 @@ public: private: const std::string path; - const std::string preprocessed_path; + std::string preprocessed_path; bool throw_on_bad_incl; @@ -127,3 +136,5 @@ private: zkutil::ZooKeeperNodeCache * zk_node_cache, std::unordered_set & contributing_zk_paths); }; + +} diff --git a/dbms/src/Common/Config/ConfigReloader.cpp b/dbms/src/Common/Config/ConfigReloader.cpp index f798569d0c0..41ea22b65de 100644 --- a/dbms/src/Common/Config/ConfigReloader.cpp +++ b/dbms/src/Common/Config/ConfigReloader.cpp @@ -15,10 +15,12 @@ constexpr decltype(ConfigReloader::reload_interval) ConfigReloader::reload_inter ConfigReloader::ConfigReloader( const std::string & path_, const std::string & include_from_path_, + const std::string & preprocessed_dir_, zkutil::ZooKeeperNodeCache && zk_node_cache_, Updater && updater_, bool already_loaded) : path(path_), include_from_path(include_from_path_) + , preprocessed_dir(preprocessed_dir_) , zk_node_cache(std::move(zk_node_cache_)) , updater(std::move(updater_)) { @@ -98,7 +100,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac tryLogCurrentException(log, "Error loading config from `" + path + "'"); return; } - config_processor.savePreprocessedConfig(loaded_config); + config_processor.savePreprocessedConfig(loaded_config, preprocessed_dir); /** We should remember last modification time if and only if config was sucessfully loaded * Otherwise a race condition could occur during config files update: diff --git a/dbms/src/Common/Config/ConfigReloader.h b/dbms/src/Common/Config/ConfigReloader.h index 63ed18a9105..cb5ecdae7da 100644 --- a/dbms/src/Common/Config/ConfigReloader.h +++ b/dbms/src/Common/Config/ConfigReloader.h @@ -33,6 +33,7 @@ public: ConfigReloader( const std::string & path, const std::string & include_from_path, + const std::string & preprocessed_dir, zkutil::ZooKeeperNodeCache && zk_node_cache, Updater && updater, bool already_loaded); @@ -70,6 +71,7 @@ private: std::string path; std::string include_from_path; + std::string preprocessed_dir; FilesChangesTracker files; zkutil::ZooKeeperNodeCache zk_node_cache; diff --git a/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp b/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp index b1d1d5d3101..c707d91d302 100644 --- a/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp +++ b/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp @@ -23,7 +23,7 @@ int main(int argc, char ** argv) return 3; } - ConfigProcessor processor(argv[1], false, true); + DB::ConfigProcessor processor(argv[1], false, true); auto config = processor.loadConfig().configuration; zkutil::ZooKeeper zk(*config, "zookeeper"); zkutil::EventPtr watch = std::make_shared(); diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 570c108b2a6..0fd332113e7 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -66,6 +66,8 @@ /// the number is unmotivated #define DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT 15 +#define DBMS_DEFAULT_PATH "/var/lib/clickhouse/" + // more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html #if defined(_MSC_VER) diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 2827e9ea283..8e0fe1436d9 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -1,4 +1,5 @@ -#include +#include "ExternalLoader.h" +#include #include #include #include @@ -42,12 +43,12 @@ void ExternalLoader::reloadPeriodically() } -ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config, +ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config_main, const ExternalLoaderUpdateSettings & update_settings, const ExternalLoaderConfigSettings & config_settings, std::unique_ptr config_repository, Logger * log, const std::string & loadable_object_name) - : config(config) + : config_main(config_main) , update_settings(update_settings) , config_settings(config_settings) , config_repository(std::move(config_repository)) @@ -214,7 +215,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error) void ExternalLoader::reloadFromConfigFiles(const bool throw_on_error, const bool force_reload, const std::string & only_dictionary) { - const auto config_paths = config_repository->list(config, config_settings.path_setting_name); + const auto config_paths = config_repository->list(config_main, config_settings.path_setting_name); for (const auto & config_path : config_paths) { @@ -262,7 +263,7 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const const auto last_modified = config_repository->getLastModificationTime(config_path); if (force_reload || last_modified > config_last_modified) { - auto loaded_config = config_repository->load(config_path); + auto loaded_config = config_repository->load(config_path, config_main.getString("path", DBMS_DEFAULT_PATH)); loadable_objects_defined_in_config[config_path].clear(); diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index 76d6cfc6f96..7a27f8a81da 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -91,7 +91,7 @@ public: using ObjectsMap = std::unordered_map; /// Objects will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. - ExternalLoader(const Configuration & config, + ExternalLoader(const Configuration & config_main, const ExternalLoaderUpdateSettings & update_settings, const ExternalLoaderConfigSettings & config_settings, std::unique_ptr config_repository, @@ -151,7 +151,7 @@ private: pcg64 rnd_engine{randomSeed()}; - const Configuration & config; + const Configuration & config_main; const ExternalLoaderUpdateSettings & update_settings; const ExternalLoaderConfigSettings & config_settings; diff --git a/dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp b/dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp index bfc7965a5ad..bb8dd61ee2d 100644 --- a/dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp +++ b/dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp @@ -61,11 +61,11 @@ Poco::Timestamp ExternalLoaderConfigRepository::getLastModificationTime( } Poco::AutoPtr ExternalLoaderConfigRepository::load( - const std::string & config_file) const + const std::string & config_file, const std::string & preprocessed_dir) const { ConfigProcessor config_processor{config_file}; ConfigProcessor::LoadedConfig preprocessed = config_processor.loadConfig(); - config_processor.savePreprocessedConfig(preprocessed); + config_processor.savePreprocessedConfig(preprocessed, preprocessed_dir); return preprocessed.configuration; } diff --git a/dbms/src/Interpreters/ExternalLoaderConfigRepository.h b/dbms/src/Interpreters/ExternalLoaderConfigRepository.h index dd5c53329b5..a1b1606dd2c 100644 --- a/dbms/src/Interpreters/ExternalLoaderConfigRepository.h +++ b/dbms/src/Interpreters/ExternalLoaderConfigRepository.h @@ -19,7 +19,7 @@ public: Poco::Timestamp getLastModificationTime(const std::string & config_file) const override; - Poco::AutoPtr load(const std::string & config_file) const override; + Poco::AutoPtr load(const std::string & config_file, const std::string & preprocessed_dir = "") const override; }; } diff --git a/dbms/src/Interpreters/IExternalLoaderConfigRepository.h b/dbms/src/Interpreters/IExternalLoaderConfigRepository.h index 79615780242..d0caaf1b921 100644 --- a/dbms/src/Interpreters/IExternalLoaderConfigRepository.h +++ b/dbms/src/Interpreters/IExternalLoaderConfigRepository.h @@ -23,7 +23,7 @@ public: virtual Poco::Timestamp getLastModificationTime(const std::string & config_file) const = 0; - virtual Poco::AutoPtr load(const std::string & config_file) const = 0; + virtual Poco::AutoPtr load(const std::string & config_file, const std::string & preprocessed_dir = "") const = 0; virtual ~IExternalLoaderConfigRepository() {} }; diff --git a/dbms/src/Interpreters/tests/users.cpp b/dbms/src/Interpreters/tests/users.cpp index f4905398262..af07629ef62 100644 --- a/dbms/src/Interpreters/tests/users.cpp +++ b/dbms/src/Interpreters/tests/users.cpp @@ -189,11 +189,11 @@ void runOneTest(const TestDescriptor & test_descriptor) const auto path_name = createTmpPath("users.xml"); createFile(path_name, test_descriptor.config_content); - ConfigurationPtr config; + DB::ConfigurationPtr config; try { - config = ConfigProcessor(path_name).loadConfig().configuration; + config = DB::ConfigProcessor(path_name).loadConfig().configuration; } catch (const Poco::Exception & ex) { diff --git a/debian/clickhouse-client.postinst b/debian/clickhouse-client.postinst index ff54e3a58fc..38d2450dcbe 100644 --- a/debian/clickhouse-client.postinst +++ b/debian/clickhouse-client.postinst @@ -4,6 +4,3 @@ set -e CLICKHOUSE_USER=${CLICKHOUSE_USER=clickhouse} mkdir -p /etc/clickhouse-client/conf.d - -# user created by clickhouse-server package -chown -R ${CLICKHOUSE_USER} /etc/clickhouse-client || true diff --git a/debian/clickhouse-server.cron.d b/debian/clickhouse-server.cron.d index 90431886613..03bbd620aa7 100644 --- a/debian/clickhouse-server.cron.d +++ b/debian/clickhouse-server.cron.d @@ -1 +1 @@ -#*/10 * * * * root (which service > /dev/null 2>&1 && (service clickhouse-server condstart || true)) || /etc/init.d/clickhouse-server condstart > /dev/null 2>&1 +#*/10 * * * * root (which service > /dev/null 2>&1 && (service clickhouse-server condstart ||:)) || /etc/init.d/clickhouse-server condstart > /dev/null 2>&1 diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 0df35f55952..9044567b2bd 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -100,10 +100,6 @@ check_config() initdb() { - if [ -d ${SYSCONFDIR} ]; then - su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${SYSCONFDIR}" || chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${SYSCONFDIR} - fi - if [ -x "$BINDIR/$EXTRACT_FROM_CONFIG" ]; then CLICKHOUSE_DATADIR_FROM_CONFIG=$(su -s $SHELL ${CLICKHOUSE_USER} -c "$BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path") if [ "(" "$?" -ne "0" ")" -o "(" -z "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ")" ]; then @@ -128,7 +124,7 @@ initdb() fi if ! $(su -s $SHELL ${CLICKHOUSE_USER} -c "test -O \"${CLICKHOUSE_DATADIR_FROM_CONFIG}\" && test -G \"${CLICKHOUSE_DATADIR_FROM_CONFIG}\""); then - if [ $(dirname "${CLICKHOUSE_DATADIR_FROM_CONFIG}") == "/" ]; then + if [ $(dirname "${CLICKHOUSE_DATADIR_FROM_CONFIG}") = "/" ]; then echo "Directory ${CLICKHOUSE_DATADIR_FROM_CONFIG} seems too dangerous to chown." else if [ ! -e "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ]; then diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst index 53e9248b71f..e17c47df6dd 100644 --- a/debian/clickhouse-server.postinst +++ b/debian/clickhouse-server.postinst @@ -9,7 +9,7 @@ CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR=/var/log/clickhouse-server} CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR=/usr/bin} CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM=clickhouse} -OS=${OS=`lsb_release -is 2>/dev/null || uname -s || true`} +OS=${OS=`lsb_release -is 2>/dev/null || uname -s ||:`} test -f /etc/default/clickhouse && . /etc/default/clickhouse @@ -68,9 +68,6 @@ Please fix this and reinstall this package." >&2 exit 1 fi - if [ -d ${CLICKHOUSE_CONFDIR} ]; then - su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_CONFDIR}" || chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_CONFDIR} - fi if [ ! -d ${CLICKHOUSE_DATADIR} ]; then mkdir -p ${CLICKHOUSE_DATADIR} @@ -78,6 +75,12 @@ Please fix this and reinstall this package." >&2 chmod 700 ${CLICKHOUSE_DATADIR} fi + if [ -d ${CLICKHOUSE_CONFDIR} ]; then + rm -v ${CLICKHOUSE_CONFDIR}/*-preprocessed.xml ||: + fi + + ln -s ${CLICKHOUSE_DATADIR}/preprocessed_configs ${CLICKHOUSE_CONFDIR}/preprocessed ||: + if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then mkdir -p ${CLICKHOUSE_LOGDIR} chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR} diff --git a/debian/clickhouse-server.preinst b/debian/clickhouse-server.preinst index 603a67a12a5..1435d8db8da 100644 --- a/debian/clickhouse-server.preinst +++ b/debian/clickhouse-server.preinst @@ -2,5 +2,5 @@ if [ "$1" = "upgrade" ]; then # Return etc/cron.d/clickhouse-server to original state - service clickhouse-server disable_cron || true + service clickhouse-server disable_cron ||: fi diff --git a/debian/clickhouse-server.prerm b/debian/clickhouse-server.prerm index d9846c38085..02e855a7125 100644 --- a/debian/clickhouse-server.prerm +++ b/debian/clickhouse-server.prerm @@ -2,5 +2,5 @@ if [ "$1" = "upgrade" ] || [ "$1" = "remove" ]; then # Return etc/cron.d/clickhouse-server to original state - service clickhouse-server disable_cron || true + service clickhouse-server disable_cron ||: fi diff --git a/debian/clickhouse-server.service b/debian/clickhouse-server.service index fa556f31951..d26a543a146 100644 --- a/debian/clickhouse-server.service +++ b/debian/clickhouse-server.service @@ -8,8 +8,6 @@ Group=clickhouse PermissionsStartOnly=true Restart=always RestartSec=30 -ExecStartPre=-/usr/bin/chown clickhouse:clickhouse -R /etc/clickhouse-server -ExecStartPre=-/bin/chown clickhouse:clickhouse -R /etc/clickhouse-server ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml LimitCORE=infinity LimitNOFILE=500000 diff --git a/debian/pbuilder-hooks/A00ccache b/debian/pbuilder-hooks/A00ccache index ab122ecf82f..53510f9d325 100755 --- a/debian/pbuilder-hooks/A00ccache +++ b/debian/pbuilder-hooks/A00ccache @@ -7,9 +7,9 @@ echo "CCACHEDIR=$CCACHEDIR CCACHE_DIR=$CCACHE_DIR SET_CCACHEDIR=$SET_CCACHEDIR" [ -z "$CCACHE_DIR" ] && export CCACHE_DIR=${CCACHEDIR:=${SET_CCACHEDIR=/var/cache/pbuilder/ccache}} if [ -n "$CCACHE_DIR" ]; then - mkdir -p $CCACHE_DIR $DISTCC_DIR || true - chown -R $BUILDUSERID:$BUILDUSERID $CCACHE_DIR $DISTCC_DIR || true - chmod -R a+rwx $CCACHE_DIR $DISTCC_DIR || true + mkdir -p $CCACHE_DIR $DISTCC_DIR ||: + chown -R $BUILDUSERID:$BUILDUSERID $CCACHE_DIR $DISTCC_DIR ||: + chmod -R a+rwx $CCACHE_DIR $DISTCC_DIR ||: fi df -h diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server index ea1d3d78a9a..1110de53c5b 100755 --- a/debian/pbuilder-hooks/B90test-server +++ b/debian/pbuilder-hooks/B90test-server @@ -9,13 +9,13 @@ TEST_PORT_RANDOM=${TEST_PORT_RANDOM=1} if [ "${PACKAGE_INSTALL}" ]; then for PKG in $(ls /tmp/buildd/*.deb | sed -e's,.*/,,;s,_.*,,' ); do - apt-get install -y --force-yes "$PKG" || true - apt-get remove -y "$PKG" || true + apt-get install -y --force-yes "$PKG" ||: + apt-get remove -y "$PKG" ||: done - dpkg --auto-deconfigure -i /tmp/buildd/*.deb || true - apt install -y -f --allow-downgrades || true - dpkg -l | grep clickhouse || true + dpkg --auto-deconfigure -i /tmp/buildd/*.deb ||: + apt install -y -f --allow-downgrades ||: + dpkg -l | grep clickhouse ||: # Some test references uses specific timezone ln -fs /usr/share/zoneinfo/Europe/Moscow /etc/localtime @@ -49,20 +49,20 @@ if [ "${TEST_CONNECT}" ]; then echo "${CLICKHOUSE_PORT_TCP}${CLICKHOUSE_PORT_TCP_SECURE}${CLICKHOUSE_SSL_CONFIG}" > /etc/clickhouse-client/config.xml openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256 openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt - chmod a+r /etc/clickhouse-server/* /etc/clickhouse-client/* + chmod a+r /etc/clickhouse-server/* /etc/clickhouse-client/* ||: CLIENT_ADD+="--secure --port ${CLICKHOUSE_PORT_TCP_SECURE}" else CLIENT_ADD+="--port ${CLICKHOUSE_PORT_TCP}" fi # For debug - # tail -n +1 -- /etc/clickhouse-server/*.xml /etc/clickhouse-server/config.d/*.xml || true + # tail -n +1 -- /etc/clickhouse-server/*.xml /etc/clickhouse-server/config.d/*.xml ||: function finish { service clickhouse-server stop - tail -n 100 /var/log/clickhouse-server/*.log || true + tail -n 100 /var/log/clickhouse-server/*.log ||: sleep 1 - killall -9 clickhouse-server || true + killall -9 clickhouse-server ||: } trap finish EXIT SIGINT SIGQUIT SIGTERM @@ -70,7 +70,7 @@ if [ "${TEST_CONNECT}" ]; then sleep ${TEST_SERVER_STARTUP_WAIT:=5} # TODO: remove me or make only on error: - tail -n100 /var/log/clickhouse-server/*.log || true + tail -n100 /var/log/clickhouse-server/*.log ||: clickhouse-client --port $CLICKHOUSE_PORT_TCP -q "SELECT * from system.build_options;" clickhouse-client ${CLIENT_ADD} -q "SELECT toDateTime(1);" diff --git a/debian/pbuilder-hooks/C99kill-make b/debian/pbuilder-hooks/C99kill-make index 863356d4724..60be8e0d402 100755 --- a/debian/pbuilder-hooks/C99kill-make +++ b/debian/pbuilder-hooks/C99kill-make @@ -2,4 +2,4 @@ # Try stop parallel build after timeout -killall make gcc gcc-7 g++-7 gcc-8 g++-8 clang clang-5.0 clang++-5.0 clang-6.0 clang++-6.0 clang-7 clang++-7 || true +killall make gcc gcc-7 g++-7 gcc-8 g++-8 clang clang-5.0 clang++-5.0 clang-6.0 clang++-6.0 clang-7 clang++-7 ||: diff --git a/libs/libdaemon/include/daemon/BaseDaemon.h b/libs/libdaemon/include/daemon/BaseDaemon.h index 65d20927322..7a16761f51c 100644 --- a/libs/libdaemon/include/daemon/BaseDaemon.h +++ b/libs/libdaemon/include/daemon/BaseDaemon.h @@ -224,7 +224,7 @@ protected: std::atomic_size_t sigint_signals_counter{0}; std::string config_path; - ConfigProcessor::LoadedConfig loaded_config; + DB::ConfigProcessor::LoadedConfig loaded_config; Poco::Util::AbstractConfiguration * last_configuration = nullptr; private: diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index f65634f1e2f..bad38c78529 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -586,7 +586,10 @@ void BaseDaemon::reloadConfiguration() * (It's convenient to log in console when you start server without any command line parameters.) */ config_path = config().getString("config-file", "config.xml"); - loaded_config = ConfigProcessor(config_path, false, true).loadConfig(/* allow_zk_includes = */ true); + DB::ConfigProcessor config_processor(config_path, false, true); + config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); + loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true); + if (last_configuration != nullptr) config().removeConfiguration(last_configuration); last_configuration = loaded_config.configuration.duplicate(); @@ -895,7 +898,7 @@ void BaseDaemon::initialize(Application & self) umask(umask_num); } - ConfigProcessor(config_path).savePreprocessedConfig(loaded_config); + DB::ConfigProcessor(config_path).savePreprocessedConfig(loaded_config, ""); /// Write core dump on crash. { diff --git a/utils/config-processor/config-processor.cpp b/utils/config-processor/config-processor.cpp index 31eaf8522a4..242a6782b3b 100644 --- a/utils/config-processor/config-processor.cpp +++ b/utils/config-processor/config-processor.cpp @@ -11,8 +11,8 @@ int main(int argc, char ** argv) return 3; } - ConfigProcessor processor(argv[1], false, true); - XMLDocumentPtr document = processor.processConfig(); + DB::ConfigProcessor processor(argv[1], false, true); + DB::XMLDocumentPtr document = processor.processConfig(); Poco::XML::DOMWriter().writeNode(std::cout, document); } catch (Poco::Exception & e)