thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 00:22:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Do not write preprocessed configs to /etc/ (#2443)

Browse Source
This commit is contained in:
proller 2018-11-27 19:11:46 +03:00 committed by GitHub
parent 1ea31e0491
commit f1791e94e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 133 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.draft.md
View File

@ -1 +1,4 @@
* Настройка `enable_optimize_predicate_expression` выключена по-умолчанию.
### Улучшения:
* Файлы *-preprocessed.xml записываются в директорию с данными (/var/lib/clickhouse/preprocessed_configs). Для /etc/clickhouse-server больше не нужен +w для пользователя clickhouse. Для удобства создан симлинк /var/lib/clickhouse/preprocessed_configs -> /etc/clickhouse-server/preprocessed

8
dbms/programs/extract-from-config/ExtractFromConfig.cpp
View File

@ -26,18 +26,18 @@ static void setupLogging(const std::string & log_level)
static std::string extractFromConfig(
const std::string & config_path, const std::string & key, bool process_zk_includes, bool try_get = false)
{
ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
DB::ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
bool has_zk_includes;
XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes);
DB::XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes);
if (has_zk_includes && process_zk_includes)
{
ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml));
DB::ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml));
zkutil::ZooKeeperPtr zookeeper = std::make_shared<zkutil::ZooKeeper>(
*bootstrap_configuration, "zookeeper");
zkutil::ZooKeeperNodeCache zk_node_cache([&] { return zookeeper; });
config_xml = processor.processConfig(&has_zk_includes, &zk_node_cache);
}
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
DB::ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
// do not throw exception if not found
if (try_get)
return configuration->getString(key, "");

8
dbms/programs/local/LocalServer.cpp
View File

@ -115,9 +115,11 @@ try
/// Load config files if exists
if (config().has("config-file") || Poco::File("config.xml").exists())
{
ConfigProcessor config_processor(config().getString("config-file", "config.xml"), false, true);
const auto config_path = config().getString("config-file", "config.xml");
ConfigProcessor config_processor(config_path, false, true);
config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
auto loaded_config = config_processor.loadConfig();
config_processor.savePreprocessedConfig(loaded_config);
config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", DBMS_DEFAULT_PATH));
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
}
@ -348,7 +350,7 @@ void LocalServer::setupUsers()
const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
ConfigProcessor config_processor(users_config_path);
const auto loaded_config = config_processor.loadConfig();
config_processor.savePreprocessedConfig(loaded_config);
config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
users_config = loaded_config.configuration;
}
else

8
dbms/programs/server/Server.cpp
View File

@ -96,7 +96,7 @@ void Server::initialize(Poco::Util::Application & self)
std::string Server::getDefaultCorePath() const
{
return getCanonicalPath(config().getString("path")) + "cores";
return getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)) + "cores";
}
int Server::main(const std::vector<std::string> & /*args*/)
@ -129,7 +129,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
ConfigProcessor config_processor(config_path);
loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
main_config_zk_node_cache, /* fallback_to_preprocessed = */ true);
config_processor.savePreprocessedConfig(loaded_config);
config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
config().removeConfiguration(old_configuration.get());
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
}
@ -160,7 +160,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
}
#endif
std::string path = getCanonicalPath(config().getString("path"));
std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH));
std::string default_database = config().getString("default_database", "default");
global_context->setPath(path);
@ -301,6 +301,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
auto main_config_reloader = std::make_unique<ConfigReloader>(config_path,
include_from_path,
config().getString("path", ""),
std::move(main_config_zk_node_cache),
[&](ConfigurationPtr config)
{
@ -322,6 +323,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
}
auto users_config_reloader = std::make_unique<ConfigReloader>(users_config_path,
include_from_path,
config().getString("path", ""),
zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }),
[&](ConfigurationPtr config) { global_context->setUsersConfig(config); },
/* already_loaded = */ false);

1
dbms/src/Client/Connection.cpp
View File

@ -27,7 +27,6 @@
#include <Poco/Net/SecureStreamSocket.h>
#endif
namespace CurrentMetrics
{
extern const Metric SendExternalTables;

59
dbms/src/Common/Config/ConfigProcessor.cpp
View File

@ -20,6 +20,11 @@
using namespace Poco::XML;
namespace DB
{
/// For cutting prerpocessed path to this base
std::string main_config_path;
/// Extracts from a string the first encountered number consisting of at least two digits.
static std::string numberFromHost(const std::string & s)
@ -40,13 +45,6 @@ static std::string numberFromHost(const std::string & s)
return "";
}
static std::string preprocessedConfigPath(const std::string & path)
{
Poco::Path preprocessed_path(path);
preprocessed_path.setBaseName(preprocessed_path.getBaseName() + PREPROCESSED_SUFFIX);
return preprocessed_path.toString();
}
bool ConfigProcessor::isPreprocessedFile(const std::string & path)
{
return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX);
@ -59,7 +57,6 @@ ConfigProcessor::ConfigProcessor(
bool log_to_console,
const Substitutions & substitutions_)
: path(path_)
, preprocessed_path(preprocessedConfigPath(path))
, throw_on_bad_incl(throw_on_bad_incl_)
, substitutions(substitutions_)
/// We need larger name pool to allow to support vast amount of users in users.xml files for ClickHouse.
@ -522,7 +519,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml};
return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml, path};
}
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
@ -556,11 +553,44 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml};
return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml, path};
}
void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir)
{
if (preprocessed_path.empty())
{
auto new_path = loaded_config.config_path;
if (new_path.substr(0, main_config_path.size()) == main_config_path)
new_path.replace(0, main_config_path.size(), "");
std::replace(new_path.begin(), new_path.end(), '/', '_');
if (preprocessed_dir.empty())
{
if (!loaded_config.configuration->has("path"))
{
// Will use current directory
auto parent_path = Poco::Path(loaded_config.config_path).makeParent();
preprocessed_dir = parent_path.toString();
Poco::Path poco_new_path(new_path);
poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX);
new_path = poco_new_path.toString();
}
else
{
preprocessed_dir = loaded_config.configuration->getString("path") + "/preprocessed_configs/";
}
}
else
{
preprocessed_dir += "/preprocessed_configs/";
}
preprocessed_path = preprocessed_dir + new_path;
auto path = Poco::Path(preprocessed_path).makeParent();
if (!path.toString().empty())
Poco::File(path).createDirectories();
}
try
{
DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml);
@ -570,3 +600,10 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
LOG_WARNING(log, "Couldn't save preprocessed config to " << preprocessed_path << ": " << e.displayText());
}
}
void ConfigProcessor::setConfigPath(const std::string & config_path)
{
main_config_path = config_path;
}
}

15
dbms/src/Common/Config/ConfigProcessor.h
View File

@ -24,6 +24,9 @@ namespace zkutil
class ZooKeeperNodeCache;
}
namespace DB
{
using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
using XMLDocumentPtr = Poco::AutoPtr<Poco::XML::Document>;
@ -72,6 +75,7 @@ public:
bool has_zk_includes;
bool loaded_from_preprocessed;
XMLDocumentPtr preprocessed_xml;
std::string config_path;
};
/// If allow_zk_includes is true, expect that the configuration XML can contain from_zk nodes.
@ -85,7 +89,12 @@ public:
zkutil::ZooKeeperNodeCache & zk_node_cache,
bool fallback_to_preprocessed = false);
void savePreprocessedConfig(const LoadedConfig & loaded_config);
/// Save preprocessed config to specified directory.
/// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/
void savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir);
/// Set path of main config.xml . It will be cutted from all configs placed to preprocessed_configs/
void setConfigPath(const std::string & config_path);
public:
using Files = std::vector<std::string>;
@ -99,7 +108,7 @@ public:
private:
const std::string path;
const std::string preprocessed_path;
std::string preprocessed_path;
bool throw_on_bad_incl;
@ -127,3 +136,5 @@ private:
zkutil::ZooKeeperNodeCache * zk_node_cache,
std::unordered_set<std::string> & contributing_zk_paths);
};
}

4
dbms/src/Common/Config/ConfigReloader.cpp
View File

@ -15,10 +15,12 @@ constexpr decltype(ConfigReloader::reload_interval) ConfigReloader::reload_inter
ConfigReloader::ConfigReloader(
const std::string & path_,
const std::string & include_from_path_,
const std::string & preprocessed_dir_,
zkutil::ZooKeeperNodeCache && zk_node_cache_,
Updater && updater_,
bool already_loaded)
: path(path_), include_from_path(include_from_path_)
, preprocessed_dir(preprocessed_dir_)
, zk_node_cache(std::move(zk_node_cache_))
, updater(std::move(updater_))
{
@ -98,7 +100,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac
tryLogCurrentException(log, "Error loading config from `" + path + "'");
return;
}
config_processor.savePreprocessedConfig(loaded_config);
config_processor.savePreprocessedConfig(loaded_config, preprocessed_dir);
/** We should remember last modification time if and only if config was sucessfully loaded
* Otherwise a race condition could occur during config files update:

2
dbms/src/Common/Config/ConfigReloader.h
View File

@ -33,6 +33,7 @@ public:
ConfigReloader(
const std::string & path,
const std::string & include_from_path,
const std::string & preprocessed_dir,
zkutil::ZooKeeperNodeCache && zk_node_cache,
Updater && updater,
bool already_loaded);
@ -70,6 +71,7 @@ private:
std::string path;
std::string include_from_path;
std::string preprocessed_dir;
FilesChangesTracker files;
zkutil::ZooKeeperNodeCache zk_node_cache;

2
dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp
View File

@ -23,7 +23,7 @@ int main(int argc, char ** argv)
return 3;
}
ConfigProcessor processor(argv[1], false, true);
DB::ConfigProcessor processor(argv[1], false, true);
auto config = processor.loadConfig().configuration;
zkutil::ZooKeeper zk(*config, "zookeeper");
zkutil::EventPtr watch = std::make_shared<Poco::Event>();

2
dbms/src/Core/Defines.h
View File

@ -66,6 +66,8 @@
/// the number is unmotivated
#define DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT 15
#define DBMS_DEFAULT_PATH "/var/lib/clickhouse/"
// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
#if defined(_MSC_VER)

11
dbms/src/Interpreters/ExternalLoader.cpp
View File

@ -1,4 +1,5 @@
#include <Interpreters/ExternalLoader.h>
#include "ExternalLoader.h"
#include <Core/Defines.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
@ -42,12 +43,12 @@ void ExternalLoader::reloadPeriodically()
}
ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config,
ExternalLoader::ExternalLoader(const Poco::Util::AbstractConfiguration & config_main,
const ExternalLoaderUpdateSettings & update_settings,
const ExternalLoaderConfigSettings & config_settings,
std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
Logger * log, const std::string & loadable_object_name)
: config(config)
: config_main(config_main)
, update_settings(update_settings)
, config_settings(config_settings)
, config_repository(std::move(config_repository))
@ -214,7 +215,7 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
void ExternalLoader::reloadFromConfigFiles(const bool throw_on_error, const bool force_reload, const std::string & only_dictionary)
{
const auto config_paths = config_repository->list(config, config_settings.path_setting_name);
const auto config_paths = config_repository->list(config_main, config_settings.path_setting_name);
for (const auto & config_path : config_paths)
{
@ -262,7 +263,7 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const
const auto last_modified = config_repository->getLastModificationTime(config_path);
if (force_reload || last_modified > config_last_modified)
{
auto loaded_config = config_repository->load(config_path);
auto loaded_config = config_repository->load(config_path, config_main.getString("path", DBMS_DEFAULT_PATH));
loadable_objects_defined_in_config[config_path].clear();

4
dbms/src/Interpreters/ExternalLoader.h
View File

@ -91,7 +91,7 @@ public:
using ObjectsMap = std::unordered_map<std::string, LoadableInfo>;
/// Objects will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
ExternalLoader(const Configuration & config,
ExternalLoader(const Configuration & config_main,
const ExternalLoaderUpdateSettings & update_settings,
const ExternalLoaderConfigSettings & config_settings,
std::unique_ptr<IExternalLoaderConfigRepository> config_repository,
@ -151,7 +151,7 @@ private:
pcg64 rnd_engine{randomSeed()};
const Configuration & config;
const Configuration & config_main;
const ExternalLoaderUpdateSettings & update_settings;
const ExternalLoaderConfigSettings & config_settings;

4
dbms/src/Interpreters/ExternalLoaderConfigRepository.cpp
View File

@ -61,11 +61,11 @@ Poco::Timestamp ExternalLoaderConfigRepository::getLastModificationTime(
}
Poco::AutoPtr<Poco::Util::AbstractConfiguration> ExternalLoaderConfigRepository::load(
const std::string & config_file) const
const std::string & config_file, const std::string & preprocessed_dir) const
{
ConfigProcessor config_processor{config_file};
ConfigProcessor::LoadedConfig preprocessed = config_processor.loadConfig();
config_processor.savePreprocessedConfig(preprocessed);
config_processor.savePreprocessedConfig(preprocessed, preprocessed_dir);
return preprocessed.configuration;
}

2
dbms/src/Interpreters/ExternalLoaderConfigRepository.h
View File

@ -19,7 +19,7 @@ public:
Poco::Timestamp getLastModificationTime(const std::string & config_file) const override;
Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file) const override;
Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file, const std::string & preprocessed_dir = "") const override;
};
}

2
dbms/src/Interpreters/IExternalLoaderConfigRepository.h
View File

@ -23,7 +23,7 @@ public:
virtual Poco::Timestamp getLastModificationTime(const std::string & config_file) const = 0;
virtual Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file) const = 0;
virtual Poco::AutoPtr<Poco::Util::AbstractConfiguration> load(const std::string & config_file, const std::string & preprocessed_dir = "") const = 0;
virtual ~IExternalLoaderConfigRepository() {}
};

4
dbms/src/Interpreters/tests/users.cpp
View File

@ -189,11 +189,11 @@ void runOneTest(const TestDescriptor & test_descriptor)
const auto path_name = createTmpPath("users.xml");
createFile(path_name, test_descriptor.config_content);
ConfigurationPtr config;
DB::ConfigurationPtr config;
try
{
config = ConfigProcessor(path_name).loadConfig().configuration;
config = DB::ConfigProcessor(path_name).loadConfig().configuration;
}
catch (const Poco::Exception & ex)
{

3
debian/clickhouse-client.postinst vendored
View File

@ -4,6 +4,3 @@ set -e
CLICKHOUSE_USER=${CLICKHOUSE_USER=clickhouse}
mkdir -p /etc/clickhouse-client/conf.d
# user created by clickhouse-server package
chown -R ${CLICKHOUSE_USER} /etc/clickhouse-client || true

2
debian/clickhouse-server.cron.d vendored
View File

@ -1 +1 @@
#*/10 * * * * root (which service > /dev/null 2>&1 && (service clickhouse-server condstart || true)) || /etc/init.d/clickhouse-server condstart > /dev/null 2>&1
#*/10 * * * * root (which service > /dev/null 2>&1 && (service clickhouse-server condstart ||:)) || /etc/init.d/clickhouse-server condstart > /dev/null 2>&1

6
debian/clickhouse-server.init vendored
View File

@ -100,10 +100,6 @@ check_config()
initdb()
{
if [ -d ${SYSCONFDIR} ]; then
su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${SYSCONFDIR}" || chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${SYSCONFDIR}
fi
if [ -x "$BINDIR/$EXTRACT_FROM_CONFIG" ]; then
CLICKHOUSE_DATADIR_FROM_CONFIG=$(su -s $SHELL ${CLICKHOUSE_USER} -c "$BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path")
if [ "(" "$?" -ne "0" ")" -o "(" -z "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ")" ]; then
@ -128,7 +124,7 @@ initdb()
fi
if ! $(su -s $SHELL ${CLICKHOUSE_USER} -c "test -O \"${CLICKHOUSE_DATADIR_FROM_CONFIG}\" && test -G \"${CLICKHOUSE_DATADIR_FROM_CONFIG}\""); then
if [ $(dirname "${CLICKHOUSE_DATADIR_FROM_CONFIG}") == "/" ]; then
if [ $(dirname "${CLICKHOUSE_DATADIR_FROM_CONFIG}") = "/" ]; then
echo "Directory ${CLICKHOUSE_DATADIR_FROM_CONFIG} seems too dangerous to chown."
else
if [ ! -e "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ]; then

11
debian/clickhouse-server.postinst vendored
View File

@ -9,7 +9,7 @@ CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR=/var/log/clickhouse-server}
CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR=/usr/bin}
CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM=clickhouse}
OS=${OS=`lsb_release -is 2>/dev/null || uname -s || true`}
OS=${OS=`lsb_release -is 2>/dev/null || uname -s ||:`}
test -f /etc/default/clickhouse && . /etc/default/clickhouse
@ -68,9 +68,6 @@ Please fix this and reinstall this package." >&2
exit 1
fi
if [ -d ${CLICKHOUSE_CONFDIR} ]; then
su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_CONFDIR}" || chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_CONFDIR}
fi
if [ ! -d ${CLICKHOUSE_DATADIR} ]; then
mkdir -p ${CLICKHOUSE_DATADIR}
@ -78,6 +75,12 @@ Please fix this and reinstall this package." >&2
chmod 700 ${CLICKHOUSE_DATADIR}
fi
if [ -d ${CLICKHOUSE_CONFDIR} ]; then
rm -v ${CLICKHOUSE_CONFDIR}/*-preprocessed.xml ||:
fi
ln -s ${CLICKHOUSE_DATADIR}/preprocessed_configs ${CLICKHOUSE_CONFDIR}/preprocessed ||:
if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then
mkdir -p ${CLICKHOUSE_LOGDIR}
chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}

2
debian/clickhouse-server.preinst vendored
View File

@ -2,5 +2,5 @@
if [ "$1" = "upgrade" ]; then
# Return etc/cron.d/clickhouse-server to original state
service clickhouse-server disable_cron || true
service clickhouse-server disable_cron ||:
fi

2
debian/clickhouse-server.prerm vendored
View File

@ -2,5 +2,5 @@
if [ "$1" = "upgrade" ] || [ "$1" = "remove" ]; then
# Return etc/cron.d/clickhouse-server to original state
service clickhouse-server disable_cron || true
service clickhouse-server disable_cron ||:
fi

2
debian/clickhouse-server.service vendored
View File

@ -8,8 +8,6 @@ Group=clickhouse
PermissionsStartOnly=true
Restart=always
RestartSec=30
ExecStartPre=-/usr/bin/chown clickhouse:clickhouse -R /etc/clickhouse-server
ExecStartPre=-/bin/chown clickhouse:clickhouse -R /etc/clickhouse-server
ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml
LimitCORE=infinity
LimitNOFILE=500000

6
debian/pbuilder-hooks/A00ccache vendored
View File

@ -7,9 +7,9 @@ echo "CCACHEDIR=$CCACHEDIR CCACHE_DIR=$CCACHE_DIR SET_CCACHEDIR=$SET_CCACHEDIR"
[ -z "$CCACHE_DIR" ] && export CCACHE_DIR=${CCACHEDIR:=${SET_CCACHEDIR=/var/cache/pbuilder/ccache}}
if [ -n "$CCACHE_DIR" ]; then
mkdir -p $CCACHE_DIR $DISTCC_DIR || true
chown -R $BUILDUSERID:$BUILDUSERID $CCACHE_DIR $DISTCC_DIR || true
chmod -R a+rwx $CCACHE_DIR $DISTCC_DIR || true
mkdir -p $CCACHE_DIR $DISTCC_DIR ||:
chown -R $BUILDUSERID:$BUILDUSERID $CCACHE_DIR $DISTCC_DIR ||:
chmod -R a+rwx $CCACHE_DIR $DISTCC_DIR ||:
fi
df -h

20
debian/pbuilder-hooks/B90test-server vendored
View File

@ -9,13 +9,13 @@ TEST_PORT_RANDOM=${TEST_PORT_RANDOM=1}
if [ "${PACKAGE_INSTALL}" ]; then
for PKG in $(ls /tmp/buildd/*.deb | sed -e's,.*/,,;s,_.*,,' ); do
apt-get install -y --force-yes "$PKG" || true
apt-get remove -y "$PKG" || true
apt-get install -y --force-yes "$PKG" ||:
apt-get remove -y "$PKG" ||:
done
dpkg --auto-deconfigure -i /tmp/buildd/*.deb || true
apt install -y -f --allow-downgrades || true
dpkg -l | grep clickhouse || true
dpkg --auto-deconfigure -i /tmp/buildd/*.deb ||:
apt install -y -f --allow-downgrades ||:
dpkg -l | grep clickhouse ||:
# Some test references uses specific timezone
ln -fs /usr/share/zoneinfo/Europe/Moscow /etc/localtime
@ -49,20 +49,20 @@ if [ "${TEST_CONNECT}" ]; then
echo "<yandex><tcp_port>${CLICKHOUSE_PORT_TCP}</tcp_port><tcp_port_secure>${CLICKHOUSE_PORT_TCP_SECURE}</tcp_port_secure>${CLICKHOUSE_SSL_CONFIG}</yandex>" > /etc/clickhouse-client/config.xml
openssl dhparam -out /etc/clickhouse-server/dhparam.pem 256
openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
chmod a+r /etc/clickhouse-server/* /etc/clickhouse-client/*
chmod a+r /etc/clickhouse-server/* /etc/clickhouse-client/* ||:
CLIENT_ADD+="--secure --port ${CLICKHOUSE_PORT_TCP_SECURE}"
else
CLIENT_ADD+="--port ${CLICKHOUSE_PORT_TCP}"
fi
# For debug
# tail -n +1 -- /etc/clickhouse-server/*.xml /etc/clickhouse-server/config.d/*.xml || true
# tail -n +1 -- /etc/clickhouse-server/*.xml /etc/clickhouse-server/config.d/*.xml ||:
function finish {
service clickhouse-server stop
tail -n 100 /var/log/clickhouse-server/*.log || true
tail -n 100 /var/log/clickhouse-server/*.log ||:
sleep 1
killall -9 clickhouse-server || true
killall -9 clickhouse-server ||:
}
trap finish EXIT SIGINT SIGQUIT SIGTERM
@ -70,7 +70,7 @@ if [ "${TEST_CONNECT}" ]; then
sleep ${TEST_SERVER_STARTUP_WAIT:=5}
# TODO: remove me or make only on error:
tail -n100 /var/log/clickhouse-server/*.log || true
tail -n100 /var/log/clickhouse-server/*.log ||:
clickhouse-client --port $CLICKHOUSE_PORT_TCP -q "SELECT * from system.build_options;"
clickhouse-client ${CLIENT_ADD} -q "SELECT toDateTime(1);"

2
debian/pbuilder-hooks/C99kill-make vendored
View File

@ -2,4 +2,4 @@
# Try stop parallel build after timeout
killall make gcc gcc-7 g++-7 gcc-8 g++-8 clang clang-5.0 clang++-5.0 clang-6.0 clang++-6.0 clang-7 clang++-7 || true
killall make gcc gcc-7 g++-7 gcc-8 g++-8 clang clang-5.0 clang++-5.0 clang-6.0 clang++-6.0 clang-7 clang++-7 ||:

2
libs/libdaemon/include/daemon/BaseDaemon.h
View File

@ -224,7 +224,7 @@ protected:
std::atomic_size_t sigint_signals_counter{0};
std::string config_path;
ConfigProcessor::LoadedConfig loaded_config;
DB::ConfigProcessor::LoadedConfig loaded_config;
Poco::Util::AbstractConfiguration * last_configuration = nullptr;
private:

7
libs/libdaemon/src/BaseDaemon.cpp
View File

@ -586,7 +586,10 @@ void BaseDaemon::reloadConfiguration()
* (It's convenient to log in console when you start server without any command line parameters.)
*/
config_path = config().getString("config-file", "config.xml");
loaded_config = ConfigProcessor(config_path, false, true).loadConfig(/* allow_zk_includes = */ true);
DB::ConfigProcessor config_processor(config_path, false, true);
config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
if (last_configuration != nullptr)
config().removeConfiguration(last_configuration);
last_configuration = loaded_config.configuration.duplicate();
@ -895,7 +898,7 @@ void BaseDaemon::initialize(Application & self)
umask(umask_num);
}
ConfigProcessor(config_path).savePreprocessedConfig(loaded_config);
DB::ConfigProcessor(config_path).savePreprocessedConfig(loaded_config, "");
/// Write core dump on crash.
{

4
utils/config-processor/config-processor.cpp
View File

@ -11,8 +11,8 @@ int main(int argc, char ** argv)
return 3;
}
ConfigProcessor processor(argv[1], false, true);
XMLDocumentPtr document = processor.processConfig();
DB::ConfigProcessor processor(argv[1], false, true);
DB::XMLDocumentPtr document = processor.processConfig();
Poco::XML::DOMWriter().writeNode(std::cout, document);
}
catch (Poco::Exception & e)