ClickHouse/src/Common/Config/ConfigProcessor.cpp

665 lines
23 KiB
C++
Raw Normal View History

#include "ConfigProcessor.h"
2015-10-05 01:26:43 +00:00
#include <sys/utsname.h>
#include <cerrno>
2018-07-27 17:12:01 +00:00
#include <cstdlib>
2015-10-05 01:26:43 +00:00
#include <cstring>
#include <algorithm>
#include <functional>
2020-08-08 03:42:42 +00:00
#include <filesystem>
2015-10-05 01:26:43 +00:00
#include <Poco/DOM/Text.h>
#include <Poco/DOM/Attr.h>
#include <Poco/DOM/Comment.h>
#include <Poco/Util/XMLConfiguration.h>
#include <Common/ZooKeeper/ZooKeeperNodeCache.h>
2018-04-03 17:35:48 +00:00
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/StringUtils/StringUtils.h>
2020-08-08 03:42:42 +00:00
#include <Common/Exception.h>
#include <common/getResource.h>
2020-12-17 18:08:42 +00:00
#include <common/errnoToString.h>
2020-11-10 18:22:26 +00:00
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#define PREPROCESSED_SUFFIX "-preprocessed"
namespace fs = std::filesystem;
2015-10-05 01:26:43 +00:00
using namespace Poco::XML;
namespace DB
{
2020-08-08 03:42:42 +00:00
namespace ErrorCodes
{
extern const int FILE_DOESNT_EXIST;
}
2019-01-22 19:56:53 +00:00
/// For cutting preprocessed path to this base
2018-12-19 13:11:06 +00:00
static std::string main_config_path;
2015-10-05 01:26:43 +00:00
2017-03-16 22:39:52 +00:00
/// Extracts from a string the first encountered number consisting of at least two digits.
2015-10-05 01:26:43 +00:00
static std::string numberFromHost(const std::string & s)
{
for (size_t i = 0; i < s.size(); ++i)
{
std::string res;
size_t j = i;
while (j < s.size() && isNumericASCII(s[j]))
res += s[j++];
if (res.size() >= 2)
{
while (res[0] == '0')
res.erase(res.begin());
return res;
}
}
return "";
2015-10-05 01:26:43 +00:00
}
bool ConfigProcessor::isPreprocessedFile(const std::string & path)
{
return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX);
}
ConfigProcessor::ConfigProcessor(
const std::string & path_,
bool throw_on_bad_incl_,
bool log_to_console,
const Substitutions & substitutions_)
: path(path_)
, throw_on_bad_incl(throw_on_bad_incl_)
, substitutions(substitutions_)
/// We need larger name pool to allow to support vast amount of users in users.xml files for ClickHouse.
/// Size is prime because Poco::XML::NamePool uses bad (inefficient, low quality)
/// hash function internally, and its size was prime by default.
, name_pool(new Poco::XML::NamePool(65521))
, dom_parser(name_pool)
2015-10-05 01:26:43 +00:00
{
2020-05-30 21:57:37 +00:00
if (log_to_console && !Poco::Logger::has("ConfigProcessor"))
{
channel_ptr = new Poco::ConsoleChannel;
2020-05-30 21:57:37 +00:00
log = &Poco::Logger::create("ConfigProcessor", channel_ptr.get(), Poco::Message::PRIO_TRACE);
}
else
{
2020-05-30 21:57:37 +00:00
log = &Poco::Logger::get("ConfigProcessor");
}
2015-10-05 01:26:43 +00:00
}
ConfigProcessor::~ConfigProcessor()
{
if (channel_ptr) /// This means we have created a new console logger in the constructor.
2020-05-30 21:57:37 +00:00
Poco::Logger::destroy("ConfigProcessor");
}
2017-03-16 22:39:52 +00:00
/// Vector containing the name of the element and a sorted list of attribute names and values
/// (except "remove" and "replace" attributes).
/// Serves as a unique identifier of the element contents for comparison.
using ElementIdentifier = std::vector<std::string>;
2015-10-05 01:26:43 +00:00
using NamedNodeMapPtr = Poco::AutoPtr<Poco::XML::NamedNodeMap>;
2017-03-16 22:39:52 +00:00
/// NOTE getting rid of iterating over the result of Node.childNodes() call is a good idea
/// because accessing the i-th element of this list takes O(i) time.
using NodeListPtr = Poco::AutoPtr<Poco::XML::NodeList>;
2015-10-05 01:26:43 +00:00
static ElementIdentifier getElementIdentifier(Node * element)
{
const NamedNodeMapPtr attrs = element->attributes();
2017-08-30 18:13:32 +00:00
std::vector<std::pair<std::string, std::string>> attrs_kv;
for (size_t i = 0, size = attrs->length(); i < size; ++i)
{
const Node * node = attrs->item(i);
std::string name = node->nodeName();
2020-04-22 05:39:31 +00:00
const auto * subst_name_pos = std::find(ConfigProcessor::SUBSTITUTION_ATTRS.begin(), ConfigProcessor::SUBSTITUTION_ATTRS.end(), name);
if (name == "replace" || name == "remove" ||
2018-07-27 17:12:01 +00:00
subst_name_pos != ConfigProcessor::SUBSTITUTION_ATTRS.end())
continue;
std::string value = node->nodeValue();
attrs_kv.push_back(std::make_pair(name, value));
}
std::sort(attrs_kv.begin(), attrs_kv.end());
ElementIdentifier res;
res.push_back(element->nodeName());
for (const auto & attr : attrs_kv)
{
res.push_back(attr.first);
res.push_back(attr.second);
}
return res;
2015-10-05 01:26:43 +00:00
}
static Node * getRootNode(Document * document)
{
const NodeListPtr children = document->childNodes();
for (size_t i = 0, size = children->length(); i < size; ++i)
{
Node * child = children->item(i);
/// Besides the root element there can be comment nodes on the top level.
/// Skip them.
if (child->nodeType() == Node::ELEMENT_NODE)
return child;
}
throw Poco::Exception("No root node in document");
2015-10-05 01:26:43 +00:00
}
static bool allWhitespace(const std::string & s)
{
return s.find_first_not_of(" \t\n\r") == std::string::npos;
2015-10-05 01:26:43 +00:00
}
void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root, const Node * with_root)
2015-10-05 01:26:43 +00:00
{
const NodeListPtr with_nodes = with_root->childNodes();
using ElementsByIdentifier = std::multimap<ElementIdentifier, Node *>;
ElementsByIdentifier config_element_by_id;
for (Node * node = config_root->firstChild(); node;)
{
Node * next_node = node->nextSibling();
/// Remove text from the original config node.
if (node->nodeType() == Node::TEXT_NODE && !allWhitespace(node->getNodeValue()))
{
config_root->removeChild(node);
}
else if (node->nodeType() == Node::ELEMENT_NODE)
{
config_element_by_id.insert(ElementsByIdentifier::value_type(getElementIdentifier(node), node));
}
node = next_node;
}
for (size_t i = 0, size = with_nodes->length(); i < size; ++i)
{
Node * with_node = with_nodes->item(i);
bool merged = false;
bool remove = false;
if (with_node->nodeType() == Node::ELEMENT_NODE)
{
Element & with_element = dynamic_cast<Element &>(*with_node);
remove = with_element.hasAttribute("remove");
bool replace = with_element.hasAttribute("replace");
if (remove && replace)
throw Poco::Exception("both remove and replace attributes set for element <" + with_node->nodeName() + ">");
ElementsByIdentifier::iterator it = config_element_by_id.find(getElementIdentifier(with_node));
if (it != config_element_by_id.end())
{
Node * config_node = it->second;
config_element_by_id.erase(it);
if (remove)
{
config_root->removeChild(config_node);
}
else if (replace)
{
with_element.removeAttribute("replace");
NodePtr new_node = config->importNode(with_node, true);
config_root->replaceChild(new_node, config_node);
}
else
{
mergeRecursive(config, config_node, with_node);
}
merged = true;
}
}
if (!merged && !remove)
{
NodePtr new_node = config->importNode(with_node, true);
config_root->appendChild(new_node);
}
}
2015-10-05 01:26:43 +00:00
}
void ConfigProcessor::merge(XMLDocumentPtr config, XMLDocumentPtr with)
2015-10-05 01:26:43 +00:00
{
2019-07-23 14:41:41 +00:00
Node * config_root = getRootNode(config.get());
Node * with_root = getRootNode(with.get());
if (config_root->nodeName() != with_root->nodeName())
throw Poco::Exception("Root element doesn't have the corresponding root element as the config file. It must be <" + config_root->nodeName() + ">");
mergeRecursive(config, config_root, with_root);
2015-10-05 01:26:43 +00:00
}
2020-03-18 00:57:00 +00:00
static std::string layerFromHost()
2015-10-05 01:26:43 +00:00
{
utsname buf;
if (uname(&buf))
2020-12-17 18:08:42 +00:00
throw Poco::Exception(std::string("uname failed: ") + errnoToString(errno));
2015-10-05 01:26:43 +00:00
std::string layer = numberFromHost(buf.nodename);
if (layer.empty())
throw Poco::Exception(std::string("no layer in host name: ") + buf.nodename);
2015-10-05 01:26:43 +00:00
return layer;
2015-10-05 01:26:43 +00:00
}
void ConfigProcessor::doIncludesRecursive(
XMLDocumentPtr config,
XMLDocumentPtr include_from,
Node * node,
zkutil::ZooKeeperNodeCache * zk_node_cache,
const zkutil::EventPtr & zk_changed_event,
std::unordered_set<std::string> & contributing_zk_paths)
2015-10-05 01:26:43 +00:00
{
if (node->nodeType() == Node::TEXT_NODE)
{
for (auto & substitution : substitutions)
{
std::string value = node->nodeValue();
bool replace_occured = false;
size_t pos;
while ((pos = value.find(substitution.first)) != std::string::npos)
{
value.replace(pos, substitution.first.length(), substitution.second);
replace_occured = true;
}
if (replace_occured)
node->setNodeValue(value);
}
}
if (node->nodeType() != Node::ELEMENT_NODE)
return;
/// Substitute <layer> for the number extracted from the hostname only if there is an
/// empty <layer> tag without attributes in the original file.
2018-11-24 01:48:06 +00:00
if (node->nodeName() == "layer"
&& !node->hasAttributes()
&& !node->hasChildNodes()
&& node->nodeValue().empty())
{
NodePtr new_node = config->createTextNode(layerFromHost());
node->appendChild(new_node);
return;
}
2018-07-27 17:12:01 +00:00
std::map<std::string, const Node *> attr_nodes;
NamedNodeMapPtr attributes = node->attributes();
2018-07-27 17:12:01 +00:00
size_t substs_count = 0;
for (const auto & attr_name : SUBSTITUTION_ATTRS)
2018-07-27 17:12:01 +00:00
{
2020-04-22 05:39:31 +00:00
const auto * subst = attributes->getNamedItem(attr_name);
2018-07-27 17:12:01 +00:00
attr_nodes[attr_name] = subst;
substs_count += static_cast<size_t>(subst == nullptr);
}
2018-07-27 17:12:01 +00:00
if (substs_count < SUBSTITUTION_ATTRS.size() - 1) /// only one substitution is allowed
throw Poco::Exception("several substitutions attributes set for element <" + node->nodeName() + ">");
/// Replace the original contents, not add to it.
bool replace = attributes->getNamedItem("replace");
bool included_something = false;
auto process_include = [&](const Node * include_attr, const std::function<const Node * (const std::string &)> & get_node, const char * error_msg)
{
2020-03-18 03:27:32 +00:00
const std::string & name = include_attr->getNodeValue();
const Node * node_to_include = get_node(name);
if (!node_to_include)
{
if (attributes->getNamedItem("optional"))
node->parentNode()->removeChild(node);
else if (throw_on_bad_incl)
throw Poco::Exception(error_msg + name);
else
2020-05-23 22:24:01 +00:00
LOG_WARNING(log, "{}{}", error_msg, name);
}
else
{
Element & element = dynamic_cast<Element &>(*node);
for (const auto & attr_name : SUBSTITUTION_ATTRS)
element.removeAttribute(attr_name);
if (replace)
{
while (Node * child = node->firstChild())
node->removeChild(child);
element.removeAttribute("replace");
}
const NodeListPtr children = node_to_include->childNodes();
for (size_t i = 0, size = children->length(); i < size; ++i)
{
NodePtr new_node = config->importNode(children->item(i), true);
node->appendChild(new_node);
}
const NamedNodeMapPtr from_attrs = node_to_include->attributes();
for (size_t i = 0, size = from_attrs->length(); i < size; ++i)
{
element.setAttributeNode(dynamic_cast<Attr *>(config->importNode(from_attrs->item(i), true)));
}
included_something = true;
}
};
2018-07-27 17:12:01 +00:00
if (attr_nodes["incl"]) // we have include subst
{
auto get_incl_node = [&](const std::string & name)
{
return include_from ? include_from->getNodeByPath("yandex/" + name) : nullptr;
};
2018-07-27 17:12:01 +00:00
process_include(attr_nodes["incl"], get_incl_node, "Include not found: ");
}
2018-07-27 17:12:01 +00:00
if (attr_nodes["from_zk"]) /// we have zookeeper subst
{
2018-07-27 17:12:01 +00:00
contributing_zk_paths.insert(attr_nodes["from_zk"]->getNodeValue());
if (zk_node_cache)
{
XMLDocumentPtr zk_document;
auto get_zk_node = [&](const std::string & name) -> const Node *
{
zkutil::ZooKeeperNodeCache::ZNode znode = zk_node_cache->get(name, zk_changed_event);
if (!znode.exists)
return nullptr;
/// Enclose contents into a fake <from_zk> tag to allow pure text substitutions.
zk_document = dom_parser.parseString("<from_zk>" + znode.contents + "</from_zk>");
return getRootNode(zk_document.get());
};
2018-07-27 17:12:01 +00:00
process_include(attr_nodes["from_zk"], get_zk_node, "Could not get ZooKeeper node: ");
}
}
2018-07-27 17:12:01 +00:00
if (attr_nodes["from_env"]) /// we have env subst
{
XMLDocumentPtr env_document;
auto get_env_node = [&](const std::string & name) -> const Node *
{
2018-07-27 17:12:01 +00:00
const char * env_val = std::getenv(name.c_str());
if (env_val == nullptr)
return nullptr;
2018-07-27 17:12:01 +00:00
env_document = dom_parser.parseString("<from_env>" + std::string{env_val} + "</from_env>");
return getRootNode(env_document.get());
};
2018-07-27 17:12:01 +00:00
process_include(attr_nodes["from_env"], get_env_node, "Env variable is not set: ");
}
if (included_something)
doIncludesRecursive(config, include_from, node, zk_node_cache, zk_changed_event, contributing_zk_paths);
else
{
NodeListPtr children = node->childNodes();
Node * child = nullptr;
for (size_t i = 0; (child = children->item(i)); ++i)
doIncludesRecursive(config, include_from, child, zk_node_cache, zk_changed_event, contributing_zk_paths);
}
2015-10-05 01:26:43 +00:00
}
ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string & config_path)
2015-10-05 01:26:43 +00:00
{
Files files;
Poco::Path merge_dir_path(config_path);
std::set<std::string> merge_dirs;
/// Add path_to_config/config_name.d dir
merge_dir_path.setExtension("d");
merge_dirs.insert(merge_dir_path.toString());
/// Add path_to_config/conf.d dir
merge_dir_path.setBaseName("conf");
merge_dirs.insert(merge_dir_path.toString());
for (const std::string & merge_dir_name : merge_dirs)
{
Poco::File merge_dir(merge_dir_name);
if (!merge_dir.exists() || !merge_dir.isDirectory())
continue;
for (Poco::DirectoryIterator it(merge_dir_name); it != Poco::DirectoryIterator(); ++it)
{
Poco::File & file = *it;
Poco::Path path(file.path());
std::string extension = path.getExtension();
std::string base_name = path.getBaseName();
// Skip non-config and temporary files
if (file.isFile() && (extension == "xml" || extension == "conf") && !startsWith(base_name, "."))
files.push_back(file.path());
}
2020-02-07 19:37:24 +00:00
}
std::sort(files.begin(), files.end());
return files;
}
XMLDocumentPtr ConfigProcessor::processConfig(
bool * has_zk_includes,
zkutil::ZooKeeperNodeCache * zk_node_cache,
const zkutil::EventPtr & zk_changed_event)
{
2020-08-08 03:42:42 +00:00
XMLDocumentPtr config;
2020-05-23 22:24:01 +00:00
LOG_DEBUG(log, "Processing configuration file '{}'.", path);
if (fs::exists(path))
2020-08-08 03:42:42 +00:00
{
config = dom_parser.parse(path);
}
else
{
/// When we can use config embedded in binary.
if (path == "config.xml")
{
auto resource = getResource("embedded.xml");
if (resource.empty())
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Configuration file {} doesn't exist and there is no embedded config", path);
LOG_DEBUG(log, "There is no file '{}', will use embedded config.", path);
config = dom_parser.parseMemory(resource.data(), resource.size());
}
else
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Configuration file {} doesn't exist", path);
}
std::vector<std::string> contributing_files;
contributing_files.push_back(path);
for (auto & merge_file : getConfigMergeFiles(path))
{
try
{
2020-05-23 22:24:01 +00:00
LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file);
XMLDocumentPtr with = dom_parser.parse(merge_file);
merge(config, with);
contributing_files.push_back(merge_file);
}
catch (Exception & e)
{
e.addMessage("while merging config '" + path + "' with '" + merge_file + "'");
throw;
}
catch (Poco::Exception & e)
{
2017-11-24 22:52:49 +00:00
throw Poco::Exception("Failed to merge config with '" + merge_file + "': " + e.displayText());
}
}
std::unordered_set<std::string> contributing_zk_paths;
try
{
Node * node = config->getNodeByPath("yandex/include_from");
XMLDocumentPtr include_from;
std::string include_from_path;
if (node)
{
2018-10-26 09:22:33 +00:00
/// if we include_from env or zk.
doIncludesRecursive(config, nullptr, node, zk_node_cache, zk_changed_event, contributing_zk_paths);
include_from_path = node->innerText();
}
else
{
std::string default_path = "/etc/metrika.xml";
if (Poco::File(default_path).exists())
include_from_path = default_path;
}
if (!include_from_path.empty())
{
2020-05-23 22:24:01 +00:00
LOG_DEBUG(log, "Including configuration file '{}'.", include_from_path);
contributing_files.push_back(include_from_path);
include_from = dom_parser.parse(include_from_path);
}
doIncludesRecursive(config, include_from, getRootNode(config.get()), zk_node_cache, zk_changed_event, contributing_zk_paths);
}
catch (Exception & e)
{
e.addMessage("while preprocessing config '" + path + "'");
throw;
}
catch (Poco::Exception & e)
{
2017-11-24 22:52:49 +00:00
throw Poco::Exception("Failed to preprocess config '" + path + "': " + e.displayText(), e);
}
if (has_zk_includes)
*has_zk_includes = !contributing_zk_paths.empty();
2020-11-10 18:22:26 +00:00
WriteBufferFromOwnString comment;
comment << " This file was generated automatically.\n";
comment << " Do not edit it: it is likely to be discarded and generated again before it's read next time.\n";
comment << " Files used to generate this file:";
for (const std::string & contributing_file : contributing_files)
{
comment << "\n " << contributing_file;
}
if (zk_node_cache && !contributing_zk_paths.empty())
{
comment << "\n ZooKeeper nodes used to generate this file:";
for (const std::string & contributing_zk_path : contributing_zk_paths)
comment << "\n " << contributing_zk_path;
}
comment << " ";
NodePtr new_node = config->createTextNode("\n\n");
config->insertBefore(new_node, config->firstChild());
new_node = config->createComment(comment.str());
config->insertBefore(new_node, config->firstChild());
return config;
2015-10-05 01:26:43 +00:00
}
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes)
2015-10-05 01:26:43 +00:00
{
bool has_zk_includes;
XMLDocumentPtr config_xml = processConfig(&has_zk_includes);
2015-10-05 01:26:43 +00:00
if (has_zk_includes && !allow_zk_includes)
throw Poco::Exception("Error while loading config '" + path + "': from_zk includes are not allowed!");
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml, path};
}
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
zkutil::ZooKeeperNodeCache & zk_node_cache,
const zkutil::EventPtr & zk_changed_event,
bool fallback_to_preprocessed)
{
XMLDocumentPtr config_xml;
bool has_zk_includes;
bool processed_successfully = false;
try
{
config_xml = processConfig(&has_zk_includes, &zk_node_cache, zk_changed_event);
processed_successfully = true;
}
catch (const Poco::Exception & ex)
{
if (!fallback_to_preprocessed)
throw;
const auto * zk_exception = dynamic_cast<const Coordination::Exception *>(ex.nested());
if (!zk_exception)
throw;
2020-05-23 22:24:01 +00:00
LOG_WARNING(log, "Error while processing from_zk config includes: {}. Config will be loaded from preprocessed file: {}", zk_exception->message(), preprocessed_path);
config_xml = dom_parser.parse(preprocessed_path);
}
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml, path};
}
void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir)
{
try
{
if (preprocessed_path.empty())
{
fs::path preprocessed_configs_path("preprocessed_configs/");
auto new_path = loaded_config.config_path;
if (new_path.substr(0, main_config_path.size()) == main_config_path)
new_path.replace(0, main_config_path.size(), "");
std::replace(new_path.begin(), new_path.end(), '/', '_');
if (preprocessed_dir.empty())
{
if (!loaded_config.configuration->has("path"))
{
// Will use current directory
auto parent_path = Poco::Path(loaded_config.config_path).makeParent();
preprocessed_dir = parent_path.toString();
Poco::Path poco_new_path(new_path);
poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX);
new_path = poco_new_path.toString();
}
else
{
fs::path loaded_config_path(loaded_config.configuration->getString("path"));
preprocessed_dir = loaded_config_path / preprocessed_configs_path;
}
}
else
{
fs::path preprocessed_dir_path(preprocessed_dir);
preprocessed_dir = (preprocessed_dir_path / preprocessed_configs_path).string();
}
preprocessed_path = (fs::path(preprocessed_dir) / fs::path(new_path)).string();
auto preprocessed_path_parent = Poco::Path(preprocessed_path).makeParent();
if (!preprocessed_path_parent.toString().empty())
Poco::File(preprocessed_path_parent).createDirectories();
}
DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml);
2020-05-23 22:24:01 +00:00
LOG_DEBUG(log, "Saved preprocessed configuration to '{}'.", preprocessed_path);
}
catch (Poco::Exception & e)
{
2020-05-23 22:24:01 +00:00
LOG_WARNING(log, "Couldn't save preprocessed config to {}: {}", preprocessed_path, e.displayText());
}
2015-10-05 01:26:43 +00:00
}
void ConfigProcessor::setConfigPath(const std::string & config_path)
{
main_config_path = config_path;
}
}