thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

working version with comments

Browse Source
This commit is contained in:
alesapin 2024-03-13 13:12:48 +01:00
parent a8028b0bdc
commit 6203d45b96
5 changed files with 104 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
programs/server/Server.cpp
View File

@ -733,8 +733,6 @@ try
LOG_INFO(log, "Available CPU instruction sets: {}", cpu_info);
#endif
sanityChecks(*this);
// Initialize global thread pool. Do it before we fetch configs from zookeeper
// nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
// ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well.
@ -904,6 +902,7 @@ try
config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
config().removeConfiguration(old_configuration.get());
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
global_context->setConfig(loaded_config.configuration);
}
Settings::checkNoSettingNamesAtTopLevel(config(), config_path);
@ -911,6 +910,9 @@ try
/// We need to reload server settings because config could be updated via zookeeper.
server_settings.loadSettingsFromConfig(config());
/// NOTE: Do sanity checks after we loaded all possible substituions from ZK
sanityChecks(*this);
#if defined(OS_LINUX)
std::string executable_path = getExecutablePath();

71
src/Common/Config/ConfigProcessor.cpp
View File

@ -272,6 +272,7 @@ void ConfigProcessor::hideRecursive(Poco::XML::Node * config_root)
void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root, const Node * with_root)
{
//LOG_DEBUG(log, "WITH ROOT {}", with_root->nodeName());
const NodeListPtr with_nodes = with_root->childNodes();
using ElementsByIdentifier = std::multimap<ElementIdentifier, Node *>;
ElementsByIdentifier config_element_by_id;
@ -287,7 +288,7 @@ void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root,
}
else if (node->nodeType() == Node::ELEMENT_NODE)
{
std::cerr << "NODES IN SOURCE: " << node->nodeName() << std::endl;
//LOG_DEBUG(log, "NODES IN SOURCE: {}", node->nodeName());
config_element_by_id.insert(ElementsByIdentifier::value_type(getElementIdentifier(node), node));
}
node = next_node;
@ -301,6 +302,7 @@ void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root,
bool remove = false;
if (with_node->nodeType() == Node::ELEMENT_NODE)
{
//LOG_DEBUG(log, "WITH NODE: {}", with_node->nodeName());
//std::cerr << "WITH NODE: " << with_node->nodeName() << std::endl;
Element & with_element = dynamic_cast<Element &>(*with_node);
remove = with_element.hasAttribute("remove");
@ -315,6 +317,7 @@ void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root,
if (it != config_element_by_id.end())
{
Node * config_node = it->second;
//LOG_DEBUG(log, "SUBNODE NODE: {}", config_node->nodeName());
//std::cerr << "SUBNODE NODE: " << config_node->nodeName() << std::endl;
config_element_by_id.erase(it);
@ -324,6 +327,7 @@ void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root,
}
else if (replace)
{
//LOG_DEBUG(log, "REPLACE: {}", config_node->nodeName());
//std::cerr << "REPLACE!!!" << std::endl;
with_element.removeAttribute("replace");
NodePtr new_node = config->importNode(with_node, true);
@ -331,6 +335,7 @@ void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root,
}
else
{
//LOG_DEBUG(log, "SUBNODE NODE HERE: {}", config_node->nodeName());
//std::cerr << "SUBNODE NODE HERE: " << config_node->nodeName() << std::endl;
Element & config_element = dynamic_cast<Element &>(*config_node);
@ -346,13 +351,15 @@ void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root,
//std::cerr << "DONE\n";
merged = true;
}
}
else
{
//std::cerr << "ELEMENT NOT FOUND\n";
//else
//{
// LOG_DEBUG(log, "ELEMENT NOT FOUND");
// //std::cerr << "ELEMENT NOT FOUND\n";
//}
}
if (!merged && !remove)
{
//LOG_DEBUG(log, "NOTHING HAPPENED");
//std::cerr << "NOTHING hAPPENED\n";
/// Since we didn't find a pair to this node in default config, we will paste it as is.
/// But it may have some child nodes which have attributes like "replace" or "remove".
@ -443,13 +450,14 @@ void ConfigProcessor::doIncludesRecursive(
/// Replace the original contents, not add to it.
bool replace = attributes->getNamedItem("replace");
bool merge = attributes->getNamedItem("merge");
bool included_something = false;
auto process_include = [&](const Node * include_attr, const std::function<const Node * (const std::string &)> & get_node, const char * error_msg)
{
const std::string & name = include_attr->getNodeValue();
LOG_DEBUG(log, "PROCESS INCLUDE {}", name);
//LOG_DEBUG(log, "PROCESS INCLUDE {}", name);
const Node * node_to_include = get_node(name);
if (!node_to_include)
{
@ -467,26 +475,38 @@ void ConfigProcessor::doIncludesRecursive(
}
else
{
Element & element = dynamic_cast<Element &>(*node);
/// Replace the whole node not just contents.
if (node->nodeName() == "include")
{
LOG_DEBUG(log, "Include here for node {}", name);
//LOG_DEBUG(log, "Include here for node {}", name);
const NodeListPtr children = node_to_include->childNodes();
Node * next_child = nullptr;
for (Node * child = children->item(0); child; child = next_child)
{
next_child = child->nextSibling();
NodePtr new_node = config->importNode(child, true);
//node->parentNode()->insertBefore(new_node, node);
mergeRecursive(config, node->parentNode(), new_node);
//LOG_DEBUG(log, "MERGEEEE {} PARENT NODE {} NODE {}", merge, node->parentNode()->nodeName(), node->nodeName());
if (merge)
{
//LOG_DEBUG(log, "MERGEEEE");
NodePtr new_node = config->importNode(child->parentNode(), true);
//LOG_DEBUG(log, "CHILD {} NEW NODE NAME {}", child->nodeName(), new_node->nodeName());
mergeRecursive(config, node->parentNode(), new_node);
}
else
{
NodePtr new_node = config->importNode(child, true);
node->parentNode()->insertBefore(new_node, node);
}
}
node->parentNode()->removeChild(node);
}
else
{
Element & element = dynamic_cast<Element &>(*node);
for (const auto & attr_name : SUBSTITUTION_ATTRS)
element.removeAttribute(attr_name);
@ -792,13 +812,19 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
//LOG_DEBUG(log, "MIN BYTES FOR WIDE PART {}", configuration->getUInt64("merge_tree.min_bytes_for_wide_part"));
//Poco::Util::AbstractConfiguration::Keys config_keys;
//configuration->keys("merge_tree", config_keys);
//for (const String & key : config_keys)
//{
// LOG_DEBUG(log, "CONFIG KEY {} LEFT in merge_tree before ZK", key);
//}
return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml, path};
}
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
zkutil::ZooKeeperNodeCache & zk_node_cache,
const zkutil::EventPtr & zk_changed_event,
bool fallback_to_preprocessed)
zkutil::ZooKeeperNodeCache & zk_node_cache, const zkutil::EventPtr & zk_changed_event, bool fallback_to_preprocessed)
{
XMLDocumentPtr config_xml;
bool has_zk_includes;
@ -817,13 +843,26 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
if (!zk_exception)
throw;
LOG_WARNING(log, "Error while processing from_zk config includes: {}. Config will be loaded from preprocessed file: {}", zk_exception->message(), preprocessed_path);
LOG_WARNING(
log,
"Error while processing from_zk config includes: {}. Config will be loaded from preprocessed file: {}",
zk_exception->message(),
preprocessed_path);
config_xml = dom_parser.parse(preprocessed_path);
}
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
//LOG_DEBUG(log, "MIN BYTES FOR WIDE PART {} WITH ZK INCLUDE", configuration->getUInt64("merge_tree.min_bytes_for_wide_part"));
//Poco::Util::AbstractConfiguration::Keys config_keys;
//configuration->keys("merge_tree", config_keys);
//for (const String & key : config_keys)
//{
// LOG_DEBUG(log, "CONFIG KEY {} LEFT in merge_tree after ZK", key);
//}
return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml, path};
}

4
src/Core/BaseSettings.cpp
View File

@ -41,13 +41,13 @@ BaseSettingsHelpers::Flags BaseSettingsHelpers::readFlags(ReadBuffer & in)
void BaseSettingsHelpers::throwSettingNotFound(std::string_view name)
{
throw Exception(ErrorCodes::UNKNOWN_SETTING, "Unknown setting {}", String{name});
throw Exception(ErrorCodes::UNKNOWN_SETTING, "Unknown setting '{}'", String{name});
}
void BaseSettingsHelpers::warningSettingNotFound(std::string_view name)
{
LOG_WARNING(getLogger("Settings"), "Unknown setting {}, skipping", name);
LOG_WARNING(getLogger("Settings"), "Unknown setting '{}', skipping", name);
}
}

6
tests/integration/test_config_substitutions/configs/config_zk_include_test.xml
View File

@ -1,8 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<clickhouse>
<background_pool_size>44</background_pool_size>
<merge_tree>
<include from_zk="/merge_max_block_size" merge="true"/>
<merge_max_block_size>99</merge_max_block_size>
<min_bytes_for_wide_part>1</min_bytes_for_wide_part>
<min_rows_for_wide_part>1111</min_rows_for_wide_part>
</merge_tree>
<include from_zk="/min_bytes_for_wide_part"/>
<include from_zk="/min_bytes_for_wide_part" merge="true"/>
</clickhouse>

40
tests/integration/test_config_substitutions/test.py
View File

@ -67,6 +67,12 @@ def start_cluster():
value=b"<merge_tree><min_bytes_for_wide_part>33</min_bytes_for_wide_part></merge_tree>",
makepath=True,
)
zk.create(
path="/merge_max_block_size",
value=b"<merge_max_block_size>8888</merge_max_block_size>",
makepath=True,
)
cluster.add_zookeeper_startup_command(create_zk_roots)
@ -244,5 +250,35 @@ def test_allow_databases(start_cluster):
)
def test_config_multiple_zk_substitutions(start_cluster):
#print(node3.query("SELECT * FROM system.merge_tree_settings"))
print(node3.query("SELECT * FROM system.merge_tree_settings WHERE changed=1"))
assert node3.query("SELECT value FROM system.merge_tree_settings WHERE name='min_bytes_for_wide_part'") == "33\n"
assert node3.query("SELECT value FROM system.merge_tree_settings WHERE name='min_rows_for_wide_part'") == "1111\n"
assert node3.query("SELECT value FROM system.merge_tree_settings WHERE name='merge_max_block_size'") == "8888\n"
assert node3.query("SELECT value FROM system.server_settings WHERE name='background_pool_size'") == "44\n"
zk = cluster.get_kazoo_client("zoo1")
zk.create(
path="/background_pool_size",
value=b"<background_pool_size>72</background_pool_size>",
makepath=True,
)
node3.replace_config(
"/etc/clickhouse-server/config.d/config_zk_include_test.xml",
"""
<clickhouse>
<include from_zk="/background_pool_size" merge="true"/>
<background_pool_size>44</background_pool_size>
<merge_tree>
<include from_zk="/merge_max_block_size" merge="true"/>
<min_bytes_for_wide_part>1</min_bytes_for_wide_part>
<min_rows_for_wide_part>1111</min_rows_for_wide_part>
</merge_tree>
<include from_zk="/min_bytes_for_wide_part" merge="true"/>
</clickhouse>
""",
)
node3.query("SYSTEM RELOAD CONFIG")
assert node3.query("SELECT value FROM system.server_settings WHERE name='background_pool_size'") == "72\n"