thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-28 18:42:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make conversion YAML->XML more conventional.

Browse Source
This commit is contained in:
Vitaly Baranov 2022-09-03 17:30:08 +02:00
parent 0d7cc82267
commit e9b75deeba
2 changed files with 228 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

185
src/Common/Config/YAMLParser.cpp
View File

@ -26,114 +26,107 @@ namespace ErrorCodes
extern const int CANNOT_PARSE_YAML; extern const int CANNOT_PARSE_YAML;
} }
/// A prefix symbol in yaml key
/// We add attributes to nodes by using a prefix symbol in the key part.
/// Currently we use @ as a prefix symbol. Note, that @ is reserved
/// by YAML standard, so we need to write a key-value pair like this: "@attribute": attr_value
const char YAML_ATTRIBUTE_PREFIX = '@';
namespace namespace
{ {
/// A prefix symbol in yaml key
/// We add attributes to nodes by using a prefix symbol in the key part.
/// Currently we use @ as a prefix symbol. Note, that @ is reserved
/// by YAML standard, so we need to write a key-value pair like this: "@attribute": attr_value
const char YAML_ATTRIBUTE_PREFIX = '@';
Poco::AutoPtr<Poco::XML::Element> createCloneNode(Poco::XML::Element & original_node) Poco::AutoPtr<Poco::XML::Element> cloneXMLNode(const Poco::XML::Element & original_node)
{
Poco::AutoPtr<Poco::XML::Element> clone_node = original_node.ownerDocument()->createElement(original_node.nodeName());
original_node.parentNode()->appendChild(clone_node);
return clone_node;
}
void processNode(const YAML::Node & node, Poco::XML::Element & parent_xml_element)
{
auto * xml_document = parent_xml_element.ownerDocument();
switch (node.Type())
{ {
case YAML::NodeType::Scalar: Poco::AutoPtr<Poco::XML::Element> clone_node = original_node.ownerDocument()->createElement(original_node.nodeName());
{ original_node.parentNode()->appendChild(clone_node);
std::string value = node.as<std::string>(); return clone_node;
Poco::AutoPtr<Poco::XML::Text> xml_value = xml_document->createTextNode(value); }
parent_xml_element.appendChild(xml_value);
break;
}
/// We process YAML Sequences as a void processNode(const YAML::Node & node, Poco::XML::Element & parent_xml_node)
/// list of <key>value</key> tags with same key and different values. {
/// For example, we translate this sequence auto * xml_document = parent_xml_node.ownerDocument();
/// seq: switch (node.Type())
/// - val1
/// - val2
///
/// into this:
/// <seq>val1</seq>
/// <seq>val2</seq>
case YAML::NodeType::Sequence:
{ {
for (const auto & child_node : node) case YAML::NodeType::Scalar:
/// For sequences it depends how we want to process them.
/// Sequences of key-value pairs such as:
/// seq:
/// - k1: val1
/// - k2: val2
/// into xml like this:
/// <seq>
/// <k1>val1</k1>
/// <k2>val2</k2>
/// </seq>
///
/// But, if the sequence is just a list, the root-node needs to be repeated, such as:
/// seq:
/// - val1
/// - val2
/// into xml like this:
/// <seq>val1</seq>
/// <seq>val2</seq>
///
/// Therefore check what type the child is, for further processing.
/// Mixing types (values list or map) will lead to strange results but should not happen.
if (parent_xml_element.hasChildNodes() && !child_node.IsMap())
{
/// Create a new parent node with same tag for each child node
processNode(child_node, *createCloneNode(parent_xml_element));
}
else
{
/// Map, so don't recreate the parent node but add directly
processNode(child_node, parent_xml_element);
}
break;
}
case YAML::NodeType::Map:
{
for (const auto & key_value_pair : node)
{ {
const auto & key_node = key_value_pair.first; std::string value = node.as<std::string>();
const auto & value_node = key_value_pair.second; Poco::AutoPtr<Poco::XML::Text> xml_value = xml_document->createTextNode(value);
std::string key = key_node.as<std::string>(); parent_xml_node.appendChild(xml_value);
bool is_attribute = (key.starts_with(YAML_ATTRIBUTE_PREFIX) && value_node.IsScalar()); break;
if (is_attribute) }
{
/// we use substr(1) here to remove YAML_ATTRIBUTE_PREFIX from key /// For sequences we repeat the parent xml node. For example,
auto attribute_name = key.substr(1); /// seq:
std::string value = value_node.as<std::string>(); /// - val1
parent_xml_element.setAttribute(attribute_name, value); /// - val2
} /// is converted into the following xml:
else /// <seq>val1</seq>
{ /// <seq>val2</seq>
Poco::AutoPtr<Poco::XML::Element> xml_key = xml_document->createElement(key); ///
parent_xml_element.appendChild(xml_key); /// A sequence of mappings is converted in the same way:
processNode(value_node, *xml_key); /// seq:
} /// - k1: val1
/// k2: val2
/// - k3: val3
/// is converted into the following xml:
/// <seq><k1>val1</k1><k2>val2</k2></seq>
/// <seq><k3>val3</k3></seq>
case YAML::NodeType::Sequence:
{
size_t i = 0;
for (auto it = node.begin(); it != node.end(); ++it, ++i)
{
const auto & child_node = *it;
bool need_clone_parent_xml_node = (i > 0);
if (need_clone_parent_xml_node)
{
/// Create a new parent node with same tag for each child node
processNode(child_node, *cloneXMLNode(parent_xml_node));
}
else
{
/// Map, so don't recreate the parent node but add directly
processNode(child_node, parent_xml_node);
}
}
break;
}
case YAML::NodeType::Map:
{
for (const auto & key_value_pair : node)
{
const auto & key_node = key_value_pair.first;
const auto & value_node = key_value_pair.second;
std::string key = key_node.as<std::string>();
bool is_attribute = (key.starts_with(YAML_ATTRIBUTE_PREFIX) && value_node.IsScalar());
if (is_attribute)
{
/// we use substr(1) here to remove YAML_ATTRIBUTE_PREFIX from key
auto attribute_name = key.substr(1);
std::string value = value_node.as<std::string>();
parent_xml_node.setAttribute(attribute_name, value);
}
else
{
Poco::AutoPtr<Poco::XML::Element> xml_key = xml_document->createElement(key);
parent_xml_node.appendChild(xml_key);
processNode(value_node, *xml_key);
}
}
break;
}
case YAML::NodeType::Null: break;
case YAML::NodeType::Undefined:
{
throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "YAMLParser has encountered node with undefined type and cannot continue parsing of the file");
} }
break;
}
case YAML::NodeType::Null: break;
case YAML::NodeType::Undefined:
{
throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "YAMLParser has encountered node with undefined type and cannot continue parsing of the file");
} }
} }
} }
}
Poco::AutoPtr<Poco::XML::Document> YAMLParser::parse(const String& path) Poco::AutoPtr<Poco::XML::Document> YAMLParser::parse(const String& path)
{ {

169
src/Common/tests/gtest_yaml_parser.cpp
View File

@ -13,40 +13,12 @@
using namespace DB; using namespace DB;
TEST(Common, YamlParserInvalidFile) TEST(YamlParser, InvalidFile)
{ {
ASSERT_THROW(YAMLParser::parse("some-non-existing-file.yaml"), Exception); ASSERT_THROW(YAMLParser::parse("some-non-existing-file.yaml"), Exception);
} }
TEST(Common, YamlParserProcessKeysList) TEST(YamlParser, ProcessValuesList)
{
auto yaml_file = getFileWithContents("keys-list.yaml", R"YAML(
operator:
access_management: "1"
networks:
- ip: "10.1.6.168"
- ip: "::1"
- ip: "127.0.0.1"
)YAML");
SCOPE_EXIT({ yaml_file->remove(); });
Poco::AutoPtr<Poco::XML::Document> xml = YAMLParser::parse(yaml_file->path());
auto *p_node = xml->getNodeByPath("/clickhouse");
EXPECT_EQ(xmlNodeAsString(p_node), R"CONFIG(<clickhouse>
<operator>
<access_management>1</access_management>
<networks>
<ip>10.1.6.168</ip>
<ip>::1</ip>
<ip>127.0.0.1</ip>
</networks>
</operator>
</clickhouse>
)CONFIG");
}
TEST(Common, YamlParserProcessValuesList)
{ {
auto yaml_file = getFileWithContents("values-list.yaml", R"YAML( auto yaml_file = getFileWithContents("values-list.yaml", R"YAML(
rules: rules:
@ -75,4 +47,141 @@ rules:
)CONFIG"); )CONFIG");
} }
TEST(YamlParser, ProcessKeysList)
{
auto yaml_file = getFileWithContents("keys-list.yaml", R"YAML(
operator:
access_management: 1
networks:
ip:
- 10.1.6.168
- ::1
- 127.0.0.1
)YAML");
SCOPE_EXIT({ yaml_file->remove(); });
Poco::AutoPtr<Poco::XML::Document> xml = YAMLParser::parse(yaml_file->path());
auto *p_node = xml->getNodeByPath("/clickhouse");
EXPECT_EQ(xmlNodeAsString(p_node), R"CONFIG(<clickhouse>
<operator>
<access_management>1</access_management>
<networks>
<ip>10.1.6.168</ip>
<ip>::1</ip>
<ip>127.0.0.1</ip>
</networks>
</operator>
</clickhouse>
)CONFIG");
}
TEST(YamlParser, ProcessListAttributes)
{
auto yaml_file = getFileWithContents("list_attributes.yaml", R"YAML(
seq:
- "@attr1": x
- k1: val1
k2: val2
"@attr2": y
- k3: val3
"@attr3": z
)YAML");
SCOPE_EXIT({ yaml_file->remove(); });
Poco::AutoPtr<Poco::XML::Document> xml = YAMLParser::parse(yaml_file->path());
auto *p_node = xml->getNodeByPath("/clickhouse");
EXPECT_EQ(xmlNodeAsString(p_node), R"CONFIG(<clickhouse>
<seq attr1="x"></seq>
<seq attr2="y">
<k1>val1</k1>
<k2>val2</k2>
</seq>
<seq attr3="z">
<k3>val3</k3>
</seq>
</clickhouse>
)CONFIG");
}
TEST(YamlParser, ProcessMapAttributes)
{
auto yaml_file = getFileWithContents("map_attributes.yaml", R"YAML(
map:
"@attr1": x
k1: val1
k2: val2
"@attr2": y
k3: val3
"@attr3": z
)YAML");
SCOPE_EXIT({ yaml_file->remove(); });
Poco::AutoPtr<Poco::XML::Document> xml = YAMLParser::parse(yaml_file->path());
auto *p_node = xml->getNodeByPath("/clickhouse");
EXPECT_EQ(xmlNodeAsString(p_node), R"CONFIG(<clickhouse>
<map attr1="x" attr2="y" attr3="z">
<k1>val1</k1>
<k2>val2</k2>
<k3>val3</k3>
</map>
</clickhouse>
)CONFIG");
}
TEST(YamlParser, ClusterDef)
{
auto yaml_file = getFileWithContents("cluster_def.yaml", R"YAML(
test_cluster:
shard:
- internal_replication: false
replica:
- host: 127.0.0.1
port: 9000
- host: 127.0.0.2
port: 9000
- internal_replication: true
replica:
- host: 127.0.0.3
port: 9000
- host: 127.0.0.4
port: 9000
)YAML");
SCOPE_EXIT({ yaml_file->remove(); });
Poco::AutoPtr<Poco::XML::Document> xml = YAMLParser::parse(yaml_file->path());
auto *p_node = xml->getNodeByPath("/clickhouse");
EXPECT_EQ(xmlNodeAsString(p_node), R"CONFIG(<clickhouse>
<test_cluster>
<shard>
<internal_replication>false</internal_replication>
<replica>
<host>127.0.0.1</host>
<port>9000</port>
</replica>
<replica>
<host>127.0.0.2</host>
<port>9000</port>
</replica>
</shard>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>127.0.0.3</host>
<port>9000</port>
</replica>
<replica>
<host>127.0.0.4</host>
<port>9000</port>
</replica>
</shard>
</test_cluster>
</clickhouse>
)CONFIG");
}
#endif #endif