thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-16 19:32:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Autodetect configuration file format if is not .xml, .yml or .yaml

Browse Source
This commit is contained in:
sakulali 2024-07-27 22:17:53 +08:00
parent 7c7783e2ce
commit 4606ec96d3
4 changed files with 89 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
src/Common/Config/ConfigProcessor.cpp
View File

@ -138,9 +138,14 @@ static Node * getRootNode(Document * document)
return XMLUtils::getRootNode(document);
}
static size_t firstNonWhitespacePos(const std::string & s)
{
return s.find_first_not_of(" \t\n\r");
}
static bool allWhitespace(const std::string & s)
{
return s.find_first_not_of(" \t\n\r") == std::string::npos;
return firstNonWhitespacePos(s) == std::string::npos;
}
static void deleteAttributesRecursive(Node * root)
@ -622,6 +627,52 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
return files;
}
XMLDocumentPtr ConfigProcessor::parseConfig(const std::string & config_path)
{
fs::path p(config_path);
std::string extension = p.extension();
boost::algorithm::to_lower(extension);
if (extension == ".xml")
return dom_parser.parse(config_path);
else if (extension == ".yaml" || extension == ".yml")
return YAMLParser::parse(config_path);
else
{
/// Suppose non regular file parsed as XML, such as pipe: /dev/fd/X (regardless it has .xml extension or not)
if (!fs::is_regular_file(config_path))
return dom_parser.parse(config_path);
/// If the regular file begins with < it might be XML, otherwise it might be YAML.
bool maybe_xml = false;
{
std::ifstream file(config_path);
if (!file.is_open())
throw Exception(ErrorCodes::CANNOT_LOAD_CONFIG, "Unknown format of '{}' config", config_path);
std::string line;
while (std::getline(file, line))
{
const size_t pos = firstNonWhitespacePos(line);
if (pos == std::string::npos)
continue;
if (pos < line.size() && '<' == line[pos])
{
maybe_xml = true;
}
break;
}
}
if (maybe_xml)
return dom_parser.parse(config_path);
else
return YAMLParser::parse(config_path);
}
}
XMLDocumentPtr ConfigProcessor::processConfig(
bool * has_zk_includes,
zkutil::ZooKeeperNodeCache * zk_node_cache,
@ -633,23 +684,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
if (fs::exists(path))
{
fs::path p(path);
std::string extension = p.extension();
boost::algorithm::to_lower(extension);
if (extension == ".yaml" || extension == ".yml")
{
config = YAMLParser::parse(path);
}
else if (extension == ".xml" || extension == ".conf" || extension.empty())
{
config = dom_parser.parse(path);
}
else
{
throw Exception(ErrorCodes::CANNOT_LOAD_CONFIG, "Unknown format of '{}' config", path);
}
config = parseConfig(path);
}
else
{
@ -673,20 +708,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file);
XMLDocumentPtr with;
fs::path p(merge_file);
std::string extension = p.extension();
boost::algorithm::to_lower(extension);
if (extension == ".yaml" || extension == ".yml")
{
with = YAMLParser::parse(merge_file);
}
else
{
with = dom_parser.parse(merge_file);
}
with = parseConfig(merge_file);
if (!merge(config, with))
{
LOG_DEBUG(log, "Merging bypassed - configuration file '{}' doesn't belong to configuration '{}' - merging root node name '{}' doesn't match '{}'",
@ -730,19 +752,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
{
LOG_DEBUG(log, "Including configuration file '{}'.", include_from_path);
fs::path p(include_from_path);
std::string extension = p.extension();
boost::algorithm::to_lower(extension);
if (extension == ".yaml" || extension == ".yml")
{
include_from = YAMLParser::parse(include_from_path);
}
else
{
include_from = dom_parser.parse(include_from_path);
}
include_from = parseConfig(include_from_path);
contributing_files.push_back(include_from_path);
}

2
src/Common/Config/ConfigProcessor.h
View File

@ -65,6 +65,8 @@ public:
zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr,
const zkutil::EventPtr & zk_changed_event = nullptr);
XMLDocumentPtr parseConfig(const std::string & config_path);
/// These configurations will be used if there is no configuration file.
static void registerEmbeddedConfig(std::string name, std::string_view content);

8
tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
View File

@ -12,5 +12,9 @@ yml
2
yaml
2
ini
Code: 347. Unknown format of '/config_default.ini' config. (CANNOT_LOAD_CONFIG)
autodetect xml (with leading whitespaces)
2
autodetect xml (non leading whitespaces)
2
autodetect yaml
2

33
tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
View File

@ -12,7 +12,9 @@ XML_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.XML
conf_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.conf
yml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yml
yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yaml
ini_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.ini
autodetect_xml_with_leading_whitespace_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.config
autodetect_xml_non_leading_whitespace_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.cfg
autodetect_yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.properties
function cleanup()
{
@ -22,7 +24,9 @@ function cleanup()
rm "${conf_config:?}"
rm "${yml_config:?}"
rm "${yaml_config:?}"
rm "${ini_config:?}"
rm "${autodetect_xml_with_leading_whitespace_config:?}"
rm "${autodetect_xml_non_leading_whitespace_config:?}"
rm "${autodetect_yaml_config:?}"
}
trap cleanup EXIT
@ -52,9 +56,19 @@ EOL
cat > "$yaml_config" <<EOL
max_threads: 2
EOL
cat > "$ini_config" <<EOL
[config]
max_threads=2
cat > "$autodetect_xml_with_leading_whitespace_config" <<EOL
<config>
<max_threads>2</max_threads>
</config>
EOL
cat > "$autodetect_xml_non_leading_whitespace_config" <<EOL
<config>
<max_threads>2</max_threads>
</config>
EOL
cat > "$autodetect_yaml_config" <<EOL
max_threads: 2
EOL
echo 'default'
@ -74,5 +88,10 @@ echo 'yml'
$CLICKHOUSE_CLIENT --config "$yml_config" -q "select getSetting('max_threads')"
echo 'yaml'
$CLICKHOUSE_CLIENT --config "$yaml_config" -q "select getSetting('max_threads')"
echo 'ini'
$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
echo 'autodetect xml (with leading whitespaces)'
$CLICKHOUSE_CLIENT --config "$autodetect_xml_with_leading_whitespace_config" -q "select getSetting('max_threads')"
echo 'autodetect xml (non leading whitespaces)'
$CLICKHOUSE_CLIENT --config "$autodetect_xml_non_leading_whitespace_config" -q "select getSetting('max_threads')"
echo 'autodetect yaml'
$CLICKHOUSE_CLIENT --config "$autodetect_yaml_config" -q "select getSetting('max_threads')"