Allow globs in keys for clickhouse-extract-from-config tool (#38966)

docker/server/entrypoint.sh

@@ -36,6 +36,9 @@ ERROR_LOG_DIR=""
 if [ -n "$ERROR_LOG_PATH" ]; then ERROR_LOG_DIR="$(dirname "$ERROR_LOG_PATH")"; fi
 FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=format_schema_path || true)"
 
+# There could be many disks declared in config
+readarray -t FILESYSTEM_CACHE_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.data_cache_path' || true)
+
 CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
 CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
 CLICKHOUSE_DB="${CLICKHOUSE_DB:-}"
@@ -46,7 +49,8 @@ for dir in "$DATA_DIR" \
   "$LOG_DIR" \
   "$TMP_DIR" \
   "$USER_PATH" \
-  "$FORMAT_SCHEMA_PATH"
+  "$FORMAT_SCHEMA_PATH" \
+  "${FILESYSTEM_CACHE_PATHS[@]}"
 do
     # check if variable not empty
     [ -z "$dir" ] && continue

programs/extract-from-config/ExtractFromConfig.cpp

@@ -1,6 +1,11 @@
 #include <iostream>
+#include <string>
+#include <vector>
+#include <memory>
+#include <queue>
 
 #include <boost/program_options.hpp>
+#include <boost/algorithm/string.hpp>
 #include <Poco/Logger.h>
 #include <Poco/ConsoleChannel.h>
 #include <Poco/FormattingChannel.h>
@@ -11,7 +16,9 @@
 #include <Common/ZooKeeper/ZooKeeperNodeCache.h>
 #include <Common/Config/ConfigProcessor.h>
 #include <Common/Exception.h>
+#include <Common/parseGlobs.h>
 
+#include <re2/re2.h>
 
 static void setupLogging(const std::string & log_level)
 {
@@ -23,7 +30,57 @@ static void setupLogging(const std::string & log_level)
     Poco::Logger::root().setLevel(log_level);
 }
 
-static std::string extractFromConfig(
+
+static std::vector<std::string> extactFromConfigAccordingToGlobs(DB::ConfigurationPtr configuration, const std::string & pattern, bool try_get)
+{
+    auto pattern_prefix = pattern.substr(0, pattern.find_first_of("*?{"));
+    boost::algorithm::trim_if(pattern_prefix, [](char s){ return s == '.'; });
+
+    auto matcher = std::make_unique<re2::RE2>(DB::makeRegexpPatternFromGlobs(pattern));
+
+    std::vector<std::string> result;
+    std::queue<std::string> working_queue;
+    working_queue.emplace(pattern_prefix);
+
+    while (!working_queue.empty())
+    {
+        auto node = working_queue.front();
+        working_queue.pop();
+
+        /// Disclose one more layer
+        Poco::Util::AbstractConfiguration::Keys keys;
+        configuration->keys(node, keys);
+
+        /// This is a leave
+        if (keys.empty())
+        {
+            if (!re2::RE2::FullMatch(node, *matcher))
+                continue;
+
+
+            if (try_get)
+            {
+                auto value = configuration->getString(node, "");
+                if (!value.empty())
+                    result.emplace_back(value);
+            }
+            else
+            {
+                result.emplace_back(configuration->getString(node));
+            }
+            continue;
+        }
+
+        /// Add new nodes to working queue
+        for (const auto & key : keys)
+            working_queue.emplace(fmt::format("{}.{}", node, key));
+    }
+
+    return result;
+}
+
+
+static std::vector<std::string> extractFromConfig(
         const std::string & config_path, const std::string & key, bool process_zk_includes, bool try_get = false)
 {
     DB::ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
@@ -38,10 +95,15 @@ static std::string extractFromConfig(
         config_xml = processor.processConfig(&has_zk_includes, &zk_node_cache);
     }
     DB::ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
-    // do not throw exception if not found
+
+    /// Check if a key has globs.
+    if (key.find_first_of("*?{") != std::string::npos)
+        return extactFromConfigAccordingToGlobs(configuration, key, try_get);
+
+    /// Do not throw exception if not found.
     if (try_get)
-        return configuration->getString(key, "");
-    return configuration->getString(key);
+        return {configuration->getString(key, "")};
+    return {configuration->getString(key)};
 }
 
 #pragma GCC diagnostic ignored "-Wunused-function"
@@ -91,7 +153,8 @@ int mainEntryClickHouseExtractFromConfig(int argc, char ** argv)
         po::notify(options);
 
         setupLogging(log_level);
-        std::cout << extractFromConfig(config_path, key, process_zk_includes, try_get) << std::endl;
+        for (const auto & value : extractFromConfig(config_path, key, process_zk_includes, try_get))
+            std::cout << value << std::endl;
     }
     catch (...)
     {