ClickHouse/programs/extract-from-config/ExtractFromConfig.cpp

#include <iostream>
#include <string>
#include <vector>
#include <memory>
#include <queue>

#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <Poco/Logger.h>
#include <Poco/ConsoleChannel.h>
#include <Poco/FormattingChannel.h>
#include <Poco/PatternFormatter.h>
#include <Poco/AutoPtr.h>
#include <Poco/Util/XMLConfiguration.h>

#include <Common/ZooKeeper/ZooKeeperNodeCache.h>
#include <Common/Config/ConfigProcessor.h>
#include <Common/Exception.h>
#include <Common/parseGlobs.h>

#ifdef __clang__
#  pragma clang diagnostic push
#  pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#endif
#include <re2/re2.h>
#ifdef __clang__
#  pragma clang diagnostic pop
#endif

static void setupLogging(const std::string & log_level)
{
    Poco::AutoPtr<Poco::ConsoleChannel> channel(new Poco::ConsoleChannel);
    Poco::AutoPtr<Poco::PatternFormatter> formatter(new Poco::PatternFormatter);
    formatter->setProperty("pattern", "%L%Y-%m-%d %H:%M:%S.%i <%p> %s: %t");
    Poco::AutoPtr<Poco::FormattingChannel> formatting_channel(new Poco::FormattingChannel(formatter, channel));
    Poco::Logger::root().setChannel(formatting_channel);
    Poco::Logger::root().setLevel(log_level);
}


static std::vector<std::string> extactFromConfigAccordingToGlobs(DB::ConfigurationPtr configuration, const std::string & pattern, bool try_get)
{
    auto pattern_prefix = pattern.substr(0, pattern.find_first_of("*?{"));
    boost::algorithm::trim_if(pattern_prefix, [](char s){ return s == '.'; });

    auto matcher = std::make_unique<re2::RE2>(DB::makeRegexpPatternFromGlobs(pattern));

    std::vector<std::string> result;
    std::queue<std::string> working_queue;
    working_queue.emplace(pattern_prefix);

    while (!working_queue.empty())
    {
        auto node = working_queue.front();
        working_queue.pop();

        /// Disclose one more layer
        Poco::Util::AbstractConfiguration::Keys keys;
        configuration->keys(node, keys);

        /// This is a leave
        if (keys.empty())
        {
            if (!re2::RE2::FullMatch(node, *matcher))
                continue;


            if (try_get)
            {
                auto value = configuration->getString(node, "");
                if (!value.empty())
                    result.emplace_back(value);
            }
            else
            {
                result.emplace_back(configuration->getString(node));
            }
            continue;
        }

        /// Add new nodes to working queue
        for (const auto & key : keys)
            working_queue.emplace(fmt::format("{}.{}", node, key));
    }

    return result;
}


static std::vector<std::string> extractFromConfig(
        const std::string & config_path, const std::string & key, bool process_zk_includes, bool try_get = false)
{
    DB::ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
    bool has_zk_includes;
    DB::XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes);
    if (has_zk_includes && process_zk_includes)
    {
        DB::ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml));

        zkutil::validateZooKeeperConfig(*bootstrap_configuration);

        zkutil::ZooKeeperPtr zookeeper = std::make_shared<zkutil::ZooKeeper>(
            *bootstrap_configuration, bootstrap_configuration->has("zookeeper") ? "zookeeper" : "keeper", nullptr);

        zkutil::ZooKeeperNodeCache zk_node_cache([&] { return zookeeper; });
        config_xml = processor.processConfig(&has_zk_includes, &zk_node_cache);
    }
    DB::ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));

    /// Check if a key has globs.
    if (key.find_first_of("*?{") != std::string::npos)
        return extactFromConfigAccordingToGlobs(configuration, key, try_get);

    /// Do not throw exception if not found.
    if (try_get)
        return {configuration->getString(key, "")};
    return {configuration->getString(key)};
}

#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wmissing-declarations"

int mainEntryClickHouseExtractFromConfig(int argc, char ** argv)
{
    bool print_stacktrace = false;
    bool process_zk_includes = false;
    bool try_get = false;
    std::string log_level;
    std::string config_path;
    std::string key;

    namespace po = boost::program_options;

    po::options_description options_desc("Allowed options");
    options_desc.add_options()
        ("help", "produce this help message")
        ("stacktrace", po::bool_switch(&print_stacktrace), "print stack traces of exceptions")
        ("process-zk-includes", po::bool_switch(&process_zk_includes),
         "if there are from_zk elements in config, connect to ZooKeeper and process them")
        ("try", po::bool_switch(&try_get), "Do not warn about missing keys")
        ("log-level", po::value<std::string>(&log_level)->default_value("error"), "log level")
        ("config-file,c", po::value<std::string>(&config_path)->required(), "path to config file")
        ("key,k", po::value<std::string>(&key)->required(), "key to get value for");

    po::positional_options_description positional_desc;
    positional_desc.add("config-file", 1);
    positional_desc.add("key", 1);

    try
    {
        po::variables_map options;
        po::store(po::command_line_parser(argc, argv).options(options_desc).positional(positional_desc).run(), options);

        if (options.count("help"))
        {
            std::cerr << "Preprocess config file and extract value of the given key." << std::endl
                << std::endl;
            std::cerr << "Usage: clickhouse extract-from-config [options]" << std::endl
                << std::endl;
            std::cerr << options_desc << std::endl;
            return 0;
        }

        po::notify(options);

        setupLogging(log_level);
        for (const auto & value : extractFromConfig(config_path, key, process_zk_includes, try_get))
            std::cout << value << std::endl;
    }
    catch (...)
    {
        std::cerr << DB::getCurrentExceptionMessage(print_stacktrace, true) << std::endl;
        return DB::getCurrentExceptionCode();
    }

    return 0;
}