thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-29 02:52:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

deduce hdfs config path

Browse Source
This commit is contained in:
taiyang-li 2022-03-24 18:04:29 +08:00
parent d6f5583820
commit 6eccd09e2b
2 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/Storages/HDFS/HDFSCommon.cpp
View File

@ -2,6 +2,7 @@
#include <Poco/URI.h>
#include <boost/algorithm/string/replace.hpp>
#include <re2/re2.h>
#include <filesystem>
#if USE_HDFS
#include <Common/ShellCommand.h>
@ -25,14 +26,14 @@ const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs";
const String HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration & config,
const String & config_path, bool isUser)
const String & prefix, bool isUser)
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(config_path, keys);
config.keys(prefix, keys);
for (const auto & key : keys)
{
const String key_path = config_path + "." + key;
const String key_path = prefix + "." + key;
String key_name;
if (key == "hadoop_kerberos_keytab")
@ -122,9 +123,16 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
throw Exception("Illegal HDFS URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
// Shall set env LIBHDFS3_CONF *before* HDFSBuilderWrapper construction.
const String & libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
String libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
if (!libhdfs3_conf.empty())
{
const String config_path = config.getString("config-file", "config.xml");
const auto config_dir = std::filesystem::path{config_path}.remove_filename();
if (std::filesystem::path{libhdfs3_conf}.is_relative() && std::filesystem::exists(config_dir / libhdfs3_conf))
libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
std::cout << "config_path:" << config_path << std::endl;
std::cout << "config_dir:" << config_dir << std::endl;
std::cout << "libhdfs3_conf:" << libhdfs3_conf << std::endl;
setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1);
}
HDFSBuilderWrapper builder;

2
src/Storages/HDFS/HDFSCommon.h
View File

@ -67,7 +67,7 @@ public:
hdfsBuilder * get() { return hdfs_builder; }
private:
void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & config_path, bool isUser = false);
void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & prefix, bool isUser = false);
String getKinitCmd();