thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #35572 from bigo-sg/hdfs_config_prefix

Browse Source 
Deduce absolute hdfs config path
This commit is contained in:
Kseniia Sumarokova 2022-03-31 10:08:32 +02:00 committed by GitHub
commit a711fc0402
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/Storages/HDFS/HDFSCommon.cpp
View File

@ -2,6 +2,7 @@
#include <Poco/URI.h> #include <Poco/URI.h>
#include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/replace.hpp>
#include <re2/re2.h> #include <re2/re2.h>
#include <filesystem>
#if USE_HDFS #if USE_HDFS
#include <Common/ShellCommand.h> #include <Common/ShellCommand.h>
@ -25,14 +26,14 @@ const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs";
const String HDFS_URL_REGEXP = "^hdfs://[^/]*/.*"; const String HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration & config, void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration & config,
const String & config_path, bool isUser) const String & prefix, bool isUser)
{ {
Poco::Util::AbstractConfiguration::Keys keys; Poco::Util::AbstractConfiguration::Keys keys;
config.keys(config_path, keys); config.keys(prefix, keys);
for (const auto & key : keys) for (const auto & key : keys)
{ {
const String key_path = config_path + "." + key; const String key_path = prefix + "." + key;
String key_name; String key_name;
if (key == "hadoop_kerberos_keytab") if (key == "hadoop_kerberos_keytab")
@ -122,9 +123,17 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
throw Exception("Illegal HDFS URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS); throw Exception("Illegal HDFS URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
// Shall set env LIBHDFS3_CONF *before* HDFSBuilderWrapper construction. // Shall set env LIBHDFS3_CONF *before* HDFSBuilderWrapper construction.
const String & libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", ""); String libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
if (!libhdfs3_conf.empty()) if (!libhdfs3_conf.empty())
{ {
if (std::filesystem::path{libhdfs3_conf}.is_relative() && !std::filesystem::exists(libhdfs3_conf))
{
const String config_path = config.getString("config-file", "config.xml");
const auto config_dir = std::filesystem::path{config_path}.remove_filename();
if (std::filesystem::exists(config_dir / libhdfs3_conf))
libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
}
setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1); setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1);
} }
HDFSBuilderWrapper builder; HDFSBuilderWrapper builder;

2
src/Storages/HDFS/HDFSCommon.h
View File

@ -67,7 +67,7 @@ public:
hdfsBuilder * get() { return hdfs_builder; } hdfsBuilder * get() { return hdfs_builder; }
private: private:
void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & config_path, bool isUser = false); void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & prefix, bool isUser = false);
String getKinitCmd(); String getKinitCmd();