thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #57387 from evillique/better-disks

Browse Source 
Initialize only required disks in clickhouse-disks
This commit is contained in:
jsc0218 2023-12-05 13:59:04 -05:00 committed by GitHub
commit cdd5280272
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 77 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
programs/disks/CommandCopy.cpp
View File

@ -36,7 +36,7 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) override
{
if (command_arguments.size() != 2)
@ -51,8 +51,8 @@ public:
const String & path_from = command_arguments[0];
const String & path_to = command_arguments[1];
DiskPtr disk_from = global_context->getDisk(disk_name_from);
DiskPtr disk_to = global_context->getDisk(disk_name_to);
DiskPtr disk_from = disk_selector->get(disk_name_from);
DiskPtr disk_to = disk_selector->get(disk_name_to);
String relative_path_from = validatePathAndGetAsRelative(path_from);
String relative_path_to = validatePathAndGetAsRelative(path_to);

4
programs/disks/CommandLink.cpp
View File

@ -27,7 +27,7 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) override
{
if (command_arguments.size() != 2)
@ -41,7 +41,7 @@ public:
const String & path_from = command_arguments[0];
const String & path_to = command_arguments[1];
DiskPtr disk = global_context->getDisk(disk_name);
DiskPtr disk = disk_selector->get(disk_name);
String relative_path_from = validatePathAndGetAsRelative(path_from);
String relative_path_to = validatePathAndGetAsRelative(path_to);

4
programs/disks/CommandList.cpp
View File

@ -33,7 +33,7 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) override
{
if (command_arguments.size() != 1)
@ -46,7 +46,7 @@ public:
const String & path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
DiskPtr disk = disk_selector->get(disk_name);
String relative_path = validatePathAndGetAsRelative(path);

29
programs/disks/CommandListDisks.cpp
View File

@ -26,8 +26,8 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
Poco::Util::LayeredConfiguration &) override
std::shared_ptr<DiskSelector> &,
Poco::Util::LayeredConfiguration & config) override
{
if (!command_arguments.empty())
{
@ -35,8 +35,29 @@ public:
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Bad Arguments");
}
for (const auto & [disk_name, _] : global_context->getDisksMap())
std::cout << disk_name << '\n';
constexpr auto config_prefix = "storage_configuration.disks";
constexpr auto default_disk_name = "default";
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(config_prefix, keys);
bool has_default_disk = false;
/// For the output to be ordered
std::set<String> disks;
for (const auto & disk_name : keys)
{
if (disk_name == default_disk_name)
has_default_disk = true;
disks.insert(disk_name);
}
if (!has_default_disk)
disks.insert(default_disk_name);
for (const auto & disk : disks)
std::cout << disk << '\n';
}
};
}

4
programs/disks/CommandMkDir.cpp
View File

@ -34,7 +34,7 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) override
{
if (command_arguments.size() != 1)
@ -47,7 +47,7 @@ public:
const String & path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
DiskPtr disk = disk_selector->get(disk_name);
String relative_path = validatePathAndGetAsRelative(path);
bool recursive = config.getBool("recursive", false);

4
programs/disks/CommandMove.cpp
View File

@ -26,7 +26,7 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) override
{
if (command_arguments.size() != 2)
@ -40,7 +40,7 @@ public:
const String & path_from = command_arguments[0];
const String & path_to = command_arguments[1];
DiskPtr disk = global_context->getDisk(disk_name);
DiskPtr disk = disk_selector->get(disk_name);
String relative_path_from = validatePathAndGetAsRelative(path_from);
String relative_path_to = validatePathAndGetAsRelative(path_to);

4
programs/disks/CommandRead.cpp
View File

@ -36,7 +36,7 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) override
{
if (command_arguments.size() != 1)
@ -47,7 +47,7 @@ public:
String disk_name = config.getString("disk", "default");
DiskPtr disk = global_context->getDisk(disk_name);
DiskPtr disk = disk_selector->get(disk_name);
String relative_path = validatePathAndGetAsRelative(command_arguments[0]);

4
programs/disks/CommandRemove.cpp
View File

@ -26,7 +26,7 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) override
{
if (command_arguments.size() != 1)
@ -39,7 +39,7 @@ public:
const String & path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
DiskPtr disk = disk_selector->get(disk_name);
String relative_path = validatePathAndGetAsRelative(path);

4
programs/disks/CommandWrite.cpp
View File

@ -37,7 +37,7 @@ public:
void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) override
{
if (command_arguments.size() != 1)
@ -50,7 +50,7 @@ public:
const String & path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
DiskPtr disk = disk_selector->get(disk_name);
String relative_path = validatePathAndGetAsRelative(path);

30
programs/disks/DisksApp.cpp
View File

@ -209,7 +209,35 @@ int DisksApp::main(const std::vector<String> & /*args*/)
po::parsed_options parsed = parser.run();
args = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::include_positional);
}
command->execute(args, global_context, config());
std::unordered_set<std::string> disks
{
config().getString("disk", "default"),
config().getString("disk-from", config().getString("disk", "default")),
config().getString("disk-to", config().getString("disk", "default")),
};
auto validator = [&disks](
const Poco::Util::AbstractConfiguration & config,
const std::string & disk_config_prefix,
const std::string & disk_name)
{
if (!disks.contains(disk_name))
return false;
const auto disk_type = config.getString(disk_config_prefix + ".type", "local");
if (disk_type == "cache")
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk type 'cache' of disk {} is not supported by clickhouse-disks", disk_name);
return true;
};
constexpr auto config_prefix = "storage_configuration.disks";
auto disk_selector = std::make_shared<DiskSelector>();
disk_selector->initialize(config(), config_prefix, global_context, validator);
command->execute(args, disk_selector, config());
return Application::EXIT_OK;
}

3
programs/disks/ICommand.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Disks/IDisk.h>
#include <Disks/DiskSelector.h>
#include <boost/program_options.hpp>
@ -25,7 +26,7 @@ public:
virtual void execute(
const std::vector<String> & command_arguments,
DB::ContextMutablePtr & global_context,
std::shared_ptr<DiskSelector> & disk_selector,
Poco::Util::LayeredConfiguration & config) = 0;
const std::optional<ProgramOptionsDescription> & getCommandOptions() const { return command_option_description; }

2
src/Coordination/KeeperContext.cpp
View File

@ -69,7 +69,7 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config,
namespace
{
bool diskValidator(const Poco::Util::AbstractConfiguration & config, const std::string & disk_config_prefix)
bool diskValidator(const Poco::Util::AbstractConfiguration & config, const std::string & disk_config_prefix, const std::string &)
{
const auto disk_type = config.getString(disk_config_prefix + ".type", "local");

4
src/Disks/DiskSelector.cpp
View File

@ -44,9 +44,9 @@ void DiskSelector::initialize(const Poco::Util::AbstractConfiguration & config,
if (disk_name == default_disk_name)
has_default_disk = true;
auto disk_config_prefix = config_prefix + "." + disk_name;
const auto disk_config_prefix = config_prefix + "." + disk_name;
if (disk_validator && !disk_validator(config, disk_config_prefix))
if (disk_validator && !disk_validator(config, disk_config_prefix, disk_name))
continue;
disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context, disks));

2
src/Disks/DiskSelector.h
View File

@ -23,7 +23,7 @@ public:
DiskSelector() = default;
DiskSelector(const DiskSelector & from) = default;
using DiskValidator = std::function<bool(const Poco::Util::AbstractConfiguration & config, const String & disk_config_prefix)>;
using DiskValidator = std::function<bool(const Poco::Util::AbstractConfiguration & config, const String & disk_config_prefix, const String & disk_name)>;
void initialize(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context, DiskValidator disk_validator = {});
DiskSelectorPtr updateFromConfig(