ClickHouse/programs/disks/DisksApp.cpp

568 lines
17 KiB
C++
Raw Normal View History

#include "DisksApp.h"
2024-05-27 11:44:45 +00:00
#include <Client/ClientBase.h>
#include <Client/ReplxxLineReader.h>
#include "Common/Exception.h"
2024-05-31 13:10:42 +00:00
#include "Common/filesystemHelpers.h"
2024-05-27 11:44:45 +00:00
#include <Common/Config/ConfigProcessor.h>
#include "DisksClient.h"
2022-12-08 17:53:05 +00:00
#include "ICommand.h"
2024-06-10 16:17:36 +00:00
#include "ICommand_fwd.h"
2024-05-27 11:44:45 +00:00
#include <cstring>
#include <filesystem>
#include <memory>
#include <optional>
#include <Disks/registerDisks.h>
#include <Formats/registerFormats.h>
2024-05-27 11:44:45 +00:00
#include <Common/TerminalSize.h>
namespace DB
{
2024-05-27 12:45:05 +00:00
namespace ErrorCodes
{
2024-06-14 11:06:56 +00:00
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
2024-05-27 12:45:05 +00:00
};
2024-05-31 13:10:42 +00:00
LineReader::Patterns DisksApp::query_extenders = {"\\"};
LineReader::Patterns DisksApp::query_delimiters = {""};
String DisksApp::word_break_characters = " \t\v\f\a\b\r\n";
2024-05-27 12:45:05 +00:00
2024-05-31 13:10:42 +00:00
CommandPtr DisksApp::getCommandByName(const String & command) const
{
2024-05-27 11:44:45 +00:00
try
{
2024-05-31 13:10:42 +00:00
if (auto it = aliases.find(command); it != aliases.end())
return command_descriptions.at(it->second);
2024-05-27 11:44:45 +00:00
return command_descriptions.at(command);
}
2024-05-31 13:10:42 +00:00
catch (std::out_of_range &)
2024-05-27 11:44:45 +00:00
{
2024-05-29 13:57:29 +00:00
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The command `{}` is unknown", command);
2024-05-27 11:44:45 +00:00
}
}
2024-06-10 16:17:36 +00:00
std::vector<String> DisksApp::getEmptyCompletion(String command_name) const
{
2024-06-10 16:17:36 +00:00
auto command_ptr = command_descriptions.at(command_name);
2024-06-17 17:40:52 +00:00
std::vector<String> answer{};
if (multidisk_commands.contains(command_ptr->command_name))
2024-05-27 11:44:45 +00:00
{
2024-06-17 17:40:52 +00:00
answer = client->getAllFilesByPatternFromAllDisks("");
}
else
{
answer = client->getCurrentDiskWithPath().getAllFilesByPattern("");
}
2024-05-27 11:44:45 +00:00
for (const auto & disk_name : client->getAllDiskNames())
{
answer.push_back(disk_name);
}
2024-06-10 16:17:36 +00:00
for (const auto & option : command_ptr->options_description.options())
2024-05-27 11:44:45 +00:00
{
answer.push_back("--" + option->long_name());
}
2024-06-10 16:17:36 +00:00
if (command_name == "help")
{
for (const auto & [current_command_name, description] : command_descriptions)
{
answer.push_back(current_command_name);
}
}
2024-05-27 11:44:45 +00:00
std::sort(answer.begin(), answer.end());
return answer;
}
2024-06-10 16:17:36 +00:00
std::vector<String> DisksApp::getCommandsToComplete(const String & command_prefix) const
{
std::vector<String> answer{};
for (const auto & [word, _] : command_descriptions)
{
if (word.starts_with(command_prefix))
{
answer.push_back(word);
}
}
if (!answer.empty())
{
2024-06-26 15:59:15 +00:00
std::sort(answer.begin(), answer.end());
2024-06-10 16:17:36 +00:00
return answer;
}
for (const auto & [word, _] : aliases)
{
if (word.starts_with(command_prefix))
{
answer.push_back(word);
}
}
if (!answer.empty())
{
2024-06-26 15:59:15 +00:00
std::sort(answer.begin(), answer.end());
2024-06-10 16:17:36 +00:00
return answer;
}
return {command_prefix};
}
2024-05-27 11:44:45 +00:00
std::vector<String> DisksApp::getCompletions(const String & prefix) const
{
2024-05-31 13:10:42 +00:00
auto arguments = po::split_unix(prefix, word_break_characters);
2024-05-27 11:44:45 +00:00
if (arguments.empty())
{
return {};
}
if (word_break_characters.contains(prefix.back()))
{
CommandPtr command;
try
{
command = getCommandByName(arguments[0]);
}
catch (...)
{
return {arguments.back()};
}
2024-06-10 16:17:36 +00:00
return getEmptyCompletion(command->command_name);
2024-05-27 11:44:45 +00:00
}
else if (arguments.size() == 1)
{
String command_prefix = arguments[0];
2024-06-10 16:17:36 +00:00
return getCommandsToComplete(command_prefix);
2024-05-27 11:44:45 +00:00
}
else
{
String last_token = arguments.back();
CommandPtr command;
try
{
command = getCommandByName(arguments[0]);
}
catch (...)
{
return {last_token};
}
2024-06-10 16:17:36 +00:00
std::vector<String> answer = {};
if (command->command_name == "help")
2024-05-27 11:44:45 +00:00
{
2024-06-10 16:17:36 +00:00
return getCommandsToComplete(last_token);
}
else
{
answer = [&]() -> std::vector<String>
2024-05-27 11:44:45 +00:00
{
2024-06-10 16:17:36 +00:00
if (multidisk_commands.contains(command->command_name))
{
return client->getAllFilesByPatternFromAllDisks(last_token);
}
else
{
return client->getCurrentDiskWithPath().getAllFilesByPattern(last_token);
}
}();
2024-05-27 11:44:45 +00:00
2024-06-10 16:17:36 +00:00
for (const auto & disk_name : client->getAllDiskNames())
2024-05-27 11:44:45 +00:00
{
2024-06-10 16:17:36 +00:00
if (disk_name.starts_with(last_token))
{
answer.push_back(disk_name);
}
2024-05-27 11:44:45 +00:00
}
2024-06-10 16:17:36 +00:00
for (const auto & option : command->options_description.options())
2024-05-27 11:44:45 +00:00
{
2024-06-10 16:17:36 +00:00
String option_sign = "--" + option->long_name();
if (option_sign.starts_with(last_token))
{
answer.push_back(option_sign);
}
2024-05-27 11:44:45 +00:00
}
}
if (!answer.empty())
{
2024-06-26 15:59:15 +00:00
std::sort(answer.begin(), answer.end());
2024-05-27 11:44:45 +00:00
return answer;
}
else
{
return {last_token};
}
}
}
2022-06-16 17:38:33 +00:00
2024-05-31 13:10:42 +00:00
bool DisksApp::processQueryText(const String & text)
2024-05-27 11:44:45 +00:00
{
2024-06-13 19:14:16 +00:00
if (text.find_first_not_of(word_break_characters) == std::string::npos)
{
return true;
}
2024-05-27 11:44:45 +00:00
if (exit_strings.find(text) != exit_strings.end())
return false;
CommandPtr command;
try
{
2024-05-31 13:10:42 +00:00
auto arguments = po::split_unix(text, word_break_characters);
2024-05-27 11:44:45 +00:00
command = getCommandByName(arguments[0]);
arguments.erase(arguments.begin());
command->execute(arguments, *client);
}
catch (DB::Exception & err)
{
int code = getCurrentExceptionCode();
if (code == ErrorCodes::LOGICAL_ERROR)
{
2024-06-17 17:40:52 +00:00
throw std::move(err);
2024-05-27 11:44:45 +00:00
}
else if (code == ErrorCodes::BAD_ARGUMENTS)
{
std::cerr << err.message() << "\n"
<< "\n";
if (command.get())
{
std::cerr << "COMMAND: " << command->command_name << "\n";
std::cerr << command->options_description << "\n";
}
2024-05-29 13:57:29 +00:00
else
{
printAvailableCommandsHelpMessage();
}
2024-05-27 11:44:45 +00:00
}
else
{
std::cerr << err.message() << "\n";
}
}
catch (std::exception & err)
{
std::cerr << err.what() << "\n";
}
2022-06-16 17:38:33 +00:00
2024-05-27 11:44:45 +00:00
return true;
}
2024-05-27 11:44:45 +00:00
void DisksApp::runInteractiveReplxx()
{
ReplxxLineReader lr(
suggest,
history_file,
/* multiline= */ false,
query_extenders,
query_delimiters,
word_break_characters.c_str(),
/* highlighter_= */ {});
lr.enableBracketedPaste();
while (true)
{
2024-05-27 14:26:20 +00:00
DiskWithPath disk_with_path = client->getCurrentDiskWithPath();
String prompt = "\x1b[1;34m" + disk_with_path.getDisk()->getName() + "\x1b[0m:" + "\x1b[1;31m" + disk_with_path.getCurrentPath()
+ "\x1b[0m$ ";
2024-05-27 11:44:45 +00:00
2024-05-27 14:26:20 +00:00
auto input = lr.readLine(prompt, "\x1b[1;31m:-] \x1b[0m");
2024-05-27 11:44:45 +00:00
if (input.empty())
break;
2024-05-27 11:44:45 +00:00
if (!processQueryText(input))
break;
}
}
2024-05-27 11:44:45 +00:00
void DisksApp::parseAndCheckOptions(
const std::vector<String> & arguments, const ProgramOptionsDescription & options_description, CommandLineOptions & options)
{
2024-05-27 11:44:45 +00:00
auto parser = po::command_line_parser(arguments).options(options_description).allow_unregistered();
po::parsed_options parsed = parser.run();
po::store(parsed, options);
}
2024-05-27 11:44:45 +00:00
void DisksApp::addOptions()
{
2024-05-27 11:44:45 +00:00
options_description.add_options()("help,h", "Print common help message")("config-file,C", po::value<String>(), "Set config file")(
"disk", po::value<String>(), "Set disk name")("save-logs", "Save logs to a file")(
2024-06-13 19:14:16 +00:00
"log-level", po::value<String>(), "Logging level")("query,q", po::value<String>(), "Query for a non-interactive mode")(
"test-mode", "Interactive interface in test regyme");
command_descriptions.emplace("list-disks", makeCommandListDisks());
2024-05-27 11:44:45 +00:00
command_descriptions.emplace("copy", makeCommandCopy());
command_descriptions.emplace("list", makeCommandList());
2024-05-27 11:44:45 +00:00
command_descriptions.emplace("cd", makeCommandChangeDirectory());
command_descriptions.emplace("move", makeCommandMove());
command_descriptions.emplace("remove", makeCommandRemove());
command_descriptions.emplace("link", makeCommandLink());
command_descriptions.emplace("write", makeCommandWrite());
command_descriptions.emplace("read", makeCommandRead());
2022-09-02 17:30:35 +00:00
command_descriptions.emplace("mkdir", makeCommandMkDir());
2024-05-27 11:44:45 +00:00
command_descriptions.emplace("switch-disk", makeCommandSwitchDisk());
2024-06-13 19:14:16 +00:00
command_descriptions.emplace("current_disk_with_path", makeCommandGetCurrentDiskAndPath());
2024-06-26 15:59:15 +00:00
command_descriptions.emplace("touch", makeCommandTouch());
2024-06-10 16:17:36 +00:00
command_descriptions.emplace("help", makeCommandHelp(*this));
2024-02-14 02:31:31 +00:00
#ifdef CLICKHOUSE_CLOUD
2024-05-27 12:17:04 +00:00
command_descriptions.emplace("packed-io", makeCommandPackedIO());
2024-02-14 02:31:31 +00:00
#endif
2024-05-31 13:10:42 +00:00
for (const auto & [command_name, command_ptr] : command_descriptions)
{
if (command_name != command_ptr->command_name)
{
throw Exception(ErrorCodes::LOGICAL_ERROR, "Command name inside map doesn't coincide with actual command name");
}
}
}
void DisksApp::processOptions()
{
if (options.count("config-file"))
config().setString("config-file", options["config-file"].as<String>());
if (options.count("disk"))
config().setString("disk", options["disk"].as<String>());
2022-10-20 23:14:19 +00:00
if (options.count("save-logs"))
config().setBool("save-logs", true);
2022-06-21 14:40:22 +00:00
if (options.count("log-level"))
2022-10-20 23:14:19 +00:00
config().setString("log-level", options["log-level"].as<String>());
2024-06-13 19:14:16 +00:00
if (options.count("test-mode"))
config().setBool("test-mode", true);
2024-05-29 14:59:50 +00:00
if (options.count("query"))
query = std::optional{options["query"].as<String>()};
}
2024-05-29 13:57:29 +00:00
2024-06-10 16:17:36 +00:00
void DisksApp::printEntryHelpMessage() const
2024-02-14 02:01:03 +00:00
{
2024-06-10 16:17:36 +00:00
std::cout << "\x1b[1;33m ClickHouse disk management tool \x1b[0m \n";
2024-05-29 13:57:29 +00:00
std::cout << options_description << '\n';
}
2024-02-14 02:01:03 +00:00
2024-05-29 13:57:29 +00:00
2024-06-10 16:17:36 +00:00
void DisksApp::printAvailableCommandsHelpMessage() const
2024-05-29 13:57:29 +00:00
{
2024-06-10 16:17:36 +00:00
std::cout << "\x1b[1;32mAvailable commands:\x1b[0m\n";
std::vector<std::pair<String, CommandPtr>> commands_with_aliases_and_descrtiptions{};
2024-05-31 13:10:42 +00:00
size_t maximal_command_length = 0;
2024-06-10 16:17:36 +00:00
for (const auto & [command_name, command_ptr] : command_descriptions)
2024-05-27 11:44:45 +00:00
{
2024-06-10 16:17:36 +00:00
std::string command_string = getCommandLineWithAliases(command_ptr);
2024-05-31 13:10:42 +00:00
maximal_command_length = std::max(maximal_command_length, command_string.size());
2024-06-10 16:17:36 +00:00
commands_with_aliases_and_descrtiptions.push_back({std::move(command_string), command_descriptions.at(command_name)});
}
for (const auto & [command_with_aliases, command_ptr] : commands_with_aliases_and_descrtiptions)
{
std::cout << "\x1b[1;33m" << command_with_aliases << "\x1b[0m" << std::string(5, ' ') << "\x1b[1;33m" << command_ptr->description
<< "\x1b[0m \n";
std::cout << command_ptr->options_description;
std::cout << std::endl;
2024-05-31 13:10:42 +00:00
}
2024-06-10 16:17:36 +00:00
}
void DisksApp::printCommandHelpMessage(CommandPtr command) const
{
String command_name_with_aliases = getCommandLineWithAliases(command);
std::cout << "\x1b[1;32m" << command_name_with_aliases << "\x1b[0m" << std::string(2, ' ') << command->description << "\n";
std::cout << command->options_description;
}
void DisksApp::printCommandHelpMessage(String command_name) const
{
printCommandHelpMessage(getCommandByName(command_name));
}
String DisksApp::getCommandLineWithAliases(CommandPtr command) const
{
String command_string = command->command_name;
bool need_comma = false;
for (const auto & [alias_name, alias_command_name] : aliases)
2024-05-31 13:10:42 +00:00
{
2024-06-10 16:17:36 +00:00
if (alias_command_name == command->command_name)
{
if (std::exchange(need_comma, true))
command_string += ",";
else
command_string += "(";
command_string += alias_name;
}
2024-05-27 11:44:45 +00:00
}
2024-06-10 16:17:36 +00:00
command_string += (need_comma ? ")" : "");
return command_string;
2024-05-27 11:44:45 +00:00
}
2024-05-27 11:44:45 +00:00
void DisksApp::initializeHistoryFile()
{
String home_path;
const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
if (home_path_cstr)
home_path = home_path_cstr;
if (config().has("history-file"))
history_file = config().getString("history-file");
else
history_file = home_path + "/.disks-file-history";
2024-05-27 11:44:45 +00:00
if (!history_file.empty() && !fs::exists(history_file))
{
try
{
FS::createFile(history_file);
}
catch (const ErrnoException & e)
{
if (e.getErrno() != EEXIST)
throw;
}
}
}
2024-05-27 11:44:45 +00:00
void DisksApp::init(const std::vector<String> & common_arguments)
{
addOptions();
parseAndCheckOptions(common_arguments, options_description, options);
po::notify(options);
if (options.count("help"))
{
2024-05-29 13:57:29 +00:00
printEntryHelpMessage();
printAvailableCommandsHelpMessage();
exit(0); // NOLINT(concurrency-mt-unsafe)
}
processOptions();
}
2024-05-27 11:44:45 +00:00
String DisksApp::getDefaultConfigFileName()
{
2024-05-27 11:44:45 +00:00
return "/etc/clickhouse-server/config.xml";
}
int DisksApp::main(const std::vector<String> & /*args*/)
{
2024-05-27 11:44:45 +00:00
std::vector<std::string> keys;
config().keys(keys);
2022-06-16 17:38:33 +00:00
if (config().has("config-file") || fs::exists(getDefaultConfigFileName()))
{
String config_path = config().getString("config-file", getDefaultConfigFileName());
ConfigProcessor config_processor(config_path, false, false);
ConfigProcessor::setConfigPath(fs::path(config_path).parent_path());
2022-06-16 17:38:33 +00:00
auto loaded_config = config_processor.loadConfig();
config().add(loaded_config.configuration.duplicate(), false, false);
}
else
{
2024-05-29 13:57:29 +00:00
printEntryHelpMessage();
throw Exception(ErrorCodes::BAD_ARGUMENTS, "No config-file specified");
2022-06-16 17:38:33 +00:00
}
2024-05-27 11:44:45 +00:00
config().keys(keys);
initializeHistoryFile();
2022-10-20 23:14:19 +00:00
if (config().has("save-logs"))
{
auto log_level = config().getString("log-level", "trace");
Poco::Logger::root().setLevel(Poco::Logger::parseLevel(log_level));
auto log_path = config().getString("logger.clickhouse-disks", "/var/log/clickhouse-server/clickhouse-disks.log");
Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::FileChannel>(new Poco::FileChannel(log_path)));
}
else
{
auto log_level = config().getString("log-level", "none");
Poco::Logger::root().setLevel(Poco::Logger::parseLevel(log_level));
}
registerDisks(/* global_skip_access_check= */ true);
registerFormats();
2024-06-17 17:40:52 +00:00
shared_context = Context::createShared();
global_context = Context::createGlobal(shared_context.get());
global_context->makeGlobalContext();
global_context->setApplicationType(Context::ApplicationType::DISKS);
2022-06-16 17:38:33 +00:00
String path = config().getString("path", DBMS_DEFAULT_PATH);
2024-06-10 16:17:36 +00:00
2022-06-16 17:38:33 +00:00
global_context->setPath(path);
2024-05-27 11:44:45 +00:00
String main_disk = config().getString("disk", "default");
2022-06-16 17:38:33 +00:00
2024-05-27 11:44:45 +00:00
auto validator = [](const Poco::Util::AbstractConfiguration &, const std::string &, const std::string &) { return true; };
2022-06-16 17:38:33 +00:00
2024-05-27 11:44:45 +00:00
constexpr auto config_prefix = "storage_configuration.disks";
auto disk_selector = std::make_shared<DiskSelector>(std::unordered_set<String>{"cache", "encrypted"});
2024-05-27 11:44:45 +00:00
disk_selector->initialize(config(), config_prefix, global_context, validator);
2023-11-30 03:09:55 +00:00
2024-05-27 11:44:45 +00:00
std::vector<std::pair<DiskPtr, std::optional<String>>> disks_with_path;
2023-11-30 03:09:55 +00:00
2024-05-27 11:44:45 +00:00
for (const auto & [_, disk_ptr] : disk_selector->getDisksMap())
{
disks_with_path.emplace_back(
disk_ptr, (disk_ptr->getName() == "local") ? std::optional{fs::current_path().string()} : std::nullopt);
}
2023-11-30 03:09:55 +00:00
2024-05-27 11:44:45 +00:00
client = std::make_unique<DisksClient>(std::move(disks_with_path), main_disk);
2023-11-30 03:09:55 +00:00
2024-05-27 11:44:45 +00:00
suggest.setCompletionsCallback([&](const String & prefix, size_t /* prefix_length */) { return getCompletions(prefix); });
2023-11-30 03:09:55 +00:00
2024-05-29 14:59:50 +00:00
if (!query.has_value())
{
2024-06-13 19:14:16 +00:00
runInteractive();
2024-05-29 14:59:50 +00:00
}
else
{
processQueryText(query.value());
}
return Application::EXIT_OK;
}
2024-05-27 11:44:45 +00:00
DisksApp::~DisksApp()
{
2024-07-04 13:40:36 +00:00
client.reset(nullptr);
2024-05-27 11:44:45 +00:00
if (global_context)
global_context->shutdown();
}
2024-06-13 19:14:16 +00:00
void DisksApp::runInteractiveTestMode()
{
for (String input; std::getline(std::cin, input);)
{
if (!processQueryText(input))
break;
std::cout << "\a\a\a\a" << std::endl;
std::cerr << std::flush;
}
}
void DisksApp::runInteractive()
{
if (config().hasOption("test-mode"))
runInteractiveTestMode();
else
runInteractiveReplxx();
}
}
int mainEntryClickHouseDisks(int argc, char ** argv)
{
try
{
DB::DisksApp app;
std::vector<String> common_arguments{argv + 1, argv + argc};
app.init(common_arguments);
return app.run();
}
catch (const DB::Exception & e)
{
std::cerr << DB::getExceptionMessage(e, false) << std::endl;
2024-05-27 11:44:45 +00:00
return 0;
}
catch (const boost::program_options::error & e)
{
std::cerr << "Bad arguments: " << e.what() << std::endl;
2024-05-27 11:44:45 +00:00
return 0;
}
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
2024-05-27 11:44:45 +00:00
return 0;
}
}