|
|
|
@ -2,6 +2,7 @@
|
|
|
|
|
#include <Poco/Util/XMLConfiguration.h>
|
|
|
|
|
#include <Poco/Util/HelpFormatter.h>
|
|
|
|
|
#include <Poco/Util/OptionCallback.h>
|
|
|
|
|
#include <Poco/String.h>
|
|
|
|
|
|
|
|
|
|
#include <DB/Databases/DatabaseOrdinary.h>
|
|
|
|
|
|
|
|
|
@ -19,7 +20,9 @@
|
|
|
|
|
#include <DB/Interpreters/Context.h>
|
|
|
|
|
#include <DB/Interpreters/ProcessList.h>
|
|
|
|
|
#include <DB/Interpreters/executeQuery.h>
|
|
|
|
|
#include <DB/Interpreters/loadMetadata.h>
|
|
|
|
|
|
|
|
|
|
#include <DB/Common/Exception.h>
|
|
|
|
|
#include <DB/Common/Macros.h>
|
|
|
|
|
#include <DB/Common/ConfigProcessor.h>
|
|
|
|
|
#include <DB/Common/escapeForFileName.h>
|
|
|
|
@ -33,6 +36,8 @@
|
|
|
|
|
#include <common/ErrorHandlers.h>
|
|
|
|
|
#include <common/ApplicationServerExt.h>
|
|
|
|
|
|
|
|
|
|
#include "StatusFile.h"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
|
{
|
|
|
|
@ -46,30 +51,28 @@ namespace ErrorCodes
|
|
|
|
|
|
|
|
|
|
LocalServer::LocalServer() = default;
|
|
|
|
|
|
|
|
|
|
LocalServer::~LocalServer() = default;
|
|
|
|
|
LocalServer::~LocalServer()
|
|
|
|
|
{
|
|
|
|
|
if (context)
|
|
|
|
|
context->shutdown(); /// required for properly exception handling
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void LocalServer::initialize(Poco::Util::Application & self)
|
|
|
|
|
{
|
|
|
|
|
Poco::Util::Application::initialize(self);
|
|
|
|
|
|
|
|
|
|
/// Load config files if exists
|
|
|
|
|
if (config().has("config-file") || Poco::File("config.xml").exists())
|
|
|
|
|
{
|
|
|
|
|
ConfigurationPtr processed_config = ConfigProcessor(false, true).loadConfig(config().getString("config-file", "config.xml"));
|
|
|
|
|
config().add(processed_config.duplicate(), PRIO_DEFAULT, false);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
|
|
|
|
|
{
|
|
|
|
|
Poco::Util::Application::defineOptions (_options);
|
|
|
|
|
|
|
|
|
|
_options.addOption(
|
|
|
|
|
Poco::Util::Option ("config-file", "C", "Load configuration from a given file")
|
|
|
|
|
Poco::Util::Option ("config-file", "", "Load configuration from a given file")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.argument(" config.xml")
|
|
|
|
|
.argument("[config.xml]")
|
|
|
|
|
.binding("config-file")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
@ -79,7 +82,7 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
|
|
|
|
|
Poco::Util::Option ("structure", "S", "Structe of initial table(list columns names with their types)")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.argument(" <struct>")
|
|
|
|
|
.argument("[name Type]")
|
|
|
|
|
.binding("table-structure")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
@ -87,12 +90,12 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
|
|
|
|
|
Poco::Util::Option ("table", "N", "Name of intial table")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.argument(" table")
|
|
|
|
|
.argument("[table]")
|
|
|
|
|
.binding("table-name")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
_options.addOption(
|
|
|
|
|
Poco::Util::Option ("file", "F", "Path to file with data of initial table (stdin if not specified)")
|
|
|
|
|
Poco::Util::Option ("file", "f", "Path to file with data of initial table (stdin if not specified)")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.argument(" stdin")
|
|
|
|
@ -103,16 +106,16 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
|
|
|
|
|
Poco::Util::Option ("input-format", "if", "Input format of intial table data")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.argument(" TabSeparated")
|
|
|
|
|
.argument("[TabSeparated]")
|
|
|
|
|
.binding("table-data-format")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
/// List of queries to execute
|
|
|
|
|
_options.addOption(
|
|
|
|
|
Poco::Util::Option ("query", "Q", "Queries to execute")
|
|
|
|
|
Poco::Util::Option ("query", "q", "Queries to execute")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.argument(" <query>", true)
|
|
|
|
|
.argument("<query>", true)
|
|
|
|
|
.binding("query")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
@ -121,19 +124,26 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
|
|
|
|
|
Poco::Util::Option ("output-format", "of", "Default output format")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.argument(" TabSeparated", true)
|
|
|
|
|
.argument("[TabSeparated]", true)
|
|
|
|
|
.binding("output-format")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
/// Alias for previous one, required for clickhouse-client compability
|
|
|
|
|
_options.addOption(
|
|
|
|
|
Poco::Util::Option ("format", "f", "Default ouput format")
|
|
|
|
|
Poco::Util::Option ("format", "", "Default ouput format")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.argument(" TabSeparated", true)
|
|
|
|
|
.argument("[TabSeparated]", true)
|
|
|
|
|
.binding("format")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
_options.addOption(
|
|
|
|
|
Poco::Util::Option ("stacktrace", "", "Print stack traces of exceptions")
|
|
|
|
|
.required(false)
|
|
|
|
|
.repeatable(false)
|
|
|
|
|
.binding("stacktrace")
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
_options.addOption(
|
|
|
|
|
Poco::Util::Option ("verbose", "", "Print info about execution of queries")
|
|
|
|
|
.required(false)
|
|
|
|
@ -151,17 +161,13 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
|
|
|
|
|
.callback(Poco::Util::OptionCallback<LocalServer>(this, &LocalServer::handleHelp)));
|
|
|
|
|
|
|
|
|
|
#define DECLARE_SETTING(TYPE, NAME, DEFAULT) \
|
|
|
|
|
{ \
|
|
|
|
|
_options.addOption(Poco::Util::Option(#NAME, "", "Settings.h").group("settings").required(false).repeatable(false).binding(#NAME)); \
|
|
|
|
|
APPLY_FOR_SETTINGS(DECLARE_SETTING) \
|
|
|
|
|
}
|
|
|
|
|
_options.addOption(Poco::Util::Option(#NAME, "", "Settings.h").group("Settings").required(false).repeatable(false).binding(#NAME));
|
|
|
|
|
APPLY_FOR_SETTINGS(DECLARE_SETTING)
|
|
|
|
|
#undef DECLARE_SETTING
|
|
|
|
|
|
|
|
|
|
#define DECLARE_SETTING(TYPE, NAME, DEFAULT) \
|
|
|
|
|
{ \
|
|
|
|
|
_options.addOption(Poco::Util::Option(#NAME, "", "Limits.h").group("limits").required(false).repeatable(false).binding(#NAME)); \
|
|
|
|
|
APPLY_FOR_LIMITS(DECLARE_SETTING) \
|
|
|
|
|
}
|
|
|
|
|
_options.addOption(Poco::Util::Option(#NAME, "", "Limits.h").group("Limits").required(false).repeatable(false).binding(#NAME));
|
|
|
|
|
APPLY_FOR_LIMITS(DECLARE_SETTING)
|
|
|
|
|
#undef DECLARE_SETTING
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -204,7 +210,7 @@ void LocalServer::displayHelp()
|
|
|
|
|
helpFormatter.format(std::cerr);
|
|
|
|
|
std::cerr << "Example printing memory used by each Unix user:\n"
|
|
|
|
|
"ps aux | tail -n +2 | awk '{ printf(\"%s\\t%s\\n\", $1, $4) }' | "
|
|
|
|
|
"clickhouse-local -S \"user String, mem Float64\" -Q \"SELECT user, round(sum(mem), 2) as memTotal FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty;\"\n";
|
|
|
|
|
"clickhouse-local -S \"user String, mem Float64\" -q \"SELECT user, round(sum(mem), 2) as memTotal FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty\"\n";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -215,19 +221,41 @@ void LocalServer::handleHelp(const std::string & name, const std::string & value
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
int LocalServer::main(const std::vector<std::string> & args)
|
|
|
|
|
/// If path is specified and not empty, will try to setup server environment and load existing metadata
|
|
|
|
|
void LocalServer::tryInitPath()
|
|
|
|
|
{
|
|
|
|
|
if (!config().has("path") || (path = config().getString("path")).empty())
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
Poco::trimInPlace(path);
|
|
|
|
|
if (path.empty())
|
|
|
|
|
return;
|
|
|
|
|
if (path.back() != '/')
|
|
|
|
|
path += '/';
|
|
|
|
|
|
|
|
|
|
context->setPath(path);
|
|
|
|
|
|
|
|
|
|
StatusFile status{path + "status"};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
int LocalServer::main(const std::vector<std::string> & args)
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
Logger * log = &logger();
|
|
|
|
|
|
|
|
|
|
if (!config().has("query") && !config().has("table-structure")) /// Nothing to process
|
|
|
|
|
{
|
|
|
|
|
if (!config().has("help"))
|
|
|
|
|
if (!config().hasOption("help"))
|
|
|
|
|
{
|
|
|
|
|
std::cerr << "There are no queries to process.\n";
|
|
|
|
|
std::cerr << "There are no queries to process." << std::endl;
|
|
|
|
|
displayHelp();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return Application::EXIT_OK;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Load config files if exists
|
|
|
|
|
if (config().has("config-file") || Poco::File("config.xml").exists())
|
|
|
|
|
{
|
|
|
|
|
ConfigurationPtr processed_config = ConfigProcessor(false, true).loadConfig(config().getString("config-file", "config.xml"));
|
|
|
|
@ -237,10 +265,11 @@ int LocalServer::main(const std::vector<std::string> & args)
|
|
|
|
|
context = std::make_unique<Context>();
|
|
|
|
|
context->setGlobalContext(*context);
|
|
|
|
|
context->setApplicationType(Context::ApplicationType::LOCAL_SERVER);
|
|
|
|
|
tryInitPath();
|
|
|
|
|
|
|
|
|
|
applyOptions();
|
|
|
|
|
|
|
|
|
|
/// Skip path, temp path, flag's path installation
|
|
|
|
|
/// Skip temp path installation
|
|
|
|
|
|
|
|
|
|
/// We will terminate process on error
|
|
|
|
|
static KillingErrorHandler error_handler;
|
|
|
|
@ -274,10 +303,22 @@ int LocalServer::main(const std::vector<std::string> & args)
|
|
|
|
|
/// Load global settings from default profile.
|
|
|
|
|
context->setSetting("profile", config().getString("default_profile", "default"));
|
|
|
|
|
|
|
|
|
|
/// Init dummy default DB
|
|
|
|
|
const std::string default_database = "default";
|
|
|
|
|
/** Init dummy default DB
|
|
|
|
|
* NOTE: We force using isolated default database to avoid conflicts with default database from server enviroment
|
|
|
|
|
* Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory;
|
|
|
|
|
* if such tables will not be dropped, clickhouse-server can not load them due to security reasons.
|
|
|
|
|
*/
|
|
|
|
|
const std::string default_database = "_local";
|
|
|
|
|
context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
|
|
|
|
|
context->setCurrentDatabase(default_database);
|
|
|
|
|
|
|
|
|
|
if (!path.empty())
|
|
|
|
|
{
|
|
|
|
|
LOG_DEBUG(log, "Loading metadata from " << path);
|
|
|
|
|
loadMetadata(*context);
|
|
|
|
|
LOG_DEBUG(log, "Loaded metadata.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
attachSystemTables();
|
|
|
|
|
|
|
|
|
|
processQueries();
|
|
|
|
@ -287,6 +328,26 @@ int LocalServer::main(const std::vector<std::string> & args)
|
|
|
|
|
|
|
|
|
|
return Application::EXIT_OK;
|
|
|
|
|
}
|
|
|
|
|
catch (const Exception & e)
|
|
|
|
|
{
|
|
|
|
|
bool print_stack_trace = config().has("stacktrace");
|
|
|
|
|
|
|
|
|
|
std::string text = e.displayText();
|
|
|
|
|
|
|
|
|
|
auto embedded_stack_trace_pos = text.find("Stack trace");
|
|
|
|
|
if (std::string::npos != embedded_stack_trace_pos && !print_stack_trace)
|
|
|
|
|
text.resize(embedded_stack_trace_pos);
|
|
|
|
|
|
|
|
|
|
std::cerr << "Code: " << e.code() << ". " << text << std::endl << std::endl;
|
|
|
|
|
|
|
|
|
|
if (print_stack_trace && std::string::npos == embedded_stack_trace_pos)
|
|
|
|
|
{
|
|
|
|
|
std::cerr << "Stack trace:" << std::endl << e.getStackTrace().toString();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// If exception code isn't zero, we should return non-zero return code anyway.
|
|
|
|
|
return e.code() ? e.code() : -1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
inline String getQuotedString(const String & s)
|
|
|
|
@ -323,10 +384,14 @@ std::string LocalServer::getInitialCreateTableQuery()
|
|
|
|
|
|
|
|
|
|
void LocalServer::attachSystemTables()
|
|
|
|
|
{
|
|
|
|
|
/// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
|
|
|
|
|
DatabasePtr system_database = context->tryGetDatabase("system");
|
|
|
|
|
if (!system_database)
|
|
|
|
|
{
|
|
|
|
|
/// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
|
|
|
|
|
system_database = std::make_shared<DatabaseMemory>("system");
|
|
|
|
|
context->addDatabase("system", system_database);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
DatabasePtr system_database = std::make_shared<DatabaseMemory>("system");
|
|
|
|
|
context->addDatabase("system", system_database);
|
|
|
|
|
system_database->attachTable("one", StorageSystemOne::create("one"));
|
|
|
|
|
system_database->attachTable("numbers", StorageSystemNumbers::create("numbers"));
|
|
|
|
|
system_database->attachTable("numbers_mt", StorageSystemNumbers::create("numbers_mt", true));
|
|
|
|
@ -346,7 +411,7 @@ void LocalServer::processQueries()
|
|
|
|
|
String initial_create_query = getInitialCreateTableQuery();
|
|
|
|
|
String queries_str = initial_create_query + config().getString("query");
|
|
|
|
|
|
|
|
|
|
bool verbose = config().getBool("verbose", false);
|
|
|
|
|
bool verbose = config().hasOption("verbose");
|
|
|
|
|
|
|
|
|
|
std::vector<String> queries;
|
|
|
|
|
auto parse_res = splitMultipartQuery(queries_str, queries);
|
|
|
|
@ -358,22 +423,14 @@ void LocalServer::processQueries()
|
|
|
|
|
|
|
|
|
|
for (const auto & query : queries)
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
ReadBufferFromString read_buf(query);
|
|
|
|
|
WriteBufferFromFileDescriptor write_buf(STDOUT_FILENO);
|
|
|
|
|
BlockInputStreamPtr plan;
|
|
|
|
|
ReadBufferFromString read_buf(query);
|
|
|
|
|
WriteBufferFromFileDescriptor write_buf(STDOUT_FILENO);
|
|
|
|
|
BlockInputStreamPtr plan;
|
|
|
|
|
|
|
|
|
|
if (verbose)
|
|
|
|
|
LOG_INFO(log, "Executing query: " << query);
|
|
|
|
|
if (verbose)
|
|
|
|
|
LOG_INFO(log, "Executing query: " << query);
|
|
|
|
|
|
|
|
|
|
executeQuery(read_buf, write_buf, *context, plan, nullptr);
|
|
|
|
|
}
|
|
|
|
|
catch (...)
|
|
|
|
|
{
|
|
|
|
|
tryLogCurrentException(log, "An error ocurred while executing query");
|
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
executeQuery(read_buf, write_buf, *context, plan, nullptr);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|