ClickHouse/programs/local/LocalServer.cpp

784 lines
28 KiB
C++
Raw Normal View History

#include "LocalServer.h"
#include <Poco/Util/XMLConfiguration.h>
#include <Poco/String.h>
#include <Poco/Logger.h>
#include <Poco/NullChannel.h>
2021-10-30 12:28:50 +00:00
#include <Poco/SimpleFileChannel.h>
#include <Databases/DatabaseMemory.h>
#include <Storages/System/attachSystemTables.h>
#include <Storages/System/attachInformationSchemaTables.h>
#include <Interpreters/ProcessList.h>
#include <Interpreters/loadMetadata.h>
2020-02-03 12:54:36 +00:00
#include <Interpreters/DatabaseCatalog.h>
2021-10-02 07:13:14 +00:00
#include <base/getFQDNOrHostName.h>
#include <base/scope_guard_safe.h>
#include <Interpreters/UserDefinedSQLObjectsLoader.h>
2021-08-12 15:16:55 +00:00
#include <Interpreters/Session.h>
#include <Common/Exception.h>
#include <Common/Macros.h>
#include <Common/Config/ConfigProcessor.h>
2019-01-28 11:18:00 +00:00
#include <Common/ThreadStatus.h>
#include <Common/TLDListsHolder.h>
#include <Common/quoteString.h>
#include <Common/randomSeed.h>
2021-07-11 11:36:27 +00:00
#include <loggers/Loggers.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromFileDescriptor.h>
2021-09-04 18:19:01 +00:00
#include <IO/UseSSL.h>
#include <Parsers/IAST.h>
2021-10-02 07:13:14 +00:00
#include <base/ErrorHandlers.h>
2017-04-21 17:47:27 +00:00
#include <Functions/registerFunctions.h>
#include <AggregateFunctions/registerAggregateFunctions.h>
#include <TableFunctions/registerTableFunctions.h>
#include <Storages/registerStorages.h>
#include <Dictionaries/registerDictionaries.h>
2019-11-27 09:39:44 +00:00
#include <Disks/registerDisks.h>
2020-10-29 03:39:43 +00:00
#include <Formats/registerFormats.h>
#include <boost/algorithm/string/replace.hpp>
#include <boost/program_options/options_description.hpp>
2021-10-02 07:13:14 +00:00
#include <base/argsToConfig.h>
#include <filesystem>
2021-05-16 22:06:09 +00:00
namespace fs = std::filesystem;
2021-07-11 23:17:14 +00:00
namespace DB
{
namespace ErrorCodes
{
2020-06-24 22:07:01 +00:00
extern const int BAD_ARGUMENTS;
extern const int CANNOT_LOAD_CONFIG;
extern const int FILE_ALREADY_EXISTS;
}
2021-10-04 15:23:08 +00:00
void LocalServer::processError(const String &) const
2021-08-21 10:55:54 +00:00
{
2021-09-05 09:27:27 +00:00
if (ignore_error)
2021-08-21 15:29:28 +00:00
return;
2021-09-11 11:34:22 +00:00
if (is_interactive)
2021-08-21 10:55:54 +00:00
{
2021-10-04 15:23:08 +00:00
String message;
2021-09-11 11:34:22 +00:00
if (server_exception)
{
2021-10-04 15:23:08 +00:00
message = getExceptionMessage(*server_exception, print_stack_trace, true);
2021-09-11 11:34:22 +00:00
}
2021-10-04 15:23:08 +00:00
else if (client_exception)
2021-09-11 11:34:22 +00:00
{
2021-10-04 15:23:08 +00:00
message = client_exception->message();
2021-09-11 11:34:22 +00:00
}
2021-10-04 15:23:08 +00:00
fmt::print(stderr, "Received exception:\n{}\n", message);
fmt::print(stderr, "\n");
2021-08-21 10:55:54 +00:00
}
2021-09-11 11:34:22 +00:00
else
2021-08-21 10:55:54 +00:00
{
2021-09-11 11:34:22 +00:00
if (server_exception)
server_exception->rethrow();
if (client_exception)
client_exception->rethrow();
2021-08-21 10:55:54 +00:00
}
}
2021-09-04 18:19:01 +00:00
bool LocalServer::executeMultiQuery(const String & all_queries_text)
{
2021-08-21 10:55:54 +00:00
bool echo_query = echo_queries;
/// Several queries separated by ';'.
/// INSERT data is ended by the end of line, not ';'.
/// An exception is VALUES format where we also support semicolon in
/// addition to end of line.
const char * this_query_begin = all_queries_text.data();
const char * this_query_end;
const char * all_queries_end = all_queries_text.data() + all_queries_text.size();
String full_query; // full_query is the query + inline INSERT data + trailing comments (the latter is our best guess for now).
String query_to_execute;
ASTPtr parsed_query;
std::optional<Exception> current_exception;
while (true)
{
auto stage = analyzeMultiQueryText(this_query_begin, this_query_end, all_queries_end,
query_to_execute, parsed_query, all_queries_text, current_exception);
switch (stage)
{
case MultiQueryProcessingStage::QUERIES_END:
case MultiQueryProcessingStage::PARSING_FAILED:
{
return true;
}
case MultiQueryProcessingStage::CONTINUE_PARSING:
{
continue;
}
case MultiQueryProcessingStage::PARSING_EXCEPTION:
{
2021-10-16 12:18:00 +00:00
if (current_exception)
current_exception->rethrow();
2021-10-16 14:34:31 +00:00
return true;
2021-08-21 10:55:54 +00:00
}
case MultiQueryProcessingStage::EXECUTE_QUERY:
{
full_query = all_queries_text.substr(this_query_begin - all_queries_text.data(), this_query_end - this_query_begin);
try
{
processParsedSingleQuery(full_query, query_to_execute, parsed_query, echo_query, false);
}
catch (...)
{
2021-10-16 19:30:35 +00:00
if (!is_interactive && !ignore_error)
2021-10-16 11:28:57 +00:00
throw;
2021-08-21 10:55:54 +00:00
// Surprisingly, this is a client error. A server error would
// have been reported w/o throwing (see onReceiveSeverException()).
2021-10-16 11:28:57 +00:00
client_exception = std::make_unique<Exception>(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode());
2021-08-21 10:55:54 +00:00
have_error = true;
}
// For INSERTs with inline data: use the end of inline data as
// reported by the format parser (it is saved in sendData()).
// This allows us to handle queries like:
// insert into t values (1); select 1
// , where the inline data is delimited by semicolon and not by a
// newline.
auto * insert_ast = parsed_query->as<ASTInsertQuery>();
if (insert_ast && insert_ast->data)
{
this_query_end = insert_ast->end;
adjustQueryEnd(this_query_end, all_queries_end, global_context->getSettingsRef().max_parser_depth);
}
// Report error.
if (have_error)
processError(full_query);
// Stop processing queries if needed.
if (have_error && !ignore_error)
return is_interactive;
this_query_begin = this_query_end;
break;
}
}
}
}
void LocalServer::initialize(Poco::Util::Application & self)
{
Poco::Util::Application::initialize(self);
/// Load config files if exists
2021-05-16 22:06:09 +00:00
if (config().has("config-file") || fs::exists("config.xml"))
{
const auto config_path = config().getString("config-file", "config.xml");
ConfigProcessor config_processor(config_path, false, true);
2021-05-16 22:06:09 +00:00
config_processor.setConfigPath(fs::path(config_path).parent_path());
auto loaded_config = config_processor.loadConfig();
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
}
}
static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const String & database_name)
{
DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(database_name);
if (!system_database)
{
/// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
system_database = std::make_shared<DatabaseMemory>(database_name, context);
DatabaseCatalog::instance().attachDatabase(database_name, system_database);
}
return system_database;
}
/// If path is specified and not empty, will try to setup server environment and load existing metadata
void LocalServer::tryInitPath()
{
std::string path;
if (config().has("path"))
{
// User-supplied path.
path = config().getString("path");
Poco::trimInPlace(path);
if (path.empty())
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
2020-08-08 01:21:04 +00:00
"Cannot work with empty storage path that is explicitly specified"
2020-06-24 22:07:01 +00:00
" by the --path option. Please check the program options and"
" correct the --path.");
}
}
else
{
2020-10-26 07:59:15 +00:00
// The path is not provided explicitly - use a unique path in the system temporary directory
// (or in the current dir if temporary don't exist)
Poco::Logger * log = &logger();
std::filesystem::path parent_folder;
std::filesystem::path default_path;
try
{
// try to guess a tmp folder name, and check if it's a directory (throw exception otherwise)
parent_folder = std::filesystem::temp_directory_path();
}
2021-07-23 20:56:47 +00:00
catch (const fs::filesystem_error& e)
{
// tmp folder don't exists? misconfiguration? chroot?
LOG_DEBUG(log, "Can not get temporary folder: {}", e.what());
parent_folder = std::filesystem::current_path();
std::filesystem::is_directory(parent_folder); // that will throw an exception if it's not a directory
LOG_DEBUG(log, "Will create working directory inside current directory: {}", parent_folder.string());
}
2020-10-26 07:59:15 +00:00
/// we can have another clickhouse-local running simultaneously, even with the same PID (for ex. - several dockers mounting the same folder)
/// or it can be some leftovers from other clickhouse-local runs
/// as we can't accurately distinguish those situations we don't touch any existent folders
/// we just try to pick some free name for our working folder
default_path = parent_folder / fmt::format("clickhouse-local-{}-{}-{}", getpid(), time(nullptr), randomSeed());
if (exists(default_path))
2020-10-27 11:04:03 +00:00
throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Unsuccessful attempt to create working directory: {} exist!", default_path.string());
create_directory(default_path);
temporary_directory_to_delete = default_path;
path = default_path.string();
LOG_DEBUG(log, "Working directory created: {}", path);
}
if (path.back() != '/')
path += '/';
2020-10-22 07:37:03 +00:00
global_context->setPath(path);
2020-10-22 07:37:03 +00:00
global_context->setTemporaryStorage(path + "tmp");
global_context->setFlagsPath(path + "flags");
2020-10-22 07:37:03 +00:00
global_context->setUserFilesPath(""); // user's files are everywhere
/// top_level_domains_lists
const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/");
if (!top_level_domains_path.empty())
TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config());
}
2021-07-11 23:17:14 +00:00
void LocalServer::cleanup()
2020-03-18 00:57:00 +00:00
{
2021-10-16 11:28:57 +00:00
try
2020-03-18 00:57:00 +00:00
{
2021-10-16 11:28:57 +00:00
connection.reset();
if (global_context)
{
global_context->shutdown();
global_context.reset();
}
2020-03-18 00:57:00 +00:00
2021-10-16 11:28:57 +00:00
status.reset();
2021-09-04 18:19:01 +00:00
2021-10-16 11:28:57 +00:00
// Delete the temporary directory if needed.
if (temporary_directory_to_delete)
{
const auto dir = *temporary_directory_to_delete;
temporary_directory_to_delete.reset();
LOG_DEBUG(&logger(), "Removing temporary directory: {}", dir.string());
remove_all(dir);
}
}
catch (...)
2021-07-11 23:17:14 +00:00
{
2021-10-16 11:28:57 +00:00
tryLogCurrentException(__PRETTY_FUNCTION__);
2021-07-11 23:17:14 +00:00
}
2020-03-18 00:57:00 +00:00
}
2021-07-11 23:17:14 +00:00
std::string LocalServer::getInitialCreateTableQuery()
{
2021-07-11 23:17:14 +00:00
if (!config().has("table-structure"))
return {};
2021-07-11 23:17:14 +00:00
auto table_name = backQuoteIfNeed(config().getString("table-name", "table"));
auto table_structure = config().getString("table-structure");
auto data_format = backQuoteIfNeed(config().getString("table-data-format", "TSV"));
2021-07-11 23:17:14 +00:00
String table_file;
if (!config().has("table-file") || config().getString("table-file") == "-")
{
/// Use Unix tools stdin naming convention
table_file = "stdin";
}
2021-07-11 23:17:14 +00:00
else
{
/// Use regular file
table_file = quoteString(config().getString("table-file"));
}
return fmt::format("CREATE TABLE {} ({}) ENGINE = File({}, {});",
table_name, table_structure, data_format, table_file);
}
static ConfigurationPtr getConfigurationFromXMLString(const char * xml_data)
{
std::stringstream ss{std::string{xml_data}}; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
Poco::XML::InputSource input_source{ss};
return {new Poco::Util::XMLConfiguration{&input_source}};
}
void LocalServer::setupUsers()
{
static const char * minimal_default_user_xml =
"<clickhouse>"
2021-07-11 23:17:14 +00:00
" <profiles>"
" <default></default>"
" </profiles>"
" <users>"
" <default>"
" <password></password>"
" <networks>"
" <ip>::/0</ip>"
" </networks>"
" <profile>default</profile>"
" <quota>default</quota>"
" </default>"
" </users>"
" <quotas>"
" <default></default>"
" </quotas>"
"</clickhouse>";
2021-07-11 23:17:14 +00:00
ConfigurationPtr users_config;
2021-07-11 23:17:14 +00:00
if (config().has("users_config") || config().has("config-file") || fs::exists("config.xml"))
{
const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
ConfigProcessor config_processor(users_config_path);
const auto loaded_config = config_processor.loadConfig();
users_config = loaded_config.configuration;
}
else
{
2021-07-11 23:17:14 +00:00
users_config = getConfigurationFromXMLString(minimal_default_user_xml);
}
2021-07-11 23:17:14 +00:00
if (users_config)
global_context->setUsersConfig(users_config);
else
throw Exception("Can't load config for users", ErrorCodes::CANNOT_LOAD_CONFIG);
}
2021-07-29 12:48:07 +00:00
String LocalServer::getQueryTextPrefix()
{
return getInitialCreateTableQuery();
}
2021-08-23 08:50:12 +00:00
void LocalServer::connect()
{
connection_parameters = ConnectionParameters(config());
2021-09-11 17:16:37 +00:00
connection = LocalConnection::createConnection(connection_parameters, global_context, need_render_progress);
2021-08-23 08:50:12 +00:00
}
2021-09-04 18:19:01 +00:00
int LocalServer::main(const std::vector<std::string> & /*args*/)
2021-07-31 12:34:29 +00:00
try
{
2021-09-04 18:19:01 +00:00
UseSSL use_ssl;
2021-07-31 12:34:29 +00:00
ThreadStatus thread_status;
2021-10-02 08:10:34 +00:00
setupSignalHandler();
2021-07-31 12:34:29 +00:00
2021-09-04 18:19:01 +00:00
std::cout << std::fixed << std::setprecision(3);
std::cerr << std::fixed << std::setprecision(3);
2021-10-29 12:04:08 +00:00
is_interactive = stdin_is_a_tty
&& (config().hasOption("interactive")
|| (!config().has("query") && !config().has("table-structure") && queries_files.empty()));
2021-10-01 13:47:39 +00:00
if (!is_interactive)
2021-09-29 19:17:26 +00:00
{
/// We will terminate process on error
static KillingErrorHandler error_handler;
Poco::ErrorHandler::set(&error_handler);
}
2021-07-31 12:34:29 +00:00
/// Don't initialize DateLUT
2017-04-21 17:47:27 +00:00
registerFunctions();
registerAggregateFunctions();
registerTableFunctions();
registerStorages();
registerDictionaries();
2019-11-27 09:39:44 +00:00
registerDisks();
2020-10-29 03:39:43 +00:00
registerFormats();
2017-04-21 17:47:27 +00:00
2021-07-31 12:34:29 +00:00
processConfig();
2021-08-19 11:07:47 +00:00
applyCmdSettings(global_context);
2021-11-14 07:19:59 +00:00
if (is_interactive)
{
clearTerminal();
showClientVersion();
std::cerr << std::endl;
}
2021-08-23 08:50:12 +00:00
connect();
2021-08-17 19:59:51 +00:00
2021-10-29 12:04:08 +00:00
if (is_interactive && !delayed_interactive)
2021-07-31 12:34:29 +00:00
{
runInteractive();
2021-07-31 12:34:29 +00:00
}
else
{
runNonInteractive();
2021-10-29 12:04:08 +00:00
if (delayed_interactive)
runInteractive();
2021-07-31 12:34:29 +00:00
}
cleanup();
return Application::EXIT_OK;
}
2021-10-16 11:28:57 +00:00
catch (const DB::Exception & e)
2021-07-31 12:34:29 +00:00
{
2021-10-16 11:28:57 +00:00
cleanup();
2021-07-23 20:54:49 +00:00
2021-10-25 08:00:55 +00:00
bool need_print_stack_trace = config().getBool("stacktrace", false);
std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl;
2021-10-16 11:28:57 +00:00
return e.code() ? e.code() : -1;
}
catch (...)
{
cleanup();
2021-07-30 07:46:22 +00:00
2021-10-16 11:28:57 +00:00
std::cerr << getCurrentExceptionMessage(false) << std::endl;
return getCurrentExceptionCode();
2021-07-23 20:54:49 +00:00
}
void LocalServer::processConfig()
{
2021-10-29 12:04:08 +00:00
delayed_interactive = config().has("interactive") && (config().has("query") || config().has("queries-file"));
if (is_interactive && !delayed_interactive)
2021-07-23 20:54:49 +00:00
{
if (config().has("query") && config().has("queries-file"))
throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS);
2021-07-29 12:48:07 +00:00
if (config().has("multiquery"))
is_multiquery = true;
2021-09-11 11:34:22 +00:00
load_suggestions = true;
2021-07-23 20:54:49 +00:00
}
else
{
2021-10-29 12:04:08 +00:00
if (delayed_interactive)
{
load_suggestions = true;
}
2021-07-23 20:54:49 +00:00
need_render_progress = config().getBool("progress", false);
echo_queries = config().hasOption("echo") || config().hasOption("verbose");
2021-07-23 20:54:49 +00:00
ignore_error = config().getBool("ignore-error", false);
2021-07-29 12:48:07 +00:00
is_multiquery = true;
2021-07-23 20:54:49 +00:00
}
2021-10-16 11:28:57 +00:00
print_stack_trace = config().getBool("stacktrace", false);
2021-07-23 20:54:49 +00:00
2021-10-30 12:28:50 +00:00
auto logging = (config().has("logger.console")
|| config().has("logger.level")
|| config().has("log-level")
|| config().has("logger.log"));
auto file_logging = config().has("server_logs_file");
if (is_interactive && logging && !file_logging)
throw Exception("For interactive mode logging is allowed only with --server_logs_file option",
ErrorCodes::BAD_ARGUMENTS);
if (file_logging)
{
auto level = Poco::Logger::parseLevel(config().getString("log-level", "trace"));
Poco::Logger::root().setLevel(level);
Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::SimpleFileChannel>(new Poco::SimpleFileChannel(server_logs_file)));
}
else if (logging)
{
// force enable logging
config().setString("logger", "logger");
// sensitive data rules are not used here
buildLoggers(config(), logger(), "clickhouse-local");
}
else
{
Poco::Logger::root().setLevel("none");
Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::NullChannel>(new Poco::NullChannel()));
}
2021-07-23 20:54:49 +00:00
shared_context = Context::createShared();
global_context = Context::createGlobal(shared_context.get());
global_context->makeGlobalContext();
global_context->setApplicationType(Context::ApplicationType::LOCAL);
tryInitPath();
Poco::Logger * log = &logger();
/// Maybe useless
if (config().has("macros"))
2020-10-22 07:37:03 +00:00
global_context->setMacros(std::make_unique<Macros>(config(), "macros", log));
2021-08-20 08:38:50 +00:00
format = config().getString("output-format", config().getString("format", is_interactive ? "PrettyCompact" : "TSV"));
2021-08-18 14:39:04 +00:00
insert_format = "Values";
2021-08-18 14:39:04 +00:00
/// Setting value from cmd arg overrides one from config
if (global_context->getSettingsRef().max_insert_block_size.changed)
insert_format_max_block_size = global_context->getSettingsRef().max_insert_block_size;
else
insert_format_max_block_size = config().getInt("insert_format_max_block_size", global_context->getSettingsRef().max_insert_block_size);
/// Sets external authenticators config (LDAP, Kerberos).
2020-10-22 07:37:03 +00:00
global_context->setExternalAuthenticatorsConfig(config());
setupUsers();
/// Limit on total number of concurrently executing queries.
2020-03-28 03:02:26 +00:00
/// There is no need for concurrent queries, override max_concurrent_queries.
2020-10-22 07:37:03 +00:00
global_context->getProcessList().setMaxSize(0);
/// Size of cache for uncompressed blocks. Zero means disabled.
size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0);
if (uncompressed_cache_size)
2020-10-22 07:37:03 +00:00
global_context->setUncompressedCache(uncompressed_cache_size);
/// Size of cache for marks (index of MergeTree family of tables). It is necessary.
/// Specify default value for mark_cache_size explicitly!
size_t mark_cache_size = config().getUInt64("mark_cache_size", 5368709120);
if (mark_cache_size)
2020-10-22 07:37:03 +00:00
global_context->setMarkCache(mark_cache_size);
/// Size of cache for uncompressed blocks of MergeTree indices. Zero means disabled.
size_t index_uncompressed_cache_size = config().getUInt64("index_uncompressed_cache_size", 0);
if (index_uncompressed_cache_size)
global_context->setIndexUncompressedCache(index_uncompressed_cache_size);
/// Size of cache for index marks (index of MergeTree skip indices). It is necessary.
/// Specify default value for index_mark_cache_size explicitly!
size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", 0);
if (index_mark_cache_size)
global_context->setIndexMarkCache(index_mark_cache_size);
/// A cache for mmapped files.
size_t mmap_cache_size = config().getUInt64("mmap_cache_size", 1000); /// The choice of default is arbitrary.
if (mmap_cache_size)
2021-03-28 19:24:28 +00:00
global_context->setMMappedFileCache(mmap_cache_size);
/// Load global settings from default_profile and system_profile.
2020-10-22 07:37:03 +00:00
global_context->setDefaultProfiles(config());
2021-08-26 11:01:15 +00:00
/// We load temporary database first, because projections need it.
DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
/** Init dummy default DB
2020-01-11 09:50:41 +00:00
* NOTE: We force using isolated default database to avoid conflicts with default database from server environment
* Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory;
* if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
*/
std::string default_database = config().getString("default_database", "_local");
DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, global_context));
2020-10-22 07:37:03 +00:00
global_context->setCurrentDatabase(default_database);
applyCmdOptions(global_context);
bool enable_objects_loader = false;
2021-01-16 15:08:21 +00:00
if (config().has("path"))
{
2021-01-16 15:08:21 +00:00
String path = global_context->getPath();
/// Lock path directory before read
2021-07-23 20:54:49 +00:00
status.emplace(fs::path(path) / "status", StatusFile::write_full_info);
fs::create_directories(fs::path(path) / "user_defined/");
LOG_DEBUG(log, "Loading user defined objects from {}", path);
Poco::File(path + "user_defined/").createDirectories();
UserDefinedSQLObjectsLoader::instance().loadObjects(global_context);
enable_objects_loader = true;
LOG_DEBUG(log, "Loaded user defined objects.");
2020-08-13 19:41:06 +00:00
LOG_DEBUG(log, "Loading metadata from {}", path);
2021-05-16 22:06:09 +00:00
fs::create_directories(fs::path(path) / "data/");
fs::create_directories(fs::path(path) / "metadata/");
2021-07-23 20:54:49 +00:00
loadMetadataSystem(global_context);
attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
loadMetadata(global_context);
2021-09-01 19:42:49 +00:00
startupSystemTables();
DatabaseCatalog::instance().loadDatabases();
2021-07-23 20:54:49 +00:00
2020-05-23 22:24:01 +00:00
LOG_DEBUG(log, "Loaded metadata.");
}
else if (!config().has("no-system-tables"))
{
attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
}
/// Persist SQL user defined objects only if user_defined folder was created
UserDefinedSQLObjectsLoader::instance().enable(enable_objects_loader);
2021-07-23 20:54:49 +00:00
server_display_name = config().getString("display_name", getFQDNOrHostName());
2021-07-28 12:56:11 +00:00
prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) ");
std::map<String, String> prompt_substitutions{{"display_name", server_display_name}};
for (const auto & [key, value] : prompt_substitutions)
boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value);
2021-08-17 19:59:51 +00:00
ClientInfo & client_info = global_context->getClientInfo();
client_info.setInitialQuery();
}
2020-03-18 00:57:00 +00:00
static std::string getHelpHeader()
{
return
"usage: clickhouse-local [initial table definition] [--query <query>]\n"
"clickhouse-local allows to execute SQL queries on your data files via single command line call."
" To do so, initially you need to define your data source and its format."
" After you can execute your SQL queries in usual manner.\n"
"There are two ways to define initial table keeping your data."
" Either just in first query like this:\n"
" CREATE TABLE <table> (<structure>) ENGINE = File(<input-format>, <file>);\n"
"Either through corresponding command line parameters --table --structure --input-format and --file.";
}
2021-07-11 23:17:14 +00:00
2020-03-18 00:57:00 +00:00
static std::string getHelpFooter()
{
return
"Example printing memory used by each Unix user:\n"
"ps aux | tail -n +2 | awk '{ printf(\"%s\\t%s\\n\", $1, $4) }' | "
"clickhouse-local -S \"user String, mem Float64\" -q"
" \"SELECT user, round(sum(mem), 2) as mem_total FROM table GROUP BY user ORDER"
" BY mem_total DESC FORMAT PrettyCompact\"";
}
2021-07-11 23:17:14 +00:00
2021-07-11 11:36:27 +00:00
void LocalServer::printHelpMessage(const OptionsDescription & options_description)
{
2021-07-11 11:36:27 +00:00
std::cout << getHelpHeader() << "\n";
std::cout << options_description.main_description.value() << "\n";
std::cout << getHelpFooter() << "\n";
}
void LocalServer::addOptions(OptionsDescription & options_description)
2021-07-11 11:36:27 +00:00
{
options_description.main_description->add_options()
("database,d", po::value<std::string>(), "database")
("table,N", po::value<std::string>(), "name of the initial table")
2021-08-20 21:19:06 +00:00
/// If structure argument is omitted then initial query is not generated
("structure,S", po::value<std::string>(), "structure of the initial table (list of column and type names)")
("file,f", po::value<std::string>(), "path to file with data of the initial table (stdin if not specified)")
2021-08-20 21:19:06 +00:00
("input-format", po::value<std::string>(), "input format of the initial table data")
("output-format", po::value<std::string>(), "default output format")
2020-06-28 23:40:43 +00:00
("logger.console", po::value<bool>()->implicit_value(true), "Log to console")
("logger.log", po::value<std::string>(), "Log file name")
("logger.level", po::value<std::string>(), "Log level")
2021-07-29 12:48:07 +00:00
("no-system-tables", "do not attach system tables (better startup time)")
("path", po::value<std::string>(), "Storage path")
("top_level_domains_path", po::value<std::string>(), "Path to lists with custom TLDs")
;
2021-07-11 11:36:27 +00:00
}
2021-07-11 23:17:14 +00:00
void LocalServer::applyCmdSettings(ContextMutablePtr context)
{
context->applySettingsChanges(cmd_settings.changes());
}
void LocalServer::applyCmdOptions(ContextMutablePtr context)
{
context->setDefaultFormat(config().getString("output-format", config().getString("format", is_interactive ? "PrettyCompact" : "TSV")));
2021-07-11 23:17:14 +00:00
applyCmdSettings(context);
}
2021-07-11 11:36:27 +00:00
void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector<Arguments> &)
{
if (options.count("table"))
config().setString("table-name", options["table"].as<std::string>());
if (options.count("file"))
config().setString("table-file", options["file"].as<std::string>());
if (options.count("structure"))
config().setString("table-structure", options["structure"].as<std::string>());
2021-08-20 21:19:06 +00:00
if (options.count("no-system-tables"))
config().setBool("no-system-tables", true);
if (options.count("input-format"))
config().setString("table-data-format", options["input-format"].as<std::string>());
if (options.count("output-format"))
config().setString("output-format", options["output-format"].as<std::string>());
2020-06-28 23:40:43 +00:00
if (options.count("logger.console"))
config().setBool("logger.console", options["logger.console"].as<bool>());
if (options.count("logger.log"))
config().setString("logger.log", options["logger.log"].as<std::string>());
if (options.count("logger.level"))
config().setString("logger.level", options["logger.level"].as<std::string>());
}
2018-05-11 14:35:32 +00:00
}
2019-12-15 06:34:43 +00:00
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wmissing-declarations"
2017-12-02 02:47:12 +00:00
int mainEntryClickHouseLocal(int argc, char ** argv)
{
DB::LocalServer app;
try
{
app.init(argc, argv);
return app.run();
}
catch (const DB::Exception & e)
{
std::cerr << DB::getExceptionMessage(e, false) << std::endl;
auto code = DB::getCurrentExceptionCode();
return code ? code : 1;
}
catch (const boost::program_options::error & e)
{
std::cerr << "Bad arguments: " << e.what() << std::endl;
return DB::ErrorCodes::BAD_ARGUMENTS;
}
2017-12-02 02:47:12 +00:00
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
2017-12-02 02:47:12 +00:00
auto code = DB::getCurrentExceptionCode();
return code ? code : 1;
}
}