mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-24 16:42:05 +00:00
Added requested PR changes, 2nd iteration.
This commit is contained in:
Vitaliy Lyudvichenko
parent
c05f512637
commit
107f7b34c7
@ -17,6 +17,7 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int FUNCTION_CANNOT_HAVE_PARAMETERS;
|
||||
extern const int TOO_LESS_ARGUMENTS_FOR_FUNCTION;
|
||||
}
|
||||
|
||||
|
||||
@ -274,6 +274,16 @@ public:
|
||||
|
||||
void shutdown();
|
||||
|
||||
enum class ApplicationType
|
||||
{
|
||||
SERVER, /// The program is run as clickhouse-server daemon (default behavior)
|
||||
CLIENT, /// clickhouse-client
|
||||
LOCAL_SERVER /// clickhouse-local
|
||||
};
|
||||
|
||||
ApplicationType getApplicationType() const;
|
||||
void setApplicationType(ApplicationType type);
|
||||
|
||||
private:
|
||||
/** Проверить, имеет ли текущий клиент доступ к заданной базе данных.
|
||||
* Если доступ запрещён, кинуть исключение.
|
||||
|
||||
@ -33,8 +33,10 @@ ASTPtr parseQuery(
|
||||
const std::string & description);
|
||||
|
||||
|
||||
/// Split queries separated by ; on to list of single queries
|
||||
/// Returns pointer to the end of last sucessfuly parsed query (first), and true if all queries are sucessfuly parsed (second)
|
||||
/** Split queries separated by ; on to list of single queries
|
||||
* Returns pointer to the end of last sucessfuly parsed query (first), and true if all queries are sucessfuly parsed (second)
|
||||
* NOTE: INSERT's data should be placed in single line.
|
||||
*/
|
||||
std::pair<const char *, bool> splitMultipartQuery(const std::string & queries, std::vector<std::string> & queries_list);
|
||||
|
||||
}
|
||||
|
||||
@ -56,7 +56,7 @@ public:
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return "File(" + format_name + ")";
|
||||
return "File";
|
||||
}
|
||||
|
||||
std::string getTableName() const override
|
||||
|
||||
@ -407,7 +407,7 @@ private:
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
int mainEntryClickhouseBenchmark(int argc, char ** argv)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
@ -502,4 +502,6 @@ int main(int argc, char ** argv)
|
||||
std::cerr << "Unknown exception\n";
|
||||
return ErrorCodes::UNKNOWN_EXCEPTION;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -22,9 +22,8 @@ add_library(clickhouse-client Client.cpp)
|
||||
target_link_libraries (clickhouse-client dbms ${LINE_EDITING_LIBS} ${BOOST_PROGRAM_OPTIONS_LIB})
|
||||
INSTALL(FILES config.xml DESTINATION /etc/clickhouse-client COMPONENT clickhouse)
|
||||
|
||||
add_executable(clickhouse-benchmark Benchmark.cpp)
|
||||
add_library(clickhouse-benchmark Benchmark.cpp)
|
||||
target_link_libraries (clickhouse-benchmark dbms ${BOOST_PROGRAM_OPTIONS_LIB})
|
||||
INSTALL(TARGETS clickhouse-benchmark RUNTIME DESTINATION bin COMPONENT clickhouse-benchmark)
|
||||
|
||||
IF(TESTS)
|
||||
add_subdirectory (tests)
|
||||
|
||||
@ -203,6 +203,8 @@ private:
|
||||
else if (Poco::File("/etc/clickhouse-client/config.xml").exists())
|
||||
loadConfiguration("/etc/clickhouse-client/config.xml");
|
||||
|
||||
context.setApplicationType(Context::ApplicationType::CLIENT);
|
||||
|
||||
/// settings and limits could be specified in config file, but passed settings has higher priority
|
||||
#define EXTRACT_SETTING(TYPE, NAME, DEFAULT) \
|
||||
if (config().has(#NAME) && !context.getSettingsRef().NAME.changed) \
|
||||
@ -1304,7 +1306,7 @@ public:
|
||||
}
|
||||
|
||||
|
||||
int main_clickhouse_client(int argc, char ** argv)
|
||||
int mainEntryClickhouseClient(int argc, char ** argv)
|
||||
{
|
||||
DB::Client client;
|
||||
|
||||
|
||||
@ -350,6 +350,7 @@ namespace ErrorCodes
|
||||
extern const int SUPPORT_IS_DISABLED = 344;
|
||||
extern const int TABLE_DIFFERS_TOO_MUCH = 345;
|
||||
extern const int CANNOT_ICONV = 346;
|
||||
extern const int CANNOT_LOAD_CONFIG = 347;
|
||||
|
||||
extern const int KEEPER_EXCEPTION = 999;
|
||||
extern const int POCO_EXCEPTION = 1000;
|
||||
|
||||
@ -80,7 +80,7 @@ void DatabaseMemory::removeTable(const String & table_name)
|
||||
void DatabaseMemory::renameTable(
|
||||
const Context & context, const String & table_name, IDatabase & to_database, const String & to_table_name)
|
||||
{
|
||||
throw Exception("renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception("DatabaseMemory: renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
time_t DatabaseMemory::getTableMetadataModificationTime(const String & table_name)
|
||||
@ -90,7 +90,7 @@ time_t DatabaseMemory::getTableMetadataModificationTime(const String & table_nam
|
||||
|
||||
ASTPtr DatabaseMemory::getCreateQuery(const String & table_name) const
|
||||
{
|
||||
throw Exception("getCreateQuery() is not supported", ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception("DatabaseMemory: getCreateQuery() is not supported", ErrorCodes::NOT_IMPLEMENTED);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@ -120,7 +120,7 @@ void DatabaseMemory::alterTable(
|
||||
const ColumnDefaults & column_defaults,
|
||||
const ASTModifier & engine_modifier)
|
||||
{
|
||||
throw Exception("alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
|
||||
throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -129,6 +129,8 @@ struct ContextShared
|
||||
|
||||
Stopwatch uptime_watch;
|
||||
|
||||
Context::ApplicationType application_type = Context::ApplicationType::SERVER;
|
||||
|
||||
|
||||
~ContextShared()
|
||||
{
|
||||
@ -1061,4 +1063,17 @@ void Context::shutdown()
|
||||
shared->shutdown();
|
||||
}
|
||||
|
||||
|
||||
Context::ApplicationType Context::getApplicationType() const
|
||||
{
|
||||
return shared->application_type;
|
||||
}
|
||||
|
||||
void Context::setApplicationType(ApplicationType type)
|
||||
{
|
||||
/// Lock isn't required, you should set it at start
|
||||
shared->application_type = type;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -152,9 +152,12 @@ void QuotaForIntervals::initFromConfig(const String & config_elem, Poco::Util::A
|
||||
continue;
|
||||
|
||||
String interval_config_elem = config_elem + "." + *it;
|
||||
time_t duration = config.getInt(interval_config_elem + ".duration");
|
||||
time_t duration = config.getInt(interval_config_elem + ".duration", 0);
|
||||
time_t offset = 0;
|
||||
|
||||
if (!duration) /// Skip quaotas with zero duration
|
||||
continue;
|
||||
|
||||
bool randomize = config.getBool(interval_config_elem + ".randomize", false);
|
||||
if (randomize)
|
||||
offset = std::uniform_int_distribution<decltype(duration)>(0, duration - 1)(rng);
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
#include <DB/Interpreters/Settings.h>
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -195,10 +195,11 @@ std::pair<const char *, bool> splitMultipartQuery(const std::string & queries, s
|
||||
if (!ast)
|
||||
break;
|
||||
|
||||
ASTInsertQuery * insert = typeid_cast<ASTInsertQuery *>(&*ast);
|
||||
ASTInsertQuery * insert = typeid_cast<ASTInsertQuery *>(ast.get());
|
||||
|
||||
if (insert && insert->data)
|
||||
{
|
||||
/// Inserting data is broken on new line
|
||||
pos = insert->data;
|
||||
while (*pos && *pos != '\n')
|
||||
++pos;
|
||||
|
||||
@ -14,13 +14,14 @@ add_library(clickhouse-local LocalServer.cpp)
|
||||
target_link_libraries(clickhouse-local dbms)
|
||||
|
||||
add_executable(clickhouse main.cpp)
|
||||
target_link_libraries(clickhouse clickhouse-server clickhouse-client clickhouse-local)
|
||||
target_link_libraries(clickhouse clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark)
|
||||
INSTALL(TARGETS clickhouse RUNTIME DESTINATION bin COMPONENT clickhouse)
|
||||
|
||||
#make symbolic links to concrete clickhouse applications
|
||||
# make symbolic links to concrete clickhouse applications
|
||||
INSTALL(CODE "execute_process(COMMAND ln -s -f clickhouse clickhouse-server WORKING_DIRECTORY \"${CMAKE_INSTALL_PREFIX}/bin\")")
|
||||
INSTALL(CODE "execute_process(COMMAND ln -s -f clickhouse clickhouse-client WORKING_DIRECTORY \"${CMAKE_INSTALL_PREFIX}/bin\")")
|
||||
INSTALL(CODE "execute_process(COMMAND ln -s -f clickhouse clickhouse-local WORKING_DIRECTORY \"${CMAKE_INSTALL_PREFIX}/bin\")")
|
||||
INSTALL(CODE "execute_process(COMMAND ln -s -f clickhouse clickhouse-benchmark WORKING_DIRECTORY \"${CMAKE_INSTALL_PREFIX}/bin\")")
|
||||
|
||||
INSTALL(
|
||||
FILES config.xml users.xml
|
||||
|
||||
@ -6,7 +6,15 @@
|
||||
#include <DB/Databases/DatabaseOrdinary.h>
|
||||
|
||||
#include <DB/Storages/System/StorageSystemNumbers.h>
|
||||
#include <DB/Storages/System/StorageSystemTables.h>
|
||||
#include <DB/Storages/System/StorageSystemDatabases.h>
|
||||
#include <DB/Storages/System/StorageSystemProcesses.h>
|
||||
#include <DB/Storages/System/StorageSystemEvents.h>
|
||||
#include <DB/Storages/System/StorageSystemOne.h>
|
||||
#include <DB/Storages/System/StorageSystemSettings.h>
|
||||
#include <DB/Storages/System/StorageSystemDictionaries.h>
|
||||
#include <DB/Storages/System/StorageSystemColumns.h>
|
||||
#include <DB/Storages/System/StorageSystemFunctions.h>
|
||||
|
||||
#include <DB/Interpreters/Context.h>
|
||||
#include <DB/Interpreters/ProcessList.h>
|
||||
@ -29,6 +37,18 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int SYNTAX_ERROR;
|
||||
extern const int CANNOT_LOAD_CONFIG;
|
||||
}
|
||||
|
||||
|
||||
LocalServer::LocalServer() = default;
|
||||
|
||||
LocalServer::~LocalServer() = default;
|
||||
|
||||
|
||||
void LocalServer::initialize(Poco::Util::Application & self)
|
||||
{
|
||||
Poco::Util::Application::initialize(self);
|
||||
@ -47,77 +67,77 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
|
||||
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("config-file", "C", "Load configuration from a given file")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
.argument (" config.xml")
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.argument(" config.xml")
|
||||
.binding("config-file")
|
||||
);
|
||||
|
||||
/// Arguments that define first query creating initial table:
|
||||
/// (If structure argument is omitted then initial query is not generated)
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("structure", "S", "Structe of initial table (list columns names with their types)")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
.argument (" <struct>")
|
||||
Poco::Util::Option ("structure", "S", "Structe of initial table(list columns names with their types)")
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.argument(" <struct>")
|
||||
.binding("table-structure")
|
||||
);
|
||||
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("table", "N", "Name of intial table")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
.argument (" table")
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.argument(" table")
|
||||
.binding("table-name")
|
||||
);
|
||||
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("file", "F", "Path to file with data of initial table (stdin if not specified)")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
.argument (" stdin")
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.argument(" stdin")
|
||||
.binding("table-file")
|
||||
);
|
||||
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("iformat", "fi", "Input format of intial table data. Note, --format is output one")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
.argument (" TabSeparated")
|
||||
Poco::Util::Option ("input-format", "if", "Input format of intial table data")
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.argument(" TabSeparated")
|
||||
.binding("table-data-format")
|
||||
);
|
||||
|
||||
/// List of queries to execute
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("query", "Q", "Queries to execute")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
.argument (" <query>", true)
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.argument(" <query>", true)
|
||||
.binding("query")
|
||||
);
|
||||
|
||||
/// Default Output format
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("oformat", "fo", "Default ouput format")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
.argument (" TabSeparated", true)
|
||||
.binding("oformat")
|
||||
Poco::Util::Option ("output-format", "of", "Default output format")
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.argument(" TabSeparated", true)
|
||||
.binding("output-format")
|
||||
);
|
||||
|
||||
/// Alias for previous one, required for clickhouse-client compability
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("format", "f", "Default ouput format")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
.argument (" TabSeparated", true)
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.argument(" TabSeparated", true)
|
||||
.binding("format")
|
||||
);
|
||||
|
||||
_options.addOption(
|
||||
Poco::Util::Option ("verbose", "", "Print info about execution queries")
|
||||
.required (false)
|
||||
.repeatable (false)
|
||||
Poco::Util::Option ("verbose", "", "Print info about execution of queries")
|
||||
.required(false)
|
||||
.repeatable(false)
|
||||
.noArgument()
|
||||
.binding("verbose")
|
||||
);
|
||||
@ -144,7 +164,7 @@ void LocalServer::defineOptions(Poco::Util::OptionSet& _options)
|
||||
|
||||
void LocalServer::applyOptions()
|
||||
{
|
||||
context->setDefaultFormat(config().getString("oformat", config().getString("format", "TabSeparated")));
|
||||
context->setDefaultFormat(config().getString("output-format", config().getString("format", "TabSeparated")));
|
||||
|
||||
/// settings and limits could be specified in config file, but passed settings has higher priority
|
||||
#define EXTRACT_SETTING(TYPE, NAME, DEFAULT) \
|
||||
@ -208,6 +228,7 @@ int LocalServer::main(const std::vector<std::string> & args)
|
||||
|
||||
context = std::make_unique<Context>();
|
||||
context->setGlobalContext(*context);
|
||||
context->setApplicationType(Context::ApplicationType::LOCAL_SERVER);
|
||||
|
||||
applyOptions();
|
||||
|
||||
@ -227,7 +248,7 @@ int LocalServer::main(const std::vector<std::string> & args)
|
||||
|
||||
setupUsers();
|
||||
|
||||
/// Limit on total number of coucurrently executed queries.
|
||||
/// Limit on total number of concurrently executing queries.
|
||||
/// Threre are no need for concurrent threads, override max_concurrent_queries.
|
||||
context->getProcessList().setMaxSize(0);
|
||||
|
||||
@ -237,7 +258,7 @@ int LocalServer::main(const std::vector<std::string> & args)
|
||||
context->setUncompressedCache(uncompressed_cache_size);
|
||||
|
||||
/// Size of cache for marks (index of MergeTree family of tables). It is necessary.
|
||||
/// Specify default value if mark_cache_size explicitly!
|
||||
/// Specify default value for mark_cache_size explicitly!
|
||||
size_t mark_cache_size = parse<size_t>(config().getString("mark_cache_size", "5368709120"));
|
||||
if (mark_cache_size)
|
||||
context->setMarkCache(mark_cache_size);
|
||||
@ -257,34 +278,53 @@ int LocalServer::main(const std::vector<std::string> & args)
|
||||
}
|
||||
|
||||
|
||||
inline String getQuotedString(const String & s)
|
||||
{
|
||||
String res;
|
||||
WriteBufferFromString buf(res);
|
||||
writeQuotedString(s, buf);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
std::string LocalServer::getInitialCreateTableQuery()
|
||||
{
|
||||
if (!config().has("table-structure"))
|
||||
return std::string();
|
||||
return {};
|
||||
|
||||
auto table_name = backQuoteIfNeed(config().getString("table-name", "table"));
|
||||
auto table_structure = config().getString("table-structure");
|
||||
auto data_format = backQuoteIfNeed(config().getString("table-data-format", "TabSeparated"));
|
||||
auto table_file = config().getString("table-file", "stdin");
|
||||
String table_file;
|
||||
if (!config().has("table-file") || config().getString("table-file") == "-") /// Use Unix tools stdin naming convention
|
||||
table_file = "stdin";
|
||||
else /// Use regular file
|
||||
table_file = getQuotedString(config().getString("table-file"));
|
||||
|
||||
return
|
||||
"CREATE TABLE " + table_name +
|
||||
" (" + table_structure + ") " +
|
||||
"ENGINE = "
|
||||
"File(" + data_format + ", '" + table_file + "')"
|
||||
"File(" + data_format + ", " + table_file + ")"
|
||||
"; ";
|
||||
}
|
||||
|
||||
|
||||
void LocalServer::attachSystemTables()
|
||||
{
|
||||
/// Only numbers an one table make sense
|
||||
/// TODO: add attachTableDelayed into DatabaseMemory
|
||||
/// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
|
||||
|
||||
DatabasePtr system_database = std::make_shared<DatabaseMemory>("system");
|
||||
context->addDatabase("system", system_database);
|
||||
system_database->attachTable("one", StorageSystemOne::create("one"));
|
||||
system_database->attachTable("numbers", StorageSystemNumbers::create("numbers"));
|
||||
system_database->attachTable("numbers_mt", StorageSystemNumbers::create("numbers_mt", true));
|
||||
system_database->attachTable("databases", StorageSystemDatabases::create("databases"));
|
||||
system_database->attachTable("tables", StorageSystemTables::create("tables"));
|
||||
system_database->attachTable("columns", StorageSystemColumns::create("columns"));
|
||||
system_database->attachTable("functions", StorageSystemFunctions::create("functions"));
|
||||
system_database->attachTable("events", StorageSystemEvents::create("events"));
|
||||
system_database->attachTable("settings", StorageSystemSettings::create("settings"));
|
||||
}
|
||||
|
||||
|
||||
@ -296,12 +336,13 @@ void LocalServer::processQueries()
|
||||
String queries_str = initial_create_query + config().getString("query");
|
||||
|
||||
bool verbose = config().getBool("verbose", false);
|
||||
if (verbose)
|
||||
LOG_INFO(log, "Executing queries: " << queries_str);
|
||||
|
||||
std::vector<String> queries;
|
||||
auto parse_res = splitMultipartQuery(queries_str, queries);
|
||||
|
||||
if (!parse_res.second)
|
||||
throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR);
|
||||
|
||||
context->setUser("default", "", Poco::Net::SocketAddress{}, "");
|
||||
|
||||
for (const auto & query : queries)
|
||||
@ -323,14 +364,29 @@ void LocalServer::processQueries()
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute while queries are valid
|
||||
if (!parse_res.second)
|
||||
{
|
||||
LOG_ERROR(log, "Cannot parse and execute the following part of query: '" << parse_res.first << "'");
|
||||
}
|
||||
}
|
||||
|
||||
static const char * minimal_default_user_xml =
|
||||
"<yandex>"
|
||||
" <profiles>"
|
||||
" <default></default>"
|
||||
" </profiles>"
|
||||
" <users>"
|
||||
" <default>"
|
||||
" <password></password>"
|
||||
" <networks>"
|
||||
" <ip>::/0</ip>"
|
||||
" </networks>"
|
||||
" <profile>default</profile>"
|
||||
" <quota>default</quota>"
|
||||
" </default>"
|
||||
" </users>"
|
||||
" <quotas>"
|
||||
" <default></default>"
|
||||
" </quotas>"
|
||||
"</yandex>";
|
||||
|
||||
|
||||
void LocalServer::setupUsers()
|
||||
{
|
||||
ConfigurationPtr users_config;
|
||||
@ -343,7 +399,7 @@ void LocalServer::setupUsers()
|
||||
else
|
||||
{
|
||||
std::stringstream default_user_stream;
|
||||
default_user_stream << default_user_xml;
|
||||
default_user_stream << minimal_default_user_xml;
|
||||
|
||||
Poco::XML::InputSource default_user_source(default_user_stream);
|
||||
users_config = ConfigurationPtr(new Poco::Util::XMLConfiguration(&default_user_source));
|
||||
@ -352,48 +408,9 @@ void LocalServer::setupUsers()
|
||||
if (users_config)
|
||||
context->setUsersConfig(users_config);
|
||||
else
|
||||
throw Exception("Can't load config for users");
|
||||
throw Exception("Can't load config for users", ErrorCodes::CANNOT_LOAD_CONFIG);
|
||||
}
|
||||
|
||||
|
||||
const char * LocalServer::default_user_xml =
|
||||
"<yandex>\n"
|
||||
" <profiles>\n"
|
||||
" <default>\n"
|
||||
" <max_memory_usage>10000000000</max_memory_usage>\n"
|
||||
" <use_uncompressed_cache>0</use_uncompressed_cache>\n"
|
||||
" <load_balancing>random</load_balancing>\n"
|
||||
" </default>\n"
|
||||
" <readonly>\n"
|
||||
" <readonly>1</readonly>\n"
|
||||
" </readonly>\n"
|
||||
" </profiles>\n"
|
||||
"\n"
|
||||
" <users>\n"
|
||||
" <default>\n"
|
||||
" <password></password>\n"
|
||||
" <networks>\n"
|
||||
" <ip>::/0</ip>\n"
|
||||
" </networks>\n"
|
||||
" <profile>default</profile>\n"
|
||||
" <quota>default</quota>\n"
|
||||
" </default>\n"
|
||||
" </users>\n"
|
||||
"\n"
|
||||
" <quotas>\n"
|
||||
" <default>\n"
|
||||
" <interval>\n"
|
||||
" <duration>3600</duration>\n"
|
||||
" <queries>0</queries>\n"
|
||||
" <errors>0</errors>\n"
|
||||
" <result_rows>0</result_rows>\n"
|
||||
" <read_rows>0</read_rows>\n"
|
||||
" <execution_time>0</execution_time>\n"
|
||||
" </interval>\n"
|
||||
" </default>\n"
|
||||
" </quotas>\n"
|
||||
"</yandex>\n";
|
||||
|
||||
}
|
||||
|
||||
YANDEX_APP_MAIN_FUNC(DB::LocalServer, main_clickhouse_local);
|
||||
YANDEX_APP_MAIN_FUNC(DB::LocalServer, mainEntryClickhouseLocal);
|
||||
|
||||
@ -15,7 +15,7 @@ class LocalServer : public Poco::Util::Application
|
||||
{
|
||||
public:
|
||||
|
||||
LocalServer() = default;
|
||||
LocalServer();
|
||||
|
||||
void initialize(Poco::Util::Application & self) override;
|
||||
|
||||
@ -23,10 +23,14 @@ public:
|
||||
|
||||
int main(const std::vector<std::string> & args) override;
|
||||
|
||||
~LocalServer() = default;
|
||||
~LocalServer();
|
||||
|
||||
private:
|
||||
|
||||
/** Composes CREATE subquery based on passed arguments (--structure --file --table and --input-format)
|
||||
* This query will be executed first, before queries passed through --query argument
|
||||
* Returns empty string if it cannot compose that query.
|
||||
*/
|
||||
std::string getInitialCreateTableQuery();
|
||||
|
||||
void applyOptions();
|
||||
@ -41,8 +45,6 @@ private:
|
||||
|
||||
void handleHelp(const std::string & name, const std::string & value);
|
||||
|
||||
static const char * default_user_xml;
|
||||
|
||||
protected:
|
||||
|
||||
std::unique_ptr<Context> context;
|
||||
|
||||
@ -193,8 +193,8 @@ int Server::main(const std::vector<std::string> & args)
|
||||
* settings, available functions, data types, aggregate functions, databases...
|
||||
*/
|
||||
global_context = std::make_unique<Context>();
|
||||
|
||||
global_context->setGlobalContext(*global_context);
|
||||
global_context->setApplicationType(Context::ApplicationType::SERVER);
|
||||
global_context->setPath(path);
|
||||
|
||||
std::string default_database = config().getString("default_database", "default");
|
||||
@ -520,4 +520,4 @@ int Server::main(const std::vector<std::string> & args)
|
||||
|
||||
}
|
||||
|
||||
YANDEX_APP_SERVER_MAIN_FUNC(DB::Server, main_clickhouse_server);
|
||||
YANDEX_APP_SERVER_MAIN_FUNC(DB::Server, mainEntryClickHouseServer);
|
||||
|
||||
@ -3,17 +3,18 @@
|
||||
#include <DB/Common/StringUtils.h>
|
||||
|
||||
/// Universal executable for various clickhouse applications
|
||||
int main_clickhouse_server(int argc, char ** argv);
|
||||
int main_clickhouse_client(int argc, char ** argv);
|
||||
int main_clickhouse_local(int argc, char ** argv);
|
||||
int mainEntryClickHouseServer(int argc, char ** argv);
|
||||
int mainEntryClickhouseClient(int argc, char ** argv);
|
||||
int mainEntryClickhouseLocal(int argc, char ** argv);
|
||||
int mainEntryClickhouseBenchmark(int argc, char ** argv);
|
||||
|
||||
|
||||
static bool is_clickhouse_app(const std::string & app_suffix, std::vector<char *> & argv)
|
||||
static bool isClickhouseApp(const std::string & app_suffix, std::vector<char *> & argv)
|
||||
{
|
||||
std::string arg_mode_app = "--" + app_suffix;
|
||||
|
||||
/// Use local mode if --local arg is passed (the arg should be quietly removed)
|
||||
auto arg_it = std::find_if(argv.begin(), argv.end(), [&](char * arg) { return !arg_mode_app.compare(arg); } );
|
||||
/// Use app if --app arg is passed (the arg should be quietly removed)
|
||||
auto arg_it = std::find_if(argv.begin(), argv.end(), [&](const char * arg) { return !arg_mode_app.compare(arg); } );
|
||||
if (arg_it != argv.end())
|
||||
{
|
||||
argv.erase(arg_it);
|
||||
@ -22,8 +23,8 @@ static bool is_clickhouse_app(const std::string & app_suffix, std::vector<char *
|
||||
|
||||
std::string app_name = "clickhouse-" + app_suffix;
|
||||
|
||||
/// Use local mode if app is run through symbolic link with name clickhouse-local
|
||||
if (!app_name.compare(argv[0]) || endsWith(argv[0], "/" + app_name))
|
||||
/// Use app if clickhouse binary is run through symbolic link with name clickhouse-app
|
||||
if (!argv.empty() && (!app_name.compare(argv[0]) || endsWith(argv[0], "/" + app_name)))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
@ -34,14 +35,16 @@ int main(int argc_, char ** argv_)
|
||||
{
|
||||
std::vector<char *> argv(argv_, argv_ + argc_);
|
||||
|
||||
auto main_func = main_clickhouse_server;
|
||||
auto main_func = mainEntryClickHouseServer;
|
||||
|
||||
if (is_clickhouse_app("local", argv))
|
||||
main_func = main_clickhouse_local;
|
||||
else if (is_clickhouse_app("client", argv))
|
||||
main_func = main_clickhouse_client;
|
||||
else if (is_clickhouse_app("server", argv)) /// --server arg should be cut
|
||||
main_func = main_clickhouse_server;
|
||||
if (isClickhouseApp("local", argv))
|
||||
main_func = mainEntryClickhouseLocal;
|
||||
else if (isClickhouseApp("client", argv))
|
||||
main_func = mainEntryClickhouseClient;
|
||||
else if (isClickhouseApp("benchmark", argv))
|
||||
main_func = mainEntryClickhouseBenchmark;
|
||||
else if (isClickhouseApp("server", argv)) /// --server arg should be cut
|
||||
main_func = mainEntryClickHouseServer;
|
||||
|
||||
return main_func(static_cast<int>(argv.size()), argv.data());
|
||||
}
|
||||
|
||||
@ -47,6 +47,7 @@ namespace ErrorCodes
|
||||
extern const int NO_REPLICA_NAME_GIVEN;
|
||||
extern const int NO_ELEMENTS_IN_CONFIG;
|
||||
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
|
||||
extern const int UNKNOWN_IDENTIFIER;
|
||||
}
|
||||
|
||||
|
||||
@ -260,16 +261,13 @@ StoragePtr StorageFactory::get(
|
||||
}
|
||||
else if (name == "File")
|
||||
{
|
||||
ASTs & args_func = typeid_cast<ASTFunction &>(*typeid_cast<ASTCreateQuery &>(*query).storage).children;
|
||||
auto & func = typeid_cast<ASTFunction &>(*typeid_cast<ASTCreateQuery &>(*query).storage);
|
||||
auto & args = typeid_cast<ASTExpressionList &>(*func.arguments).children;
|
||||
|
||||
constexpr auto error_msg = "Storage File requires exactly 1 parameter - name of using format.";
|
||||
constexpr auto error_msg = "Storage File requires 1 or 2 arguments: name of used format and source.";
|
||||
|
||||
/// TODO: Maybe some my misunderstanding of ASTs
|
||||
|
||||
if (args_func.size() != 1)
|
||||
throw Exception(error_msg, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
ASTs & args = typeid_cast<ASTExpressionList &>(*args_func.at(0)).children;
|
||||
if (func.parameters)
|
||||
throw Exception(error_msg, ErrorCodes::FUNCTION_CANNOT_HAVE_PARAMETERS);
|
||||
|
||||
if (args.empty() || args.size() > 2)
|
||||
throw Exception(error_msg, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
@ -281,13 +279,32 @@ StoragePtr StorageFactory::get(
|
||||
String source_path;
|
||||
if (args.size() >= 2)
|
||||
{
|
||||
args[1] = evaluateConstantExpressionOrIdentidierAsLiteral(args[1], local_context);
|
||||
String table_source = static_cast<const ASTLiteral &>(*args[1]).value.safeGet<String>();
|
||||
/// Will use FD if args[1] is int literal or identifier with std* name
|
||||
|
||||
if (table_source == "stdin")
|
||||
source_fd = STDIN_FILENO;
|
||||
else
|
||||
source_path = std::move(table_source);
|
||||
if (ASTIdentifier * identifier = typeid_cast<ASTIdentifier *>(args[1].get()))
|
||||
{
|
||||
if (identifier->name == "stdin")
|
||||
source_fd = STDIN_FILENO;
|
||||
else if (identifier->name == "stdout")
|
||||
source_fd = STDOUT_FILENO;
|
||||
else if (identifier->name == "stderr")
|
||||
source_fd = STDERR_FILENO;
|
||||
else
|
||||
throw Exception("Unknown identifier '" + identifier->name + "' in second arg of File storage constructor",
|
||||
ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||
}
|
||||
|
||||
if (ASTLiteral * literal = typeid_cast<ASTLiteral *>(args[1].get()))
|
||||
{
|
||||
auto type = literal->value.getType();
|
||||
if (type == Field::Types::Int64)
|
||||
source_fd = static_cast<int>(literal->value.get<Int64>());
|
||||
else if (type == Field::Types::UInt64)
|
||||
source_fd = static_cast<int>(literal->value.get<UInt64>());
|
||||
}
|
||||
|
||||
args[1] = evaluateConstantExpressionOrIdentidierAsLiteral(args[1], local_context);
|
||||
source_path = static_cast<const ASTLiteral &>(*args[1]).value.safeGet<String>();
|
||||
}
|
||||
|
||||
return StorageFile::create(
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
#include <DB/Storages/StorageFile.h>
|
||||
|
||||
#include <DB/DataStreams/FormatFactory.h>
|
||||
#include <DB/Interpreters/Context.h>
|
||||
#include <DB/IO/ReadBufferFromFile.h>
|
||||
#include <DB/IO/WriteBufferFromFile.h>
|
||||
#include <DB/IO/WriteHelpers.h>
|
||||
|
||||
#include <DB/DataStreams/FormatFactory.h>
|
||||
#include <DB/DataStreams/IProfilingBlockInputStream.h>
|
||||
#include <DB/DataStreams/IBlockOutputStream.h>
|
||||
|
||||
@ -19,12 +19,19 @@ namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_WRITE_TO_FILE_DESCRIPTOR;
|
||||
extern const int CANNOT_SEEK_THROUGH_FILE;
|
||||
extern const int DATABASE_ACCESS_DENIED;
|
||||
};
|
||||
|
||||
|
||||
static std::string getTablePath(const std::string & db_dir_path, const std::string & table_name, const std::string & format_name)
|
||||
{
|
||||
return db_dir_path + escapeForFileName(table_name) + "/main." + escapeForFileName(format_name);
|
||||
return db_dir_path + escapeForFileName(table_name) + "/data." + escapeForFileName(format_name);
|
||||
}
|
||||
|
||||
static void checkCreationIsAllowed(Context & context_global)
|
||||
{
|
||||
if (context_global.getApplicationType() == Context::ApplicationType::SERVER)
|
||||
throw Exception("Using file descriptor or user specified path as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED);
|
||||
}
|
||||
|
||||
|
||||
@ -42,26 +49,31 @@ StorageFile::StorageFile(
|
||||
: IStorage(materialized_columns_, alias_columns_, column_defaults_),
|
||||
table_name(table_name_), format_name(format_name_), columns(columns_), context_global(context_), table_fd(table_fd_)
|
||||
{
|
||||
if (table_fd < 0) // Will use file
|
||||
if (table_fd < 0) /// Will use file
|
||||
{
|
||||
use_table_fd = false;
|
||||
|
||||
if (!table_path_.empty()) // Is user's file
|
||||
if (!table_path_.empty()) /// Is user's file
|
||||
{
|
||||
checkCreationIsAllowed(context_global);
|
||||
path = Poco::Path(table_path_).absolute().toString();
|
||||
is_db_table = false;
|
||||
}
|
||||
else // Is DB's file
|
||||
else /// Is DB's file
|
||||
{
|
||||
path = getTablePath(db_dir_path, table_name, format_name);
|
||||
is_db_table = true;
|
||||
Poco::File(Poco::Path(path).parent()).createDirectories();
|
||||
}
|
||||
}
|
||||
else
|
||||
else /// Will use FD
|
||||
{
|
||||
checkCreationIsAllowed(context_global);
|
||||
is_db_table = false;
|
||||
use_table_fd = true;
|
||||
|
||||
/// Save initial offset, it will be used for repeating SELECTs
|
||||
/// If FD isn't seekable (lseek returns -1), then the second and subsequent SELECTs will fail.
|
||||
table_fd_init_offset = lseek(table_fd, 0, SEEK_CUR);
|
||||
}
|
||||
}
|
||||
@ -72,7 +84,7 @@ class StorageFileBlockInputStream : public IProfilingBlockInputStream
|
||||
public:
|
||||
|
||||
StorageFileBlockInputStream(StorageFile & storage_, const Context & context, size_t max_block_size)
|
||||
: storage(storage_), lock(storage.rwlock, storage.use_table_fd)
|
||||
: storage(storage_), lock(storage.rwlock, storage.use_table_fd)
|
||||
{
|
||||
if (storage.use_table_fd)
|
||||
{
|
||||
@ -86,7 +98,7 @@ public:
|
||||
|
||||
/// ReadBuffer's seek() doesn't make sence, since cache is empty
|
||||
if (lseek(storage.table_fd, storage.table_fd_init_offset, SEEK_SET) < 0)
|
||||
throw Exception("Cannot seek file descriptor, inside " + storage.getName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
||||
throwFromErrno("Cannot seek file descriptor, inside " + storage.getName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
||||
}
|
||||
|
||||
storage.table_fd_was_used = true;
|
||||
@ -97,7 +109,7 @@ public:
|
||||
read_buf = std::make_unique<ReadBufferFromFile>(storage.path);
|
||||
}
|
||||
|
||||
input_formatter = FormatFactory().getInput(storage.format_name, *read_buf, storage.getSampleBlock(), context, max_block_size);
|
||||
reader = FormatFactory().getInput(storage.format_name, *read_buf, storage.getSampleBlock(), context, max_block_size);
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
@ -108,23 +120,28 @@ public:
|
||||
String getID() const override
|
||||
{
|
||||
std::stringstream res_stream;
|
||||
res_stream << "File(" << &storage << ")";
|
||||
res_stream << "File(" << storage.format_name << ", ";
|
||||
if (!storage.path.empty())
|
||||
res_stream << storage.path;
|
||||
else
|
||||
res_stream << storage.table_fd;
|
||||
res_stream << ")";
|
||||
return res_stream.str();
|
||||
}
|
||||
|
||||
Block readImpl() override
|
||||
{
|
||||
return input_formatter->read();
|
||||
return reader->read();
|
||||
}
|
||||
|
||||
void readPrefixImpl() override
|
||||
{
|
||||
input_formatter->readPrefix();
|
||||
reader->readPrefix();
|
||||
}
|
||||
|
||||
void readSuffixImpl() override
|
||||
{
|
||||
input_formatter->readSuffix();
|
||||
reader->readSuffix();
|
||||
}
|
||||
|
||||
private:
|
||||
@ -132,7 +149,7 @@ private:
|
||||
Poco::ScopedRWLock lock;
|
||||
Block sample_block;
|
||||
std::unique_ptr<ReadBufferFromFileDescriptor> read_buf;
|
||||
BlockInputStreamPtr input_formatter;
|
||||
BlockInputStreamPtr reader;
|
||||
};
|
||||
|
||||
|
||||
@ -154,15 +171,14 @@ class StorageFileBlockOutputStream : public IBlockOutputStream
|
||||
public:
|
||||
|
||||
StorageFileBlockOutputStream(StorageFile & storage_)
|
||||
: storage(storage_), lock(storage.rwlock)
|
||||
: storage(storage_), lock(storage.rwlock)
|
||||
{
|
||||
if (storage.use_table_fd)
|
||||
{
|
||||
/// TODO: more detail checks: 1) fd is writeable 2) points to the end of data (if possible)
|
||||
if (storage.table_fd_was_used)
|
||||
throw Exception("Write to file descriptor after use, inside " + storage.getName(),
|
||||
ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR);
|
||||
|
||||
/** NOTE: Using real file binded to FD may be misleading:
|
||||
* SELECT *; INSERT insert_data; SELECT *; last SELECT returns initil_fd_data + insert_data
|
||||
* INSERT data; SELECT *; last SELECT returns only insert_data
|
||||
*/
|
||||
storage.table_fd_was_used = true;
|
||||
write_buf = std::make_unique<WriteBufferFromFileDescriptor>(storage.table_fd);
|
||||
}
|
||||
@ -171,34 +187,34 @@ public:
|
||||
write_buf = std::make_unique<WriteBufferFromFile>(storage.path, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT);
|
||||
}
|
||||
|
||||
output_formatter = FormatFactory().getOutput(storage.format_name, *write_buf, storage.getSampleBlock(), storage.context_global);
|
||||
writer = FormatFactory().getOutput(storage.format_name, *write_buf, storage.getSampleBlock(), storage.context_global);
|
||||
}
|
||||
|
||||
void write(const Block & block) override
|
||||
{
|
||||
output_formatter->write(block);
|
||||
writer->write(block);
|
||||
}
|
||||
|
||||
void writePrefix() override
|
||||
{
|
||||
output_formatter->writePrefix();
|
||||
writer->writePrefix();
|
||||
}
|
||||
|
||||
void writeSuffix() override
|
||||
{
|
||||
output_formatter->writeSuffix();
|
||||
writer->writeSuffix();
|
||||
}
|
||||
|
||||
void flush() override
|
||||
{
|
||||
output_formatter->flush();
|
||||
writer->flush();
|
||||
}
|
||||
|
||||
private:
|
||||
StorageFile & storage;
|
||||
Poco::ScopedWriteRWLock lock;
|
||||
std::unique_ptr<WriteBufferFromFileDescriptor> write_buf;
|
||||
BlockOutputStreamPtr output_formatter;
|
||||
BlockOutputStreamPtr writer;
|
||||
};
|
||||
|
||||
BlockOutputStreamPtr StorageFile::write(
|
||||
@ -218,7 +234,7 @@ void StorageFile::drop()
|
||||
void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
|
||||
{
|
||||
if (!is_db_table)
|
||||
throw Exception("Can't rename table '" + table_name + "' stored in user-defined file (or FD)");
|
||||
throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED);
|
||||
|
||||
Poco::ScopedWriteRWLock lock(rwlock);
|
||||
|
||||
|
||||
@ -82,7 +82,10 @@ def main(args):
|
||||
break
|
||||
|
||||
suite_dir = os.path.join(base_dir, suite)
|
||||
suite = re.search('^[0-9]+_(.*)$', suite).group(1)
|
||||
suite_re_obj = re.search('^[0-9]+_(.*)$', suite)
|
||||
if not suite_re_obj: #skip .gitignore and so on
|
||||
continue
|
||||
suite = suite_re_obj.group(1)
|
||||
if os.path.isdir(suite_dir):
|
||||
print("\nRunning {} tests.\n".format(suite))
|
||||
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
0 0 0
|
||||
0 0 0
|
||||
0 0 0
|
||||
|
||||
0 0 0
|
||||
0 0 0
|
||||
0 0 0
|
||||
@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
TABLE_HASH="cityHash64(groupArray(cityHash64(*)))"
|
||||
|
||||
function pack_unpack_compare()
|
||||
{
|
||||
local buf_file="test.buf.'.$3"
|
||||
|
||||
clickhouse-client --query "DROP TABLE IF EXISTS test.buf"
|
||||
clickhouse-client --query "DROP TABLE IF EXISTS test.buf_file"
|
||||
|
||||
clickhouse-client --query "CREATE TABLE test.buf ENGINE = Memory AS $1"
|
||||
local res_orig=$(clickhouse-client --query "SELECT $TABLE_HASH FROM test.buf")
|
||||
|
||||
clickhouse-client --max_threads=1 --query "CREATE TABLE test.buf_file ENGINE = File($3) AS SELECT * FROM test.buf"
|
||||
local res_db_file=$(clickhouse-client --max_threads=1 --query "SELECT $TABLE_HASH FROM test.buf_file")
|
||||
|
||||
clickhouse-client --max_threads=1 --query "SELECT * FROM test.buf FORMAT $3" > "$buf_file"
|
||||
local res_ch_local1=$(clickhouse-local --structure "$2" --file "$buf_file" --table "my super table" --input-format "$3" --output-format TabSeparated --query "SELECT $TABLE_HASH FROM \`my super table\`" 2>/dev/null)
|
||||
local res_ch_local2=$(clickhouse-local --structure "$2" --table "my super table" --input-format "$3" --output-format TabSeparated --query "SELECT $TABLE_HASH FROM \`my super table\`" < "$buf_file" 2>/dev/null)
|
||||
|
||||
clickhouse-client --query "DROP TABLE IF EXISTS test.buf"
|
||||
clickhouse-client --query "DROP TABLE IF EXISTS test.buf_file"
|
||||
rm -f "$buf_file"
|
||||
|
||||
echo $((res_orig - res_db_file)) $((res_orig - res_ch_local1)) $((res_orig - res_ch_local2))
|
||||
}
|
||||
|
||||
pack_unpack_compare "SELECT number FROM system.numbers LIMIT 10000" "number UInt64" "TabSeparated"
|
||||
pack_unpack_compare "SELECT number FROM system.numbers LIMIT 10000" "number UInt64" "Native"
|
||||
pack_unpack_compare "SELECT number FROM system.numbers LIMIT 10000" "number UInt64" "JSONEachRow"
|
||||
echo
|
||||
pack_unpack_compare "SELECT name, is_aggregate FROM system.functions" "name String, is_aggregate UInt8" "TabSeparated"
|
||||
pack_unpack_compare "SELECT name, is_aggregate FROM system.functions" "name String, is_aggregate UInt8" "Native"
|
||||
pack_unpack_compare "SELECT name, is_aggregate FROM system.functions" "name String, is_aggregate UInt8" "TSKV"
|
||||
1
debian/daemons
vendored
1
debian/daemons
vendored
@ -2,4 +2,5 @@ compressor
|
||||
clickhouse-client
|
||||
clickhouse-server
|
||||
clickhouse-benchmark
|
||||
clickhouse-local
|
||||
config-processor
|
||||
|
||||
Loading…
Reference in New Issue
Block a user