thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Something working

Browse Source
This commit is contained in:
alesapin 2021-05-12 16:04:34 +03:00
parent ffb4e1af8f
commit 41eadf9127
8 changed files with 85 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
base/daemon/BaseDaemon.cpp
View File

@ -468,7 +468,7 @@ void BaseDaemon::reloadConfiguration()
* instead of using files specified in config.xml.
* (It's convenient to log in console when you start server without any command line parameters.)
*/
config_path = config().getString("config-file", "config.xml");
config_path = config().getString("config-file", getDefaultConfigFileName());
DB::ConfigProcessor config_processor(config_path, false, true);
config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
@ -516,6 +516,11 @@ std::string BaseDaemon::getDefaultCorePath() const
return "/opt/cores/";
}
std::string BaseDaemon::getDefaultConfigFileName() const
{
return "config.xml";
}
void BaseDaemon::closeFDs()
{
#if defined(OS_FREEBSD) || defined(OS_DARWIN)

2
base/daemon/BaseDaemon.h
View File

@ -149,6 +149,8 @@ protected:
virtual std::string getDefaultCorePath() const;
virtual std::string getDefaultConfigFileName() const;
std::optional<DB::StatusFile> pid_file;
std::atomic_bool is_cancelled{false};

2
programs/keeper/CMakeLists.txt
View File

@ -12,3 +12,5 @@ set (CLICKHOUSE_KEEPER_LINK
)
clickhouse_program_add(keeper)
install (FILES clickhouse-keeper.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-keeper" COMPONENT clickhouse-keeper RENAME keeper_config.xml)

115
programs/keeper/Keeper.cpp
View File

@ -1,5 +1,6 @@
#include "Keeper.h"
#include <Common/ClickHouseRevision.h>
#include <Server/ProtocolServerAdapter.h>
#include <Common/DNSResolver.h>
#include <Interpreters/DNSCacheUpdater.h>
@ -122,16 +123,6 @@ Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port
return socket_address;
}
std::string getCanonicalPath(std::string && path)
{
Poco::trimInPlace(path);
if (path.empty())
throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER);
if (path.back() != '/')
path += '/';
return std::move(path);
}
[[noreturn]] void forceShutdown()
{
#if defined(THREAD_SANITIZER) && defined(OS_LINUX)
@ -146,7 +137,6 @@ std::string getCanonicalPath(std::string && path)
}
Poco::Net::SocketAddress Keeper::socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure) const
{
auto address = makeSocketAddress(host, port, &logger());
@ -218,11 +208,10 @@ int Keeper::run()
}
if (config().hasOption("version"))
{
std::cout << DBMS_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
std::cout << DBMS_NAME << " keeper version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
return 0;
}
return Application::run(); // NOLINT
}
@ -237,9 +226,9 @@ void Keeper::initialize(Poco::Util::Application & self)
Poco::Environment::osArchitecture());
}
std::string Keeper::getDefaultCorePath() const
std::string Keeper::getDefaultConfigFileName() const
{
return getCanonicalPath(config().getString("path", KEEPER_DEFAULT_PATH)) + "cores";
return "keeper_config.xml";
}
void Keeper::defineOptions(Poco::Util::OptionSet & options)
@ -266,11 +255,11 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
MainThreadStatus::getInstance();
#if !defined(NDEBUG) || !defined(__OPTIMIZE__)
LOG_WARNING(log, "Server was built in debug mode. It will work slowly.");
LOG_WARNING(log, "Keeper was built in debug mode. It will work slowly.");
#endif
#if defined(SANITIZER)
LOG_WARNING(log, "Server was built with sanitizer. It will work slowly.");
LOG_WARNING(log, "Keeper was built with sanitizer. It will work slowly.");
#endif
auto shared_context = Context::createShared();
@ -278,6 +267,10 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
global_context->makeGlobalContext();
global_context->setApplicationType(Context::ApplicationType::SERVER);
if (!config().has("keeper_server"))
throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Keeper configuration (<keeper_server> section) not found in config");
const Settings & settings = global_context->getSettingsRef();
GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 500));
@ -290,9 +283,10 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
DateLUT::instance();
LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone());
Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024));
/// Don't want to use DNS cache
DNSResolver::instance().setDisableCacheFlag();
Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0);
Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024));
std::vector<std::string> listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host");
@ -306,54 +300,51 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
auto servers = std::make_shared<std::vector<ProtocolServerAdapter>>();
if (config().has("keeper_server"))
{
#if USE_NURAFT
/// Initialize test keeper RAFT. Do nothing if no nu_keeper_server in config.
global_context->initializeKeeperStorageDispatcher();
for (const auto & listen_host : listen_hosts)
/// Initialize test keeper RAFT. Do nothing if no nu_keeper_server in config.
global_context->initializeKeeperStorageDispatcher();
for (const auto & listen_host : listen_hosts)
{
/// TCP Keeper
const char * port_name = "keeper_server.tcp_port";
createServer(listen_host, port_name, listen_try, [&](UInt16 port)
{
/// TCP Keeper
const char * port_name = "keeper_server.tcp_port";
createServer(listen_host, port_name, listen_try, [&](UInt16 port)
{
Poco::Net::ServerSocket socket;
auto address = socketBindListen(socket, listen_host, port);
socket.setReceiveTimeout(settings.receive_timeout);
socket.setSendTimeout(settings.send_timeout);
servers->emplace_back(
port_name,
std::make_unique<Poco::Net::TCPServer>(
new KeeperTCPHandlerFactory(*this, false), server_pool, socket, new Poco::Net::TCPServerParams));
Poco::Net::ServerSocket socket;
auto address = socketBindListen(socket, listen_host, port);
socket.setReceiveTimeout(settings.receive_timeout);
socket.setSendTimeout(settings.send_timeout);
servers->emplace_back(
port_name,
std::make_unique<Poco::Net::TCPServer>(
new KeeperTCPHandlerFactory(*this, false), server_pool, socket, new Poco::Net::TCPServerParams));
LOG_INFO(log, "Listening for connections to Keeper (tcp): {}", address.toString());
});
LOG_INFO(log, "Listening for connections to Keeper (tcp): {}", address.toString());
});
const char * secure_port_name = "keeper_server.tcp_port_secure";
createServer(listen_host, secure_port_name, listen_try, [&](UInt16 port)
{
const char * secure_port_name = "keeper_server.tcp_port_secure";
createServer(listen_host, secure_port_name, listen_try, [&](UInt16 port)
{
#if USE_SSL
Poco::Net::SecureServerSocket socket;
auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
socket.setReceiveTimeout(settings.receive_timeout);
socket.setSendTimeout(settings.send_timeout);
servers->emplace_back(
secure_port_name,
std::make_unique<Poco::Net::TCPServer>(
new KeeperTCPHandlerFactory(*this, true), server_pool, socket, new Poco::Net::TCPServerParams));
LOG_INFO(log, "Listening for connections to Keeper with secure protocol (tcp_secure): {}", address.toString());
Poco::Net::SecureServerSocket socket;
auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
socket.setReceiveTimeout(settings.receive_timeout);
socket.setSendTimeout(settings.send_timeout);
servers->emplace_back(
secure_port_name,
std::make_unique<Poco::Net::TCPServer>(
new KeeperTCPHandlerFactory(*this, true), server_pool, socket, new Poco::Net::TCPServerParams));
LOG_INFO(log, "Listening for connections to Keeper with secure protocol (tcp_secure): {}", address.toString());
#else
UNUSED(port);
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
ErrorCodes::SUPPORT_IS_DISABLED};
UNUSED(port);
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
ErrorCodes::SUPPORT_IS_DISABLED};
#endif
});
}
});
}
#else
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "ClickHouse server built without NuRaft library. Cannot use internal coordination.");
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "ClickHouse keeper built without NuRaft library. Cannot use coordination.");
#endif
}
for (auto & server : *servers)
server.start();
@ -418,4 +409,14 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
return Application::EXIT_OK;
}
void Keeper::logRevision() const
{
Poco::Logger::root().information("Starting ClickHouse Keeper " + std::string{VERSION_STRING}
+ " with revision " + std::to_string(ClickHouseRevision::getVersionRevision())
+ ", " + build_id_info
+ ", PID " + std::to_string(getpid()));
}
}

4
programs/keeper/Keeper.h
View File

@ -42,6 +42,8 @@ public:
void defineOptions(Poco::Util::OptionSet & _options) override;
protected:
void logRevision() const override;
int run() override;
void initialize(Application & self) override;
@ -50,7 +52,7 @@ protected:
int main(const std::vector<std::string> & args) override;
std::string getDefaultCorePath() const override;
std::string getDefaultConfigFileName() const override;
private:
ContextPtr global_context;

2
programs/server/CMakeLists.txt
View File

@ -38,7 +38,7 @@ if (OS_LINUX)
# 1. Allow to run the binary without download of any other files.
# 2. Allow to implement "sudo clickhouse install" tool.
foreach(RESOURCE_FILE config.xml users.xml embedded.xml play.html)
foreach(RESOURCE_FILE config.xml users.xml embedded.xml play.html keeper_embedded.xml)
set(RESOURCE_OBJ ${RESOURCE_FILE}.o)
set(RESOURCE_OBJS ${RESOURCE_OBJS} ${RESOURCE_OBJ})

13
src/Common/Config/ConfigProcessor.cpp
View File

@ -462,10 +462,19 @@ XMLDocumentPtr ConfigProcessor::processConfig(
}
else
{
/// When we can use config embedded in binary.
/// These embedded files added during build with some cmake magic.
/// Look at the end of programs/sever/CMakeLists.txt.
std::string embedded_name;
if (path == "config.xml")
embedded_name = "embedded.xml";
if (path == "keeper_config.xml")
embedded_name = "keeper_embedded.xml";
/// When we can use config embedded in binary.
if (!embedded_name.empty())
{
auto resource = getResource("embedded.xml");
auto resource = getResource(embedded_name);
if (resource.empty())
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Configuration file {} doesn't exist and there is no embedded config", path);
LOG_DEBUG(log, "There is no file '{}', will use embedded config.", path);

2
src/Core/Defines.h
View File

@ -98,6 +98,8 @@
#define DBMS_DEFAULT_PATH "/var/lib/clickhouse/"
#define KEEPER_DEFAULT_PATH "/var/lib/clickhouse/coordination"
// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html
/// Marks that extra information is sent to a shard. It could be any magic numbers.