thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow to use mysql format without ssl - try2 with mysql interface split (#7524)

Browse Source 
* Allow use mysql format without ssl

* fix

* fix

*  fix

* Also disable poco's optional libraries

* clean

* fix

* fix

* Requested changes

* clean

* Requested fixes

* Update MySQLHandler.cpp
This commit is contained in:
proller 2019-11-02 13:20:46 +03:00 committed by alexey-milovidov
parent 6e4d63e394
commit 6a573b4092
12 changed files with 131 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmake/find/poco.cmake
View File

@ -8,6 +8,14 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/CMakeLists.txt")
set (MISSING_INTERNAL_POCO_LIBRARY 1)
endif ()
if (NOT ENABLE_LIBRARIES)
set (ENABLE_POCO_NETSSL ${ENABLE_LIBRARIES} CACHE BOOL "")
set (ENABLE_POCO_MONGODB ${ENABLE_LIBRARIES} CACHE BOOL "")
set (ENABLE_POCO_REDIS ${ENABLE_LIBRARIES} CACHE BOOL "")
set (ENABLE_POCO_ODBC ${ENABLE_LIBRARIES} CACHE BOOL "")
set (ENABLE_POCO_SQL ${ENABLE_LIBRARIES} CACHE BOOL "")
endif ()
set (POCO_COMPONENTS Net XML SQL Data)
if (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL)
list (APPEND POCO_COMPONENTS Crypto NetSSL)

12
dbms/programs/server/CMakeLists.txt
View File

@ -10,13 +10,11 @@ set(CLICKHOUSE_SERVER_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp
)
if (USE_SSL)
set(CLICKHOUSE_SERVER_SOURCES
${CLICKHOUSE_SERVER_SOURCES}
${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp
)
endif ()
set(CLICKHOUSE_SERVER_SOURCES
${CLICKHOUSE_SERVER_SOURCES}
${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp
)
set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_common_config clickhouse_common_zookeeper clickhouse_parsers string_utils PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
if (USE_POCO_NETSSL)

85
dbms/programs/server/MySQLHandler.cpp
View File

@ -1,7 +1,6 @@
#include <Common/config.h>
#if USE_SSL
#include "MySQLHandler.h"
#include "MySQLHandler.h"
#include <limits>
#include <ext/scope_guard.h>
#include <Columns/ColumnVector.h>
@ -15,37 +14,39 @@
#include <IO/ReadBufferFromPocoSocket.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromPocoSocket.h>
#include <Poco/Crypto/CipherFactory.h>
#include <Poco/Crypto/RSAKey.h>
#include <Poco/Net/SecureStreamSocket.h>
#include <Poco/Net/SSLManager.h>
#include <Storages/IStorage.h>
#if USE_POCO_NETSSL
#include <Poco/Net/SecureStreamSocket.h>
#include <Poco/Net/SSLManager.h>
#include <Poco/Crypto/CipherFactory.h>
#include <Poco/Crypto/RSAKey.h>
#endif
namespace DB
{
using namespace MySQLProtocol;
#if USE_POCO_NETSSL
using Poco::Net::SecureStreamSocket;
using Poco::Net::SSLManager;
#endif
namespace ErrorCodes
{
extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES;
extern const int OPENSSL_ERROR;
extern const int SUPPORT_IS_DISABLED;
}
MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, RSA & public_key_, RSA & private_key_, bool ssl_enabled, size_t connection_id_)
MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_,
bool ssl_enabled, size_t connection_id_)
: Poco::Net::TCPServerConnection(socket_)
, server(server_)
, log(&Poco::Logger::get("MySQLHandler"))
, connection_context(server.context())
, connection_id(connection_id_)
, public_key(public_key_)
, private_key(private_key_)
, auth_plugin(new MySQLProtocol::Authentication::Native41())
{
server_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_CONNECT_WITH_DB | CLIENT_DEPRECATE_EOF;
@ -197,21 +198,7 @@ void MySQLHandler::finishHandshake(MySQLProtocol::HandshakeResponse & packet)
if (payload_size == SSL_REQUEST_PAYLOAD_SIZE)
{
read_bytes(packet_size); /// Reading rest SSLRequest.
SSLRequest ssl_request;
ReadBufferFromMemory payload(buf, pos);
payload.ignore(PACKET_HEADER_SIZE);
ssl_request.readPayload(payload);
connection_context.mysql.client_capabilities = ssl_request.capability_flags;
connection_context.mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH;
secure_connection = true;
ss = std::make_shared<SecureStreamSocket>(SecureStreamSocket::attach(socket(), SSLManager::instance().defaultServerContext()));
in = std::make_shared<ReadBufferFromPocoSocket>(*ss);
out = std::make_shared<WriteBufferFromPocoSocket>(*ss);
connection_context.mysql.sequence_id = 2;
packet_sender = std::make_shared<PacketSender>(*in, *out, connection_context.mysql.sequence_id);
packet_sender->max_packet_size = connection_context.mysql.max_packet_size;
packet_sender->receivePacket(packet); /// Reading HandshakeResponse from secure socket.
finishHandshakeSSL(packet_size, buf, pos, read_bytes, packet);
}
else
{
@ -232,7 +219,9 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl
// For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used.
auto user = connection_context.getUser(user_name);
if (user->authentication.getType() != DB::Authentication::DOUBLE_SHA1_PASSWORD)
auth_plugin = std::make_unique<MySQLProtocol::Authentication::Sha256Password>(public_key, private_key, log);
{
authPluginSSL();
}
try {
std::optional<String> auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional<String>(initial_auth_response) : std::nullopt;
@ -302,5 +291,47 @@ void MySQLHandler::comQuery(ReadBuffer & payload)
packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true);
}
void MySQLHandler::authPluginSSL()
{
throw Exception("Compiled without SSL", ErrorCodes::SUPPORT_IS_DISABLED);
}
void MySQLHandler::finishHandshakeSSL([[maybe_unused]] size_t packet_size, [[maybe_unused]] char * buf, [[maybe_unused]] size_t pos, [[maybe_unused]] std::function<void(size_t)> read_bytes, [[maybe_unused]] MySQLProtocol::HandshakeResponse & packet)
{
throw Exception("Compiled without SSL", ErrorCodes::SUPPORT_IS_DISABLED);
}
#if USE_SSL && USE_POCO_NETSSL
MySQLHandlerSSL::MySQLHandlerSSL(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_, RSA & public_key_, RSA & private_key_)
: MySQLHandler(server_, socket_, ssl_enabled, connection_id_)
, public_key(public_key_)
, private_key(private_key_)
{}
void MySQLHandlerSSL::authPluginSSL()
{
auth_plugin = std::make_unique<MySQLProtocol::Authentication::Sha256Password>(public_key, private_key, log);
}
void MySQLHandlerSSL::finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function<void(size_t)> read_bytes, MySQLProtocol::HandshakeResponse & packet)
{
read_bytes(packet_size); /// Reading rest SSLRequest.
SSLRequest ssl_request;
ReadBufferFromMemory payload(buf, pos);
payload.ignore(PACKET_HEADER_SIZE);
ssl_request.readPayload(payload);
connection_context.mysql.client_capabilities = ssl_request.capability_flags;
connection_context.mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH;
secure_connection = true;
ss = std::make_shared<SecureStreamSocket>(SecureStreamSocket::attach(socket(), SSLManager::instance().defaultServerContext()));
in = std::make_shared<ReadBufferFromPocoSocket>(*ss);
out = std::make_shared<WriteBufferFromPocoSocket>(*ss);
connection_context.mysql.sequence_id = 2;
packet_sender = std::make_shared<PacketSender>(*in, *out, connection_context.mysql.sequence_id);
packet_sender->max_packet_size = connection_context.mysql.max_packet_size;
packet_sender->receivePacket(packet); /// Reading HandshakeResponse from secure socket.
}
#endif
}

37
dbms/programs/server/MySQLHandler.h
View File

@ -1,13 +1,13 @@
#pragma once
#include <Common/config.h>
#if USE_SSL
#include <Poco/Net/TCPServerConnection.h>
#include <Poco/Net/SecureStreamSocket.h>
#include <Common/getFQDNOrHostName.h>
#include <Core/MySQLProtocol.h>
#include "IServer.h"
#if USE_POCO_NETSSL
#include <Poco/Net/SecureStreamSocket.h>
#endif
namespace DB
{
@ -16,7 +16,7 @@ namespace DB
class MySQLHandler : public Poco::Net::TCPServerConnection
{
public:
MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, RSA & public_key_, RSA & private_key_, bool ssl_enabled, size_t connection_id_);
MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_);
void run() final;
@ -34,28 +34,47 @@ private:
void authenticate(const String & user_name, const String & auth_plugin_name, const String & auth_response);
virtual void authPluginSSL();
virtual void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function<void(size_t)> read_bytes, MySQLProtocol::HandshakeResponse & packet);
IServer & server;
protected:
Poco::Logger * log;
Context connection_context;
std::shared_ptr<MySQLProtocol::PacketSender> packet_sender;
private:
size_t connection_id = 0;
size_t server_capability_flags = 0;
size_t client_capability_flags = 0;
RSA & public_key;
RSA & private_key;
protected:
std::unique_ptr<MySQLProtocol::Authentication::IPlugin> auth_plugin;
std::shared_ptr<Poco::Net::SecureStreamSocket> ss;
std::shared_ptr<ReadBuffer> in;
std::shared_ptr<WriteBuffer> out;
bool secure_connection = false;
};
}
#if USE_SSL && USE_POCO_NETSSL
class MySQLHandlerSSL : public MySQLHandler
{
public:
MySQLHandlerSSL(IServer & server_, const Poco::Net::StreamSocket & socket_, bool ssl_enabled, size_t connection_id_, RSA & public_key_, RSA & private_key_);
private:
void authPluginSSL() override;
void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function<void(size_t)> read_bytes, MySQLProtocol::HandshakeResponse & packet) override;
RSA & public_key;
RSA & private_key;
std::shared_ptr<Poco::Net::SecureStreamSocket> ss;
};
#endif
}

21
dbms/programs/server/MySQLHandlerFactory.cpp
View File

@ -1,7 +1,5 @@
#include "MySQLHandlerFactory.h"
#if USE_POCO_NETSSL && USE_SSL
#include <Common/OpenSSLHelpers.h>
#include <Poco/Net/SSLManager.h>
#include <Poco/Net/TCPServerConnectionFactory.h>
#include <Poco/Util/Application.h>
#include <common/logger_useful.h>
@ -9,6 +7,10 @@
#include "IServer.h"
#include "MySQLHandler.h"
#if USE_POCO_NETSSL
#include <Poco/Net/SSLManager.h>
#endif
namespace DB
{
@ -24,6 +26,8 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
: server(server_)
, log(&Logger::get("MySQLHandlerFactory"))
{
#if USE_POCO_NETSSL
try
{
Poco::Net::SSLManager::instance().defaultServerContext();
@ -33,7 +37,9 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
LOG_INFO(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false));
ssl_enabled = false;
}
#endif
#if USE_SSL
/// Reading rsa keys for SHA256 authentication plugin.
try
{
@ -44,8 +50,10 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
LOG_WARNING(log, "Failed to read RSA keys. Error: " << getCurrentExceptionMessage(false));
generateRSAKeys();
}
#endif
}
#if USE_SSL
void MySQLHandlerFactory::readRSAKeys()
{
const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
@ -113,13 +121,18 @@ void MySQLHandlerFactory::generateRSAKeys()
if (!private_key)
throw Exception("Failed to copy RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
}
#endif
Poco::Net::TCPServerConnection * MySQLHandlerFactory::createConnection(const Poco::Net::StreamSocket & socket)
{
size_t connection_id = last_connection_id++;
LOG_TRACE(log, "MySQL connection. Id: " << connection_id << ". Address: " << socket.peerAddress().toString());
return new MySQLHandler(server, socket, *public_key, *private_key, ssl_enabled, connection_id);
#if USE_POCO_NETSSL && USE_SSL
return new MySQLHandlerSSL(server, socket, ssl_enabled, connection_id, *public_key, *private_key);
#else
return new MySQLHandler(server, socket, ssl_enabled, connection_id);
#endif
}
}
#endif

11
dbms/programs/server/MySQLHandlerFactory.h
View File

@ -1,12 +1,12 @@
#pragma once
#include <Common/config.h>
#if USE_POCO_NETSSL && USE_SSL
#include <Poco/Net/TCPServerConnectionFactory.h>
#include <atomic>
#include <openssl/rsa.h>
#include "IServer.h"
#if USE_SSL
#include <openssl/rsa.h>
#endif
namespace DB
{
@ -17,6 +17,7 @@ private:
IServer & server;
Poco::Logger * log;
#if USE_SSL
struct RSADeleter
{
void operator()(RSA * ptr) { RSA_free(ptr); }
@ -27,6 +28,9 @@ private:
RSAPtr private_key;
bool ssl_enabled = true;
#else
bool ssl_enabled = false;
#endif
std::atomic<size_t> last_connection_id = 0;
public:
@ -40,4 +44,3 @@ public:
};
}
#endif

3
dbms/programs/server/Server.cpp
View File

@ -57,7 +57,7 @@
#include "TCPHandlerFactory.h"
#include "Common/config_version.h"
#include <Common/SensitiveDataMasker.h>
#include "MySQLHandlerFactory.h"
#if defined(OS_LINUX)
#include <Common/hasLinuxCapability.h>
@ -65,7 +65,6 @@
#endif
#if USE_POCO_NETSSL
#include "MySQLHandlerFactory.h"
#include <Poco/Net/Context.h>
#include <Poco/Net/SecureServerSocket.h>
#endif

5
dbms/src/Core/MySQLProtocol.cpp
View File

@ -1,7 +1,4 @@
#include "MySQLProtocol.h"
#if USE_SSL
#include <IO/WriteBuffer.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromString.h>
@ -104,5 +101,3 @@ size_t getLengthEncodedStringSize(const String & s)
}
}
#endif // USE_SSL

17
dbms/src/Core/MySQLProtocol.h
View File

@ -1,12 +1,5 @@
#pragma once
#include "config_core.h"
#if USE_SSL
#include <ext/scope_guard.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <random>
#include <sstream>
#include <Common/MemoryTracker.h>
@ -27,6 +20,11 @@
#include <Poco/Net/StreamSocket.h>
#include <Poco/RandomStream.h>
#include <Poco/SHA1Engine.h>
#include "config_core.h"
#if USE_SSL
#include <openssl/pem.h>
#include <openssl/rsa.h>
#endif
/// Implementation of MySQL wire protocol.
/// Works only on little-endian architecture.
@ -941,6 +939,7 @@ private:
String scramble;
};
#if USE_SSL
/// Caching SHA2 plugin is not used because it would be possible to authenticate knowing hash from users.xml.
/// https://dev.mysql.com/doc/internals/en/sha256.html
class Sha256Password : public IPlugin
@ -1001,7 +1000,6 @@ public:
if (auth_response == "\1")
{
LOG_TRACE(log, "Client requests public key.");
BIO * mem = BIO_new(BIO_s_mem());
SCOPE_EXIT(BIO_free(mem));
if (PEM_write_bio_RSA_PUBKEY(mem, &public_key) != 1)
@ -1074,10 +1072,9 @@ private:
Logger * log;
String scramble;
};
#endif
}
}
}
#endif // USE_SSL

4
dbms/src/Formats/FormatFactory.cpp
View File

@ -264,9 +264,7 @@ void registerOutputFormatProcessorXML(FormatFactory & factory);
void registerOutputFormatProcessorODBCDriver(FormatFactory & factory);
void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory);
void registerOutputFormatProcessorNull(FormatFactory & factory);
#if USE_SSL
void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory);
#endif
/// Input only formats.
void registerInputFormatProcessorCapnProto(FormatFactory & factory);
@ -314,9 +312,7 @@ FormatFactory::FormatFactory()
registerOutputFormatProcessorODBCDriver(*this);
registerOutputFormatProcessorODBCDriver2(*this);
registerOutputFormatProcessorNull(*this);
#if USE_SSL
registerOutputFormatProcessorMySQLWrite(*this);
#endif
}
FormatFactory & FormatFactory::instance()

6
dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp
View File

@ -1,12 +1,8 @@
#include <Processors/Formats/Impl/MySQLOutputFormat.h>
#if USE_SSL
#include <Core/MySQLProtocol.h>
#include <Interpreters/ProcessList.h>
#include <Formats/FormatFactory.h>
#include <Interpreters/Context.h>
#include <iomanip>
#include <sstream>
@ -118,5 +114,3 @@ void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory)
}
}
#endif // USE_SSL

6
dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h
View File

@ -1,9 +1,5 @@
#pragma once
#include "config_core.h"
#if USE_SSL
#include <Processors/Formats/IRowOutputFormat.h>
#include <Core/Block.h>
@ -44,5 +40,3 @@ private:
};
}
#endif