Merge branch 'master' of github.com:yandex/ClickHouse

This commit is contained in:
Alexey Milovidov 2018-05-14 22:12:44 +03:00
commit 73c3dd81d1
21 changed files with 72 additions and 57 deletions

View File

@ -3,6 +3,9 @@ set -e -x
source default-config
ccache -M 32G
ccache -s
[[ -d "${WORKSPACE}/sources" ]] || die "Run get-sources.sh first"
mkdir -p "${WORKSPACE}/build"

View File

@ -6,7 +6,7 @@ source default-config
./install-os-packages.sh libssl-dev
./install-os-packages.sh libicu-dev
./install-os-packages.sh libreadline-dev
./install-os-packages.sh libmysqlclient-dev
./install-os-packages.sh libmariadbclient-dev
./install-os-packages.sh libunixodbc-dev
if [[ "$ENABLE_EMBEDDED_COMPILER" == 1 && "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 1 ]]; then

View File

@ -21,8 +21,7 @@ BUILD_TARGETS=clickhouse
BUILD_TYPE=Debug
ENABLE_EMBEDDED_COMPILER=0
CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_TCMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0"
# TODO it doesn't build with -D ENABLE_NETSSL=0 -D ENABLE_MONGODB=0 -D ENABLE_MYSQL=0 -D ENABLE_DATA_ODBC=0
CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_TCMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_MYSQL=0"
[[ $(uname) == "FreeBSD" ]] && COMPILER_PACKAGE_VERSION=devel

View File

@ -1,6 +1,9 @@
#!/usr/bin/env bash
set -e -x
mkdir -p /var/cache/ccache
DOCKER_ENV+=" --mount=type=bind,source=/var/cache/ccache,destination=/ccache -e CCACHE_DIR=/ccache "
PROJECT_ROOT="$(cd "$(dirname "$0")/.."; pwd -P)"
[[ -n "$CONFIG" ]] && DOCKER_ENV="--env=CONFIG"
docker run -t --network=host --mount=type=bind,source=${PROJECT_ROOT},destination=/ClickHouse --workdir=/ClickHouse/ci $DOCKER_ENV "$1" "$2"

View File

@ -8,8 +8,21 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/CMakeLists.txt")
set (MISSING_INTERNAL_POCO_LIBRARY 1)
endif ()
set (POCO_COMPONENTS Net XML SQL Data)
if (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL)
list (APPEND POCO_COMPONENTS Crypto NetSSL)
endif ()
if (NOT DEFINED ENABLE_POCO_MONGODB OR ENABLE_POCO_MONGODB)
list (APPEND POCO_COMPONENTS MongoDB)
endif ()
# TODO: after new poco release with SQL library rename ENABLE_POCO_ODBC -> ENABLE_POCO_SQLODBC
if (NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC)
list (APPEND POCO_COMPONENTS DataODBC)
#list (APPEND POCO_COMPONENTS SQLODBC) # future
endif ()
if (NOT USE_INTERNAL_POCO_LIBRARY)
find_package (Poco COMPONENTS Net NetSSL XML SQL Data Crypto DataODBC MongoDB)
find_package (Poco COMPONENTS ${POCO_COMPONENTS})
endif ()
if (Poco_INCLUDE_DIRS AND Poco_Foundation_LIBRARY)
@ -46,13 +59,12 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
"${ClickHouse_SOURCE_DIR}/contrib/poco/Util/include/"
)
if (NOT DEFINED POCO_ENABLE_MONGODB OR POCO_ENABLE_MONGODB)
set (Poco_MongoDB_FOUND 1)
if (NOT DEFINED ENABLE_POCO_MONGODB OR ENABLE_POCO_MONGODB)
set (USE_POCO_MONGODB 1)
set (Poco_MongoDB_LIBRARY PocoMongoDB)
set (Poco_MongoDB_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/MongoDB/include/")
endif ()
if (EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/")
set (Poco_SQL_FOUND 1)
set (Poco_SQL_LIBRARY PocoSQL)
@ -60,8 +72,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
"${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/include"
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/include"
)
if (ODBC_FOUND)
set (Poco_SQLODBC_FOUND 1)
if ((NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC) AND ODBC_FOUND)
set (USE_POCO_SQLODBC 1)
set (Poco_SQLODBC_INCLUDE_DIRS
"${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/"
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/"
@ -73,8 +85,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
set (Poco_Data_FOUND 1)
set (Poco_Data_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/include")
set (Poco_Data_LIBRARY PocoData)
if (ODBC_FOUND)
set (Poco_DataODBC_FOUND 1)
if ((NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC) AND ODBC_FOUND)
set (USE_POCO_DATAODBC 1)
set (Poco_DataODBC_INCLUDE_DIRS
"${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/"
${ODBC_INCLUDE_DIRECTORIES}
@ -84,8 +96,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
endif ()
# TODO! fix internal ssl
if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY)
set (Poco_NetSSL_FOUND 1)
if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
set (USE_POCO_NETSSL 1)
set (Poco_NetSSL_LIBRARY PocoNetSSL)
set (Poco_Crypto_LIBRARY PocoCrypto)
endif ()
@ -103,7 +115,7 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
set (Poco_XML_LIBRARY PocoXML)
endif ()
message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_MongoDB_LIBRARY}; MongoDB=${Poco_MongoDB_FOUND}, DataODBC=${Poco_DataODBC_FOUND}, NetSSL=${Poco_NetSSL_FOUND}")
message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_MongoDB_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, DataODBC=${Poco_DataODBC_FOUND}, NetSSL=${USE_POCO_NETSSL}")
# How to make sutable poco:
# use branch:

View File

@ -128,7 +128,7 @@ if (USE_INTERNAL_POCO_LIBRARY)
set (_save ${ENABLE_TESTS})
set (ENABLE_TESTS 0)
set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1)
if (USE_INTERNAL_SSL_LIBRARY)
if (USE_INTERNAL_SSL_LIBRARY OR (DEFINED ENABLE_POCO_NETSSL AND NOT ENABLE_POCO_NETSSL))
set (DISABLE_INTERNAL_OPENSSL 1 CACHE INTERNAL "")
set (ENABLE_NETSSL 0 CACHE INTERNAL "") # TODO!
set (ENABLE_CRYPTO 0 CACHE INTERNAL "") # TODO!
@ -141,7 +141,8 @@ if (USE_INTERNAL_POCO_LIBRARY)
set (ENABLE_TESTS ${_save})
set (CMAKE_CXX_FLAGS ${save_CMAKE_CXX_FLAGS})
set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS})
if (OPENSSL_FOUND AND TARGET Crypto)
if (OPENSSL_FOUND AND TARGET Crypto AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
# Bug in poco https://github.com/pocoproject/poco/pull/2100 found on macos
target_include_directories(Crypto PUBLIC ${OPENSSL_INCLUDE_DIR})
endif ()

View File

@ -170,7 +170,7 @@ if (NOT USE_INTERNAL_BOOST_LIBRARY)
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
endif ()
if (Poco_SQLODBC_FOUND)
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
@ -184,7 +184,7 @@ if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (dbms PRIVATE ${Poco_Data_INCLUDE_DIRS})
endif()
if (Poco_DataODBC_FOUND)
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
@ -192,12 +192,11 @@ if (Poco_DataODBC_FOUND)
endif()
endif()
if (Poco_MongoDB_FOUND)
if (USE_POCO_MONGODB)
target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
endif()
if (Poco_NetSSL_FOUND)
if (USE_POCO_NETSSL)
target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY})
endif()

View File

@ -21,7 +21,7 @@
#include <Interpreters/ClientInfo.h>
#include <Common/config.h>
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
#include <Poco/Net/SecureStreamSocket.h>
#endif
@ -57,7 +57,7 @@ void Connection::connect()
if (static_cast<bool>(secure))
{
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
socket = std::make_unique<Poco::Net::SecureStreamSocket>();
#else
throw Exception{"tcp_secure protocol is disabled because poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED};

View File

@ -10,7 +10,7 @@
#cmakedefine01 USE_CAPNP
#cmakedefine01 USE_EMBEDDED_COMPILER
#cmakedefine01 LLVM_HAS_RTTI
#cmakedefine01 Poco_SQLODBC_FOUND
#cmakedefine01 Poco_DataODBC_FOUND
#cmakedefine01 Poco_MongoDB_FOUND
#cmakedefine01 Poco_NetSSL_FOUND
#cmakedefine01 USE_POCO_SQLODBC
#cmakedefine01 USE_POCO_DATAODBC
#cmakedefine01 USE_POCO_MONGODB
#cmakedefine01 USE_POCO_NETSSL

View File

@ -35,10 +35,10 @@ const char * auto_config_build[]
"USE_VECTORCLASS", "@USE_VECTORCLASS@",
"USE_RDKAFKA", "@USE_RDKAFKA@",
"USE_CAPNP", "@USE_CAPNP@",
"USE_Poco_SQLODBC", "@Poco_SQLODBC_FOUND@",
"USE_Poco_DataODBC", "@Poco_DataODBC_FOUND@",
"USE_Poco_MongoDB", "@Poco_MongoDB_FOUND@",
"USE_Poco_NetSSL", "@Poco_NetSSL_FOUND@",
"USE_POCO_SQLODBC", "@USE_POCO_SQLODBC@",
"USE_POCO_DATAODBC", "@USE_POCO_DATAODBC@",
"USE_POCO_MONGODB", "@USE_POCO_MONGODB@",
"USE_POCO_NETSSL", "@USE_POCO_NETSSL@",
nullptr, nullptr
};

View File

@ -16,10 +16,10 @@
#include <mutex>
#include <Common/config.h>
#if Poco_MongoDB_FOUND
#if USE_POCO_MONGODB
#include <Dictionaries/MongoDBDictionarySource.h>
#endif
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#include <Poco/Data/ODBC/Connector.h>
@ -88,7 +88,7 @@ Block createSampleBlock(const DictionaryStructure & dict_struct)
DictionarySourceFactory::DictionarySourceFactory()
: log(&Poco::Logger::get("DictionarySourceFactory"))
{
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
Poco::Data::ODBC::Connector::registerConnector();
#endif
}
@ -139,7 +139,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
}
else if ("mongodb" == source_type)
{
#if Poco_MongoDB_FOUND
#if USE_POCO_MONGODB
return std::make_unique<MongoDBDictionarySource>(dict_struct, config, config_prefix + ".mongodb", sample_block);
#else
throw Exception{"Dictionary source of type `mongodb` is disabled because poco library was built without mongodb support.",
@ -148,7 +148,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
}
else if ("odbc" == source_type)
{
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
return std::make_unique<ODBCDictionarySource>(dict_struct, config, config_prefix + ".odbc", sample_block, context);
#else
throw Exception{"Dictionary source of type `odbc` is disabled because poco library was built without ODBC support.",
@ -168,7 +168,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
if (dict_struct.has_expressions)
throw Exception{"Dictionary source of type `http` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
// Used for https queries
std::call_once(ssl_init_once, SSLInit);
#endif

View File

@ -1,5 +1,5 @@
#include <Common/config.h>
#if Poco_MongoDB_FOUND
#if USE_POCO_MONGODB
#include <vector>
#include <string>

View File

@ -1,5 +1,5 @@
#include <Common/config.h>
#if Poco_MongoDB_FOUND
#if USE_POCO_MONGODB
#include <Poco/Util/AbstractConfiguration.h>
#pragma GCC diagnostic push

View File

@ -1,7 +1,7 @@
#include <IO/HTTPCommon.h>
#include <Common/config.h>
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
#include <Poco/Net/AcceptCertificateHandler.h>
#include <Poco/Net/Context.h>
#include <Poco/Net/InvalidCertificateHandler.h>
@ -30,7 +30,7 @@ std::once_flag ssl_init_once;
void SSLInit()
{
// http://stackoverflow.com/questions/18315472/https-request-in-c-using-poco
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
Poco::Net::initializeSSL();
#endif
}

View File

@ -9,7 +9,7 @@
#include <Poco/Version.h>
#include <common/logger_useful.h>
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
#include <Poco/Net/HTTPSClientSession.h>
#endif
@ -36,7 +36,7 @@ ReadWriteBufferFromHTTP::ReadWriteBufferFromHTTP(const Poco::URI & uri,
session
{
std::unique_ptr<Poco::Net::HTTPClientSession>(
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
is_ssl ? new Poco::Net::HTTPSClientSession :
#endif
new Poco::Net::HTTPClientSession)

View File

@ -26,6 +26,7 @@
#include <Interpreters/DDLWorker.h>
#include <Interpreters/ProcessList.h>
#include <Interpreters/loadMetadata.h>
#include <Interpreters/DNSCacheUpdater.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Storages/System/attachSystemTables.h>
#include <AggregateFunctions/registerAggregateFunctions.h>
@ -38,12 +39,9 @@
#include "StatusFile.h"
#include "TCPHandlerFactory.h"
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
#include <Poco/Net/Context.h>
#include <Poco/Net/SecureServerSocket.h>
#include <Interpreters/DNSCacheUpdater.h>
#endif
namespace CurrentMetrics
@ -431,7 +429,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
/// HTTPS
if (config().has("https_port"))
{
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
std::call_once(ssl_init_once, SSLInit);
Poco::Net::SecureServerSocket socket;
@ -471,7 +469,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
/// TCP with SSL
if (config().has("tcp_port_secure"))
{
#if Poco_NetSSL_FOUND
#if USE_POCO_NETSSL
Poco::Net::SecureServerSocket socket;
auto address = socket_bind_listen(socket, listen_host, config().getInt("tcp_port_secure"), /* secure = */ true);
socket.setReceiveTimeout(settings.receive_timeout);

View File

@ -23,7 +23,7 @@ void registerStorageJoin(StorageFactory & factory);
void registerStorageView(StorageFactory & factory);
void registerStorageMaterializedView(StorageFactory & factory);
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
void registerStorageODBC(StorageFactory & factory);
#endif
@ -56,7 +56,7 @@ void registerStorages()
registerStorageView(factory);
registerStorageMaterializedView(factory);
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
registerStorageODBC(factory);
#endif

View File

@ -7,12 +7,12 @@ list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFuncti
add_library(clickhouse_table_functions ${clickhouse_table_functions_sources})
target_link_libraries(clickhouse_table_functions clickhouse_storages_system dbms ${Poco_Foundation_LIBRARY})
if (Poco_SQLODBC_FOUND)
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse_table_functions ${Poco_SQLODBC_LIBRARY})
target_include_directories (clickhouse_table_functions PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS})
endif ()
if (Poco_DataODBC_FOUND)
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse_table_functions ${Poco_DataODBC_LIBRARY})
target_include_directories (clickhouse_table_functions PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
endif ()

View File

@ -1,6 +1,6 @@
#include <TableFunctions/TableFunctionODBC.h>
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#include <type_traits>
#include <ext/scope_guard.h>

View File

@ -1,7 +1,7 @@
#pragma once
#include <Common/config.h>
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#include <TableFunctions/ITableFunction.h>

View File

@ -13,7 +13,7 @@ void registerTableFunctionNumbers(TableFunctionFactory & factory);
void registerTableFunctionCatBoostPool(TableFunctionFactory & factory);
void registerTableFunctionFile(TableFunctionFactory & factory);
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
void registerTableFunctionODBC(TableFunctionFactory & factory);
#endif
@ -33,7 +33,7 @@ void registerTableFunctions()
registerTableFunctionCatBoostPool(factory);
registerTableFunctionFile(factory);
#if Poco_SQLODBC_FOUND || Poco_DataODBC_FOUND
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
registerTableFunctionODBC(factory);
#endif