From 4b30b3168cd8675f50c2741bb89c90bac6b08428 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 19 May 2020 20:48:28 +0300 Subject: [PATCH] try fix build --- .gitmodules | 11 +++- cmake/find_cassandra.cmake | 8 ++- contrib/CMakeLists.txt | 1 + contrib/cassandra | 2 +- contrib/libuv | 1 + src/CMakeLists.txt | 5 ++ .../CassandraBlockInputStream.cpp | 60 +++++++++---------- .../CassandraDictionarySource.cpp | 51 ++++++++-------- src/Dictionaries/ya.make | 4 +- src/Functions/ya.make | 2 +- .../external_sources.py | 2 +- 11 files changed, 83 insertions(+), 64 deletions(-) create mode 160000 contrib/libuv diff --git a/.gitmodules b/.gitmodules index f5d4b8340fa..b3a624aaae9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -95,9 +95,6 @@ [submodule "contrib/rapidjson"] path = contrib/rapidjson url = https://github.com/Tencent/rapidjson -[submodule "contrib/cassandra"] - path = contrib/cassandra - url = https://github.com/datastax/cpp-driver.git [submodule "contrib/fastops"] path = contrib/fastops url = https://github.com/ClickHouse-Extras/fastops @@ -160,3 +157,11 @@ [submodule "contrib/openldap"] path = contrib/openldap url = https://github.com/openldap/openldap.git +[submodule "contrib/cassandra"] + path = contrib/cassandra + url = https://github.com/tavplubix/cpp-driver.git + branch = ch-tmp +[submodule "contrib/libuv"] + path = contrib/libuv + url = https://github.com/libuv/libuv.git + branch = v1.x diff --git a/cmake/find_cassandra.cmake b/cmake/find_cassandra.cmake index 7f7346ce545..951cfc88b11 100644 --- a/cmake/find_cassandra.cmake +++ b/cmake/find_cassandra.cmake @@ -1,10 +1,14 @@ if (NOT DEFINED ENABLE_CASSANDRA OR ENABLE_CASSANDRA) - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv") + message (WARNING "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive") + elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") message (WARNING "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive") else() + set(LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") set (CASSANDRA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/") - set (CASSANDRA_LIBRARY cassandra) + set (LIBUV_LIBRARY uv_a) + set (CASSANDRA_LIBRARY cassandra_static) set (USE_CASSANDRA 1) set(CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 76da288991b..ce187038e2a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -319,5 +319,6 @@ if (USE_FASTOPS) endif() if (USE_CASSANDRA) + add_subdirectory(libuv) add_subdirectory(cassandra) endif() diff --git a/contrib/cassandra b/contrib/cassandra index fd9b73d4acf..5c0f2a62bdc 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit fd9b73d4acfd85293ab304be64e2e1e2109e521d +Subproject commit 5c0f2a62bdc63dcc390d771c9afaa9dc34eb8e5b diff --git a/contrib/libuv b/contrib/libuv new file mode 160000 index 00000000000..cc51217a317 --- /dev/null +++ b/contrib/libuv @@ -0,0 +1 @@ +Subproject commit cc51217a317e96510fbb284721d5e6bc2af31e33 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 222a3e486f9..d713cec8b8a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -356,6 +356,11 @@ if (USE_OPENCL) target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${OpenCL_INCLUDE_DIRS}) endif () +if (USE_CASSANDRA) + dbms_target_link_libraries(PRIVATE ${CASSANDRA_LIBRARY}) + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${CASS_INCLUDE_DIR}) +endif() + dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index f76c9dd93f6..73028e8d2b2 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -22,14 +22,14 @@ namespace ErrorCodes } CassandraBlockInputStream::CassandraBlockInputStream( - CassSession *session, - const std::string &query_str, + CassSession *session_, + const std::string &query_str_, const DB::Block &sample_block, - const size_t max_block_size) - : session{session} - , statement{cass_statement_new(query_str.c_str(), 0)} - , query_str{query_str} - , max_block_size{max_block_size} + const size_t max_block_size_) + : session(session_) + , statement(cass_statement_new(query_str_.c_str(), 0)) + , query_str(query_str_) + , max_block_size(max_block_size_) { cass_statement_set_paging_size(statement, max_block_size); this->has_more_pages = cass_true; @@ -51,77 +51,77 @@ namespace { switch (type) { - case ValueType::UInt8: + case ValueType::vtUInt8: { cass_uint32_t _value; cass_value_get_uint32(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::UInt16: + case ValueType::vtUInt16: { cass_uint32_t _value; cass_value_get_uint32(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::UInt32: + case ValueType::vtUInt32: { cass_uint32_t _value; cass_value_get_uint32(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::UInt64: + case ValueType::vtUInt64: { cass_int64_t _value; cass_value_get_int64(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Int8: + case ValueType::vtInt8: { cass_int8_t _value; cass_value_get_int8(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Int16: + case ValueType::vtInt16: { cass_int16_t _value; cass_value_get_int16(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Int32: + case ValueType::vtInt32: { cass_int32_t _value; cass_value_get_int32(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Int64: + case ValueType::vtInt64: { cass_int64_t _value; cass_value_get_int64(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Float32: + case ValueType::vtFloat32: { cass_float_t _value; cass_value_get_float(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Float64: + case ValueType::vtFloat64: { cass_double_t _value; cass_value_get_double(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::String: + case ValueType::vtString: { const char* _value; size_t _value_length; @@ -129,21 +129,21 @@ namespace static_cast(column).insertData(_value, _value_length); break; } - case ValueType::Date: + case ValueType::vtDate: { cass_int64_t _value; cass_value_get_int64(value, &_value); static_cast(column).insertValue(UInt32{cass_date_from_epoch(_value)}); // FIXME break; } - case ValueType::DateTime: + case ValueType::vtDateTime: { cass_int64_t _value; cass_value_get_int64(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::UUID: + case ValueType::vtUUID: { CassUuid _value; cass_value_get_uuid(value, &_value); @@ -166,7 +166,7 @@ namespace MutableColumns columns(description.sample_block.columns()); CassFuture* query_future = cass_session_execute(session, statement); - const CassResult* result = cass_future_get_result(query_future); + const CassResult* result_tmp = cass_future_get_result(query_future); if (result == nullptr) { const char* error_message; @@ -176,12 +176,12 @@ namespace throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; } - const CassRow* row = cass_result_first_row(result); + const CassRow* row = cass_result_first_row(result_tmp); const CassValue* map = cass_row_get_column(row, 0); - CassIterator* iterator = cass_iterator_from_map(map); - while (cass_iterator_next(iterator)) { - const CassValue* _key = cass_iterator_get_map_key(iterator); - const CassValue* _value = cass_iterator_get_map_value(iterator); + CassIterator* iterator_tmp = cass_iterator_from_map(map); + while (cass_iterator_next(iterator_tmp)) { + const CassValue* _key = cass_iterator_get_map_key(iterator_tmp); + const CassValue* _value = cass_iterator_get_map_value(iterator_tmp); auto pair_values = {std::make_pair(_key, 0ul), std::make_pair(_value, 1ul)}; for (const auto &[value, idx]: pair_values) { if (description.types[idx].second) { @@ -194,13 +194,13 @@ namespace } } - has_more_pages = cass_result_has_more_pages(result); + has_more_pages = cass_result_has_more_pages(result_tmp); if (has_more_pages) { - cass_statement_set_paging_state(statement, result); + cass_statement_set_paging_state(statement, result_tmp); } - cass_result_free(result); + cass_result_free(result_tmp); return description.sample_block.cloneWithColumns(std::move(columns)); } diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index cfd21510e69..c51ae2877d8 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -15,7 +15,8 @@ namespace DB const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - const Context & /* context */) -> DictionarySourcePtr { + const Context & /* context */, + bool /*check_config*/) -> DictionarySourcePtr { #if USE_CASSANDRA return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); #else @@ -49,42 +50,42 @@ namespace ErrorCodes static const size_t max_block_size = 8192; CassandraDictionarySource::CassandraDictionarySource( - const DB::DictionaryStructure &dict_struct, - const std::string &host, - UInt16 port, - const std::string &user, - const std::string &password, - const std::string &method, - const std::string &db, - const DB::Block &sample_block) - : dict_struct{dict_struct} - , host{host} - , port{port} - , user{user} - , password{password} - , method{method} - , db{db} - , sample_block{sample_block} - , cluster{cass_cluster_new()} - , session{cass_session_new()} + const DB::DictionaryStructure & dict_struct_, + const std::string & host_, + UInt16 port_, + const std::string & user_, + const std::string & password_, + const std::string & method_, + const std::string & db_, + const DB::Block & sample_block_) + : dict_struct(dict_struct_) + , host(host_) + , port(port_) + , user(user_) + , password(password_) + , method(method_) + , db(db_) + , sample_block(sample_block_) + , cluster(cass_cluster_new()) + , session(cass_session_new()) { cass_cluster_set_contact_points(cluster, toConnectionString(host, port).c_str()); } CassandraDictionarySource::CassandraDictionarySource( - const DB::DictionaryStructure &dict_struct, - const Poco::Util::AbstractConfiguration &config, - const std::string &config_prefix, - DB::Block &sample_block) + const DB::DictionaryStructure & dict_struct_, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + DB::Block & sample_block_) : CassandraDictionarySource( - dict_struct, + dict_struct_, config.getString(config_prefix + ".host"), config.getUInt(config_prefix + ".port"), config.getString(config_prefix + ".user", ""), config.getString(config_prefix + ".password", ""), config.getString(config_prefix + ".method", ""), config.getString(config_prefix + ".db", ""), - sample_block) + sample_block_) { } diff --git a/src/Dictionaries/ya.make b/src/Dictionaries/ya.make index e47b55d5254..22703a3924d 100644 --- a/src/Dictionaries/ya.make +++ b/src/Dictionaries/ya.make @@ -16,6 +16,8 @@ SRCS( CacheDictionary_generate1.cpp CacheDictionary_generate2.cpp CacheDictionary_generate3.cpp + CassandraBlockInputStream.cpp + CassandraDictionarySource.cpp ClickHouseDictionarySource.cpp ComplexKeyCacheDictionary.cpp ComplexKeyCacheDictionary_createAttributeWithType.cpp @@ -24,8 +26,8 @@ SRCS( ComplexKeyCacheDictionary_generate3.cpp ComplexKeyCacheDictionary_setAttributeValue.cpp ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp - ComplexKeyHashedDictionary.cpp ComplexKeyDirectDictionary.cpp + ComplexKeyHashedDictionary.cpp DictionaryBlockInputStreamBase.cpp DictionaryFactory.cpp DictionarySourceFactory.cpp diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 8e53ffe493d..da9435148b6 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -281,10 +281,10 @@ SRCS( rand64.cpp randConstant.cpp rand.cpp + randomFixedString.cpp randomPrintableASCII.cpp randomString.cpp randomStringUTF8.cpp - randomFixedString.cpp regexpQuoteMeta.cpp registerFunctionsArithmetic.cpp registerFunctionsComparison.cpp diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py index 117eb7b7e6f..c90725f2f6a 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py @@ -2,7 +2,7 @@ import warnings import pymysql.cursors import pymongo -import cassandra +import cassandra.cluster import redis import aerospike from tzlocal import get_localzone