From 8e12eedbb33750b47f5a7023c223854b5ab067f0 Mon Sep 17 00:00:00 2001 From: Oleg Favstov Date: Sun, 27 Jan 2019 19:51:05 +0300 Subject: [PATCH 01/29] Initial commit --- .gitmodules | 3 +++ CMakeLists.txt | 1 + cmake/find_cassandra.cmake | 12 ++++++++++++ contrib/CMakeLists.txt | 9 ++++++++- contrib/cassandra | 1 + dbms/src/Common/config.h.in | 1 + dbms/src/Dictionaries/CMakeLists.txt | 6 +++++- .../Dictionaries/CassandraDBDictionarySource.cpp | 1 + .../Dictionaries/CassandraDBDictionarySource.h | 16 ++++++++++++++++ 9 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 cmake/find_cassandra.cmake create mode 160000 contrib/cassandra create mode 100644 dbms/src/Dictionaries/CassandraDBDictionarySource.cpp create mode 100644 dbms/src/Dictionaries/CassandraDBDictionarySource.h diff --git a/.gitmodules b/.gitmodules index 24211b6707e..86100baa3dc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -64,3 +64,6 @@ [submodule "contrib/cppkafka"] path = contrib/cppkafka url = https://github.com/mfontanini/cppkafka.git +[submodule "contrib/cassandra"] + path = contrib/cassandra + url = https://github.com/datastax/cpp-driver.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 98c3643f055..b8f8d803585 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -236,6 +236,7 @@ include (cmake/find_protobuf.cmake) include (cmake/find_hdfs3.cmake) include (cmake/find_consistent-hashing.cmake) include (cmake/find_base64.cmake) +include (cmake/find_cassandra.cmake) if (ENABLE_TESTS) include (cmake/find_gtest.cmake) endif () diff --git a/cmake/find_cassandra.cmake b/cmake/find_cassandra.cmake new file mode 100644 index 00000000000..3e4bb3ca373 --- /dev/null +++ b/cmake/find_cassandra.cmake @@ -0,0 +1,12 @@ +if (NOT DEFINED ENABLE_CASSANDRA OR ENABLE_CASSANDRA) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") + message (WARNING "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive") + else() + set (CASSANDRA_INCLUDE_DIR + "${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/") + set (CASSANDRA_LIBRARY cassandra) + set (USE_CASSANDRA 1) + + message(STATUS "Using cassandra: ${CASSANDRA_LIBRARY}") + endif() +endif() \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 0c4b6c15287..4a4add1f095 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -219,5 +219,12 @@ if (USE_INTERNAL_HDFS3_LIBRARY) endif () if (USE_BASE64) - add_subdirectory (base64-cmake) + add_subdirectory(base64-cmake) endif() + +if (USE_CASSANDRA) + # TODO osfavstov: cassandra/CMakeLists.txt change + # 5: set(CASS_ROOT_DIR "${CMAKE_SOURCE_DIR}/contrib/cassandra"); + # 10: include(${ClickHouse_SOURCE_DIR}/contrib/cassandra/cmake/modules/CppDriver.cmake) + add_subdirectory(cassandra) +endif() \ No newline at end of file diff --git a/contrib/cassandra b/contrib/cassandra new file mode 160000 index 00000000000..fd9b73d4acf --- /dev/null +++ b/contrib/cassandra @@ -0,0 +1 @@ +Subproject commit fd9b73d4acfd85293ab304be64e2e1e2109e521d diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index 0c756841f2e..42ff3e43ec2 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -18,6 +18,7 @@ #cmakedefine01 USE_XXHASH #cmakedefine01 USE_INTERNAL_LLVM_LIBRARY #cmakedefine01 USE_PROTOBUF +#cmakedefine01 USE_CASSANDRA #cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 LLVM_HAS_RTTI diff --git a/dbms/src/Dictionaries/CMakeLists.txt b/dbms/src/Dictionaries/CMakeLists.txt index d7f85a5c7eb..534f49bc5bb 100644 --- a/dbms/src/Dictionaries/CMakeLists.txt +++ b/dbms/src/Dictionaries/CMakeLists.txt @@ -11,7 +11,7 @@ generate_code(CacheDictionary_generate2 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 generate_code(CacheDictionary_generate3 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128) add_headers_and_sources(clickhouse_dictionaries ${CMAKE_CURRENT_BINARY_DIR}/generated/) -add_library(clickhouse_dictionaries ${LINK_MODE} ${clickhouse_dictionaries_sources}) +add_library(clickhouse_dictionaries ${LINK_MODE} ${clickhouse_dictionaries_sources} CassandraDBDictionarySource.cpp CassandraDBDictionarySource.h) target_link_libraries(clickhouse_dictionaries PRIVATE clickhouse_common_io pocoext ${MYSQLXX_LIBRARY} ${BTRIE_LIBRARIES}) if(Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY) @@ -36,4 +36,8 @@ if(USE_POCO_MONGODB) target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_MongoDB_LIBRARY}) endif() +if(USE_CASSANDRA) + target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${CASSANDRA_INCLUDE_DIR}) +endif() + add_subdirectory(Embedded) diff --git a/dbms/src/Dictionaries/CassandraDBDictionarySource.cpp b/dbms/src/Dictionaries/CassandraDBDictionarySource.cpp new file mode 100644 index 00000000000..084ef283107 --- /dev/null +++ b/dbms/src/Dictionaries/CassandraDBDictionarySource.cpp @@ -0,0 +1 @@ +#include "CassandraDBDictionarySource.h" \ No newline at end of file diff --git a/dbms/src/Dictionaries/CassandraDBDictionarySource.h b/dbms/src/Dictionaries/CassandraDBDictionarySource.h new file mode 100644 index 00000000000..a9a43b026b0 --- /dev/null +++ b/dbms/src/Dictionaries/CassandraDBDictionarySource.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +#if USE_CASSANDRA + +# include "DictionaryStructure.h" +# include "IDictionarySource.h" +# include + +namespace DB +{ + +} + +#endif From ac46a3a976b511b16501b3c3df75f56667efba61 Mon Sep 17 00:00:00 2001 From: Oleg Favstov Date: Sat, 16 Feb 2019 13:11:49 +0300 Subject: [PATCH 02/29] Add basic realisation --- dbms/src/Dictionaries/CMakeLists.txt | 2 +- .../CassandraBlockInputStream.cpp | 158 ++++++++++++++++++ .../Dictionaries/CassandraBlockInputStream.h | 38 +++++ .../CassandraDBDictionarySource.cpp | 1 - .../CassandraDBDictionarySource.h | 16 -- .../CassandraDictionarySource.cpp | 114 +++++++++++++ .../Dictionaries/CassandraDictionarySource.h | 58 +++++++ 7 files changed, 369 insertions(+), 18 deletions(-) create mode 100644 dbms/src/Dictionaries/CassandraBlockInputStream.cpp create mode 100644 dbms/src/Dictionaries/CassandraBlockInputStream.h delete mode 100644 dbms/src/Dictionaries/CassandraDBDictionarySource.cpp delete mode 100644 dbms/src/Dictionaries/CassandraDBDictionarySource.h create mode 100644 dbms/src/Dictionaries/CassandraDictionarySource.cpp create mode 100644 dbms/src/Dictionaries/CassandraDictionarySource.h diff --git a/dbms/src/Dictionaries/CMakeLists.txt b/dbms/src/Dictionaries/CMakeLists.txt index 534f49bc5bb..21b82a49154 100644 --- a/dbms/src/Dictionaries/CMakeLists.txt +++ b/dbms/src/Dictionaries/CMakeLists.txt @@ -11,7 +11,7 @@ generate_code(CacheDictionary_generate2 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 generate_code(CacheDictionary_generate3 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128) add_headers_and_sources(clickhouse_dictionaries ${CMAKE_CURRENT_BINARY_DIR}/generated/) -add_library(clickhouse_dictionaries ${LINK_MODE} ${clickhouse_dictionaries_sources} CassandraDBDictionarySource.cpp CassandraDBDictionarySource.h) +add_library(clickhouse_dictionaries ${LINK_MODE} ${clickhouse_dictionaries_sources}) target_link_libraries(clickhouse_dictionaries PRIVATE clickhouse_common_io pocoext ${MYSQLXX_LIBRARY} ${BTRIE_LIBRARIES}) if(Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY) diff --git a/dbms/src/Dictionaries/CassandraBlockInputStream.cpp b/dbms/src/Dictionaries/CassandraBlockInputStream.cpp new file mode 100644 index 00000000000..8493cf8a6e5 --- /dev/null +++ b/dbms/src/Dictionaries/CassandraBlockInputStream.cpp @@ -0,0 +1,158 @@ +#include +#include + +#if USE_CASSANDRA + +# include "CassandraBlockInputStream.h" +#include "CassandraBlockInputStream.h" + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; + extern const int CASSANDRA_INTERNAL_ERROR; +} + +CassandraBlockInputStream::CassandraBlockInputStream( + CassSession *session, + const std::string &query_str, + const DB::Block &sample_block, + const size_t max_block_size) + : session{session} + , query_str{query_str} + , max_block_size{max_block_size} +{ + CassStatement * statement = cass_statement_new(query_str.c_str(), 0); + CassFuture* future = cass_session_execute(session, statement); + + const CassResult * result = cass_future_get_result(future); + cass_statement_free(statement); + + if (result == nullptr) { +// CassError error_code = cass_future_error_code(future); + const char* error_message; + size_t error_message_length; + cass_future_error_message(future, &error_message, &error_message_length); + + throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; + } + + cass_future_free(future); + + this->result = result; + + description.init(sample_block); +} + +CassandraBlockInputStream::~CassandraBlockInputStream() { + if (iterator != nullptr) + cass_iterator_free(iterator); + cass_result_free(result); +} + +namespace +{ + using ValueType = ExternalResultDescription::ValueType; + + void insertValue(IColumn & column, const ValueType type, const CassValue * value) + { + switch (type) + { + case ValueType::UInt8: + { + cass_uint32_t _value; + cass_value_get_uint32(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::UInt16: + { + cass_uint32_t _value; + cass_value_get_uint32(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::UInt32: + { + cass_uint32_t _value; + cass_value_get_uint32(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::UInt64: + { + cass_int64_t _value; + cass_value_get_int64(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::Int8: + { + cass_int8_t _value; + cass_value_get_int8(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::Int16: + { + cass_int16_t _value; + cass_value_get_int16(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::Int32: + { + cass_int32_t _value; + cass_value_get_int32(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::Int64: + { + cass_int64_t _value; + cass_value_get_int64(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::Float32: + { + cass_float_t _value; + cass_value_get_float(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::Float64: + { + cass_double_t _value; + cass_value_get_double(value, &_value); + static_cast(column).insertValue(_value); + break; + } + case ValueType::String: + { + const char* _value; + size_t _value_length; + cass_value_get_string + static_cast(column).insertData(value.data(), value.size()); + break; + } + case ValueType::Date: + static_cast(column).insertValue(UInt16(value.getDate().getDayNum())); + break; + case ValueType::DateTime: + static_cast(column).insertValue(UInt32(value.getDateTime())); + break; + case ValueType::UUID: + static_cast(column).insert(parse(value.data(), value.size())); + break; + } + } + + void insertDefaultValue(IColumn & column, const IColumn & sample_column) { column.insertFrom(sample_column, 0); } +} + + +} +#endif diff --git a/dbms/src/Dictionaries/CassandraBlockInputStream.h b/dbms/src/Dictionaries/CassandraBlockInputStream.h new file mode 100644 index 00000000000..fdf9954d18f --- /dev/null +++ b/dbms/src/Dictionaries/CassandraBlockInputStream.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include +#include "ExternalResultDescription.h" + + +namespace DB +{ +/// Allows processing results of a Cassandra query as a sequence of Blocks, simplifies chaining + class CassandraBlockInputStream final : public IBlockInputStream + { + public: + CassandraBlockInputStream( + CassSession * session, + const std::string & query_str, + const Block & sample_block, + const size_t max_block_size); + ~CassandraBlockInputStream() override; + + String getName() const override { return "Cassandra"; } + + Block getHeader() const override { return description.sample_block.cloneEmpty(); } + + private: + Block readImpl() override; + + CassSession * session, + const std::string & query_str; + const size_t max_block_size; + ExternalResultDescription description; + const CassResult * result; + CassIterator * iterator = nullptr; + }; + +} diff --git a/dbms/src/Dictionaries/CassandraDBDictionarySource.cpp b/dbms/src/Dictionaries/CassandraDBDictionarySource.cpp deleted file mode 100644 index 084ef283107..00000000000 --- a/dbms/src/Dictionaries/CassandraDBDictionarySource.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "CassandraDBDictionarySource.h" \ No newline at end of file diff --git a/dbms/src/Dictionaries/CassandraDBDictionarySource.h b/dbms/src/Dictionaries/CassandraDBDictionarySource.h deleted file mode 100644 index a9a43b026b0..00000000000 --- a/dbms/src/Dictionaries/CassandraDBDictionarySource.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include - -#if USE_CASSANDRA - -# include "DictionaryStructure.h" -# include "IDictionarySource.h" -# include - -namespace DB -{ - -} - -#endif diff --git a/dbms/src/Dictionaries/CassandraDictionarySource.cpp b/dbms/src/Dictionaries/CassandraDictionarySource.cpp new file mode 100644 index 00000000000..81cc3e9d85c --- /dev/null +++ b/dbms/src/Dictionaries/CassandraDictionarySource.cpp @@ -0,0 +1,114 @@ +#include "CassandraDictionarySource.h" +#include "DictionarySourceFactory.h" +#include "DictionaryStructure.h" + +namespace DB +{ + namespace ErrorCodes + { + extern const int SUPPORT_IS_DISABLED; + } + + void registerDictionarySourceCassandra(DictionarySourceFactory & factory) + { + auto createTableSource = [=](const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + Block & sample_block, + const Context & /* context */) -> DictionarySourcePtr { +#if USE_CASSANDRA + return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); +#else + (void)dict_struct; + (void)config; + (void)config_prefix; + (void)sample_block; + throw Exception{"Dictionary source of type `cassandra` is disabled because library was built without cassandra support.", + ErrorCodes::SUPPORT_IS_DISABLED}; +#endif + }; + factory.registerSource("cassandra", createTableSource); + } + +} + +#if USE_CASSANDRA + +# include + +namespace DB +{ +namespace ErrorCodes { + extern const int UNSUPPORTED_METHOD; + extern const int WRONG_PASSWORD; +} + +static const size_t max_block_size = 8192; + +CassandraDictionarySource::CassandraDictionarySource( + const DB::DictionaryStructure &dict_struct, + const std::string &host, + UInt16 port, + const std::string &user, + const std::string &password, + const std::string &method, + const std::string &db, + const DB::Block &sample_block) + : dict_struct{dict_struct} + , host{host} + , port{port} + , user{user} + , password{password} + , method{method} + , db{db} + , sample_block{sample_block} + , cluster{cass_cluster_new()} + , session{cass_session_new()} +{ + cass_cluster_set_contact_points(cluster, toConnectionString(host, port).c_str()); +} + +CassandraDictionarySource::CassandraDictionarySource( + const DB::DictionaryStructure &dict_struct, + const Poco::Util::AbstractConfiguration &config, + const std::string &config_prefix, + DB::Block &sample_block) + : CassandraDictionarySource( + dict_struct, + config.getString(config_prefix + ".host"), + config.getUInt(config_prefix + ".port"), + config.getString(config_prefix + ".user", ""), + config.getString(config_prefix + ".password", ""), + config.getString(config_prefix + ".method", ""), + config.getString(config_prefix + ".db", ""), + sample_block) +{ +} + +CassandraDictionarySource::CassandraDictionarySource(const CassandraDictionarySource & other) + : CassandraDictionarySource{other.dict_struct, + other.host, + other.port, + other.user, + other.password, + other.method, + other.db, + other.sample_block} +{ +} + +CassandraDictionarySource::~CassandraDictionarySource() { + cass_session_free(session); + cass_cluster_free(cluster); +} + +std::string CassandraDictionarySource::toConnectionString(const std::string &host, const UInt16 port) { + return host + (port != 0 ? ":" + std::to_string(port) : ""); +} + +BlockInputStreamPtr CassandraDict + + +} + +#endif diff --git a/dbms/src/Dictionaries/CassandraDictionarySource.h b/dbms/src/Dictionaries/CassandraDictionarySource.h new file mode 100644 index 00000000000..2d5e81a2733 --- /dev/null +++ b/dbms/src/Dictionaries/CassandraDictionarySource.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +#if USE_CASSANDRA + +# include "DictionaryStructure.h" +# include "IDictionarySource.h" +# include + +namespace DB +{ +class CassandraDictionarySource final : public IDictionarySource { + CassandraDictionarySource( + const DictionaryStructure & dict_struct, + const std::string & host, + UInt16 port, + const std::string & user, + const std::string & password, + const std::string & method, + const std::string & db, + const Block & sample_block); + +public: + CassandraDictionarySource( + const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + Block & sample_block); + + CassandraDictionarySource(const CassandraDictionarySource & other); + + ~CassandraDictionarySource() override; + + BlockInputStreamPtr loadAll() override; + + BlockInputStreamPtr loadIds(const std::vector & ids) override; + + BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector & requested_rows) override; + +private: + static std::string toConnectionString(const std::string& host, const UInt16 port); + + const DictionaryStructure dict_struct; + const std::string host; + const UInt16 port; + const std::string user; + const std::string password; + const std::string method; + const std::string db; + Block sample_block; + + CassCluster * cluster; + CassSession * session; +}; +} + +#endif From 0e047e9abcd873f18eb0cf42471dc41e19c6ef0d Mon Sep 17 00:00:00 2001 From: Oleg Favstov Date: Thu, 11 Apr 2019 11:05:01 +0200 Subject: [PATCH 03/29] Base code for testing added --- ci/jobs/quick-build/run.sh | 2 +- .../CassandraBlockInputStream.cpp | 87 ++++++++++++++----- .../Dictionaries/CassandraBlockInputStream.h | 1 + .../src/Dictionaries/registerDictionaries.cpp | 2 + dbms/tests/integration/helpers/cluster.py | 20 ++++- .../helpers/docker_compose_cassandra.yml | 7 ++ dbms/tests/integration/image/Dockerfile | 2 +- .../external_sources.py | 20 +++++ .../test_external_dictionaries/test.py | 5 +- .../dicts/external_dicts_dict_sources.md | 16 +++- 10 files changed, 132 insertions(+), 30 deletions(-) create mode 100644 dbms/tests/integration/helpers/docker_compose_cassandra.yml diff --git a/ci/jobs/quick-build/run.sh b/ci/jobs/quick-build/run.sh index 9e8fe9353d6..c7d8470d358 100755 --- a/ci/jobs/quick-build/run.sh +++ b/ci/jobs/quick-build/run.sh @@ -21,7 +21,7 @@ BUILD_TARGETS=clickhouse BUILD_TYPE=Debug ENABLE_EMBEDDED_COMPILER=0 -CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_JEMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0 -D ENABLE_SSL=0 -D ENABLE_POCO_NETSSL=0" +CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_JEMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0 -D ENABLE_SSL=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_CASSANDRA=0" [[ $(uname) == "FreeBSD" ]] && COMPILER_PACKAGE_VERSION=devel && export COMPILER_PATH=/usr/local/bin diff --git a/dbms/src/Dictionaries/CassandraBlockInputStream.cpp b/dbms/src/Dictionaries/CassandraBlockInputStream.cpp index 8493cf8a6e5..ac90ac8b528 100644 --- a/dbms/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/dbms/src/Dictionaries/CassandraBlockInputStream.cpp @@ -1,10 +1,13 @@ #include #include +#include +#include +#include #if USE_CASSANDRA # include "CassandraBlockInputStream.h" -#include "CassandraBlockInputStream.h" +# include "CassandraBlockInputStream.h" namespace DB @@ -25,23 +28,8 @@ CassandraBlockInputStream::CassandraBlockInputStream( , max_block_size{max_block_size} { CassStatement * statement = cass_statement_new(query_str.c_str(), 0); - CassFuture* future = cass_session_execute(session, statement); - - const CassResult * result = cass_future_get_result(future); - cass_statement_free(statement); - - if (result == nullptr) { -// CassError error_code = cass_future_error_code(future); - const char* error_message; - size_t error_message_length; - cass_future_error_message(future, &error_message, &error_message_length); - - throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; - } - - cass_future_free(future); - - this->result = result; + cass_statement_set_paging_size(statement, max_block_size) + this->has_more_pages = cass_true; description.init(sample_block); } @@ -134,23 +122,78 @@ namespace { const char* _value; size_t _value_length; - cass_value_get_string - static_cast(column).insertData(value.data(), value.size()); + cass_value_get_string(value, &_value, &_value_length); + static_cast(column).insertData(_value, _value_length); break; } case ValueType::Date: - static_cast(column).insertValue(UInt16(value.getDate().getDayNum())); + { + cass_int64_t _value; + cass_value_get_int64(value, &_value); + static_cast(column).insertValue(UInt32{cass_date_from_epoch(_value)}); break; + } case ValueType::DateTime: - static_cast(column).insertValue(UInt32(value.getDateTime())); + { + cass_int64_t _value; + cass_value_get_int64(value, &_value); + static_cast(column).insertValue(_value); break; + } case ValueType::UUID: + { + CassUuid _value; + cass_value_get_uuid(value, &_value); static_cast(column).insert(parse(value.data(), value.size())); break; + } } } void insertDefaultValue(IColumn & column, const IColumn & sample_column) { column.insertFrom(sample_column, 0); } + + Block CassandraBlockInputStream::readImpl() + { + if (has_more_pages) + return {}; + + CassFuture* query_future = cass_session_execute(session, statement); + + const CassResult* result = cass_future_get_result(query_future); + + if (result == nullptr) { + const char* error_message; + size_t error_message_length; + cass_future_error_message(future, &error_message, &error_message_length); + + throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; + } + + const CassRow* row = cass_result_first_row(result); + const CassValue* map = cass_row_get_column(row, 0); + CassIterator* iterator = cass_iterator_from_map(map); + while (cass_iterator_next(iterator)) { + const CassValue* _key = cass_iterator_get_map_key(iterator); + const CassValue* _value = cass_iterator_get_map_value(iterator); + for (const auto &[value, idx]: {{_key, 0}, {_value, 1}}) { + if (description.types[idx].second) { + ColumnNullable & column_nullable = static_cast(*columns[idx]); + insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value); + column_nullable.getNullMapData().emplace_back(0); + } else { + insertValue(*columns[idx], description.types[idx].first, value); + } + } + } + + has_more_pages = cass_result_has_more_pages(result); + + if (has_more_pages) { + cass_statement_set_paging_state(statement, result); + } + + cass_result_free(result); + } } diff --git a/dbms/src/Dictionaries/CassandraBlockInputStream.h b/dbms/src/Dictionaries/CassandraBlockInputStream.h index fdf9954d18f..005c6f69b75 100644 --- a/dbms/src/Dictionaries/CassandraBlockInputStream.h +++ b/dbms/src/Dictionaries/CassandraBlockInputStream.h @@ -32,6 +32,7 @@ namespace DB const size_t max_block_size; ExternalResultDescription description; const CassResult * result; + cass_bool_t has_more_pages; CassIterator * iterator = nullptr; }; diff --git a/dbms/src/Dictionaries/registerDictionaries.cpp b/dbms/src/Dictionaries/registerDictionaries.cpp index 1a8c5a7be7b..93df888d519 100644 --- a/dbms/src/Dictionaries/registerDictionaries.cpp +++ b/dbms/src/Dictionaries/registerDictionaries.cpp @@ -7,6 +7,7 @@ void registerDictionarySourceFile(DictionarySourceFactory & source_factory); void registerDictionarySourceMysql(DictionarySourceFactory & source_factory); void registerDictionarySourceClickHouse(DictionarySourceFactory & source_factory); void registerDictionarySourceMongoDB(DictionarySourceFactory & source_factory); +void registerDictionarySourceCassandra(DictionarySourceFactory & source_factory); void registerDictionarySourceXDBC(DictionarySourceFactory & source_factory); void registerDictionarySourceJDBC(DictionarySourceFactory & source_factory); void registerDictionarySourceExecutable(DictionarySourceFactory & source_factory); @@ -30,6 +31,7 @@ void registerDictionaries() registerDictionarySourceMysql(source_factory); registerDictionarySourceClickHouse(source_factory); registerDictionarySourceMongoDB(source_factory); + registerDictionarySourceCassandra(source_factory); registerDictionarySourceXDBC(source_factory); registerDictionarySourceJDBC(source_factory); registerDictionarySourceExecutable(source_factory); diff --git a/dbms/tests/integration/helpers/cluster.py b/dbms/tests/integration/helpers/cluster.py index 7ee2c1f9f54..38f09a72524 100644 --- a/dbms/tests/integration/helpers/cluster.py +++ b/dbms/tests/integration/helpers/cluster.py @@ -92,6 +92,7 @@ class ClickHouseCluster: self.base_zookeeper_cmd = None self.base_mysql_cmd = [] self.base_kafka_cmd = [] + self.base_cassandra_cmd = [] self.pre_zookeeper_commands = [] self.instances = {} self.with_zookeeper = False @@ -101,6 +102,7 @@ class ClickHouseCluster: self.with_odbc_drivers = False self.with_hdfs = False self.with_mongo = False + self.with_cassandra = False self.docker_client = None self.is_up = False @@ -112,7 +114,7 @@ class ClickHouseCluster: cmd += " client" return cmd - def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False, ipv4_address=None, ipv6_address=None): + def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_cassandra=False, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False, ipv4_address=None, ipv6_address=None): """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -130,7 +132,7 @@ class ClickHouseCluster: instance = ClickHouseInstance( self, self.base_dir, name, config_dir, main_configs, user_configs, macros, with_zookeeper, - self.zookeeper_config_path, with_mysql, with_kafka, with_mongo, self.base_configs_dir, self.server_bin_path, + self.zookeeper_config_path, with_mysql, with_kafka, with_mongo, with_cassandra, self.base_configs_dir, self.server_bin_path, self.odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=hostname, env_variables=env_variables, image=image, stay_alive=stay_alive, ipv4_address=ipv4_address, ipv6_address=ipv6_address) @@ -185,6 +187,13 @@ class ClickHouseCluster: self.base_mongo_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_mongo.yml')] + if with_cassandra and not self.with_cassandra: + self.with_cassandra = True + self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_cassandra.yml')]) + self.base_cassandra_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', + self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_cassandra.yml')] + + return instance @@ -316,6 +325,10 @@ class ClickHouseCluster: subprocess_check_call(self.base_mongo_cmd + ['up', '-d', '--force-recreate']) self.wait_mongo_to_start(30) + if self.with_cassandra and self.base_cassandra_cmd: + subprocess_check_call(self.base_cassandra_cmd + ['up', '-d', '--force-recreate']) + time.sleep(10) + subprocess_check_call(self.base_cmd + ['up', '-d', '--no-recreate']) start_deadline = time.time() + 20.0 # seconds @@ -414,7 +427,7 @@ class ClickHouseInstance: def __init__( self, cluster, base_path, name, custom_config_dir, custom_main_configs, custom_user_configs, macros, - with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_mongo, base_configs_dir, server_bin_path, odbc_bridge_bin_path, + with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_mongo, with_cassandra, base_configs_dir, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False, ipv4_address=None, ipv6_address=None): @@ -439,6 +452,7 @@ class ClickHouseInstance: self.with_mysql = with_mysql self.with_kafka = with_kafka self.with_mongo = with_mongo + self.with_cassandra = with_cassandra self.path = p.join(self.cluster.instances_dir, name) self.docker_compose_path = p.join(self.path, 'docker_compose.yml') diff --git a/dbms/tests/integration/helpers/docker_compose_cassandra.yml b/dbms/tests/integration/helpers/docker_compose_cassandra.yml new file mode 100644 index 00000000000..bb6a0221c54 --- /dev/null +++ b/dbms/tests/integration/helpers/docker_compose_cassandra.yml @@ -0,0 +1,7 @@ +version: '2.2' +services: + cassandra1: + image: cassandra + restart: always + ports: + - 6340:6349 diff --git a/dbms/tests/integration/image/Dockerfile b/dbms/tests/integration/image/Dockerfile index 1dd5c1713b2..0b3cdd5a65f 100644 --- a/dbms/tests/integration/image/Dockerfile +++ b/dbms/tests/integration/image/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes - ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip install pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2 pymongo tzlocal +RUN pip install pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2 pymongo tzlocal cassandra-driver ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce diff --git a/dbms/tests/integration/test_external_dictionaries/external_sources.py b/dbms/tests/integration/test_external_dictionaries/external_sources.py index 71dc05ca78c..e0adb8c187b 100644 --- a/dbms/tests/integration/test_external_dictionaries/external_sources.py +++ b/dbms/tests/integration/test_external_dictionaries/external_sources.py @@ -2,6 +2,7 @@ import warnings import pymysql.cursors import pymongo +import cassandra from tzlocal import get_localzone import datetime import os @@ -372,3 +373,22 @@ class SourceHTTP(SourceHTTPBase): class SourceHTTPS(SourceHTTPBase): def _get_schema(self): return "https" + +class SourceCassandra(ExternalSource): + def get_source_str(self, table_name): + return ''' + + {host} + {port} + + '''.format( + host=self.docker_hostname, + port=self.docker_port, + ) + + def prepare(self, structure, table_name, cluster): + self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port) + self.prepared = True + + def load_data(self, data, table_name): + for ro diff --git a/dbms/tests/integration/test_external_dictionaries/test.py b/dbms/tests/integration/test_external_dictionaries/test.py index 314ec26a106..0bb76ce171e 100644 --- a/dbms/tests/integration/test_external_dictionaries/test.py +++ b/dbms/tests/integration/test_external_dictionaries/test.py @@ -5,7 +5,7 @@ import time from helpers.cluster import ClickHouseCluster from dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, SourceExecutableHashed, SourceMongo -from external_sources import SourceHTTP, SourceHTTPS +from external_sources import SourceHTTP, SourceHTTPS, SourceCassandra SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -79,6 +79,7 @@ LAYOUTS = [ ] SOURCES = [ + SourceCassandra("Cassandra", "localhost", "6340", "cassandra1", "6349", "", ""), SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse"), SourceClickHouse("RemoteClickHouse", "localhost", "9000", "clickhouse1", "9000", "default", ""), @@ -120,7 +121,7 @@ def setup_module(module): for fname in os.listdir(dict_configs_path): main_configs.append(os.path.join(dict_configs_path, fname)) cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) - node = cluster.add_instance('node', main_configs=main_configs, with_mysql=True, with_mongo=True) + node = cluster.add_instance('node', main_configs=main_configs, with_mysql=True, with_mongo=True, with_cassandra=True) cluster.add_instance('clickhouse1') @pytest.fixture(scope="module") diff --git a/docs/en/query_language/dicts/external_dicts_dict_sources.md b/docs/en/query_language/dicts/external_dicts_dict_sources.md index 8fb2145ecaf..cda41f8294b 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_sources.md +++ b/docs/en/query_language/dicts/external_dicts_dict_sources.md @@ -30,6 +30,7 @@ Types of sources (`source_type`): - [MySQL](#dicts-external_dicts_dict_sources-mysql) - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) - [MongoDB](#dicts-external_dicts_dict_sources-mongodb) + - [Cassanda](#dicts-external_dicts_dict_sources-cassandra) - [ODBC](#dicts-external_dicts_dict_sources-odbc) @@ -143,7 +144,7 @@ PASSWORD = test If you then make a query such as ``` -SELECT * FROM odbc('DSN=gregtest;Servername=some-server.com', 'test_db'); +SELECT * FROM odbc('DSN=gregtest;Servername=some-server.com', 'test_db'); ``` ODBC driver will send values of `USERNAME` and `PASSWORD` from `odbc.ini` to `some-server.com`. @@ -421,4 +422,17 @@ Setting fields: - `db` – Name of the database. - `collection` – Name of the collection. +### Cassandra {#dicts-external_dicts_dict_sources-cassandra} + +Example of settings: + +```xml + + + localhost + 6349 + + +``` + [Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict_sources/) From 3175caa1c0a4c517080b7a1c2d1ccac697318bf1 Mon Sep 17 00:00:00 2001 From: Gleb-Tretyakov Date: Thu, 23 May 2019 00:09:29 +0300 Subject: [PATCH 04/29] Fix compilation errors --- .../CassandraBlockInputStream.cpp | 37 ++++++++++++------- .../Dictionaries/CassandraBlockInputStream.h | 8 ++-- .../CassandraDictionarySource.cpp | 10 ++++- .../Dictionaries/CassandraDictionarySource.h | 17 +++++++++ 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/dbms/src/Dictionaries/CassandraBlockInputStream.cpp b/dbms/src/Dictionaries/CassandraBlockInputStream.cpp index ac90ac8b528..f76c9dd93f6 100644 --- a/dbms/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/dbms/src/Dictionaries/CassandraBlockInputStream.cpp @@ -1,11 +1,14 @@ -#include -#include -#include -#include -#include +# include +# include +# include +# include +# include +# include +# include #if USE_CASSANDRA +# include # include "CassandraBlockInputStream.h" # include "CassandraBlockInputStream.h" @@ -24,11 +27,11 @@ CassandraBlockInputStream::CassandraBlockInputStream( const DB::Block &sample_block, const size_t max_block_size) : session{session} + , statement{cass_statement_new(query_str.c_str(), 0)} , query_str{query_str} , max_block_size{max_block_size} { - CassStatement * statement = cass_statement_new(query_str.c_str(), 0); - cass_statement_set_paging_size(statement, max_block_size) + cass_statement_set_paging_size(statement, max_block_size); this->has_more_pages = cass_true; description.init(sample_block); @@ -130,33 +133,37 @@ namespace { cass_int64_t _value; cass_value_get_int64(value, &_value); - static_cast(column).insertValue(UInt32{cass_date_from_epoch(_value)}); + static_cast(column).insertValue(UInt32{cass_date_from_epoch(_value)}); // FIXME break; } case ValueType::DateTime: { cass_int64_t _value; cass_value_get_int64(value, &_value); - static_cast(column).insertValue(_value); + static_cast(column).insertValue(_value); break; } case ValueType::UUID: { CassUuid _value; cass_value_get_uuid(value, &_value); - static_cast(column).insert(parse(value.data(), value.size())); + std::array uuid_str; + cass_uuid_string(_value, uuid_str.data()); + static_cast(column).insert(parse(uuid_str.data(), uuid_str.size())); break; } } } +} - void insertDefaultValue(IColumn & column, const IColumn & sample_column) { column.insertFrom(sample_column, 0); } + // void insertDefaultValue(IColumn & column, const IColumn & sample_column) { column.insertFrom(sample_column, 0); } Block CassandraBlockInputStream::readImpl() { if (has_more_pages) return {}; + MutableColumns columns(description.sample_block.columns()); CassFuture* query_future = cass_session_execute(session, statement); const CassResult* result = cass_future_get_result(query_future); @@ -164,7 +171,7 @@ namespace if (result == nullptr) { const char* error_message; size_t error_message_length; - cass_future_error_message(future, &error_message, &error_message_length); + cass_future_error_message(query_future, &error_message, &error_message_length); throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; } @@ -175,7 +182,8 @@ namespace while (cass_iterator_next(iterator)) { const CassValue* _key = cass_iterator_get_map_key(iterator); const CassValue* _value = cass_iterator_get_map_value(iterator); - for (const auto &[value, idx]: {{_key, 0}, {_value, 1}}) { + auto pair_values = {std::make_pair(_key, 0ul), std::make_pair(_value, 1ul)}; + for (const auto &[value, idx]: pair_values) { if (description.types[idx].second) { ColumnNullable & column_nullable = static_cast(*columns[idx]); insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value); @@ -193,8 +201,9 @@ namespace } cass_result_free(result); + + return description.sample_block.cloneWithColumns(std::move(columns)); } -} } diff --git a/dbms/src/Dictionaries/CassandraBlockInputStream.h b/dbms/src/Dictionaries/CassandraBlockInputStream.h index 005c6f69b75..2b7c3b68744 100644 --- a/dbms/src/Dictionaries/CassandraBlockInputStream.h +++ b/dbms/src/Dictionaries/CassandraBlockInputStream.h @@ -1,10 +1,9 @@ #pragma once -#include #include #include #include -#include "ExternalResultDescription.h" +#include namespace DB @@ -27,8 +26,9 @@ namespace DB private: Block readImpl() override; - CassSession * session, - const std::string & query_str; + CassSession * session; + CassStatement * statement; + String query_str; const size_t max_block_size; ExternalResultDescription description; const CassResult * result; diff --git a/dbms/src/Dictionaries/CassandraDictionarySource.cpp b/dbms/src/Dictionaries/CassandraDictionarySource.cpp index 81cc3e9d85c..ad89d5e2e43 100644 --- a/dbms/src/Dictionaries/CassandraDictionarySource.cpp +++ b/dbms/src/Dictionaries/CassandraDictionarySource.cpp @@ -35,6 +35,8 @@ namespace DB #if USE_CASSANDRA # include +# include +# include "CassandraBlockInputStream.h" namespace DB { @@ -106,7 +108,13 @@ std::string CassandraDictionarySource::toConnectionString(const std::string &hos return host + (port != 0 ? ":" + std::to_string(port) : ""); } -BlockInputStreamPtr CassandraDict +BlockInputStreamPtr CassandraDictionarySource::loadAll() { + return std::make_shared(nullptr, "", sample_block, max_block_size); +} + +std::string CassandraDictionarySource::toString() const { + return "Cassandra: " + /*db + '.' + collection + ',' + (user.empty() ? " " : " " + user + '@') + */ host + ':' + DB::toString(port); +} } diff --git a/dbms/src/Dictionaries/CassandraDictionarySource.h b/dbms/src/Dictionaries/CassandraDictionarySource.h index 2d5e81a2733..60c503fc4da 100644 --- a/dbms/src/Dictionaries/CassandraDictionarySource.h +++ b/dbms/src/Dictionaries/CassandraDictionarySource.h @@ -1,6 +1,7 @@ #pragma once #include +#include #if USE_CASSANDRA @@ -34,9 +35,25 @@ public: BlockInputStreamPtr loadAll() override; + bool supportsSelectiveLoad() const override { return true; } + + bool isModified() const override { return true; } + + ///Not yet supported + bool hasUpdateField() const override { return false; } + + DictionarySourcePtr clone() const override { return std::make_unique(*this); } + BlockInputStreamPtr loadIds(const std::vector & ids) override; BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector & requested_rows) override; + + BlockInputStreamPtr loadUpdatedAll() override + { + throw Exception{"Method loadUpdatedAll is unsupported for CassandraDictionarySource", ErrorCodes::NOT_IMPLEMENTED}; + } + + std::string toString() const override; private: static std::string toConnectionString(const std::string& host, const UInt16 port); From a82f4925ec3189eb7eef1ebca48dfa65b35042c4 Mon Sep 17 00:00:00 2001 From: favstovol Date: Thu, 30 May 2019 02:01:25 +0300 Subject: [PATCH 05/29] Add stubs --- contrib/cassandra-cmake/CMakeLists.txt | 0 dbms/src/Common/ErrorCodes.cpp | 1 + dbms/src/Dictionaries/CassandraDictionarySource.cpp | 5 +++-- dbms/src/Dictionaries/CassandraDictionarySource.h | 12 +++++++++--- 4 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 contrib/cassandra-cmake/CMakeLists.txt diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 093f98d22a1..04a7e72e0c8 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -427,6 +427,7 @@ namespace ErrorCodes extern const int BAD_TTL_EXPRESSION = 450; extern const int BAD_TTL_FILE = 451; extern const int SETTING_CONSTRAINT_VIOLATION = 452; + extern const int CASSANDRA_INTERNAL_ERROR = 453; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Dictionaries/CassandraDictionarySource.cpp b/dbms/src/Dictionaries/CassandraDictionarySource.cpp index ad89d5e2e43..cfd21510e69 100644 --- a/dbms/src/Dictionaries/CassandraDictionarySource.cpp +++ b/dbms/src/Dictionaries/CassandraDictionarySource.cpp @@ -17,7 +17,7 @@ namespace DB Block & sample_block, const Context & /* context */) -> DictionarySourcePtr { #if USE_CASSANDRA - return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); + return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); #else (void)dict_struct; (void)config; @@ -40,7 +40,8 @@ namespace DB namespace DB { -namespace ErrorCodes { +namespace ErrorCodes +{ extern const int UNSUPPORTED_METHOD; extern const int WRONG_PASSWORD; } diff --git a/dbms/src/Dictionaries/CassandraDictionarySource.h b/dbms/src/Dictionaries/CassandraDictionarySource.h index 60c503fc4da..2bdd476951a 100644 --- a/dbms/src/Dictionaries/CassandraDictionarySource.h +++ b/dbms/src/Dictionaries/CassandraDictionarySource.h @@ -44,9 +44,15 @@ public: DictionarySourcePtr clone() const override { return std::make_unique(*this); } - BlockInputStreamPtr loadIds(const std::vector & ids) override; + BlockInputStreamPtr loadIds(const std::vector & /* ids */) override + { + throw Exception{"Method loadIds is not implemented yet", ErrorCodes::NOT_IMPLEMENTED}; + } - BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector & requested_rows) override; + BlockInputStreamPtr loadKeys(const Columns & /* key_columns */, const std::vector & /* requested_rows */) override + { + throw Exception{"Method loadKeys is not implemented yet", ErrorCodes::NOT_IMPLEMENTED}; + } BlockInputStreamPtr loadUpdatedAll() override { @@ -56,7 +62,7 @@ public: std::string toString() const override; private: - static std::string toConnectionString(const std::string& host, const UInt16 port); + static std::string toConnectionString(const std::string & host, const UInt16 port); const DictionaryStructure dict_struct; const std::string host; From 4b30b3168cd8675f50c2741bb89c90bac6b08428 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 19 May 2020 20:48:28 +0300 Subject: [PATCH 06/29] try fix build --- .gitmodules | 11 +++- cmake/find_cassandra.cmake | 8 ++- contrib/CMakeLists.txt | 1 + contrib/cassandra | 2 +- contrib/libuv | 1 + src/CMakeLists.txt | 5 ++ .../CassandraBlockInputStream.cpp | 60 +++++++++---------- .../CassandraDictionarySource.cpp | 51 ++++++++-------- src/Dictionaries/ya.make | 4 +- src/Functions/ya.make | 2 +- .../external_sources.py | 2 +- 11 files changed, 83 insertions(+), 64 deletions(-) create mode 160000 contrib/libuv diff --git a/.gitmodules b/.gitmodules index f5d4b8340fa..b3a624aaae9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -95,9 +95,6 @@ [submodule "contrib/rapidjson"] path = contrib/rapidjson url = https://github.com/Tencent/rapidjson -[submodule "contrib/cassandra"] - path = contrib/cassandra - url = https://github.com/datastax/cpp-driver.git [submodule "contrib/fastops"] path = contrib/fastops url = https://github.com/ClickHouse-Extras/fastops @@ -160,3 +157,11 @@ [submodule "contrib/openldap"] path = contrib/openldap url = https://github.com/openldap/openldap.git +[submodule "contrib/cassandra"] + path = contrib/cassandra + url = https://github.com/tavplubix/cpp-driver.git + branch = ch-tmp +[submodule "contrib/libuv"] + path = contrib/libuv + url = https://github.com/libuv/libuv.git + branch = v1.x diff --git a/cmake/find_cassandra.cmake b/cmake/find_cassandra.cmake index 7f7346ce545..951cfc88b11 100644 --- a/cmake/find_cassandra.cmake +++ b/cmake/find_cassandra.cmake @@ -1,10 +1,14 @@ if (NOT DEFINED ENABLE_CASSANDRA OR ENABLE_CASSANDRA) - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv") + message (WARNING "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive") + elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") message (WARNING "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive") else() + set(LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") set (CASSANDRA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/") - set (CASSANDRA_LIBRARY cassandra) + set (LIBUV_LIBRARY uv_a) + set (CASSANDRA_LIBRARY cassandra_static) set (USE_CASSANDRA 1) set(CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 76da288991b..ce187038e2a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -319,5 +319,6 @@ if (USE_FASTOPS) endif() if (USE_CASSANDRA) + add_subdirectory(libuv) add_subdirectory(cassandra) endif() diff --git a/contrib/cassandra b/contrib/cassandra index fd9b73d4acf..5c0f2a62bdc 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit fd9b73d4acfd85293ab304be64e2e1e2109e521d +Subproject commit 5c0f2a62bdc63dcc390d771c9afaa9dc34eb8e5b diff --git a/contrib/libuv b/contrib/libuv new file mode 160000 index 00000000000..cc51217a317 --- /dev/null +++ b/contrib/libuv @@ -0,0 +1 @@ +Subproject commit cc51217a317e96510fbb284721d5e6bc2af31e33 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 222a3e486f9..d713cec8b8a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -356,6 +356,11 @@ if (USE_OPENCL) target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${OpenCL_INCLUDE_DIRS}) endif () +if (USE_CASSANDRA) + dbms_target_link_libraries(PRIVATE ${CASSANDRA_LIBRARY}) + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${CASS_INCLUDE_DIR}) +endif() + dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index f76c9dd93f6..73028e8d2b2 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -22,14 +22,14 @@ namespace ErrorCodes } CassandraBlockInputStream::CassandraBlockInputStream( - CassSession *session, - const std::string &query_str, + CassSession *session_, + const std::string &query_str_, const DB::Block &sample_block, - const size_t max_block_size) - : session{session} - , statement{cass_statement_new(query_str.c_str(), 0)} - , query_str{query_str} - , max_block_size{max_block_size} + const size_t max_block_size_) + : session(session_) + , statement(cass_statement_new(query_str_.c_str(), 0)) + , query_str(query_str_) + , max_block_size(max_block_size_) { cass_statement_set_paging_size(statement, max_block_size); this->has_more_pages = cass_true; @@ -51,77 +51,77 @@ namespace { switch (type) { - case ValueType::UInt8: + case ValueType::vtUInt8: { cass_uint32_t _value; cass_value_get_uint32(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::UInt16: + case ValueType::vtUInt16: { cass_uint32_t _value; cass_value_get_uint32(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::UInt32: + case ValueType::vtUInt32: { cass_uint32_t _value; cass_value_get_uint32(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::UInt64: + case ValueType::vtUInt64: { cass_int64_t _value; cass_value_get_int64(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Int8: + case ValueType::vtInt8: { cass_int8_t _value; cass_value_get_int8(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Int16: + case ValueType::vtInt16: { cass_int16_t _value; cass_value_get_int16(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Int32: + case ValueType::vtInt32: { cass_int32_t _value; cass_value_get_int32(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Int64: + case ValueType::vtInt64: { cass_int64_t _value; cass_value_get_int64(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Float32: + case ValueType::vtFloat32: { cass_float_t _value; cass_value_get_float(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::Float64: + case ValueType::vtFloat64: { cass_double_t _value; cass_value_get_double(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::String: + case ValueType::vtString: { const char* _value; size_t _value_length; @@ -129,21 +129,21 @@ namespace static_cast(column).insertData(_value, _value_length); break; } - case ValueType::Date: + case ValueType::vtDate: { cass_int64_t _value; cass_value_get_int64(value, &_value); static_cast(column).insertValue(UInt32{cass_date_from_epoch(_value)}); // FIXME break; } - case ValueType::DateTime: + case ValueType::vtDateTime: { cass_int64_t _value; cass_value_get_int64(value, &_value); static_cast(column).insertValue(_value); break; } - case ValueType::UUID: + case ValueType::vtUUID: { CassUuid _value; cass_value_get_uuid(value, &_value); @@ -166,7 +166,7 @@ namespace MutableColumns columns(description.sample_block.columns()); CassFuture* query_future = cass_session_execute(session, statement); - const CassResult* result = cass_future_get_result(query_future); + const CassResult* result_tmp = cass_future_get_result(query_future); if (result == nullptr) { const char* error_message; @@ -176,12 +176,12 @@ namespace throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; } - const CassRow* row = cass_result_first_row(result); + const CassRow* row = cass_result_first_row(result_tmp); const CassValue* map = cass_row_get_column(row, 0); - CassIterator* iterator = cass_iterator_from_map(map); - while (cass_iterator_next(iterator)) { - const CassValue* _key = cass_iterator_get_map_key(iterator); - const CassValue* _value = cass_iterator_get_map_value(iterator); + CassIterator* iterator_tmp = cass_iterator_from_map(map); + while (cass_iterator_next(iterator_tmp)) { + const CassValue* _key = cass_iterator_get_map_key(iterator_tmp); + const CassValue* _value = cass_iterator_get_map_value(iterator_tmp); auto pair_values = {std::make_pair(_key, 0ul), std::make_pair(_value, 1ul)}; for (const auto &[value, idx]: pair_values) { if (description.types[idx].second) { @@ -194,13 +194,13 @@ namespace } } - has_more_pages = cass_result_has_more_pages(result); + has_more_pages = cass_result_has_more_pages(result_tmp); if (has_more_pages) { - cass_statement_set_paging_state(statement, result); + cass_statement_set_paging_state(statement, result_tmp); } - cass_result_free(result); + cass_result_free(result_tmp); return description.sample_block.cloneWithColumns(std::move(columns)); } diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index cfd21510e69..c51ae2877d8 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -15,7 +15,8 @@ namespace DB const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - const Context & /* context */) -> DictionarySourcePtr { + const Context & /* context */, + bool /*check_config*/) -> DictionarySourcePtr { #if USE_CASSANDRA return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); #else @@ -49,42 +50,42 @@ namespace ErrorCodes static const size_t max_block_size = 8192; CassandraDictionarySource::CassandraDictionarySource( - const DB::DictionaryStructure &dict_struct, - const std::string &host, - UInt16 port, - const std::string &user, - const std::string &password, - const std::string &method, - const std::string &db, - const DB::Block &sample_block) - : dict_struct{dict_struct} - , host{host} - , port{port} - , user{user} - , password{password} - , method{method} - , db{db} - , sample_block{sample_block} - , cluster{cass_cluster_new()} - , session{cass_session_new()} + const DB::DictionaryStructure & dict_struct_, + const std::string & host_, + UInt16 port_, + const std::string & user_, + const std::string & password_, + const std::string & method_, + const std::string & db_, + const DB::Block & sample_block_) + : dict_struct(dict_struct_) + , host(host_) + , port(port_) + , user(user_) + , password(password_) + , method(method_) + , db(db_) + , sample_block(sample_block_) + , cluster(cass_cluster_new()) + , session(cass_session_new()) { cass_cluster_set_contact_points(cluster, toConnectionString(host, port).c_str()); } CassandraDictionarySource::CassandraDictionarySource( - const DB::DictionaryStructure &dict_struct, - const Poco::Util::AbstractConfiguration &config, - const std::string &config_prefix, - DB::Block &sample_block) + const DB::DictionaryStructure & dict_struct_, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + DB::Block & sample_block_) : CassandraDictionarySource( - dict_struct, + dict_struct_, config.getString(config_prefix + ".host"), config.getUInt(config_prefix + ".port"), config.getString(config_prefix + ".user", ""), config.getString(config_prefix + ".password", ""), config.getString(config_prefix + ".method", ""), config.getString(config_prefix + ".db", ""), - sample_block) + sample_block_) { } diff --git a/src/Dictionaries/ya.make b/src/Dictionaries/ya.make index e47b55d5254..22703a3924d 100644 --- a/src/Dictionaries/ya.make +++ b/src/Dictionaries/ya.make @@ -16,6 +16,8 @@ SRCS( CacheDictionary_generate1.cpp CacheDictionary_generate2.cpp CacheDictionary_generate3.cpp + CassandraBlockInputStream.cpp + CassandraDictionarySource.cpp ClickHouseDictionarySource.cpp ComplexKeyCacheDictionary.cpp ComplexKeyCacheDictionary_createAttributeWithType.cpp @@ -24,8 +26,8 @@ SRCS( ComplexKeyCacheDictionary_generate3.cpp ComplexKeyCacheDictionary_setAttributeValue.cpp ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp - ComplexKeyHashedDictionary.cpp ComplexKeyDirectDictionary.cpp + ComplexKeyHashedDictionary.cpp DictionaryBlockInputStreamBase.cpp DictionaryFactory.cpp DictionarySourceFactory.cpp diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 8e53ffe493d..da9435148b6 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -281,10 +281,10 @@ SRCS( rand64.cpp randConstant.cpp rand.cpp + randomFixedString.cpp randomPrintableASCII.cpp randomString.cpp randomStringUTF8.cpp - randomFixedString.cpp regexpQuoteMeta.cpp registerFunctionsArithmetic.cpp registerFunctionsComparison.cpp diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py index 117eb7b7e6f..c90725f2f6a 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py @@ -2,7 +2,7 @@ import warnings import pymysql.cursors import pymongo -import cassandra +import cassandra.cluster import redis import aerospike from tzlocal import get_localzone From f40fadc3d64381d5b85cefc047b996315ffe262c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 21 May 2020 18:29:18 +0300 Subject: [PATCH 07/29] fix segfault --- .../CassandraBlockInputStream.cpp | 21 ++++++++++--------- src/Dictionaries/CassandraBlockInputStream.h | 2 +- .../helpers/docker_compose_cassandra.yml | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index 73028e8d2b2..471f6df3e28 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -40,7 +40,8 @@ CassandraBlockInputStream::CassandraBlockInputStream( CassandraBlockInputStream::~CassandraBlockInputStream() { if (iterator != nullptr) cass_iterator_free(iterator); - cass_result_free(result); + if (result) + cass_result_free(result); } namespace @@ -166,7 +167,7 @@ namespace MutableColumns columns(description.sample_block.columns()); CassFuture* query_future = cass_session_execute(session, statement); - const CassResult* result_tmp = cass_future_get_result(query_future); + result = cass_future_get_result(query_future); if (result == nullptr) { const char* error_message; @@ -176,12 +177,12 @@ namespace throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; } - const CassRow* row = cass_result_first_row(result_tmp); + const CassRow* row = cass_result_first_row(result); const CassValue* map = cass_row_get_column(row, 0); - CassIterator* iterator_tmp = cass_iterator_from_map(map); - while (cass_iterator_next(iterator_tmp)) { - const CassValue* _key = cass_iterator_get_map_key(iterator_tmp); - const CassValue* _value = cass_iterator_get_map_value(iterator_tmp); + iterator = cass_iterator_from_map(map); + while (cass_iterator_next(iterator)) { + const CassValue* _key = cass_iterator_get_map_key(iterator); + const CassValue* _value = cass_iterator_get_map_value(iterator); auto pair_values = {std::make_pair(_key, 0ul), std::make_pair(_value, 1ul)}; for (const auto &[value, idx]: pair_values) { if (description.types[idx].second) { @@ -194,13 +195,13 @@ namespace } } - has_more_pages = cass_result_has_more_pages(result_tmp); + has_more_pages = cass_result_has_more_pages(result); if (has_more_pages) { - cass_statement_set_paging_state(statement, result_tmp); + cass_statement_set_paging_state(statement, result); } - cass_result_free(result_tmp); + cass_result_free(result); return description.sample_block.cloneWithColumns(std::move(columns)); } diff --git a/src/Dictionaries/CassandraBlockInputStream.h b/src/Dictionaries/CassandraBlockInputStream.h index 2b7c3b68744..8af63745f17 100644 --- a/src/Dictionaries/CassandraBlockInputStream.h +++ b/src/Dictionaries/CassandraBlockInputStream.h @@ -31,7 +31,7 @@ namespace DB String query_str; const size_t max_block_size; ExternalResultDescription description; - const CassResult * result; + const CassResult * result = nullptr; cass_bool_t has_more_pages; CassIterator * iterator = nullptr; }; diff --git a/tests/integration/helpers/docker_compose_cassandra.yml b/tests/integration/helpers/docker_compose_cassandra.yml index bb6a0221c54..6bbedcc1130 100644 --- a/tests/integration/helpers/docker_compose_cassandra.yml +++ b/tests/integration/helpers/docker_compose_cassandra.yml @@ -1,4 +1,4 @@ -version: '2.2' +version: '2.3' services: cassandra1: image: cassandra From e6ca09e1343ad154abfd70c3e30ceb46a868f257 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 22 May 2020 17:19:33 +0300 Subject: [PATCH 08/29] fix build --- CMakeLists.txt | 2 +- .../cassandra.cmake} | 13 +- contrib/CMakeLists.txt | 2 +- contrib/cassandra | 2 +- contrib/libuv-cmake/CMakeLists.txt | 591 ++++++++++++++++++ src/CMakeLists.txt | 4 +- .../CassandraBlockInputStream.cpp | 96 +-- .../CassandraDictionarySource.cpp | 4 +- 8 files changed, 655 insertions(+), 59 deletions(-) rename cmake/{find_cassandra.cmake => find/cassandra.cmake} (63%) create mode 100644 contrib/libuv-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d774f9c9f5..54a88404579 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -364,7 +364,7 @@ include (cmake/find/fastops.cmake) include (cmake/find/orc.cmake) include (cmake/find/avro.cmake) include (cmake/find/msgpack.cmake) -include (cmake/find_cassandra.cmake) +include (cmake/find/cassandra.cmake) find_contrib_lib(cityhash) find_contrib_lib(farmhash) diff --git a/cmake/find_cassandra.cmake b/cmake/find/cassandra.cmake similarity index 63% rename from cmake/find_cassandra.cmake rename to cmake/find/cassandra.cmake index 951cfc88b11..b1d76702cfa 100644 --- a/cmake/find_cassandra.cmake +++ b/cmake/find/cassandra.cmake @@ -4,13 +4,18 @@ if (NOT DEFINED ENABLE_CASSANDRA OR ENABLE_CASSANDRA) elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") message (WARNING "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive") else() - set(LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") + set (LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") set (CASSANDRA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/") - set (LIBUV_LIBRARY uv_a) - set (CASSANDRA_LIBRARY cassandra_static) + if (USE_STATIC_LIBRARIES) + set (LIBUV_LIBRARY uv_a) + set (CASSANDRA_LIBRARY cassandra_static) + else() + set (LIBUV_LIBRARY uv) + set (CASSANDRA_LIBRARY cassandra) + endif() set (USE_CASSANDRA 1) - set(CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra") + set (CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra") message(STATUS "Using cassandra: ${CASSANDRA_LIBRARY}") endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ce187038e2a..9c42993dda7 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -319,6 +319,6 @@ if (USE_FASTOPS) endif() if (USE_CASSANDRA) - add_subdirectory(libuv) + add_subdirectory(libuv-cmake) add_subdirectory(cassandra) endif() diff --git a/contrib/cassandra b/contrib/cassandra index 5c0f2a62bdc..bc593f2644a 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit 5c0f2a62bdc63dcc390d771c9afaa9dc34eb8e5b +Subproject commit bc593f2644a6c50c4057459e242e214a6af70969 diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt new file mode 100644 index 00000000000..b84ce217f3b --- /dev/null +++ b/contrib/libuv-cmake/CMakeLists.txt @@ -0,0 +1,591 @@ +cmake_minimum_required(VERSION 3.4) +project(libuv LANGUAGES C) + +cmake_policy(SET CMP0057 NEW) # Enable IN_LIST operator +cmake_policy(SET CMP0064 NEW) # Support if (TEST) operator + +#list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") + +include(CMakePackageConfigHelpers) +include(CMakeDependentOption) +include(CheckCCompilerFlag) +include(GNUInstallDirs) +include(CTest) + +set(CMAKE_C_VISIBILITY_PRESET hidden) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS ON) +set(CMAKE_C_STANDARD 90) + +#cmake_dependent_option(LIBUV_BUILD_TESTS +# "Build the unit tests when BUILD_TESTING is enabled and we are the root project" ON +# "BUILD_TESTING;CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR" OFF) +#cmake_dependent_option(LIBUV_BUILD_BENCH +# "Build the benchmarks when building unit tests and we are the root project" ON +# "LIBUV_BUILD_TESTS" OFF) + +# Qemu Build +option(QEMU "build for qemu" OFF) +if(QEMU) + add_definitions(-D__QEMU__=1) +endif() + +# Compiler check +string(CONCAT is-msvc $, + $ +>) + +check_c_compiler_flag(/W4 UV_LINT_W4) +check_c_compiler_flag(/wd4100 UV_LINT_NO_UNUSED_PARAMETER_MSVC) +check_c_compiler_flag(/wd4127 UV_LINT_NO_CONDITIONAL_CONSTANT_MSVC) +check_c_compiler_flag(/wd4201 UV_LINT_NO_NONSTANDARD_MSVC) +check_c_compiler_flag(/wd4206 UV_LINT_NO_NONSTANDARD_EMPTY_TU_MSVC) +check_c_compiler_flag(/wd4210 UV_LINT_NO_NONSTANDARD_FILE_SCOPE_MSVC) +check_c_compiler_flag(/wd4232 UV_LINT_NO_NONSTANDARD_NONSTATIC_DLIMPORT_MSVC) +check_c_compiler_flag(/wd4456 UV_LINT_NO_HIDES_LOCAL) +check_c_compiler_flag(/wd4457 UV_LINT_NO_HIDES_PARAM) +check_c_compiler_flag(/wd4459 UV_LINT_NO_HIDES_GLOBAL) +check_c_compiler_flag(/wd4706 UV_LINT_NO_CONDITIONAL_ASSIGNMENT_MSVC) +check_c_compiler_flag(/wd4996 UV_LINT_NO_UNSAFE_MSVC) + +check_c_compiler_flag(-Wall UV_LINT_WALL) # DO NOT use this under MSVC + +# TODO: Place these into its own function +check_c_compiler_flag(-Wno-unused-parameter UV_LINT_NO_UNUSED_PARAMETER) +check_c_compiler_flag(-Wstrict-prototypes UV_LINT_STRICT_PROTOTYPES) +check_c_compiler_flag(-Wextra UV_LINT_EXTRA) + +set(lint-no-unused-parameter $<$:-Wno-unused-parameter>) +set(lint-strict-prototypes $<$:-Wstrict-prototypes>) +set(lint-extra $<$:-Wextra>) +set(lint-w4 $<$:/W4>) +set(lint-no-unused-parameter-msvc $<$:/wd4100>) +set(lint-no-conditional-constant-msvc $<$:/wd4127>) +set(lint-no-nonstandard-msvc $<$:/wd4201>) +set(lint-no-nonstandard-empty-tu-msvc $<$:/wd4206>) +set(lint-no-nonstandard-file-scope-msvc $<$:/wd4210>) +set(lint-no-nonstandard-nonstatic-dlimport-msvc $<$:/wd4232>) +set(lint-no-hides-local-msvc $<$:/wd4456>) +set(lint-no-hides-param-msvc $<$:/wd4457>) +set(lint-no-hides-global-msvc $<$:/wd4459>) +set(lint-no-conditional-assignment-msvc $<$:/wd4706>) +set(lint-no-unsafe-msvc $<$:/wd4996>) +# Unfortunately, this one is complicated because MSVC and clang-cl support -Wall +# but using it is like calling -Weverything +string(CONCAT lint-default $< + $,$>:-Wall +>) + +list(APPEND uv_cflags ${lint-strict-prototypes} ${lint-extra} ${lint-default} ${lint-w4}) +list(APPEND uv_cflags ${lint-no-unused-parameter}) +list(APPEND uv_cflags ${lint-no-unused-parameter-msvc}) +list(APPEND uv_cflags ${lint-no-conditional-constant-msvc}) +list(APPEND uv_cflags ${lint-no-nonstandard-msvc}) +list(APPEND uv_cflags ${lint-no-nonstandard-empty-tu-msvc}) +list(APPEND uv_cflags ${lint-no-nonstandard-file-scope-msvc}) +list(APPEND uv_cflags ${lint-no-nonstandard-nonstatic-dlimport-msvc}) +list(APPEND uv_cflags ${lint-no-hides-local-msvc}) +list(APPEND uv_cflags ${lint-no-hides-param-msvc}) +list(APPEND uv_cflags ${lint-no-hides-global-msvc}) +list(APPEND uv_cflags ${lint-no-conditional-assignment-msvc}) +list(APPEND uv_cflags ${lint-no-unsafe-msvc}) + +set(uv_sources + src/fs-poll.c + src/idna.c + src/inet.c + src/random.c + src/strscpy.c + src/threadpool.c + src/timer.c + src/uv-common.c + src/uv-data-getter-setters.c + src/version.c) + +#if(WIN32) +# list(APPEND uv_defines WIN32_LEAN_AND_MEAN _WIN32_WINNT=0x0600) +# list(APPEND uv_libraries +# psapi +# iphlpapi +# userenv +# ws2_32) +# list(APPEND uv_sources +# src/win/async.c +# src/win/core.c +# src/win/detect-wakeup.c +# src/win/dl.c +# src/win/error.c +# src/win/fs.c +# src/win/fs-event.c +# src/win/getaddrinfo.c +# src/win/getnameinfo.c +# src/win/handle.c +# src/win/loop-watcher.c +# src/win/pipe.c +# src/win/thread.c +# src/win/poll.c +# src/win/process.c +# src/win/process-stdio.c +# src/win/signal.c +# src/win/snprintf.c +# src/win/stream.c +# src/win/tcp.c +# src/win/tty.c +# src/win/udp.c +# src/win/util.c +# src/win/winapi.c +# src/win/winsock.c) +# list(APPEND uv_test_libraries ws2_32) +# list(APPEND uv_test_sources src/win/snprintf.c test/runner-win.c) +#else() +list(APPEND uv_defines _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE) +if(NOT CMAKE_SYSTEM_NAME MATCHES "Android|OS390") + # TODO: This should be replaced with find_package(Threads) if possible + # Android has pthread as part of its c library, not as a separate + # libpthread.so. + list(APPEND uv_libraries pthread) +endif() +list(APPEND uv_sources + src/unix/async.c + src/unix/core.c + src/unix/dl.c + src/unix/fs.c + src/unix/getaddrinfo.c + src/unix/getnameinfo.c + src/unix/loop-watcher.c + src/unix/loop.c + src/unix/pipe.c + src/unix/poll.c + src/unix/process.c + src/unix/random-devurandom.c + src/unix/signal.c + src/unix/stream.c + src/unix/tcp.c + src/unix/thread.c + src/unix/tty.c + src/unix/udp.c) +list(APPEND uv_test_sources test/runner-unix.c) +#endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "AIX") + list(APPEND uv_defines + _ALL_SOURCE + _LINUX_SOURCE_COMPAT + _THREAD_SAFE + _XOPEN_SOURCE=500 + HAVE_SYS_AHAFS_EVPRODS_H) + list(APPEND uv_libraries perfstat) + list(APPEND uv_sources + src/unix/aix.c + src/unix/aix-common.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Android") + list(APPEND uv_libraries dl) + list(APPEND uv_sources + src/unix/android-ifaddrs.c + src/unix/linux-core.c + src/unix/linux-inotify.c + src/unix/linux-syscalls.c + src/unix/procfs-exepath.c + src/unix/pthread-fixes.c + src/unix/random-getentropy.c + src/unix/random-getrandom.c + src/unix/random-sysctl-linux.c + src/unix/sysinfo-loadavg.c) +endif() + +if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "Android|Linux|OS390") + list(APPEND uv_sources src/unix/proctitle.c) +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD") + list(APPEND uv_sources src/unix/freebsd.c) +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|NetBSD|OpenBSD") + list(APPEND uv_sources src/unix/posix-hrtime.c src/unix/bsd-proctitle.c) +endif() + +if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|NetBSD|OpenBSD") + list(APPEND uv_sources src/unix/bsd-ifaddrs.c src/unix/kqueue.c) +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + list(APPEND uv_sources src/unix/random-getrandom.c) +endif() + +if(APPLE OR CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") + list(APPEND uv_sources src/unix/random-getentropy.c) +endif() + +if(APPLE) + list(APPEND uv_defines _DARWIN_UNLIMITED_SELECT=1 _DARWIN_USE_64_BIT_INODE=1) + list(APPEND uv_sources + src/unix/darwin-proctitle.c + src/unix/darwin.c + src/unix/fsevents.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND uv_defines _GNU_SOURCE _POSIX_C_SOURCE=200112) + list(APPEND uv_libraries dl rt) + list(APPEND uv_sources + src/unix/linux-core.c + src/unix/linux-inotify.c + src/unix/linux-syscalls.c + src/unix/procfs-exepath.c + src/unix/random-getrandom.c + src/unix/random-sysctl-linux.c + src/unix/sysinfo-loadavg.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") + list(APPEND uv_sources src/unix/netbsd.c) + list(APPEND uv_libraries kvm) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") + list(APPEND uv_sources src/unix/openbsd.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "OS390") + list(APPEND uv_defines PATH_MAX=255) + list(APPEND uv_defines _AE_BIMODAL) + list(APPEND uv_defines _ALL_SOURCE) + list(APPEND uv_defines _ISOC99_SOURCE) + list(APPEND uv_defines _LARGE_TIME_API) + list(APPEND uv_defines _OPEN_MSGQ_EXT) + list(APPEND uv_defines _OPEN_SYS_FILE_EXT) + list(APPEND uv_defines _OPEN_SYS_IF_EXT) + list(APPEND uv_defines _OPEN_SYS_SOCK_EXT3) + list(APPEND uv_defines _OPEN_SYS_SOCK_IPV6) + list(APPEND uv_defines _UNIX03_SOURCE) + list(APPEND uv_defines _UNIX03_THREADS) + list(APPEND uv_defines _UNIX03_WITHDRAWN) + list(APPEND uv_defines _XOPEN_SOURCE_EXTENDED) + list(APPEND uv_sources + src/unix/pthread-fixes.c + src/unix/os390.c + src/unix/os390-syscalls.c) + list(APPEND uv_cflags -Wc,DLL -Wc,exportall -Wc,xplink) + list(APPEND uv_libraries -Wl,xplink) + list(APPEND uv_test_libraries -Wl,xplink) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "OS400") + list(APPEND uv_defines + _ALL_SOURCE + _LINUX_SOURCE_COMPAT + _THREAD_SAFE + _XOPEN_SOURCE=500) + list(APPEND uv_sources + src/unix/aix-common.c + src/unix/ibmi.c + src/unix/no-fsevents.c + src/unix/no-proctitle.c + src/unix/posix-poll.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "SunOS") + list(APPEND uv_defines __EXTENSIONS__ _XOPEN_SOURCE=500) + list(APPEND uv_libraries kstat nsl sendfile socket) + list(APPEND uv_sources src/unix/no-proctitle.c src/unix/sunos.c) +endif() + +if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|Linux|NetBSD|OpenBSD") + list(APPEND uv_test_libraries util) +endif() + +set(uv_sources_tmp "") +foreach(file ${uv_sources}) + list(APPEND uv_sources_tmp "${LIBUV_ROOT_DIR}/${file}") +endforeach(file) +set(uv_sources "${uv_sources_tmp}") + +add_library(uv SHARED ${uv_sources}) +target_compile_definitions(uv + INTERFACE + USING_UV_SHARED=1 + PRIVATE + BUILDING_UV_SHARED=1 + ${uv_defines}) +target_compile_options(uv PRIVATE ${uv_cflags}) +target_include_directories(uv + PUBLIC + $ + $ + PRIVATE + $) +target_link_libraries(uv ${uv_libraries}) + +add_library(uv_a STATIC ${uv_sources}) +target_compile_definitions(uv_a PRIVATE ${uv_defines}) +target_compile_options(uv_a PRIVATE ${uv_cflags}) +target_include_directories(uv_a + PUBLIC + $ + $ + PRIVATE + $) +target_link_libraries(uv_a ${uv_libraries}) + +#if(LIBUV_BUILD_TESTS) +# # Small hack: use ${uv_test_sources} now to get the runner skeleton, +# # before the actual tests are added. +# add_executable( +# uv_run_benchmarks_a +# ${uv_test_sources} +# test/benchmark-async-pummel.c +# test/benchmark-async.c +# test/benchmark-fs-stat.c +# test/benchmark-getaddrinfo.c +# test/benchmark-loop-count.c +# test/benchmark-million-async.c +# test/benchmark-million-timers.c +# test/benchmark-multi-accept.c +# test/benchmark-ping-pongs.c +# test/benchmark-ping-udp.c +# test/benchmark-pound.c +# test/benchmark-pump.c +# test/benchmark-sizes.c +# test/benchmark-spawn.c +# test/benchmark-tcp-write-batch.c +# test/benchmark-thread.c +# test/benchmark-udp-pummel.c +# test/blackhole-server.c +# test/dns-server.c +# test/echo-server.c +# test/run-benchmarks.c +# test/runner.c) +# target_compile_definitions(uv_run_benchmarks_a PRIVATE ${uv_defines}) +# target_compile_options(uv_run_benchmarks_a PRIVATE ${uv_cflags}) +# target_link_libraries(uv_run_benchmarks_a uv_a ${uv_test_libraries}) +# +# list(APPEND uv_test_sources +# test/blackhole-server.c +# test/echo-server.c +# test/run-tests.c +# test/runner.c +# test/test-active.c +# test/test-async-null-cb.c +# test/test-async.c +# test/test-barrier.c +# test/test-callback-order.c +# test/test-callback-stack.c +# test/test-close-fd.c +# test/test-close-order.c +# test/test-condvar.c +# test/test-connect-unspecified.c +# test/test-connection-fail.c +# test/test-cwd-and-chdir.c +# test/test-default-loop-close.c +# test/test-delayed-accept.c +# test/test-dlerror.c +# test/test-eintr-handling.c +# test/test-embed.c +# test/test-emfile.c +# test/test-env-vars.c +# test/test-error.c +# test/test-fail-always.c +# test/test-fork.c +# test/test-fs-copyfile.c +# test/test-fs-event.c +# test/test-fs-poll.c +# test/test-fs.c +# test/test-fs-readdir.c +# test/test-fs-fd-hash.c +# test/test-fs-open-flags.c +# test/test-get-currentexe.c +# test/test-get-loadavg.c +# test/test-get-memory.c +# test/test-get-passwd.c +# test/test-getaddrinfo.c +# test/test-gethostname.c +# test/test-getnameinfo.c +# test/test-getsockname.c +# test/test-getters-setters.c +# test/test-gettimeofday.c +# test/test-handle-fileno.c +# test/test-homedir.c +# test/test-hrtime.c +# test/test-idle.c +# test/test-idna.c +# test/test-ip4-addr.c +# test/test-ip6-addr.c +# test/test-ipc-heavy-traffic-deadlock-bug.c +# test/test-ipc-send-recv.c +# test/test-ipc.c +# test/test-loop-alive.c +# test/test-loop-close.c +# test/test-loop-configure.c +# test/test-loop-handles.c +# test/test-loop-stop.c +# test/test-loop-time.c +# test/test-multiple-listen.c +# test/test-mutexes.c +# test/test-osx-select.c +# test/test-pass-always.c +# test/test-ping-pong.c +# test/test-pipe-bind-error.c +# test/test-pipe-close-stdout-read-stdin.c +# test/test-pipe-connect-error.c +# test/test-pipe-connect-multiple.c +# test/test-pipe-connect-prepare.c +# test/test-pipe-getsockname.c +# test/test-pipe-pending-instances.c +# test/test-pipe-sendmsg.c +# test/test-pipe-server-close.c +# test/test-pipe-set-fchmod.c +# test/test-pipe-set-non-blocking.c +# test/test-platform-output.c +# test/test-poll-close-doesnt-corrupt-stack.c +# test/test-poll-close.c +# test/test-poll-closesocket.c +# test/test-poll-oob.c +# test/test-poll.c +# test/test-process-priority.c +# test/test-process-title-threadsafe.c +# test/test-process-title.c +# test/test-queue-foreach-delete.c +# test/test-random.c +# test/test-ref.c +# test/test-run-nowait.c +# test/test-run-once.c +# test/test-semaphore.c +# test/test-shutdown-close.c +# test/test-shutdown-eof.c +# test/test-shutdown-twice.c +# test/test-signal-multiple-loops.c +# test/test-signal-pending-on-close.c +# test/test-signal.c +# test/test-socket-buffer-size.c +# test/test-spawn.c +# test/test-stdio-over-pipes.c +# test/test-strscpy.c +# test/test-tcp-alloc-cb-fail.c +# test/test-tcp-bind-error.c +# test/test-tcp-bind6-error.c +# test/test-tcp-close-accept.c +# test/test-tcp-close-while-connecting.c +# test/test-tcp-close.c +# test/test-tcp-close-reset.c +# test/test-tcp-connect-error-after-write.c +# test/test-tcp-connect-error.c +# test/test-tcp-connect-timeout.c +# test/test-tcp-connect6-error.c +# test/test-tcp-create-socket-early.c +# test/test-tcp-flags.c +# test/test-tcp-oob.c +# test/test-tcp-open.c +# test/test-tcp-read-stop.c +# test/test-tcp-shutdown-after-write.c +# test/test-tcp-try-write.c +# test/test-tcp-try-write-error.c +# test/test-tcp-unexpected-read.c +# test/test-tcp-write-after-connect.c +# test/test-tcp-write-fail.c +# test/test-tcp-write-queue-order.c +# test/test-tcp-write-to-half-open-connection.c +# test/test-tcp-writealot.c +# test/test-thread-equal.c +# test/test-thread.c +# test/test-threadpool-cancel.c +# test/test-threadpool.c +# test/test-timer-again.c +# test/test-timer-from-check.c +# test/test-timer.c +# test/test-tmpdir.c +# test/test-tty-duplicate-key.c +# test/test-tty-escape-sequence-processing.c +# test/test-tty.c +# test/test-udp-alloc-cb-fail.c +# test/test-udp-bind.c +# test/test-udp-connect.c +# test/test-udp-create-socket-early.c +# test/test-udp-dgram-too-big.c +# test/test-udp-ipv6.c +# test/test-udp-multicast-interface.c +# test/test-udp-multicast-interface6.c +# test/test-udp-multicast-join.c +# test/test-udp-multicast-join6.c +# test/test-udp-multicast-ttl.c +# test/test-udp-open.c +# test/test-udp-options.c +# test/test-udp-send-and-recv.c +# test/test-udp-send-hang-loop.c +# test/test-udp-send-immediate.c +# test/test-udp-send-unreachable.c +# test/test-udp-try-send.c +# test/test-uname.c +# test/test-walk-handles.c +# test/test-watcher-cross-stop.c) +# +# add_executable(uv_run_tests ${uv_test_sources} uv_win_longpath.manifest) +# target_compile_definitions(uv_run_tests +# PRIVATE ${uv_defines} USING_UV_SHARED=1) +# target_compile_options(uv_run_tests PRIVATE ${uv_cflags}) +# target_link_libraries(uv_run_tests uv ${uv_test_libraries}) +# add_test(NAME uv_test +# COMMAND uv_run_tests +# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +# if(CMAKE_SYSTEM_NAME STREQUAL "OS390") +# set_tests_properties(uv_test PROPERTIES ENVIRONMENT +# "LIBPATH=${CMAKE_BINARY_DIR}:$ENV{LIBPATH}") +# endif() +# add_executable(uv_run_tests_a ${uv_test_sources} uv_win_longpath.manifest) +# target_compile_definitions(uv_run_tests_a PRIVATE ${uv_defines}) +# target_compile_options(uv_run_tests_a PRIVATE ${uv_cflags}) +# if(QEMU) +# target_link_libraries(uv_run_tests_a uv_a ${uv_test_libraries} -static) +# else() +# target_link_libraries(uv_run_tests_a uv_a ${uv_test_libraries}) +# endif() +# add_test(NAME uv_test_a +# COMMAND uv_run_tests_a +# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +#endif() + +if(UNIX) + # Now for some gibbering horrors from beyond the stars... + foreach(lib IN LISTS uv_libraries) + list(APPEND LIBS "-l${lib}") + endforeach() + string(REPLACE ";" " " LIBS "${LIBS}") + # Consider setting project version via project() call? + file(STRINGS ${LIBUV_ROOT_DIR}/configure.ac configure_ac REGEX ^AC_INIT) + string(REGEX MATCH "([0-9]+)[.][0-9]+[.][0-9]+" PACKAGE_VERSION "${configure_ac}") + set(UV_VERSION_MAJOR "${CMAKE_MATCH_1}") + # The version in the filename is mirroring the behaviour of autotools. + set_target_properties(uv PROPERTIES + VERSION ${UV_VERSION_MAJOR}.0.0 + SOVERSION ${UV_VERSION_MAJOR}) + set(includedir ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) + set(libdir ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}) + set(prefix ${CMAKE_INSTALL_PREFIX}) + configure_file(${LIBUV_ROOT_DIR}/libuv.pc.in ${LIBUV_ROOT_DIR}/libuv.pc @ONLY) + + install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DOCDIR}) + install(FILES ${PROJECT_BINARY_DIR}/libuv.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + install(TARGETS uv LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + install(TARGETS uv_a ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endif() + +#if(WIN32) +# install(DIRECTORY include/ DESTINATION include) +# install(FILES LICENSE DESTINATION .) +# install(TARGETS uv uv_a +# RUNTIME DESTINATION lib/$ +# ARCHIVE DESTINATION lib/$) +#endif() +# +#message(STATUS "summary of build options: +# Install prefix: ${CMAKE_INSTALL_PREFIX} +# Target system: ${CMAKE_SYSTEM_NAME} +# Compiler: +# C compiler: ${CMAKE_C_COMPILER} +# CFLAGS: ${CMAKE_C_FLAGS_${_build_type}} ${CMAKE_C_FLAGS} +#") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d713cec8b8a..7c6b4006021 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -357,8 +357,8 @@ if (USE_OPENCL) endif () if (USE_CASSANDRA) - dbms_target_link_libraries(PRIVATE ${CASSANDRA_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${CASS_INCLUDE_DIR}) + dbms_target_link_libraries(PUBLIC ${CASSANDRA_LIBRARY}) + dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) endif() dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR}) diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index 471f6df3e28..bf43adcdc59 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -22,7 +22,7 @@ namespace ErrorCodes } CassandraBlockInputStream::CassandraBlockInputStream( - CassSession *session_, + CassSession * session_, const std::string &query_str_, const DB::Block &sample_block, const size_t max_block_size_) @@ -48,108 +48,108 @@ namespace { using ValueType = ExternalResultDescription::ValueType; - void insertValue(IColumn & column, const ValueType type, const CassValue * value) + void insertValue(IColumn & column, const ValueType type, const CassValue * cass_value) { switch (type) { case ValueType::vtUInt8: { - cass_uint32_t _value; - cass_value_get_uint32(value, &_value); - static_cast(column).insertValue(_value); + cass_uint32_t value; + cass_value_get_uint32(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtUInt16: { - cass_uint32_t _value; - cass_value_get_uint32(value, &_value); - static_cast(column).insertValue(_value); + cass_uint32_t value; + cass_value_get_uint32(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtUInt32: { - cass_uint32_t _value; - cass_value_get_uint32(value, &_value); - static_cast(column).insertValue(_value); + cass_uint32_t value; + cass_value_get_uint32(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtUInt64: { - cass_int64_t _value; - cass_value_get_int64(value, &_value); - static_cast(column).insertValue(_value); + cass_int64_t value; + cass_value_get_int64(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtInt8: { - cass_int8_t _value; - cass_value_get_int8(value, &_value); - static_cast(column).insertValue(_value); + cass_int8_t value; + cass_value_get_int8(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtInt16: { - cass_int16_t _value; - cass_value_get_int16(value, &_value); - static_cast(column).insertValue(_value); + cass_int16_t value; + cass_value_get_int16(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtInt32: { - cass_int32_t _value; - cass_value_get_int32(value, &_value); - static_cast(column).insertValue(_value); + cass_int32_t value; + cass_value_get_int32(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtInt64: { - cass_int64_t _value; - cass_value_get_int64(value, &_value); - static_cast(column).insertValue(_value); + cass_int64_t value; + cass_value_get_int64(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtFloat32: { - cass_float_t _value; - cass_value_get_float(value, &_value); - static_cast(column).insertValue(_value); + cass_float_t value; + cass_value_get_float(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtFloat64: { - cass_double_t _value; - cass_value_get_double(value, &_value); - static_cast(column).insertValue(_value); + cass_double_t value; + cass_value_get_double(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtString: { - const char* _value; - size_t _value_length; - cass_value_get_string(value, &_value, &_value_length); - static_cast(column).insertData(_value, _value_length); + const char * value; + size_t value_length; + cass_value_get_string(cass_value, &value, &value_length); + static_cast(column).insertData(value, value_length); break; } case ValueType::vtDate: { - cass_int64_t _value; - cass_value_get_int64(value, &_value); - static_cast(column).insertValue(UInt32{cass_date_from_epoch(_value)}); // FIXME + cass_int64_t value; + cass_value_get_int64(cass_value, &value); + static_cast(column).insertValue(UInt32{cass_date_from_epoch(value)}); // FIXME break; } case ValueType::vtDateTime: { - cass_int64_t _value; - cass_value_get_int64(value, &_value); - static_cast(column).insertValue(_value); + cass_int64_t value; + cass_value_get_int64(cass_value, &value); + static_cast(column).insertValue(value); break; } case ValueType::vtUUID: { - CassUuid _value; - cass_value_get_uuid(value, &_value); + CassUuid value; + cass_value_get_uuid(cass_value, &value); std::array uuid_str; - cass_uuid_string(_value, uuid_str.data()); + cass_uuid_string(value, uuid_str.data()); static_cast(column).insert(parse(uuid_str.data(), uuid_str.size())); break; } @@ -181,9 +181,9 @@ namespace const CassValue* map = cass_row_get_column(row, 0); iterator = cass_iterator_from_map(map); while (cass_iterator_next(iterator)) { - const CassValue* _key = cass_iterator_get_map_key(iterator); - const CassValue* _value = cass_iterator_get_map_value(iterator); - auto pair_values = {std::make_pair(_key, 0ul), std::make_pair(_value, 1ul)}; + const CassValue* cass_key = cass_iterator_get_map_key(iterator); + const CassValue* cass_value = cass_iterator_get_map_value(iterator); + auto pair_values = {std::make_pair(cass_key, 0ul), std::make_pair(cass_value, 1ul)}; for (const auto &[value, idx]: pair_values) { if (description.types[idx].second) { ColumnNullable & column_nullable = static_cast(*columns[idx]); diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index c51ae2877d8..3a4c7e2f2b7 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -11,7 +11,7 @@ namespace DB void registerDictionarySourceCassandra(DictionarySourceFactory & factory) { - auto createTableSource = [=](const DictionaryStructure & dict_struct, + auto create_table_source = [=](const DictionaryStructure & dict_struct, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, @@ -28,7 +28,7 @@ namespace DB ErrorCodes::SUPPORT_IS_DISABLED}; #endif }; - factory.registerSource("cassandra", createTableSource); + factory.registerSource("cassandra", create_table_source); } } From 310d5225280029b9018c6caf0a2570bb70999e99 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 26 May 2020 22:21:18 +0300 Subject: [PATCH 09/29] minimal implementation --- contrib/cassandra | 2 +- .../CassandraBlockInputStream.cpp | 124 ++++++++++++------ src/Dictionaries/CassandraBlockInputStream.h | 5 + .../CassandraDictionarySource.cpp | 74 ++++++++--- src/Dictionaries/CassandraDictionarySource.h | 52 ++++---- src/Dictionaries/ExternalQueryBuilder.cpp | 43 +++--- src/Dictionaries/ExternalQueryBuilder.h | 4 +- tests/integration/helpers/cluster.py | 15 ++- .../helpers/docker_compose_cassandra.yml | 2 +- .../external_sources.py | 50 ++++++- .../test.py | 9 +- 11 files changed, 276 insertions(+), 104 deletions(-) diff --git a/contrib/cassandra b/contrib/cassandra index bc593f2644a..9606ff1f70b 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit bc593f2644a6c50c4057459e242e214a6af70969 +Subproject commit 9606ff1f70bd3fc5d395df32e626923c012ffb5f diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index bf43adcdc59..473a42549a8 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -50,76 +50,77 @@ namespace void insertValue(IColumn & column, const ValueType type, const CassValue * cass_value) { + /// Cassandra does not support unsigned integers switch (type) { case ValueType::vtUInt8: { - cass_uint32_t value; - cass_value_get_uint32(cass_value, &value); - static_cast(column).insertValue(value); + cass_int8_t value; + cass_value_get_int8(cass_value, &value); + assert_cast(column).insertValue(value); break; } case ValueType::vtUInt16: { - cass_uint32_t value; - cass_value_get_uint32(cass_value, &value); - static_cast(column).insertValue(value); + cass_int16_t value; + cass_value_get_int16(cass_value, &value); + assert_cast(column).insertValue(value); break; } case ValueType::vtUInt32: { - cass_uint32_t value; - cass_value_get_uint32(cass_value, &value); - static_cast(column).insertValue(value); + cass_int32_t value; + cass_value_get_int32(cass_value, &value); + assert_cast(column).insertValue(value); break; } case ValueType::vtUInt64: { cass_int64_t value; cass_value_get_int64(cass_value, &value); - static_cast(column).insertValue(value); + assert_cast(column).insertValue(value); break; } case ValueType::vtInt8: { cass_int8_t value; cass_value_get_int8(cass_value, &value); - static_cast(column).insertValue(value); + assert_cast(column).insertValue(value); break; } case ValueType::vtInt16: { cass_int16_t value; cass_value_get_int16(cass_value, &value); - static_cast(column).insertValue(value); + assert_cast(column).insertValue(value); break; } case ValueType::vtInt32: { cass_int32_t value; cass_value_get_int32(cass_value, &value); - static_cast(column).insertValue(value); + assert_cast(column).insertValue(value); break; } case ValueType::vtInt64: { cass_int64_t value; cass_value_get_int64(cass_value, &value); - static_cast(column).insertValue(value); + assert_cast(column).insertValue(value); break; } case ValueType::vtFloat32: { cass_float_t value; cass_value_get_float(cass_value, &value); - static_cast(column).insertValue(value); + assert_cast(column).insertValue(value); break; } case ValueType::vtFloat64: { cass_double_t value; cass_value_get_double(cass_value, &value); - static_cast(column).insertValue(value); + assert_cast(column).insertValue(value); break; } case ValueType::vtString: @@ -127,21 +128,21 @@ namespace const char * value; size_t value_length; cass_value_get_string(cass_value, &value, &value_length); - static_cast(column).insertData(value, value_length); + assert_cast(column).insertData(value, value_length); break; } case ValueType::vtDate: { - cass_int64_t value; - cass_value_get_int64(cass_value, &value); - static_cast(column).insertValue(UInt32{cass_date_from_epoch(value)}); // FIXME + cass_uint32_t value; + cass_value_get_uint32(cass_value, &value); + assert_cast(column).insertValue(static_cast(value)); break; } case ValueType::vtDateTime: { cass_int64_t value; cass_value_get_int64(cass_value, &value); - static_cast(column).insertValue(value); + assert_cast(column).insertValue(static_cast(value / 1000)); break; } case ValueType::vtUUID: @@ -150,7 +151,7 @@ namespace cass_value_get_uuid(cass_value, &value); std::array uuid_str; cass_uuid_string(value, uuid_str.data()); - static_cast(column).insert(parse(uuid_str.data(), uuid_str.size())); + assert_cast(column).insert(parse(uuid_str.data(), uuid_str.size())); break; } } @@ -161,10 +162,10 @@ namespace Block CassandraBlockInputStream::readImpl() { - if (has_more_pages) + if (!has_more_pages) return {}; - MutableColumns columns(description.sample_block.columns()); + MutableColumns columns = description.sample_block.cloneEmptyColumns(); CassFuture* query_future = cass_session_execute(session, statement); result = cass_future_get_result(query_future); @@ -177,23 +178,49 @@ namespace throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; } - const CassRow* row = cass_result_first_row(result); - const CassValue* map = cass_row_get_column(row, 0); - iterator = cass_iterator_from_map(map); - while (cass_iterator_next(iterator)) { - const CassValue* cass_key = cass_iterator_get_map_key(iterator); - const CassValue* cass_value = cass_iterator_get_map_value(iterator); - auto pair_values = {std::make_pair(cass_key, 0ul), std::make_pair(cass_value, 1ul)}; - for (const auto &[value, idx]: pair_values) { - if (description.types[idx].second) { - ColumnNullable & column_nullable = static_cast(*columns[idx]); - insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value); + [[maybe_unused]] size_t row_count = 0; + assert(cass_result_column_count(result) == columns.size()); + CassIterator * rows_iter = cass_iterator_from_result(result); /// Points to rows[-1] + while (cass_iterator_next(rows_iter)) + { + const CassRow * row = cass_iterator_get_row(rows_iter); + for (size_t col_idx = 0; col_idx < columns.size(); ++col_idx) + { + const CassValue * val = cass_row_get_column(row, col_idx); + if (cass_value_is_null(val)) + columns[col_idx]->insertDefault(); + else if (description.types[col_idx].second) + { + ColumnNullable & column_nullable = static_cast(*columns[col_idx]); + insertValue(column_nullable.getNestedColumn(), description.types[col_idx].first, val); column_nullable.getNullMapData().emplace_back(0); - } else { - insertValue(*columns[idx], description.types[idx].first, value); } + else + insertValue(*columns[col_idx], description.types[col_idx].first, val); } + ++row_count; } + assert(cass_result_row_count(result) == row_count); + cass_iterator_free(rows_iter); + + //const CassRow* row = cass_result_first_row(result); + //const CassValue* map = cass_row_get_column(row, 0); + //const CassValue* map = cass_row_get_column(row, 0); + //iterator = cass_iterator_from_map(map); + //while (cass_iterator_next(iterator)) { + // const CassValue* cass_key = cass_iterator_get_map_key(iterator); + // const CassValue* cass_value = cass_iterator_get_map_value(iterator); + // auto pair_values = {std::make_pair(cass_key, 0ul), std::make_pair(cass_value, 1ul)}; + // for (const auto &[value, idx]: pair_values) { + // if (description.types[idx].second) { + // ColumnNullable & column_nullable = static_cast(*columns[idx]); + // insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value); + // column_nullable.getNullMapData().emplace_back(0); + // } else { + // insertValue(*columns[idx], description.types[idx].first, value); + // } + // } + //} has_more_pages = cass_result_has_more_pages(result); @@ -207,5 +234,28 @@ namespace } +void cassandraCheck(CassError code) +{ + if (code != CASS_OK) + throw Exception("Cassandra driver error " + std::to_string(code) + ": " + cass_error_desc(code), + ErrorCodes::CASSANDRA_INTERNAL_ERROR); +} + +void cassandraWaitAndCheck(CassFuture * future) +{ + auto code = cass_future_error_code(future); /// Waits if not ready + if (code == CASS_OK) + { + cass_future_free(future); + return; + } + const char * message; + size_t message_len; + cass_future_error_message(future, &message, & message_len); + String full_message = "Cassandra driver error " + std::to_string(code) + ": " + cass_error_desc(code) + ": " + message; + cass_future_free(future); /// Frees message + throw Exception(full_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR); +} + } #endif diff --git a/src/Dictionaries/CassandraBlockInputStream.h b/src/Dictionaries/CassandraBlockInputStream.h index 8af63745f17..c8476bd2c15 100644 --- a/src/Dictionaries/CassandraBlockInputStream.h +++ b/src/Dictionaries/CassandraBlockInputStream.h @@ -8,6 +8,11 @@ namespace DB { + +void cassandraCheck(CassError error); +void cassandraWaitAndCheck(CassFuture * future); + + /// Allows processing results of a Cassandra query as a sequence of Blocks, simplifies chaining class CassandraBlockInputStream final : public IBlockInputStream { diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index 3a4c7e2f2b7..c165ecea212 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -1,6 +1,8 @@ #include "CassandraDictionarySource.h" #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" +#include "ExternalQueryBuilder.h" +#include namespace DB { @@ -51,25 +53,32 @@ static const size_t max_block_size = 8192; CassandraDictionarySource::CassandraDictionarySource( const DB::DictionaryStructure & dict_struct_, - const std::string & host_, + const String & host_, UInt16 port_, - const std::string & user_, - const std::string & password_, - const std::string & method_, - const std::string & db_, + const String & user_, + const String & password_, + //const std::string & method_, + const String & db_, + const String & table_, const DB::Block & sample_block_) - : dict_struct(dict_struct_) + : log(&Logger::get("CassandraDictionarySource")) + , dict_struct(dict_struct_) , host(host_) , port(port_) , user(user_) , password(password_) - , method(method_) + //, method(method_) , db(db_) + , table(table_) , sample_block(sample_block_) - , cluster(cass_cluster_new()) + , cluster(cass_cluster_new()) //FIXME will not be freed in case of exception , session(cass_session_new()) { - cass_cluster_set_contact_points(cluster, toConnectionString(host, port).c_str()); + cassandraCheck(cass_cluster_set_contact_points(cluster, host.c_str())); + if (port) + cassandraCheck(cass_cluster_set_port(cluster, port)); + cass_cluster_set_credentials(cluster, user.c_str(), password.c_str()); + cassandraWaitAndCheck(cass_session_connect_keyspace(session, cluster, db.c_str())); } CassandraDictionarySource::CassandraDictionarySource( @@ -80,11 +89,12 @@ CassandraDictionarySource::CassandraDictionarySource( : CassandraDictionarySource( dict_struct_, config.getString(config_prefix + ".host"), - config.getUInt(config_prefix + ".port"), + config.getUInt(config_prefix + ".port", 0), config.getString(config_prefix + ".user", ""), config.getString(config_prefix + ".password", ""), - config.getString(config_prefix + ".method", ""), - config.getString(config_prefix + ".db", ""), + //config.getString(config_prefix + ".method", ""), + config.getString(config_prefix + ".keyspace", ""), + config.getString(config_prefix + ".column_family"), sample_block_) { } @@ -95,8 +105,9 @@ CassandraDictionarySource::CassandraDictionarySource(const CassandraDictionarySo other.port, other.user, other.password, - other.method, + //other.method, other.db, + other.table, other.sample_block} { } @@ -106,18 +117,45 @@ CassandraDictionarySource::~CassandraDictionarySource() { cass_cluster_free(cluster); } -std::string CassandraDictionarySource::toConnectionString(const std::string &host, const UInt16 port) { - return host + (port != 0 ? ":" + std::to_string(port) : ""); -} +//std::string CassandraDictionarySource::toConnectionString(const std::string &host, const UInt16 port) { +// return host + (port != 0 ? ":" + std::to_string(port) : ""); +//} -BlockInputStreamPtr CassandraDictionarySource::loadAll() { - return std::make_shared(nullptr, "", sample_block, max_block_size); +BlockInputStreamPtr CassandraDictionarySource::loadAll() +{ + ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes}; + String query = builder.composeLoadAllQuery(); + query.pop_back(); + query += " ALLOW FILTERING;"; + LOG_INFO(log, "Loading all using query: " << query); + return std::make_shared(session, query, sample_block, max_block_size); } std::string CassandraDictionarySource::toString() const { return "Cassandra: " + /*db + '.' + collection + ',' + (user.empty() ? " " : " " + user + '@') + */ host + ':' + DB::toString(port); } +BlockInputStreamPtr CassandraDictionarySource::loadIds(const std::vector & ids) +{ + ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes}; + String query = builder.composeLoadIdsQuery(ids); + query.pop_back(); + query += " ALLOW FILTERING;"; + LOG_INFO(log, "Loading ids using query: " << query); + return std::make_shared(session, query, sample_block, max_block_size); +} + +BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) +{ + //FIXME split conditions on partition key and clustering key + ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes}; + String query = builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::IN_WITH_TUPLES); + query.pop_back(); + query += " ALLOW FILTERING;"; + LOG_INFO(log, "Loading keys using query: " << query); + return std::make_shared(session, query, sample_block, max_block_size); +} + } diff --git a/src/Dictionaries/CassandraDictionarySource.h b/src/Dictionaries/CassandraDictionarySource.h index 2bdd476951a..400481d0a95 100644 --- a/src/Dictionaries/CassandraDictionarySource.h +++ b/src/Dictionaries/CassandraDictionarySource.h @@ -1,25 +1,29 @@ #pragma once +#if !defined(ARCADIA_BUILD) #include -#include +#endif #if USE_CASSANDRA -# include "DictionaryStructure.h" -# include "IDictionarySource.h" -# include +#include "DictionaryStructure.h" +#include "IDictionarySource.h" +#include +#include +#include namespace DB { class CassandraDictionarySource final : public IDictionarySource { CassandraDictionarySource( const DictionaryStructure & dict_struct, - const std::string & host, + const String & host, UInt16 port, - const std::string & user, - const std::string & password, - const std::string & method, - const std::string & db, + const String & user, + const String & password, + //const std::string & method, + const String & db, + const String & table, const Block & sample_block); public: @@ -44,15 +48,15 @@ public: DictionarySourcePtr clone() const override { return std::make_unique(*this); } - BlockInputStreamPtr loadIds(const std::vector & /* ids */) override - { - throw Exception{"Method loadIds is not implemented yet", ErrorCodes::NOT_IMPLEMENTED}; - } + BlockInputStreamPtr loadIds(const std::vector & ids) override; + //{ + // throw Exception{"Method loadIds is not implemented yet", ErrorCodes::NOT_IMPLEMENTED}; + //} - BlockInputStreamPtr loadKeys(const Columns & /* key_columns */, const std::vector & /* requested_rows */) override - { - throw Exception{"Method loadKeys is not implemented yet", ErrorCodes::NOT_IMPLEMENTED}; - } + BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector & requested_rows) override; + //{ + // throw Exception{"Method loadKeys is not implemented yet", ErrorCodes::NOT_IMPLEMENTED}; + //} BlockInputStreamPtr loadUpdatedAll() override { @@ -62,15 +66,17 @@ public: std::string toString() const override; private: - static std::string toConnectionString(const std::string & host, const UInt16 port); + //static std::string toConnectionString(const std::string & host, const UInt16 port); + Poco::Logger * log; const DictionaryStructure dict_struct; - const std::string host; + const String host; const UInt16 port; - const std::string user; - const std::string password; - const std::string method; - const std::string db; + const String user; + const String password; + //const std::string method; + const String db; + const String table; Block sample_block; CassCluster * cluster; diff --git a/src/Dictionaries/ExternalQueryBuilder.cpp b/src/Dictionaries/ExternalQueryBuilder.cpp index 529fb3d60fa..d55d77ea9a5 100644 --- a/src/Dictionaries/ExternalQueryBuilder.cpp +++ b/src/Dictionaries/ExternalQueryBuilder.cpp @@ -63,6 +63,13 @@ void ExternalQueryBuilder::writeQuoted(const std::string & s, WriteBuffer & out) std::string ExternalQueryBuilder::composeLoadAllQuery() const { WriteBufferFromOwnString out; + composeLoadAllQuery(out); + writeChar(';', out); + return out.str(); +} + +void ExternalQueryBuilder::composeLoadAllQuery(WriteBuffer & out) const +{ writeString("SELECT ", out); if (dict_struct.id) @@ -149,24 +156,26 @@ std::string ExternalQueryBuilder::composeLoadAllQuery() const writeString(" WHERE ", out); writeString(where, out); } - - writeChar(';', out); - - return out.str(); } std::string ExternalQueryBuilder::composeUpdateQuery(const std::string & update_field, const std::string & time_point) const { - std::string out = composeLoadAllQuery(); - std::string update_query; + WriteBufferFromOwnString out; + composeLoadAllQuery(out); if (!where.empty()) - update_query = " AND " + update_field + " >= '" + time_point + "'"; + writeString(" AND ", out); else - update_query = " WHERE " + update_field + " >= '" + time_point + "'"; + writeString(" WHERE ", out); - return out.insert(out.size() - 1, update_query); /// This is done to insert "update_query" before "out"'s semicolon + writeQuoted(update_field, out); + writeString(" >= '", out); + writeString(time_point, out); + writeChar('\'', out); + + writeChar(';', out); + return out.str(); } @@ -303,7 +312,7 @@ ExternalQueryBuilder::composeLoadKeysQuery(const Columns & key_columns, const st } else /* if (method == IN_WITH_TUPLES) */ { - writeString(composeKeyTupleDefinition(), out); + composeKeyTupleDefinition(out); writeString(" IN (", out); first = true; @@ -346,7 +355,7 @@ void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, cons const auto & key_description = (*dict_struct.key)[i]; /// key_i=value_i - writeString(key_description.name, out); + writeQuoted(key_description.name, out); writeString("=", out); key_description.type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings); } @@ -355,26 +364,24 @@ void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, cons } -std::string ExternalQueryBuilder::composeKeyTupleDefinition() const +void ExternalQueryBuilder::composeKeyTupleDefinition(WriteBuffer & out) const { if (!dict_struct.key) throw Exception{"Composite key required for method", ErrorCodes::UNSUPPORTED_METHOD}; - std::string result{"("}; + writeChar('(', out); auto first = true; for (const auto & key : *dict_struct.key) { if (!first) - result += ", "; + writeString(", ", out); first = false; - result += key.name; + writeQuoted(key.name, out); } - result += ")"; - - return result; + writeChar(')', out); } diff --git a/src/Dictionaries/ExternalQueryBuilder.h b/src/Dictionaries/ExternalQueryBuilder.h index 93e10f2d6b0..2ffc6a475ee 100644 --- a/src/Dictionaries/ExternalQueryBuilder.h +++ b/src/Dictionaries/ExternalQueryBuilder.h @@ -58,11 +58,13 @@ struct ExternalQueryBuilder private: const FormatSettings format_settings; + void composeLoadAllQuery(WriteBuffer & out) const; + /// Expression in form (x = c1 AND y = c2 ...) void composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out) const; /// Expression in form (x, y, ...) - std::string composeKeyTupleDefinition() const; + void composeKeyTupleDefinition(WriteBuffer & out) const; /// Expression in form (c1, c2, ...) void composeKeyTuple(const Columns & key_columns, const size_t row, WriteBuffer & out) const; diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 8b97d77f4c9..9e66712fd7e 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -19,6 +19,7 @@ import pprint import psycopg2 import pymongo import pymysql +import cassandra.cluster from dicttoxml import dicttoxml from kazoo.client import KazooClient from kazoo.exceptions import KazooException @@ -448,6 +449,18 @@ class ClickHouseCluster: logging.warning("Can't connect to SchemaRegistry: %s", str(ex)) time.sleep(1) + def wait_cassandra_to_start(self, timeout=15): + cass_client = cassandra.cluster.Cluster(["localhost"], port="9043") + start = time.time() + while time.time() - start < timeout: + try: + cass_client.connect().execute("drop keyspace if exists test;") + logging.info("Connected to Cassandra %s") + return + except Exception as ex: + logging.warning("Can't connect to Minio: %s", str(ex)) + time.sleep(1) + def start(self, destroy_dirs=True): if self.is_up: return @@ -526,7 +539,7 @@ class ClickHouseCluster: if self.with_cassandra and self.base_cassandra_cmd: subprocess_check_call(self.base_cassandra_cmd + ['up', '-d', '--force-recreate']) - time.sleep(10) + self.wait_cassandra_to_start() clickhouse_start_cmd = self.base_cmd + ['up', '-d', '--no-recreate'] logging.info("Trying to create ClickHouse instance by command %s", ' '.join(map(str, clickhouse_start_cmd))) diff --git a/tests/integration/helpers/docker_compose_cassandra.yml b/tests/integration/helpers/docker_compose_cassandra.yml index 6bbedcc1130..6567a352027 100644 --- a/tests/integration/helpers/docker_compose_cassandra.yml +++ b/tests/integration/helpers/docker_compose_cassandra.yml @@ -4,4 +4,4 @@ services: image: cassandra restart: always ports: - - 6340:6349 + - 9043:9042 diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py index 493cccc8482..2dad70bc913 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py @@ -8,6 +8,7 @@ import aerospike from tzlocal import get_localzone import datetime import os +import uuid class ExternalSource(object): @@ -407,23 +408,70 @@ class SourceHTTPS(SourceHTTPBase): return "https" class SourceCassandra(ExternalSource): + TYPE_MAPPING = { + 'UInt8': 'tinyint', + 'UInt16': 'smallint', + 'UInt32': 'int', + 'UInt64': 'bigint', + 'Int8': 'tinyint', + 'Int16': 'smallint', + 'Int32': 'int', + 'Int64': 'bigint', + 'UUID': 'uuid', + 'Date': 'date', + 'DateTime': 'timestamp', + 'String': 'text', + 'Float32': 'float', + 'Float64': 'double' + } + + def __init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password): + ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password) + self.structure = dict() + def get_source_str(self, table_name): return ''' {host} {port} + test + {table} '''.format( host=self.docker_hostname, port=self.docker_port, + table=table_name, ) def prepare(self, structure, table_name, cluster): self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port) + self.session = self.client.connect() + self.session.execute("create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};") + self.structure[table_name] = structure + columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()] + keys = ['"' + col.name + '"' for col in structure.keys] + # FIXME use partition key + query = 'create table test."{name}" ({columns}, primary key ("{some_col}", {pk}));'.format( + name=table_name, columns=', '.join(columns), some_col=structure.ordinary_fields[0].name, pk=', '.join(keys)) + self.session.execute(query) self.prepared = True + def get_value_to_insert(self, value, type): + if type == 'UUID': + return uuid.UUID(value) + elif type == 'DateTime': + local_datetime = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + return get_localzone().localize(local_datetime) + return value + def load_data(self, data, table_name): - pass + names_and_types = [(field.name, field.field_type) for field in self.structure[table_name].get_all_fields()] + columns = ['"' + col[0] + '"' for col in names_and_types] + insert = 'insert into test."{table}" ({columns}) values ({args})'.format( + table=table_name, columns=','.join(columns), args=','.join(['%s']*len(columns))) + for row in data: + values = [self.get_value_to_insert(row.get_value_by_name(col[0]), col[1]) for col in names_and_types] + self.session.execute(insert, values) class SourceRedis(ExternalSource): def __init__( diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index 2e1d3ed4c70..98ba191c948 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -8,7 +8,7 @@ from external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceEx from external_sources import SourceMongo, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra from external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra import math - +import time SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries') @@ -119,7 +119,7 @@ LAYOUTS = [ ] SOURCES = [ - SourceCassandra("Cassandra", "localhost", "6340", "cassandra1", "6349", "", ""), + SourceCassandra("Cassandra", "localhost", "9043", "cassandra1", "9042", "", ""), SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), SourceMongoURI("MongoDB_URI", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse"), @@ -134,7 +134,7 @@ SOURCES = [ DICTIONARIES = [] -# Key-value dictionaries with onle one possible field for key +# Key-value dictionaries with only one possible field for key SOURCES_KV = [ SourceRedis("RedisSimple", "localhost", "6380", "redis1", "6379", "", "", storage_type="simple"), SourceRedis("RedisHash", "localhost", "6380", "redis1", "6379", "", "", storage_type="hash_map"), @@ -212,6 +212,7 @@ def get_dictionaries(fold, total_folds, all_dicts): return all_dicts[fold * chunk_len : (fold + 1) * chunk_len] +#@pytest.mark.timeout(3000) @pytest.mark.parametrize("fold", list(range(10))) def test_simple_dictionaries(started_cluster, fold): fields = FIELDS["simple"] @@ -227,6 +228,8 @@ def test_simple_dictionaries(started_cluster, fold): node.query("system reload dictionaries") + #time.sleep(3000) + queries_with_answers = [] for dct in simple_dicts: for row in data: From 415c8469a355f076e327d88d48e2d820996f2962 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 27 May 2020 16:51:23 +0300 Subject: [PATCH 10/29] comaptibility with glibc 2.4 --- .gitmodules | 4 +- contrib/cassandra | 2 +- contrib/libuv | 2 +- contrib/libuv-cmake/CMakeLists.txt | 538 ++++++++++------------------- 4 files changed, 196 insertions(+), 350 deletions(-) diff --git a/.gitmodules b/.gitmodules index bbc9f15ffd8..a4c84301fc9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -163,8 +163,8 @@ branch = ch-tmp [submodule "contrib/libuv"] path = contrib/libuv - url = https://github.com/libuv/libuv.git - branch = v1.x + url = https://github.com/tavplubix/libuv.git + branch = clickhouse [submodule "contrib/fmtlib"] path = contrib/fmtlib url = https://github.com/fmtlib/fmt.git diff --git a/contrib/cassandra b/contrib/cassandra index 9606ff1f70b..58a71947d9d 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit 9606ff1f70bd3fc5d395df32e626923c012ffb5f +Subproject commit 58a71947d9dd8412f5aeb38275fa81417ea27ee0 diff --git a/contrib/libuv b/contrib/libuv index cc51217a317..379988fef9b 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit cc51217a317e96510fbb284721d5e6bc2af31e33 +Subproject commit 379988fef9b0c6ac706a624dbac6be8924a3a0da diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt index b84ce217f3b..a2869e037ff 100644 --- a/contrib/libuv-cmake/CMakeLists.txt +++ b/contrib/libuv-cmake/CMakeLists.txt @@ -1,96 +1,23 @@ cmake_minimum_required(VERSION 3.4) project(libuv LANGUAGES C) -cmake_policy(SET CMP0057 NEW) # Enable IN_LIST operator -cmake_policy(SET CMP0064 NEW) # Support if (TEST) operator - -#list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") - include(CMakePackageConfigHelpers) include(CMakeDependentOption) -include(CheckCCompilerFlag) include(GNUInstallDirs) include(CTest) -set(CMAKE_C_VISIBILITY_PRESET hidden) -set(CMAKE_C_STANDARD_REQUIRED ON) -set(CMAKE_C_EXTENSIONS ON) -set(CMAKE_C_STANDARD 90) - #cmake_dependent_option(LIBUV_BUILD_TESTS # "Build the unit tests when BUILD_TESTING is enabled and we are the root project" ON # "BUILD_TESTING;CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR" OFF) -#cmake_dependent_option(LIBUV_BUILD_BENCH -# "Build the benchmarks when building unit tests and we are the root project" ON -# "LIBUV_BUILD_TESTS" OFF) -# Qemu Build -option(QEMU "build for qemu" OFF) -if(QEMU) - add_definitions(-D__QEMU__=1) +if(MSVC) + list(APPEND uv_cflags /W4) +elseif(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") + list(APPEND uv_cflags -fvisibility=hidden --std=gnu89) + list(APPEND uv_cflags -Wall -Wextra -Wstrict-prototypes) + list(APPEND uv_cflags -Wno-unused-parameter) endif() -# Compiler check -string(CONCAT is-msvc $, - $ ->) - -check_c_compiler_flag(/W4 UV_LINT_W4) -check_c_compiler_flag(/wd4100 UV_LINT_NO_UNUSED_PARAMETER_MSVC) -check_c_compiler_flag(/wd4127 UV_LINT_NO_CONDITIONAL_CONSTANT_MSVC) -check_c_compiler_flag(/wd4201 UV_LINT_NO_NONSTANDARD_MSVC) -check_c_compiler_flag(/wd4206 UV_LINT_NO_NONSTANDARD_EMPTY_TU_MSVC) -check_c_compiler_flag(/wd4210 UV_LINT_NO_NONSTANDARD_FILE_SCOPE_MSVC) -check_c_compiler_flag(/wd4232 UV_LINT_NO_NONSTANDARD_NONSTATIC_DLIMPORT_MSVC) -check_c_compiler_flag(/wd4456 UV_LINT_NO_HIDES_LOCAL) -check_c_compiler_flag(/wd4457 UV_LINT_NO_HIDES_PARAM) -check_c_compiler_flag(/wd4459 UV_LINT_NO_HIDES_GLOBAL) -check_c_compiler_flag(/wd4706 UV_LINT_NO_CONDITIONAL_ASSIGNMENT_MSVC) -check_c_compiler_flag(/wd4996 UV_LINT_NO_UNSAFE_MSVC) - -check_c_compiler_flag(-Wall UV_LINT_WALL) # DO NOT use this under MSVC - -# TODO: Place these into its own function -check_c_compiler_flag(-Wno-unused-parameter UV_LINT_NO_UNUSED_PARAMETER) -check_c_compiler_flag(-Wstrict-prototypes UV_LINT_STRICT_PROTOTYPES) -check_c_compiler_flag(-Wextra UV_LINT_EXTRA) - -set(lint-no-unused-parameter $<$:-Wno-unused-parameter>) -set(lint-strict-prototypes $<$:-Wstrict-prototypes>) -set(lint-extra $<$:-Wextra>) -set(lint-w4 $<$:/W4>) -set(lint-no-unused-parameter-msvc $<$:/wd4100>) -set(lint-no-conditional-constant-msvc $<$:/wd4127>) -set(lint-no-nonstandard-msvc $<$:/wd4201>) -set(lint-no-nonstandard-empty-tu-msvc $<$:/wd4206>) -set(lint-no-nonstandard-file-scope-msvc $<$:/wd4210>) -set(lint-no-nonstandard-nonstatic-dlimport-msvc $<$:/wd4232>) -set(lint-no-hides-local-msvc $<$:/wd4456>) -set(lint-no-hides-param-msvc $<$:/wd4457>) -set(lint-no-hides-global-msvc $<$:/wd4459>) -set(lint-no-conditional-assignment-msvc $<$:/wd4706>) -set(lint-no-unsafe-msvc $<$:/wd4996>) -# Unfortunately, this one is complicated because MSVC and clang-cl support -Wall -# but using it is like calling -Weverything -string(CONCAT lint-default $< - $,$>:-Wall ->) - -list(APPEND uv_cflags ${lint-strict-prototypes} ${lint-extra} ${lint-default} ${lint-w4}) -list(APPEND uv_cflags ${lint-no-unused-parameter}) -list(APPEND uv_cflags ${lint-no-unused-parameter-msvc}) -list(APPEND uv_cflags ${lint-no-conditional-constant-msvc}) -list(APPEND uv_cflags ${lint-no-nonstandard-msvc}) -list(APPEND uv_cflags ${lint-no-nonstandard-empty-tu-msvc}) -list(APPEND uv_cflags ${lint-no-nonstandard-file-scope-msvc}) -list(APPEND uv_cflags ${lint-no-nonstandard-nonstatic-dlimport-msvc}) -list(APPEND uv_cflags ${lint-no-hides-local-msvc}) -list(APPEND uv_cflags ${lint-no-hides-param-msvc}) -list(APPEND uv_cflags ${lint-no-hides-global-msvc}) -list(APPEND uv_cflags ${lint-no-conditional-assignment-msvc}) -list(APPEND uv_cflags ${lint-no-unsafe-msvc}) - set(uv_sources src/fs-poll.c src/idna.c @@ -103,11 +30,172 @@ set(uv_sources src/uv-data-getter-setters.c src/version.c) +set(uv_test_sources + test/blackhole-server.c + test/echo-server.c + test/run-tests.c + test/runner.c + test/test-active.c + test/test-async-null-cb.c + test/test-async.c + test/test-barrier.c + test/test-callback-order.c + test/test-callback-stack.c + test/test-close-fd.c + test/test-close-order.c + test/test-condvar.c + test/test-connect-unspecified.c + test/test-connection-fail.c + test/test-cwd-and-chdir.c + test/test-default-loop-close.c + test/test-delayed-accept.c + test/test-dlerror.c + test/test-eintr-handling.c + test/test-embed.c + test/test-emfile.c + test/test-env-vars.c + test/test-error.c + test/test-fail-always.c + test/test-fork.c + test/test-fs-copyfile.c + test/test-fs-event.c + test/test-fs-poll.c + test/test-fs.c + test/test-fs-readdir.c + test/test-fs-fd-hash.c + test/test-fs-open-flags.c + test/test-get-currentexe.c + test/test-get-loadavg.c + test/test-get-memory.c + test/test-get-passwd.c + test/test-getaddrinfo.c + test/test-gethostname.c + test/test-getnameinfo.c + test/test-getsockname.c + test/test-getters-setters.c + test/test-gettimeofday.c + test/test-handle-fileno.c + test/test-homedir.c + test/test-hrtime.c + test/test-idle.c + test/test-idna.c + test/test-ip4-addr.c + test/test-ip6-addr.c + test/test-ipc-heavy-traffic-deadlock-bug.c + test/test-ipc-send-recv.c + test/test-ipc.c + test/test-loop-alive.c + test/test-loop-close.c + test/test-loop-configure.c + test/test-loop-handles.c + test/test-loop-stop.c + test/test-loop-time.c + test/test-multiple-listen.c + test/test-mutexes.c + test/test-osx-select.c + test/test-pass-always.c + test/test-ping-pong.c + test/test-pipe-bind-error.c + test/test-pipe-close-stdout-read-stdin.c + test/test-pipe-connect-error.c + test/test-pipe-connect-multiple.c + test/test-pipe-connect-prepare.c + test/test-pipe-getsockname.c + test/test-pipe-pending-instances.c + test/test-pipe-sendmsg.c + test/test-pipe-server-close.c + test/test-pipe-set-fchmod.c + test/test-pipe-set-non-blocking.c + test/test-platform-output.c + test/test-poll-close-doesnt-corrupt-stack.c + test/test-poll-close.c + test/test-poll-closesocket.c + test/test-poll-oob.c + test/test-poll.c + test/test-process-priority.c + test/test-process-title-threadsafe.c + test/test-process-title.c + test/test-queue-foreach-delete.c + test/test-random.c + test/test-ref.c + test/test-run-nowait.c + test/test-run-once.c + test/test-semaphore.c + test/test-shutdown-close.c + test/test-shutdown-eof.c + test/test-shutdown-twice.c + test/test-signal-multiple-loops.c + test/test-signal-pending-on-close.c + test/test-signal.c + test/test-socket-buffer-size.c + test/test-spawn.c + test/test-stdio-over-pipes.c + test/test-strscpy.c + test/test-tcp-alloc-cb-fail.c + test/test-tcp-bind-error.c + test/test-tcp-bind6-error.c + test/test-tcp-close-accept.c + test/test-tcp-close-while-connecting.c + test/test-tcp-close.c + test/test-tcp-close-reset.c + test/test-tcp-connect-error-after-write.c + test/test-tcp-connect-error.c + test/test-tcp-connect-timeout.c + test/test-tcp-connect6-error.c + test/test-tcp-create-socket-early.c + test/test-tcp-flags.c + test/test-tcp-oob.c + test/test-tcp-open.c + test/test-tcp-read-stop.c + test/test-tcp-shutdown-after-write.c + test/test-tcp-try-write.c + test/test-tcp-try-write-error.c + test/test-tcp-unexpected-read.c + test/test-tcp-write-after-connect.c + test/test-tcp-write-fail.c + test/test-tcp-write-queue-order.c + test/test-tcp-write-to-half-open-connection.c + test/test-tcp-writealot.c + test/test-thread-equal.c + test/test-thread.c + test/test-threadpool-cancel.c + test/test-threadpool.c + test/test-timer-again.c + test/test-timer-from-check.c + test/test-timer.c + test/test-tmpdir.c + test/test-tty-duplicate-key.c + test/test-tty.c + test/test-udp-alloc-cb-fail.c + test/test-udp-bind.c + test/test-udp-connect.c + test/test-udp-create-socket-early.c + test/test-udp-dgram-too-big.c + test/test-udp-ipv6.c + test/test-udp-multicast-interface.c + test/test-udp-multicast-interface6.c + test/test-udp-multicast-join.c + test/test-udp-multicast-join6.c + test/test-udp-multicast-ttl.c + test/test-udp-open.c + test/test-udp-options.c + test/test-udp-send-and-recv.c + test/test-udp-send-hang-loop.c + test/test-udp-send-immediate.c + test/test-udp-send-unreachable.c + test/test-udp-try-send.c + test/test-uname.c + test/test-walk-handles.c + test/test-watcher-cross-stop.c) + #if(WIN32) # list(APPEND uv_defines WIN32_LEAN_AND_MEAN _WIN32_WINNT=0x0600) # list(APPEND uv_libraries -# psapi +# advapi32 # iphlpapi +# psapi +# shell32 +# user32 # userenv # ws2_32) # list(APPEND uv_sources @@ -140,8 +228,7 @@ set(uv_sources # list(APPEND uv_test_sources src/win/snprintf.c test/runner-win.c) #else() list(APPEND uv_defines _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE) -if(NOT CMAKE_SYSTEM_NAME MATCHES "Android|OS390") - # TODO: This should be replaced with find_package(Threads) if possible +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") # Android has pthread as part of its c library, not as a separate # libpthread.so. list(APPEND uv_libraries pthread) @@ -173,16 +260,13 @@ if(CMAKE_SYSTEM_NAME STREQUAL "AIX") _ALL_SOURCE _LINUX_SOURCE_COMPAT _THREAD_SAFE - _XOPEN_SOURCE=500 - HAVE_SYS_AHAFS_EVPRODS_H) + _XOPEN_SOURCE=500) list(APPEND uv_libraries perfstat) - list(APPEND uv_sources - src/unix/aix.c - src/unix/aix-common.c) + list(APPEND uv_sources src/unix/aix.c) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Android") - list(APPEND uv_libraries dl) + list(APPEND uv_libs dl) list(APPEND uv_sources src/unix/android-ifaddrs.c src/unix/linux-core.c @@ -190,13 +274,12 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android") src/unix/linux-syscalls.c src/unix/procfs-exepath.c src/unix/pthread-fixes.c - src/unix/random-getentropy.c src/unix/random-getrandom.c src/unix/random-sysctl-linux.c src/unix/sysinfo-loadavg.c) endif() -if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "Android|Linux|OS390") +if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "Android|Linux|OS/390") list(APPEND uv_sources src/unix/proctitle.c) endif() @@ -250,11 +333,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") list(APPEND uv_sources src/unix/openbsd.c) endif() -if(CMAKE_SYSTEM_NAME STREQUAL "OS390") +if(CMAKE_SYSTEM_NAME STREQUAL "OS/390") list(APPEND uv_defines PATH_MAX=255) list(APPEND uv_defines _AE_BIMODAL) list(APPEND uv_defines _ALL_SOURCE) - list(APPEND uv_defines _ISOC99_SOURCE) list(APPEND uv_defines _LARGE_TIME_API) list(APPEND uv_defines _OPEN_MSGQ_EXT) list(APPEND uv_defines _OPEN_SYS_FILE_EXT) @@ -267,25 +349,9 @@ if(CMAKE_SYSTEM_NAME STREQUAL "OS390") list(APPEND uv_defines _XOPEN_SOURCE_EXTENDED) list(APPEND uv_sources src/unix/pthread-fixes.c + src/unix/pthread-barrier.c src/unix/os390.c src/unix/os390-syscalls.c) - list(APPEND uv_cflags -Wc,DLL -Wc,exportall -Wc,xplink) - list(APPEND uv_libraries -Wl,xplink) - list(APPEND uv_test_libraries -Wl,xplink) -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "OS400") - list(APPEND uv_defines - _ALL_SOURCE - _LINUX_SOURCE_COMPAT - _THREAD_SAFE - _XOPEN_SOURCE=500) - list(APPEND uv_sources - src/unix/aix-common.c - src/unix/ibmi.c - src/unix/no-fsevents.c - src/unix/no-proctitle.c - src/unix/posix-poll.c) endif() if(CMAKE_SYSTEM_NAME STREQUAL "SunOS") @@ -304,225 +370,24 @@ foreach(file ${uv_sources}) endforeach(file) set(uv_sources "${uv_sources_tmp}") +list(APPEND uv_defines CLICKHOUSE_GLIBC_COMPATIBILITY) + add_library(uv SHARED ${uv_sources}) target_compile_definitions(uv - INTERFACE - USING_UV_SHARED=1 - PRIVATE - BUILDING_UV_SHARED=1 - ${uv_defines}) + INTERFACE USING_UV_SHARED=1 + PRIVATE ${uv_defines} BUILDING_UV_SHARED=1) target_compile_options(uv PRIVATE ${uv_cflags}) -target_include_directories(uv - PUBLIC - $ - $ - PRIVATE - $) +target_include_directories(uv PUBLIC ${LIBUV_ROOT_DIR}/include PRIVATE ${LIBUV_ROOT_DIR}/src) target_link_libraries(uv ${uv_libraries}) add_library(uv_a STATIC ${uv_sources}) target_compile_definitions(uv_a PRIVATE ${uv_defines}) target_compile_options(uv_a PRIVATE ${uv_cflags}) -target_include_directories(uv_a - PUBLIC - $ - $ - PRIVATE - $) +target_include_directories(uv_a PUBLIC ${LIBUV_ROOT_DIR}/include PRIVATE ${LIBUV_ROOT_DIR}/src) target_link_libraries(uv_a ${uv_libraries}) #if(LIBUV_BUILD_TESTS) -# # Small hack: use ${uv_test_sources} now to get the runner skeleton, -# # before the actual tests are added. -# add_executable( -# uv_run_benchmarks_a -# ${uv_test_sources} -# test/benchmark-async-pummel.c -# test/benchmark-async.c -# test/benchmark-fs-stat.c -# test/benchmark-getaddrinfo.c -# test/benchmark-loop-count.c -# test/benchmark-million-async.c -# test/benchmark-million-timers.c -# test/benchmark-multi-accept.c -# test/benchmark-ping-pongs.c -# test/benchmark-ping-udp.c -# test/benchmark-pound.c -# test/benchmark-pump.c -# test/benchmark-sizes.c -# test/benchmark-spawn.c -# test/benchmark-tcp-write-batch.c -# test/benchmark-thread.c -# test/benchmark-udp-pummel.c -# test/blackhole-server.c -# test/dns-server.c -# test/echo-server.c -# test/run-benchmarks.c -# test/runner.c) -# target_compile_definitions(uv_run_benchmarks_a PRIVATE ${uv_defines}) -# target_compile_options(uv_run_benchmarks_a PRIVATE ${uv_cflags}) -# target_link_libraries(uv_run_benchmarks_a uv_a ${uv_test_libraries}) -# -# list(APPEND uv_test_sources -# test/blackhole-server.c -# test/echo-server.c -# test/run-tests.c -# test/runner.c -# test/test-active.c -# test/test-async-null-cb.c -# test/test-async.c -# test/test-barrier.c -# test/test-callback-order.c -# test/test-callback-stack.c -# test/test-close-fd.c -# test/test-close-order.c -# test/test-condvar.c -# test/test-connect-unspecified.c -# test/test-connection-fail.c -# test/test-cwd-and-chdir.c -# test/test-default-loop-close.c -# test/test-delayed-accept.c -# test/test-dlerror.c -# test/test-eintr-handling.c -# test/test-embed.c -# test/test-emfile.c -# test/test-env-vars.c -# test/test-error.c -# test/test-fail-always.c -# test/test-fork.c -# test/test-fs-copyfile.c -# test/test-fs-event.c -# test/test-fs-poll.c -# test/test-fs.c -# test/test-fs-readdir.c -# test/test-fs-fd-hash.c -# test/test-fs-open-flags.c -# test/test-get-currentexe.c -# test/test-get-loadavg.c -# test/test-get-memory.c -# test/test-get-passwd.c -# test/test-getaddrinfo.c -# test/test-gethostname.c -# test/test-getnameinfo.c -# test/test-getsockname.c -# test/test-getters-setters.c -# test/test-gettimeofday.c -# test/test-handle-fileno.c -# test/test-homedir.c -# test/test-hrtime.c -# test/test-idle.c -# test/test-idna.c -# test/test-ip4-addr.c -# test/test-ip6-addr.c -# test/test-ipc-heavy-traffic-deadlock-bug.c -# test/test-ipc-send-recv.c -# test/test-ipc.c -# test/test-loop-alive.c -# test/test-loop-close.c -# test/test-loop-configure.c -# test/test-loop-handles.c -# test/test-loop-stop.c -# test/test-loop-time.c -# test/test-multiple-listen.c -# test/test-mutexes.c -# test/test-osx-select.c -# test/test-pass-always.c -# test/test-ping-pong.c -# test/test-pipe-bind-error.c -# test/test-pipe-close-stdout-read-stdin.c -# test/test-pipe-connect-error.c -# test/test-pipe-connect-multiple.c -# test/test-pipe-connect-prepare.c -# test/test-pipe-getsockname.c -# test/test-pipe-pending-instances.c -# test/test-pipe-sendmsg.c -# test/test-pipe-server-close.c -# test/test-pipe-set-fchmod.c -# test/test-pipe-set-non-blocking.c -# test/test-platform-output.c -# test/test-poll-close-doesnt-corrupt-stack.c -# test/test-poll-close.c -# test/test-poll-closesocket.c -# test/test-poll-oob.c -# test/test-poll.c -# test/test-process-priority.c -# test/test-process-title-threadsafe.c -# test/test-process-title.c -# test/test-queue-foreach-delete.c -# test/test-random.c -# test/test-ref.c -# test/test-run-nowait.c -# test/test-run-once.c -# test/test-semaphore.c -# test/test-shutdown-close.c -# test/test-shutdown-eof.c -# test/test-shutdown-twice.c -# test/test-signal-multiple-loops.c -# test/test-signal-pending-on-close.c -# test/test-signal.c -# test/test-socket-buffer-size.c -# test/test-spawn.c -# test/test-stdio-over-pipes.c -# test/test-strscpy.c -# test/test-tcp-alloc-cb-fail.c -# test/test-tcp-bind-error.c -# test/test-tcp-bind6-error.c -# test/test-tcp-close-accept.c -# test/test-tcp-close-while-connecting.c -# test/test-tcp-close.c -# test/test-tcp-close-reset.c -# test/test-tcp-connect-error-after-write.c -# test/test-tcp-connect-error.c -# test/test-tcp-connect-timeout.c -# test/test-tcp-connect6-error.c -# test/test-tcp-create-socket-early.c -# test/test-tcp-flags.c -# test/test-tcp-oob.c -# test/test-tcp-open.c -# test/test-tcp-read-stop.c -# test/test-tcp-shutdown-after-write.c -# test/test-tcp-try-write.c -# test/test-tcp-try-write-error.c -# test/test-tcp-unexpected-read.c -# test/test-tcp-write-after-connect.c -# test/test-tcp-write-fail.c -# test/test-tcp-write-queue-order.c -# test/test-tcp-write-to-half-open-connection.c -# test/test-tcp-writealot.c -# test/test-thread-equal.c -# test/test-thread.c -# test/test-threadpool-cancel.c -# test/test-threadpool.c -# test/test-timer-again.c -# test/test-timer-from-check.c -# test/test-timer.c -# test/test-tmpdir.c -# test/test-tty-duplicate-key.c -# test/test-tty-escape-sequence-processing.c -# test/test-tty.c -# test/test-udp-alloc-cb-fail.c -# test/test-udp-bind.c -# test/test-udp-connect.c -# test/test-udp-create-socket-early.c -# test/test-udp-dgram-too-big.c -# test/test-udp-ipv6.c -# test/test-udp-multicast-interface.c -# test/test-udp-multicast-interface6.c -# test/test-udp-multicast-join.c -# test/test-udp-multicast-join6.c -# test/test-udp-multicast-ttl.c -# test/test-udp-open.c -# test/test-udp-options.c -# test/test-udp-send-and-recv.c -# test/test-udp-send-hang-loop.c -# test/test-udp-send-immediate.c -# test/test-udp-send-unreachable.c -# test/test-udp-try-send.c -# test/test-uname.c -# test/test-walk-handles.c -# test/test-watcher-cross-stop.c) -# -# add_executable(uv_run_tests ${uv_test_sources} uv_win_longpath.manifest) +# add_executable(uv_run_tests ${uv_test_sources}) # target_compile_definitions(uv_run_tests # PRIVATE ${uv_defines} USING_UV_SHARED=1) # target_compile_options(uv_run_tests PRIVATE ${uv_cflags}) @@ -530,18 +395,10 @@ target_link_libraries(uv_a ${uv_libraries}) # add_test(NAME uv_test # COMMAND uv_run_tests # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -# if(CMAKE_SYSTEM_NAME STREQUAL "OS390") -# set_tests_properties(uv_test PROPERTIES ENVIRONMENT -# "LIBPATH=${CMAKE_BINARY_DIR}:$ENV{LIBPATH}") -# endif() -# add_executable(uv_run_tests_a ${uv_test_sources} uv_win_longpath.manifest) +# add_executable(uv_run_tests_a ${uv_test_sources}) # target_compile_definitions(uv_run_tests_a PRIVATE ${uv_defines}) # target_compile_options(uv_run_tests_a PRIVATE ${uv_cflags}) -# if(QEMU) -# target_link_libraries(uv_run_tests_a uv_a ${uv_test_libraries} -static) -# else() -# target_link_libraries(uv_run_tests_a uv_a ${uv_test_libraries}) -# endif() +# target_link_libraries(uv_run_tests_a uv_a ${uv_test_libraries}) # add_test(NAME uv_test_a # COMMAND uv_run_tests_a # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) @@ -549,18 +406,15 @@ target_link_libraries(uv_a ${uv_libraries}) if(UNIX) # Now for some gibbering horrors from beyond the stars... - foreach(lib IN LISTS uv_libraries) - list(APPEND LIBS "-l${lib}") - endforeach() - string(REPLACE ";" " " LIBS "${LIBS}") - # Consider setting project version via project() call? + foreach(x ${uv_libraries}) + set(LIBS "${LIBS} -l${x}") + endforeach(x) file(STRINGS ${LIBUV_ROOT_DIR}/configure.ac configure_ac REGEX ^AC_INIT) - string(REGEX MATCH "([0-9]+)[.][0-9]+[.][0-9]+" PACKAGE_VERSION "${configure_ac}") - set(UV_VERSION_MAJOR "${CMAKE_MATCH_1}") + string(REGEX MATCH [0-9]+[.][0-9]+[.][0-9]+ PACKAGE_VERSION "${configure_ac}") + string(REGEX MATCH ^[0-9]+ UV_VERSION_MAJOR "${PACKAGE_VERSION}") # The version in the filename is mirroring the behaviour of autotools. - set_target_properties(uv PROPERTIES - VERSION ${UV_VERSION_MAJOR}.0.0 - SOVERSION ${UV_VERSION_MAJOR}) + set_target_properties(uv PROPERTIES VERSION ${UV_VERSION_MAJOR}.0.0 + SOVERSION ${UV_VERSION_MAJOR}) set(includedir ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) set(libdir ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}) set(prefix ${CMAKE_INSTALL_PREFIX}) @@ -568,7 +422,7 @@ if(UNIX) install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DOCDIR}) - install(FILES ${PROJECT_BINARY_DIR}/libuv.pc + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libuv.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) install(TARGETS uv LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) install(TARGETS uv_a ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) @@ -581,11 +435,3 @@ endif() # RUNTIME DESTINATION lib/$ # ARCHIVE DESTINATION lib/$) #endif() -# -#message(STATUS "summary of build options: -# Install prefix: ${CMAKE_INSTALL_PREFIX} -# Target system: ${CMAKE_SYSTEM_NAME} -# Compiler: -# C compiler: ${CMAKE_C_COMPILER} -# CFLAGS: ${CMAKE_C_FLAGS_${_build_type}} ${CMAKE_C_FLAGS} -#") From f099ce19a7dde2ab9c1d9e37b1f932cb32cf4480 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 27 May 2020 16:54:39 +0300 Subject: [PATCH 11/29] fix --- src/Dictionaries/CassandraBlockInputStream.h | 2 +- tests/integration/helpers/cluster.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dictionaries/CassandraBlockInputStream.h b/src/Dictionaries/CassandraBlockInputStream.h index c8476bd2c15..22e4429343d 100644 --- a/src/Dictionaries/CassandraBlockInputStream.h +++ b/src/Dictionaries/CassandraBlockInputStream.h @@ -9,7 +9,7 @@ namespace DB { -void cassandraCheck(CassError error); +void cassandraCheck(CassError code); void cassandraWaitAndCheck(CassFuture * future); diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 44e08d5bf6a..49a18d14796 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -460,7 +460,7 @@ class ClickHouseCluster: logging.warning("Can't connect to SchemaRegistry: %s", str(ex)) time.sleep(1) - def wait_cassandra_to_start(self, timeout=15): + def wait_cassandra_to_start(self, timeout=30): cass_client = cassandra.cluster.Cluster(["localhost"], port="9043") start = time.time() while time.time() - start < timeout: From 905bce4aebb6925f37f33806cfd9642a51cec69c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 27 May 2020 23:13:25 +0300 Subject: [PATCH 12/29] fix --- contrib/libuv-cmake/CMakeLists.txt | 6 ++++- .../CassandraBlockInputStream.cpp | 21 +++++++++-------- .../CassandraDictionarySource.cpp | 23 ++++++++----------- tests/integration/helpers/cluster.py | 6 ++--- .../external_sources.py | 1 + 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt index a2869e037ff..65aeabdd9cb 100644 --- a/contrib/libuv-cmake/CMakeLists.txt +++ b/contrib/libuv-cmake/CMakeLists.txt @@ -227,7 +227,11 @@ set(uv_test_sources # list(APPEND uv_test_libraries ws2_32) # list(APPEND uv_test_sources src/win/snprintf.c test/runner-win.c) #else() -list(APPEND uv_defines _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE) + +if(CMAKE_SIZEOF_VOID_P EQUAL 4) + list(APPEND uv_defines _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE) +endif() + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") # Android has pthread as part of its c library, not as a separate # libpthread.so. diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index 473a42549a8..6d8a45508ce 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -1,16 +1,17 @@ -# include -# include -# include -# include -# include -# include -# include +#if !defined(ARCADIA_BUILD) +#include +#endif #if USE_CASSANDRA -# include -# include "CassandraBlockInputStream.h" -# include "CassandraBlockInputStream.h" +#include +#include +#include +#include +#include +#include +#include +#include "CassandraBlockInputStream.h" namespace DB diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index 4150fe56f14..2aee5d44b6b 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -13,19 +13,16 @@ namespace DB void registerDictionarySourceCassandra(DictionarySourceFactory & factory) { - auto create_table_source = [=](const DictionaryStructure & dict_struct, - const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, - Block & sample_block, - const Context & /* context */, - bool /*check_config*/) -> DictionarySourcePtr { + auto create_table_source = [=]([[maybe_unused]] const DictionaryStructure & dict_struct, + [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, + [[maybe_unused]] const std::string & config_prefix, + [[maybe_unused]] Block & sample_block, + const Context & /* context */, + bool /*check_config*/) -> DictionarySourcePtr + { #if USE_CASSANDRA return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); #else - (void)dict_struct; - (void)config; - (void)config_prefix; - (void)sample_block; throw Exception{"Dictionary source of type `cassandra` is disabled because library was built without cassandra support.", ErrorCodes::SUPPORT_IS_DISABLED}; #endif @@ -37,9 +34,9 @@ namespace DB #if USE_CASSANDRA -# include -# include -# include "CassandraBlockInputStream.h" +#include +#include +#include "CassandraBlockInputStream.h" namespace DB { diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 49a18d14796..a539db1d47f 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -465,11 +465,11 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - cass_client.connect().execute("drop keyspace if exists test;") - logging.info("Connected to Cassandra %s") + cass_client.connect() + logging.info("Connected to Cassandra") return except Exception as ex: - logging.warning("Can't connect to Minio: %s", str(ex)) + logging.warning("Can't connect to Cassandra: %s", str(ex)) time.sleep(1) def start(self, destroy_dirs=True): diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py index 2dad70bc913..336f3ddc28b 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py @@ -447,6 +447,7 @@ class SourceCassandra(ExternalSource): self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port) self.session = self.client.connect() self.session.execute("create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};") + self.session.execute('drop table if exists test."{}"'.format(table_name)) self.structure[table_name] = structure columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()] keys = ['"' + col.name + '"' for col in structure.keys] From aade00130c2a1e76b83ae595be8c1ae9c0c0e39b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 1 Jun 2020 20:35:30 +0300 Subject: [PATCH 13/29] add raii wrappers --- .../CassandraBlockInputStream.cpp | 88 ++++--------------- src/Dictionaries/CassandraBlockInputStream.h | 16 ++-- .../CassandraDictionarySource.cpp | 27 ++---- src/Dictionaries/CassandraDictionarySource.h | 24 +---- src/Dictionaries/CassandraHelpers.cpp | 38 ++++++++ src/Dictionaries/CassandraHelpers.h | 72 +++++++++++++++ src/Dictionaries/ya.make | 1 + 7 files changed, 140 insertions(+), 126 deletions(-) create mode 100644 src/Dictionaries/CassandraHelpers.cpp create mode 100644 src/Dictionaries/CassandraHelpers.h diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index 6d8a45508ce..e00fd5ec3e9 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -23,26 +23,17 @@ namespace ErrorCodes } CassandraBlockInputStream::CassandraBlockInputStream( - CassSession * session_, - const std::string &query_str_, - const DB::Block &sample_block, + const CassClusterPtr & cluster, + const String & query_str, + const Block & sample_block, const size_t max_block_size_) - : session(session_) - , statement(cass_statement_new(query_str_.c_str(), 0)) - , query_str(query_str_) + : statement(query_str.c_str(), /*parameters count*/ 0) , max_block_size(max_block_size_) + , has_more_pages(cass_true) { - cass_statement_set_paging_size(statement, max_block_size); - this->has_more_pages = cass_true; - description.init(sample_block); -} - -CassandraBlockInputStream::~CassandraBlockInputStream() { - if (iterator != nullptr) - cass_iterator_free(iterator); - if (result) - cass_result_free(result); + cassandraCheck(cass_statement_set_paging_size(statement, max_block_size)); + cassandraWaitAndCheck(cass_session_connect(session, cluster)); } namespace @@ -51,7 +42,7 @@ namespace void insertValue(IColumn & column, const ValueType type, const CassValue * cass_value) { - /// Cassandra does not support unsigned integers + /// Cassandra does not support unsigned integers (cass_uint32_t is for Date) switch (type) { case ValueType::vtUInt8: @@ -159,19 +150,17 @@ namespace } } - // void insertDefaultValue(IColumn & column, const IColumn & sample_column) { column.insertFrom(sample_column, 0); } - Block CassandraBlockInputStream::readImpl() { if (!has_more_pages) return {}; MutableColumns columns = description.sample_block.cloneEmptyColumns(); - CassFuture* query_future = cass_session_execute(session, statement); + CassFuturePtr query_future = cass_session_execute(session, statement); - result = cass_future_get_result(query_future); + CassResultPtr result = cass_future_get_result(query_future); - if (result == nullptr) { + if (!result) { const char* error_message; size_t error_message_length; cass_future_error_message(query_future, &error_message, &error_message_length); @@ -181,7 +170,7 @@ namespace [[maybe_unused]] size_t row_count = 0; assert(cass_result_column_count(result) == columns.size()); - CassIterator * rows_iter = cass_iterator_from_result(result); /// Points to rows[-1] + CassIteratorPtr rows_iter = cass_iterator_from_result(result); /// Points to rows[-1] while (cass_iterator_next(rows_iter)) { const CassRow * row = cass_iterator_get_row(rows_iter); @@ -192,7 +181,7 @@ namespace columns[col_idx]->insertDefault(); else if (description.types[col_idx].second) { - ColumnNullable & column_nullable = static_cast(*columns[col_idx]); + ColumnNullable & column_nullable = assert_cast(*columns[col_idx]); insertValue(column_nullable.getNestedColumn(), description.types[col_idx].first, val); column_nullable.getNullMapData().emplace_back(0); } @@ -202,61 +191,14 @@ namespace ++row_count; } assert(cass_result_row_count(result) == row_count); - cass_iterator_free(rows_iter); - - //const CassRow* row = cass_result_first_row(result); - //const CassValue* map = cass_row_get_column(row, 0); - //const CassValue* map = cass_row_get_column(row, 0); - //iterator = cass_iterator_from_map(map); - //while (cass_iterator_next(iterator)) { - // const CassValue* cass_key = cass_iterator_get_map_key(iterator); - // const CassValue* cass_value = cass_iterator_get_map_value(iterator); - // auto pair_values = {std::make_pair(cass_key, 0ul), std::make_pair(cass_value, 1ul)}; - // for (const auto &[value, idx]: pair_values) { - // if (description.types[idx].second) { - // ColumnNullable & column_nullable = static_cast(*columns[idx]); - // insertValue(column_nullable.getNestedColumn(), description.types[idx].first, value); - // column_nullable.getNullMapData().emplace_back(0); - // } else { - // insertValue(*columns[idx], description.types[idx].first, value); - // } - // } - //} has_more_pages = cass_result_has_more_pages(result); - if (has_more_pages) { - cass_statement_set_paging_state(statement, result); - } - - cass_result_free(result); + if (has_more_pages) + cassandraCheck(cass_statement_set_paging_state(statement, result)); return description.sample_block.cloneWithColumns(std::move(columns)); } - -void cassandraCheck(CassError code) -{ - if (code != CASS_OK) - throw Exception("Cassandra driver error " + std::to_string(code) + ": " + cass_error_desc(code), - ErrorCodes::CASSANDRA_INTERNAL_ERROR); -} - -void cassandraWaitAndCheck(CassFuture * future) -{ - auto code = cass_future_error_code(future); /// Waits if not ready - if (code == CASS_OK) - { - cass_future_free(future); - return; - } - const char * message; - size_t message_len; - cass_future_error_message(future, &message, & message_len); - String full_message = "Cassandra driver error " + std::to_string(code) + ": " + cass_error_desc(code) + ": " + message; - cass_future_free(future); /// Frees message - throw Exception(full_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR); -} - } #endif diff --git a/src/Dictionaries/CassandraBlockInputStream.h b/src/Dictionaries/CassandraBlockInputStream.h index 22e4429343d..700211ebb3e 100644 --- a/src/Dictionaries/CassandraBlockInputStream.h +++ b/src/Dictionaries/CassandraBlockInputStream.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include @@ -9,8 +9,6 @@ namespace DB { -void cassandraCheck(CassError code); -void cassandraWaitAndCheck(CassFuture * future); /// Allows processing results of a Cassandra query as a sequence of Blocks, simplifies chaining @@ -18,11 +16,10 @@ void cassandraWaitAndCheck(CassFuture * future); { public: CassandraBlockInputStream( - CassSession * session, - const std::string & query_str, + const CassClusterPtr & cluster, + const String & query_str, const Block & sample_block, const size_t max_block_size); - ~CassandraBlockInputStream() override; String getName() const override { return "Cassandra"; } @@ -31,14 +28,11 @@ void cassandraWaitAndCheck(CassFuture * future); private: Block readImpl() override; - CassSession * session; - CassStatement * statement; - String query_str; + CassSessionPtr session; + CassStatementPtr statement; const size_t max_block_size; ExternalResultDescription description; - const CassResult * result = nullptr; cass_bool_t has_more_pages; - CassIterator * iterator = nullptr; }; } diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index 2aee5d44b6b..a58dd383b46 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -34,7 +34,6 @@ namespace DB #if USE_CASSANDRA -#include #include #include "CassandraBlockInputStream.h" @@ -54,28 +53,23 @@ CassandraDictionarySource::CassandraDictionarySource( UInt16 port_, const String & user_, const String & password_, - //const std::string & method_, const String & db_, const String & table_, const DB::Block & sample_block_) - : log(&Logger::get("CassandraDictionarySource")) + : log(&Poco::Logger::get("CassandraDictionarySource")) , dict_struct(dict_struct_) , host(host_) , port(port_) , user(user_) , password(password_) - //, method(method_) , db(db_) , table(table_) , sample_block(sample_block_) - , cluster(cass_cluster_new()) //FIXME will not be freed in case of exception - , session(cass_session_new()) { cassandraCheck(cass_cluster_set_contact_points(cluster, host.c_str())); if (port) cassandraCheck(cass_cluster_set_port(cluster, port)); cass_cluster_set_credentials(cluster, user.c_str(), password.c_str()); - cassandraWaitAndCheck(cass_session_connect_keyspace(session, cluster, db.c_str())); } CassandraDictionarySource::CassandraDictionarySource( @@ -89,7 +83,6 @@ CassandraDictionarySource::CassandraDictionarySource( config.getUInt(config_prefix + ".port", 0), config.getString(config_prefix + ".user", ""), config.getString(config_prefix + ".password", ""), - //config.getString(config_prefix + ".method", ""), config.getString(config_prefix + ".keyspace", ""), config.getString(config_prefix + ".column_family"), sample_block_) @@ -102,22 +95,12 @@ CassandraDictionarySource::CassandraDictionarySource(const CassandraDictionarySo other.port, other.user, other.password, - //other.method, other.db, other.table, other.sample_block} { } -CassandraDictionarySource::~CassandraDictionarySource() { - cass_session_free(session); - cass_cluster_free(cluster); -} - -//std::string CassandraDictionarySource::toConnectionString(const std::string &host, const UInt16 port) { -// return host + (port != 0 ? ":" + std::to_string(port) : ""); -//} - BlockInputStreamPtr CassandraDictionarySource::loadAll() { ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes}; @@ -125,11 +108,11 @@ BlockInputStreamPtr CassandraDictionarySource::loadAll() query.pop_back(); query += " ALLOW FILTERING;"; LOG_INFO(log, "Loading all using query: ", query); - return std::make_shared(session, query, sample_block, max_block_size); + return std::make_shared(cluster, query, sample_block, max_block_size); } std::string CassandraDictionarySource::toString() const { - return "Cassandra: " + /*db + '.' + collection + ',' + (user.empty() ? " " : " " + user + '@') + */ host + ':' + DB::toString(port); + return "Cassandra: " + db + '.' + table; } BlockInputStreamPtr CassandraDictionarySource::loadIds(const std::vector & ids) @@ -139,7 +122,7 @@ BlockInputStreamPtr CassandraDictionarySource::loadIds(const std::vector query.pop_back(); query += " ALLOW FILTERING;"; LOG_INFO(log, "Loading ids using query: ", query); - return std::make_shared(session, query, sample_block, max_block_size); + return std::make_shared(cluster, query, sample_block, max_block_size); } BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) @@ -150,7 +133,7 @@ BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_colu query.pop_back(); query += " ALLOW FILTERING;"; LOG_INFO(log, "Loading keys using query: ", query); - return std::make_shared(session, query, sample_block, max_block_size); + return std::make_shared(cluster, query, sample_block, max_block_size); } diff --git a/src/Dictionaries/CassandraDictionarySource.h b/src/Dictionaries/CassandraDictionarySource.h index 400481d0a95..564fa75c3a2 100644 --- a/src/Dictionaries/CassandraDictionarySource.h +++ b/src/Dictionaries/CassandraDictionarySource.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -#include -#endif +#include #if USE_CASSANDRA @@ -10,10 +8,10 @@ #include "IDictionarySource.h" #include #include -#include namespace DB { + class CassandraDictionarySource final : public IDictionarySource { CassandraDictionarySource( const DictionaryStructure & dict_struct, @@ -21,7 +19,6 @@ class CassandraDictionarySource final : public IDictionarySource { UInt16 port, const String & user, const String & password, - //const std::string & method, const String & db, const String & table, const Block & sample_block); @@ -35,29 +32,20 @@ public: CassandraDictionarySource(const CassandraDictionarySource & other); - ~CassandraDictionarySource() override; - BlockInputStreamPtr loadAll() override; bool supportsSelectiveLoad() const override { return true; } bool isModified() const override { return true; } - ///Not yet supported bool hasUpdateField() const override { return false; } DictionarySourcePtr clone() const override { return std::make_unique(*this); } BlockInputStreamPtr loadIds(const std::vector & ids) override; - //{ - // throw Exception{"Method loadIds is not implemented yet", ErrorCodes::NOT_IMPLEMENTED}; - //} BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector & requested_rows) override; - //{ - // throw Exception{"Method loadKeys is not implemented yet", ErrorCodes::NOT_IMPLEMENTED}; - //} - + BlockInputStreamPtr loadUpdatedAll() override { throw Exception{"Method loadUpdatedAll is unsupported for CassandraDictionarySource", ErrorCodes::NOT_IMPLEMENTED}; @@ -66,21 +54,17 @@ public: std::string toString() const override; private: - //static std::string toConnectionString(const std::string & host, const UInt16 port); - Poco::Logger * log; const DictionaryStructure dict_struct; const String host; const UInt16 port; const String user; const String password; - //const std::string method; const String db; const String table; Block sample_block; - CassCluster * cluster; - CassSession * session; + CassClusterPtr cluster; }; } diff --git a/src/Dictionaries/CassandraHelpers.cpp b/src/Dictionaries/CassandraHelpers.cpp new file mode 100644 index 00000000000..9c181abcf43 --- /dev/null +++ b/src/Dictionaries/CassandraHelpers.cpp @@ -0,0 +1,38 @@ +#include + +#if USE_CASSANDRA +#include + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int CASSANDRA_INTERNAL_ERROR; +} + +void cassandraCheck(CassError code) +{ + if (code != CASS_OK) + throw Exception("Cassandra driver error " + std::to_string(code) + ": " + cass_error_desc(code), + ErrorCodes::CASSANDRA_INTERNAL_ERROR); +} + + +void cassandraWaitAndCheck(CassFuturePtr && future) +{ + auto code = cass_future_error_code(future); /// Waits if not ready + if (code == CASS_OK) + return; + + /// `future` owns `message` and will free it on destruction + const char * message; + size_t message_len; + cass_future_error_message(future, &message, & message_len); + std::string full_message = "Cassandra driver error " + std::to_string(code) + ": " + cass_error_desc(code) + ": " + message; + throw Exception(full_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR); +} + +} + +#endif diff --git a/src/Dictionaries/CassandraHelpers.h b/src/Dictionaries/CassandraHelpers.h new file mode 100644 index 00000000000..48573c1080f --- /dev/null +++ b/src/Dictionaries/CassandraHelpers.h @@ -0,0 +1,72 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include +#endif + +#if USE_CASSANDRA +#include +#include + +namespace DB +{ + +namespace Cassandra +{ + +template +CassT * defaultCtor() { return nullptr; } + +/// RAII wrapper for raw pointers to objects from cassandra driver library +template> +class ObjectHolder +{ + CassT * ptr = nullptr; +public: + template + ObjectHolder(Args &&... args) : ptr(Ctor(std::forward(args)...)) {} + ObjectHolder(CassT * ptr_) : ptr(ptr_) {} + + ObjectHolder(const ObjectHolder &) = delete; + ObjectHolder & operator = (const ObjectHolder &) = delete; + + ObjectHolder(ObjectHolder && rhs) noexcept : ptr(rhs.ptr) { rhs.ptr = nullptr; } + ObjectHolder & operator = (ObjectHolder && rhs) noexcept + { + if (ptr) + Dtor(ptr); + ptr = rhs.ptr; + rhs.ptr = nullptr; + } + + ~ObjectHolder() + { + if (ptr) + Dtor(ptr); + } + + /// For implicit conversion when passing object to driver library functions + operator CassT * () { return ptr; } + operator const CassT * () const { return ptr; } +}; + +} + +/// These object are created on pointer construction +using CassClusterPtr = Cassandra::ObjectHolder; +using CassSessionPtr = Cassandra::ObjectHolder; +using CassStatementPtr = Cassandra::ObjectHolder; + +/// The following objects are created inside Cassandra driver library, +/// but must be freed by user code +using CassFuturePtr = Cassandra::ObjectHolder; +using CassResultPtr = Cassandra::ObjectHolder; +using CassIteratorPtr = Cassandra::ObjectHolder; + +/// Checks return code, throws exception on error +void cassandraCheck(CassError code); +void cassandraWaitAndCheck(CassFuturePtr && future); + +} + +#endif diff --git a/src/Dictionaries/ya.make b/src/Dictionaries/ya.make index 5e1af27a165..3de623a9a8b 100644 --- a/src/Dictionaries/ya.make +++ b/src/Dictionaries/ya.make @@ -19,6 +19,7 @@ SRCS( CacheDictionary_generate3.cpp CassandraBlockInputStream.cpp CassandraDictionarySource.cpp + CassandraHelpers.cpp ClickHouseDictionarySource.cpp ComplexKeyCacheDictionary.cpp ComplexKeyCacheDictionary_createAttributeWithType.cpp From 73926b2cf9c71093dac41b9fadaaa866122add1a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 3 Jun 2020 14:54:26 +0300 Subject: [PATCH 14/29] support loading from multiple partitions with complex key --- .../CassandraDictionarySource.cpp | 161 +++++++++++------- src/Dictionaries/CassandraDictionarySource.h | 52 ++++-- src/Dictionaries/CassandraHelpers.cpp | 30 ++++ src/Dictionaries/CassandraHelpers.h | 6 + src/Dictionaries/ExternalQueryBuilder.cpp | 76 ++++++--- src/Dictionaries/ExternalQueryBuilder.h | 17 +- .../external_sources.py | 6 +- 7 files changed, 236 insertions(+), 112 deletions(-) diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index a58dd383b46..fec60fe3d83 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -1,8 +1,10 @@ #include "CassandraDictionarySource.h" #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" -#include "ExternalQueryBuilder.h" #include +#include +#include +#include namespace DB { @@ -21,6 +23,7 @@ namespace DB bool /*check_config*/) -> DictionarySourcePtr { #if USE_CASSANDRA + setupCassandraDriverLibraryLogging(CASS_LOG_TRACE); return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); #else throw Exception{"Dictionary source of type `cassandra` is disabled because library was built without cassandra support.", @@ -35,107 +38,149 @@ namespace DB #if USE_CASSANDRA #include +#include +#include #include "CassandraBlockInputStream.h" namespace DB { namespace ErrorCodes { - extern const int UNSUPPORTED_METHOD; - extern const int WRONG_PASSWORD; + extern const int LOGICAL_ERROR; + extern const int INVALID_CONFIG_PARAMETER; +} + +CassandraSettings::CassandraSettings( + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix) + : host(config.getString(config_prefix + ".host")) + , port(config.getUInt(config_prefix + ".port", 0)) + , user(config.getString(config_prefix + ".user", "")) + , password(config.getString(config_prefix + ".password", "")) + , db(config.getString(config_prefix + ".keyspace", "")) + , table(config.getString(config_prefix + ".column_family")) + , allow_filtering(config.getBool(config_prefix + ".allow_filtering", false)) + , partition_key_prefix(config.getUInt(config_prefix + ".partition_key_prefix", 1)) + , max_threads(config.getUInt(config_prefix + ".max_threads", 8)) + , where(config.getString(config_prefix + ".where", "")) +{ + setConsistency(config.getString(config_prefix + ".consistency", "One")); +} + +void CassandraSettings::setConsistency(const String & config_str) +{ + if (config_str == "One") + consistency = CASS_CONSISTENCY_ONE; + else if (config_str == "Two") + consistency = CASS_CONSISTENCY_TWO; + else if (config_str == "Three") + consistency = CASS_CONSISTENCY_THREE; + else if (config_str == "All") + consistency = CASS_CONSISTENCY_ALL; + else if (config_str == "EachQuorum") + consistency = CASS_CONSISTENCY_EACH_QUORUM; + else if (config_str == "Quorum") + consistency = CASS_CONSISTENCY_QUORUM; + else if (config_str == "LocalQuorum") + consistency = CASS_CONSISTENCY_LOCAL_QUORUM; + else if (config_str == "LocalOne") + consistency = CASS_CONSISTENCY_LOCAL_ONE; + else if (config_str == "Serial") + consistency = CASS_CONSISTENCY_SERIAL; + else if (config_str == "LocalSerial") + consistency = CASS_CONSISTENCY_LOCAL_SERIAL; + else /// CASS_CONSISTENCY_ANY is only valid for writes + throw Exception("Unsupported consistency level: " + config_str, ErrorCodes::INVALID_CONFIG_PARAMETER); } static const size_t max_block_size = 8192; CassandraDictionarySource::CassandraDictionarySource( - const DB::DictionaryStructure & dict_struct_, - const String & host_, - UInt16 port_, - const String & user_, - const String & password_, - const String & db_, - const String & table_, - const DB::Block & sample_block_) + const DictionaryStructure & dict_struct_, + const CassandraSettings & settings_, + const Block & sample_block_) : log(&Poco::Logger::get("CassandraDictionarySource")) , dict_struct(dict_struct_) - , host(host_) - , port(port_) - , user(user_) - , password(password_) - , db(db_) - , table(table_) + , settings(settings_) , sample_block(sample_block_) + , query_builder(dict_struct, settings.db, settings.table, settings.where, IdentifierQuotingStyle::DoubleQuotes) { - cassandraCheck(cass_cluster_set_contact_points(cluster, host.c_str())); - if (port) - cassandraCheck(cass_cluster_set_port(cluster, port)); - cass_cluster_set_credentials(cluster, user.c_str(), password.c_str()); + cassandraCheck(cass_cluster_set_contact_points(cluster, settings.host.c_str())); + if (settings.port) + cassandraCheck(cass_cluster_set_port(cluster, settings.port)); + cass_cluster_set_credentials(cluster, settings.user.c_str(), settings.password.c_str()); + cassandraCheck(cass_cluster_set_consistency(cluster, settings.consistency)); } CassandraDictionarySource::CassandraDictionarySource( - const DB::DictionaryStructure & dict_struct_, + const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, - DB::Block & sample_block_) + const String & config_prefix, + Block & sample_block_) : CassandraDictionarySource( dict_struct_, - config.getString(config_prefix + ".host"), - config.getUInt(config_prefix + ".port", 0), - config.getString(config_prefix + ".user", ""), - config.getString(config_prefix + ".password", ""), - config.getString(config_prefix + ".keyspace", ""), - config.getString(config_prefix + ".column_family"), + CassandraSettings(config, config_prefix), sample_block_) { } -CassandraDictionarySource::CassandraDictionarySource(const CassandraDictionarySource & other) - : CassandraDictionarySource{other.dict_struct, - other.host, - other.port, - other.user, - other.password, - other.db, - other.table, - other.sample_block} +void CassandraDictionarySource::maybeAllowFiltering(String & query) { + if (!settings.allow_filtering) + return; + query.pop_back(); /// remove semicolon + query += " ALLOW FILTERING;"; } BlockInputStreamPtr CassandraDictionarySource::loadAll() { - ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes}; - String query = builder.composeLoadAllQuery(); - query.pop_back(); - query += " ALLOW FILTERING;"; - LOG_INFO(log, "Loading all using query: ", query); + String query = query_builder.composeLoadAllQuery(); + maybeAllowFiltering(query); + LOG_INFO(log, "Loading all using query: {}", query); return std::make_shared(cluster, query, sample_block, max_block_size); } std::string CassandraDictionarySource::toString() const { - return "Cassandra: " + db + '.' + table; + return "Cassandra: " + settings.db + '.' + settings.table; } BlockInputStreamPtr CassandraDictionarySource::loadIds(const std::vector & ids) { - ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes}; - String query = builder.composeLoadIdsQuery(ids); - query.pop_back(); - query += " ALLOW FILTERING;"; - LOG_INFO(log, "Loading ids using query: ", query); + String query = query_builder.composeLoadIdsQuery(ids); + maybeAllowFiltering(query); + LOG_INFO(log, "Loading ids using query: {}", query); return std::make_shared(cluster, query, sample_block, max_block_size); } BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { - //FIXME split conditions on partition key and clustering key - ExternalQueryBuilder builder{dict_struct, db, table, "", IdentifierQuotingStyle::DoubleQuotes}; - String query = builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::IN_WITH_TUPLES); - query.pop_back(); - query += " ALLOW FILTERING;"; - LOG_INFO(log, "Loading keys using query: ", query); - return std::make_shared(cluster, query, sample_block, max_block_size); -} + if (requested_rows.empty()) + throw Exception("No rows requested", ErrorCodes::LOGICAL_ERROR); + /// TODO is there a better way to load data by complex keys? + std::unordered_map> partitions; + for (const auto & row : requested_rows) + { + SipHash partition_key; + for (const auto i : ext::range(0, settings.partition_key_prefix)) + key_columns[i]->updateHashWithValue(row, partition_key); + partitions[partition_key.get64()].push_back(row); + } + + BlockInputStreams streams; + for (const auto & partition : partitions) + { + String query = query_builder.composeLoadKeysQuery(key_columns, partition.second, ExternalQueryBuilder::CASSANDRA_SEPARATE_PARTITION_KEY, settings.partition_key_prefix); + maybeAllowFiltering(query); + LOG_INFO(log, "Loading keys for partition hash {} using query: {}", partition.first, query); + streams.push_back(std::make_shared(cluster, query, sample_block, max_block_size)); + } + + if (streams.size() == 1) + return streams.front(); + + return std::make_shared(streams, nullptr, settings.max_threads); +} } diff --git a/src/Dictionaries/CassandraDictionarySource.h b/src/Dictionaries/CassandraDictionarySource.h index 564fa75c3a2..dff93fcd029 100644 --- a/src/Dictionaries/CassandraDictionarySource.h +++ b/src/Dictionaries/CassandraDictionarySource.h @@ -6,32 +6,47 @@ #include "DictionaryStructure.h" #include "IDictionarySource.h" +#include "ExternalQueryBuilder.h" #include #include namespace DB { +struct CassandraSettings +{ + String host; + UInt16 port; + String user; + String password; + String db; + String table; + + CassConsistency consistency; + bool allow_filtering; + /// TODO get information about key from the driver + size_t partition_key_prefix; + size_t max_threads; + String where; + + CassandraSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix); + + void setConsistency(const String & config_str); +}; + class CassandraDictionarySource final : public IDictionarySource { +public: CassandraDictionarySource( const DictionaryStructure & dict_struct, - const String & host, - UInt16 port, - const String & user, - const String & password, - const String & db, - const String & table, + const CassandraSettings & settings_, const Block & sample_block); -public: CassandraDictionarySource( const DictionaryStructure & dict_struct, const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, + const String & config_prefix, Block & sample_block); - CassandraDictionarySource(const CassandraDictionarySource & other); - BlockInputStreamPtr loadAll() override; bool supportsSelectiveLoad() const override { return true; } @@ -40,7 +55,10 @@ public: bool hasUpdateField() const override { return false; } - DictionarySourcePtr clone() const override { return std::make_unique(*this); } + DictionarySourcePtr clone() const override + { + return std::make_unique(dict_struct, settings, sample_block); + } BlockInputStreamPtr loadIds(const std::vector & ids) override; @@ -51,18 +69,16 @@ public: throw Exception{"Method loadUpdatedAll is unsupported for CassandraDictionarySource", ErrorCodes::NOT_IMPLEMENTED}; } - std::string toString() const override; + String toString() const override; private: + void maybeAllowFiltering(String & query); + Poco::Logger * log; const DictionaryStructure dict_struct; - const String host; - const UInt16 port; - const String user; - const String password; - const String db; - const String table; + const CassandraSettings settings; Block sample_block; + ExternalQueryBuilder query_builder; CassClusterPtr cluster; }; diff --git a/src/Dictionaries/CassandraHelpers.cpp b/src/Dictionaries/CassandraHelpers.cpp index 9c181abcf43..4f92a75a1f3 100644 --- a/src/Dictionaries/CassandraHelpers.cpp +++ b/src/Dictionaries/CassandraHelpers.cpp @@ -2,6 +2,8 @@ #if USE_CASSANDRA #include +#include +#include namespace DB { @@ -33,6 +35,34 @@ void cassandraWaitAndCheck(CassFuturePtr && future) throw Exception(full_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR); } +static std::once_flag setup_logging_flag; + +void setupCassandraDriverLibraryLogging(CassLogLevel level) +{ + std::call_once(setup_logging_flag, [level]() + { + Poco::Logger * logger = &Poco::Logger::get("CassandraDriverLibrary"); + cass_log_set_level(level); + if (level != CASS_LOG_DISABLED) + cass_log_set_callback(cassandraLogCallback, logger); + }); +} + +void cassandraLogCallback(const CassLogMessage * message, void * data) +{ + Poco::Logger * logger = static_cast(data); + if (message->severity == CASS_LOG_CRITICAL || message->severity == CASS_LOG_ERROR) + LOG_ERROR(logger, message->message); + else if (message->severity == CASS_LOG_WARN) + LOG_WARNING(logger, message->message); + else if (message->severity == CASS_LOG_INFO) + LOG_INFO(logger, message->message); + else if (message->severity == CASS_LOG_DEBUG) + LOG_DEBUG(logger, message->message); + else if (message->severity == CASS_LOG_TRACE) + LOG_TRACE(logger, message->message); +} + } #endif diff --git a/src/Dictionaries/CassandraHelpers.h b/src/Dictionaries/CassandraHelpers.h index 48573c1080f..2a91815e37d 100644 --- a/src/Dictionaries/CassandraHelpers.h +++ b/src/Dictionaries/CassandraHelpers.h @@ -67,6 +67,12 @@ using CassIteratorPtr = Cassandra::ObjectHolder std::string -ExternalQueryBuilder::composeLoadKeysQuery(const Columns & key_columns, const std::vector & requested_rows, LoadKeysMethod method) +ExternalQueryBuilder::composeLoadKeysQuery(const Columns & key_columns, const std::vector & requested_rows, LoadKeysMethod method, size_t partition_key_prefix) { if (!dict_struct.key) throw Exception{"Composite key required for method", ErrorCodes::UNSUPPORTED_METHOD}; @@ -307,25 +307,30 @@ ExternalQueryBuilder::composeLoadKeysQuery(const Columns & key_columns, const st writeString(" OR ", out); first = false; - composeKeyCondition(key_columns, row, out); + + writeString("(", out); + composeKeyCondition(key_columns, row, out, 0, key_columns.size()); + writeString(")", out); } } - else /* if (method == IN_WITH_TUPLES) */ + else if (method == IN_WITH_TUPLES) { - composeKeyTupleDefinition(out); - writeString(" IN (", out); - - first = true; - for (const auto row : requested_rows) - { - if (!first) - writeString(", ", out); - - first = false; - composeKeyTuple(key_columns, row, out); - } - - writeString(")", out); + composeInWithTuples(key_columns, requested_rows, out, 0, key_columns.size()); + } + else /* if (method == CASSANDRA_SEPARATE_PARTITION_KEY) */ + { + /// CQL does not allow using OR conditions + /// and does not allow using multi-column IN expressions with partition key columns. + /// So we have to use multiple queries with conditions like + /// (partition_key_1 = val1 AND partition_key_2 = val2 ...) AND (clustering_key_1, ...) IN ((val3, ...), ...) + /// for each partition key. + /// `partition_key_prefix` is a number of columns from partition key. + /// All `requested_rows` must have the same values of partition key. + composeKeyCondition(key_columns, requested_rows.at(0), out, 0, partition_key_prefix); + if (partition_key_prefix && partition_key_prefix < key_columns.size()) + writeString(" AND ", out); + if (partition_key_prefix < key_columns.size()) + composeInWithTuples(key_columns, requested_rows, out, partition_key_prefix, key_columns.size()); } if (!where.empty()) @@ -339,13 +344,11 @@ ExternalQueryBuilder::composeLoadKeysQuery(const Columns & key_columns, const st } -void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out) const +void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out, + size_t beg, size_t end) const { - writeString("(", out); - - const auto keys_size = key_columns.size(); auto first = true; - for (const auto i : ext::range(0, keys_size)) + for (const auto i : ext::range(beg, end)) { if (!first) writeString(" AND ", out); @@ -359,12 +362,30 @@ void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, cons writeString("=", out); key_description.type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings); } +} + + +void ExternalQueryBuilder::composeInWithTuples(const Columns & key_columns, const std::vector & requested_rows, + WriteBuffer & out, size_t beg, size_t end) +{ + composeKeyTupleDefinition(out, beg, end); + writeString(" IN (", out); + + bool first = true; + for (const auto row : requested_rows) + { + if (!first) + writeString(", ", out); + + first = false; + composeKeyTuple(key_columns, row, out, beg, end); + } writeString(")", out); } -void ExternalQueryBuilder::composeKeyTupleDefinition(WriteBuffer & out) const +void ExternalQueryBuilder::composeKeyTupleDefinition(WriteBuffer & out, size_t beg, size_t end) const { if (!dict_struct.key) throw Exception{"Composite key required for method", ErrorCodes::UNSUPPORTED_METHOD}; @@ -372,26 +393,25 @@ void ExternalQueryBuilder::composeKeyTupleDefinition(WriteBuffer & out) const writeChar('(', out); auto first = true; - for (const auto & key : *dict_struct.key) + for (const auto i : ext::range(beg, end)) { if (!first) writeString(", ", out); first = false; - writeQuoted(key.name, out); + writeQuoted((*dict_struct.key)[i].name, out); } writeChar(')', out); } -void ExternalQueryBuilder::composeKeyTuple(const Columns & key_columns, const size_t row, WriteBuffer & out) const +void ExternalQueryBuilder::composeKeyTuple(const Columns & key_columns, const size_t row, WriteBuffer & out, size_t beg, size_t end) const { writeString("(", out); - const auto keys_size = key_columns.size(); auto first = true; - for (const auto i : ext::range(0, keys_size)) + for (const auto i : ext::range(beg, end)) { if (!first) writeString(", ", out); diff --git a/src/Dictionaries/ExternalQueryBuilder.h b/src/Dictionaries/ExternalQueryBuilder.h index 2ffc6a475ee..3011efbc895 100644 --- a/src/Dictionaries/ExternalQueryBuilder.h +++ b/src/Dictionaries/ExternalQueryBuilder.h @@ -42,17 +42,19 @@ struct ExternalQueryBuilder std::string composeLoadIdsQuery(const std::vector & ids); /** Generate a query to load data by set of composite keys. - * There are two methods of specification of composite keys in WHERE: + * There are three methods of specification of composite keys in WHERE: * 1. (x = c11 AND y = c12) OR (x = c21 AND y = c22) ... * 2. (x, y) IN ((c11, c12), (c21, c22), ...) + * 3. (x = c1 AND (y, z) IN ((c2, c3), ...)) */ enum LoadKeysMethod { AND_OR_CHAIN, IN_WITH_TUPLES, + CASSANDRA_SEPARATE_PARTITION_KEY, }; - std::string composeLoadKeysQuery(const Columns & key_columns, const std::vector & requested_rows, LoadKeysMethod method); + std::string composeLoadKeysQuery(const Columns & key_columns, const std::vector & requested_rows, LoadKeysMethod method, size_t partition_key_prefix = 0); private: @@ -60,14 +62,19 @@ private: void composeLoadAllQuery(WriteBuffer & out) const; + /// In the following methods `beg` and `end` specifies which columns to write in expression + /// Expression in form (x = c1 AND y = c2 ...) - void composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out) const; + void composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out, size_t beg, size_t end) const; + + /// Expression in form (x, y, ...) IN ((c1, c2, ...), ...) + void composeInWithTuples(const Columns & key_columns, const std::vector & requested_rows, WriteBuffer & out, size_t beg, size_t end); /// Expression in form (x, y, ...) - void composeKeyTupleDefinition(WriteBuffer & out) const; + void composeKeyTupleDefinition(WriteBuffer & out, size_t beg, size_t end) const; /// Expression in form (c1, c2, ...) - void composeKeyTuple(const Columns & key_columns, const size_t row, WriteBuffer & out) const; + void composeKeyTuple(const Columns & key_columns, const size_t row, WriteBuffer & out, size_t beg, size_t end) const; /// Write string with specified quoting style. void writeQuoted(const std::string & s, WriteBuffer & out) const; diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py index 336f3ddc28b..04fe83414e7 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py @@ -436,6 +436,7 @@ class SourceCassandra(ExternalSource): {port} test {table} + 1 '''.format( host=self.docker_hostname, @@ -451,9 +452,8 @@ class SourceCassandra(ExternalSource): self.structure[table_name] = structure columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()] keys = ['"' + col.name + '"' for col in structure.keys] - # FIXME use partition key - query = 'create table test."{name}" ({columns}, primary key ("{some_col}", {pk}));'.format( - name=table_name, columns=', '.join(columns), some_col=structure.ordinary_fields[0].name, pk=', '.join(keys)) + query = 'create table test."{name}" ({columns}, primary key ({pk}));'.format( + name=table_name, columns=', '.join(columns), pk=', '.join(keys)) self.session.execute(query) self.prepared = True From e67837bc4ae82dcb5d7c23067f952d24d60252fb Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 3 Jun 2020 18:07:37 +0300 Subject: [PATCH 15/29] cleenup --- .gitmodules | 6 +- CMakeLists.txt | 1 + cmake/find/cassandra.cmake | 12 +- contrib/CMakeLists.txt | 6 +- contrib/cassandra | 2 +- contrib/cassandra-cmake/CMakeLists.txt | 0 contrib/libuv | 2 +- contrib/libuv-cmake/CMakeLists.txt | 441 ------------------ .../compose}/docker_compose_cassandra.yml | 0 .../external-dicts-dict-sources.md | 28 +- .../CassandraBlockInputStream.cpp | 390 +++++++++------- src/Dictionaries/CassandraBlockInputStream.h | 43 +- .../CassandraDictionarySource.cpp | 86 ++-- src/Dictionaries/CassandraDictionarySource.h | 13 +- src/Dictionaries/CassandraHelpers.cpp | 2 +- src/Dictionaries/CassandraHelpers.h | 10 +- src/Dictionaries/ExternalQueryBuilder.cpp | 10 +- src/Dictionaries/registerDictionaries.h | 1 - tests/integration/helpers/cluster.py | 4 +- .../external_sources.py | 1 + .../test.py | 5 +- 21 files changed, 381 insertions(+), 682 deletions(-) delete mode 100644 contrib/cassandra-cmake/CMakeLists.txt delete mode 100644 contrib/libuv-cmake/CMakeLists.txt rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_cassandra.yml (100%) diff --git a/.gitmodules b/.gitmodules index a4c84301fc9..c05da0c9ff9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -159,11 +159,11 @@ url = https://github.com/openldap/openldap.git [submodule "contrib/cassandra"] path = contrib/cassandra - url = https://github.com/tavplubix/cpp-driver.git - branch = ch-tmp + url = https://github.com/ClickHouse-Extras/cpp-driver.git + branch = clickhouse [submodule "contrib/libuv"] path = contrib/libuv - url = https://github.com/tavplubix/libuv.git + url = https://github.com/ClickHouse-Extras/libuv.git branch = clickhouse [submodule "contrib/fmtlib"] path = contrib/fmtlib diff --git a/CMakeLists.txt b/CMakeLists.txt index 54a88404579..27005bd8d87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,6 +328,7 @@ message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE include (GNUInstallDirs) include (cmake/contrib_finder.cmake) include (cmake/lib_name.cmake) + find_contrib_lib(double-conversion) # Must be before parquet include (cmake/find/ssl.cmake) include (cmake/find/ldap.cmake) # after ssl diff --git a/cmake/find/cassandra.cmake b/cmake/find/cassandra.cmake index b1d76702cfa..f41e0f645f4 100644 --- a/cmake/find/cassandra.cmake +++ b/cmake/find/cassandra.cmake @@ -1,8 +1,10 @@ -if (NOT DEFINED ENABLE_CASSANDRA OR ENABLE_CASSANDRA) +option(ENABLE_CASSANDRA "Enable Cassandra" ${ENABLE_LIBRARIES}) + +if (ENABLE_CASSANDRA) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv") - message (WARNING "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive") + message (ERROR "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive") elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") - message (WARNING "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive") + message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive") else() set (LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") set (CASSANDRA_INCLUDE_DIR @@ -17,6 +19,8 @@ if (NOT DEFINED ENABLE_CASSANDRA OR ENABLE_CASSANDRA) set (USE_CASSANDRA 1) set (CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra") - message(STATUS "Using cassandra: ${CASSANDRA_LIBRARY}") endif() endif() + +message (STATUS "Using cassandra=${USE_CASSANDRA}: ${CASSANDRA_INCLUDE_DIR} : ${CASSANDRA_LIBRARY}") +message (STATUS "Using libuv: ${LIBUV_ROOT_DIR} : ${LIBUV_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 0a907f01573..99b94d04473 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -291,7 +291,7 @@ if (USE_INTERNAL_AWS_S3_LIBRARY) endif () if (USE_BASE64) - add_subdirectory(base64-cmake) + add_subdirectory (base64-cmake) endif() if (USE_INTERNAL_HYPERSCAN_LIBRARY) @@ -315,8 +315,8 @@ if (USE_FASTOPS) endif() if (USE_CASSANDRA) - add_subdirectory(libuv-cmake) - add_subdirectory(cassandra) + add_subdirectory (libuv) + add_subdirectory (cassandra) endif() add_subdirectory (fmtlib-cmake) diff --git a/contrib/cassandra b/contrib/cassandra index 58a71947d9d..a49b4e0e269 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit 58a71947d9dd8412f5aeb38275fa81417ea27ee0 +Subproject commit a49b4e0e2696a4b8ef286a5b9538d1cbe8490509 diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/contrib/libuv b/contrib/libuv index 379988fef9b..84438304f41 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit 379988fef9b0c6ac706a624dbac6be8924a3a0da +Subproject commit 84438304f41d8ea6670ee5409f4d6c63ca784f28 diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt deleted file mode 100644 index 65aeabdd9cb..00000000000 --- a/contrib/libuv-cmake/CMakeLists.txt +++ /dev/null @@ -1,441 +0,0 @@ -cmake_minimum_required(VERSION 3.4) -project(libuv LANGUAGES C) - -include(CMakePackageConfigHelpers) -include(CMakeDependentOption) -include(GNUInstallDirs) -include(CTest) - -#cmake_dependent_option(LIBUV_BUILD_TESTS -# "Build the unit tests when BUILD_TESTING is enabled and we are the root project" ON -# "BUILD_TESTING;CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR" OFF) - -if(MSVC) - list(APPEND uv_cflags /W4) -elseif(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") - list(APPEND uv_cflags -fvisibility=hidden --std=gnu89) - list(APPEND uv_cflags -Wall -Wextra -Wstrict-prototypes) - list(APPEND uv_cflags -Wno-unused-parameter) -endif() - -set(uv_sources - src/fs-poll.c - src/idna.c - src/inet.c - src/random.c - src/strscpy.c - src/threadpool.c - src/timer.c - src/uv-common.c - src/uv-data-getter-setters.c - src/version.c) - -set(uv_test_sources - test/blackhole-server.c - test/echo-server.c - test/run-tests.c - test/runner.c - test/test-active.c - test/test-async-null-cb.c - test/test-async.c - test/test-barrier.c - test/test-callback-order.c - test/test-callback-stack.c - test/test-close-fd.c - test/test-close-order.c - test/test-condvar.c - test/test-connect-unspecified.c - test/test-connection-fail.c - test/test-cwd-and-chdir.c - test/test-default-loop-close.c - test/test-delayed-accept.c - test/test-dlerror.c - test/test-eintr-handling.c - test/test-embed.c - test/test-emfile.c - test/test-env-vars.c - test/test-error.c - test/test-fail-always.c - test/test-fork.c - test/test-fs-copyfile.c - test/test-fs-event.c - test/test-fs-poll.c - test/test-fs.c - test/test-fs-readdir.c - test/test-fs-fd-hash.c - test/test-fs-open-flags.c - test/test-get-currentexe.c - test/test-get-loadavg.c - test/test-get-memory.c - test/test-get-passwd.c - test/test-getaddrinfo.c - test/test-gethostname.c - test/test-getnameinfo.c - test/test-getsockname.c - test/test-getters-setters.c - test/test-gettimeofday.c - test/test-handle-fileno.c - test/test-homedir.c - test/test-hrtime.c - test/test-idle.c - test/test-idna.c - test/test-ip4-addr.c - test/test-ip6-addr.c - test/test-ipc-heavy-traffic-deadlock-bug.c - test/test-ipc-send-recv.c - test/test-ipc.c - test/test-loop-alive.c - test/test-loop-close.c - test/test-loop-configure.c - test/test-loop-handles.c - test/test-loop-stop.c - test/test-loop-time.c - test/test-multiple-listen.c - test/test-mutexes.c - test/test-osx-select.c - test/test-pass-always.c - test/test-ping-pong.c - test/test-pipe-bind-error.c - test/test-pipe-close-stdout-read-stdin.c - test/test-pipe-connect-error.c - test/test-pipe-connect-multiple.c - test/test-pipe-connect-prepare.c - test/test-pipe-getsockname.c - test/test-pipe-pending-instances.c - test/test-pipe-sendmsg.c - test/test-pipe-server-close.c - test/test-pipe-set-fchmod.c - test/test-pipe-set-non-blocking.c - test/test-platform-output.c - test/test-poll-close-doesnt-corrupt-stack.c - test/test-poll-close.c - test/test-poll-closesocket.c - test/test-poll-oob.c - test/test-poll.c - test/test-process-priority.c - test/test-process-title-threadsafe.c - test/test-process-title.c - test/test-queue-foreach-delete.c - test/test-random.c - test/test-ref.c - test/test-run-nowait.c - test/test-run-once.c - test/test-semaphore.c - test/test-shutdown-close.c - test/test-shutdown-eof.c - test/test-shutdown-twice.c - test/test-signal-multiple-loops.c - test/test-signal-pending-on-close.c - test/test-signal.c - test/test-socket-buffer-size.c - test/test-spawn.c - test/test-stdio-over-pipes.c - test/test-strscpy.c - test/test-tcp-alloc-cb-fail.c - test/test-tcp-bind-error.c - test/test-tcp-bind6-error.c - test/test-tcp-close-accept.c - test/test-tcp-close-while-connecting.c - test/test-tcp-close.c - test/test-tcp-close-reset.c - test/test-tcp-connect-error-after-write.c - test/test-tcp-connect-error.c - test/test-tcp-connect-timeout.c - test/test-tcp-connect6-error.c - test/test-tcp-create-socket-early.c - test/test-tcp-flags.c - test/test-tcp-oob.c - test/test-tcp-open.c - test/test-tcp-read-stop.c - test/test-tcp-shutdown-after-write.c - test/test-tcp-try-write.c - test/test-tcp-try-write-error.c - test/test-tcp-unexpected-read.c - test/test-tcp-write-after-connect.c - test/test-tcp-write-fail.c - test/test-tcp-write-queue-order.c - test/test-tcp-write-to-half-open-connection.c - test/test-tcp-writealot.c - test/test-thread-equal.c - test/test-thread.c - test/test-threadpool-cancel.c - test/test-threadpool.c - test/test-timer-again.c - test/test-timer-from-check.c - test/test-timer.c - test/test-tmpdir.c - test/test-tty-duplicate-key.c - test/test-tty.c - test/test-udp-alloc-cb-fail.c - test/test-udp-bind.c - test/test-udp-connect.c - test/test-udp-create-socket-early.c - test/test-udp-dgram-too-big.c - test/test-udp-ipv6.c - test/test-udp-multicast-interface.c - test/test-udp-multicast-interface6.c - test/test-udp-multicast-join.c - test/test-udp-multicast-join6.c - test/test-udp-multicast-ttl.c - test/test-udp-open.c - test/test-udp-options.c - test/test-udp-send-and-recv.c - test/test-udp-send-hang-loop.c - test/test-udp-send-immediate.c - test/test-udp-send-unreachable.c - test/test-udp-try-send.c - test/test-uname.c - test/test-walk-handles.c - test/test-watcher-cross-stop.c) - -#if(WIN32) -# list(APPEND uv_defines WIN32_LEAN_AND_MEAN _WIN32_WINNT=0x0600) -# list(APPEND uv_libraries -# advapi32 -# iphlpapi -# psapi -# shell32 -# user32 -# userenv -# ws2_32) -# list(APPEND uv_sources -# src/win/async.c -# src/win/core.c -# src/win/detect-wakeup.c -# src/win/dl.c -# src/win/error.c -# src/win/fs.c -# src/win/fs-event.c -# src/win/getaddrinfo.c -# src/win/getnameinfo.c -# src/win/handle.c -# src/win/loop-watcher.c -# src/win/pipe.c -# src/win/thread.c -# src/win/poll.c -# src/win/process.c -# src/win/process-stdio.c -# src/win/signal.c -# src/win/snprintf.c -# src/win/stream.c -# src/win/tcp.c -# src/win/tty.c -# src/win/udp.c -# src/win/util.c -# src/win/winapi.c -# src/win/winsock.c) -# list(APPEND uv_test_libraries ws2_32) -# list(APPEND uv_test_sources src/win/snprintf.c test/runner-win.c) -#else() - -if(CMAKE_SIZEOF_VOID_P EQUAL 4) - list(APPEND uv_defines _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE) -endif() - -if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") - # Android has pthread as part of its c library, not as a separate - # libpthread.so. - list(APPEND uv_libraries pthread) -endif() -list(APPEND uv_sources - src/unix/async.c - src/unix/core.c - src/unix/dl.c - src/unix/fs.c - src/unix/getaddrinfo.c - src/unix/getnameinfo.c - src/unix/loop-watcher.c - src/unix/loop.c - src/unix/pipe.c - src/unix/poll.c - src/unix/process.c - src/unix/random-devurandom.c - src/unix/signal.c - src/unix/stream.c - src/unix/tcp.c - src/unix/thread.c - src/unix/tty.c - src/unix/udp.c) -list(APPEND uv_test_sources test/runner-unix.c) -#endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "AIX") - list(APPEND uv_defines - _ALL_SOURCE - _LINUX_SOURCE_COMPAT - _THREAD_SAFE - _XOPEN_SOURCE=500) - list(APPEND uv_libraries perfstat) - list(APPEND uv_sources src/unix/aix.c) -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "Android") - list(APPEND uv_libs dl) - list(APPEND uv_sources - src/unix/android-ifaddrs.c - src/unix/linux-core.c - src/unix/linux-inotify.c - src/unix/linux-syscalls.c - src/unix/procfs-exepath.c - src/unix/pthread-fixes.c - src/unix/random-getrandom.c - src/unix/random-sysctl-linux.c - src/unix/sysinfo-loadavg.c) -endif() - -if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "Android|Linux|OS/390") - list(APPEND uv_sources src/unix/proctitle.c) -endif() - -if(CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD") - list(APPEND uv_sources src/unix/freebsd.c) -endif() - -if(CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|NetBSD|OpenBSD") - list(APPEND uv_sources src/unix/posix-hrtime.c src/unix/bsd-proctitle.c) -endif() - -if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|NetBSD|OpenBSD") - list(APPEND uv_sources src/unix/bsd-ifaddrs.c src/unix/kqueue.c) -endif() - -if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - list(APPEND uv_sources src/unix/random-getrandom.c) -endif() - -if(APPLE OR CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") - list(APPEND uv_sources src/unix/random-getentropy.c) -endif() - -if(APPLE) - list(APPEND uv_defines _DARWIN_UNLIMITED_SELECT=1 _DARWIN_USE_64_BIT_INODE=1) - list(APPEND uv_sources - src/unix/darwin-proctitle.c - src/unix/darwin.c - src/unix/fsevents.c) -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - list(APPEND uv_defines _GNU_SOURCE _POSIX_C_SOURCE=200112) - list(APPEND uv_libraries dl rt) - list(APPEND uv_sources - src/unix/linux-core.c - src/unix/linux-inotify.c - src/unix/linux-syscalls.c - src/unix/procfs-exepath.c - src/unix/random-getrandom.c - src/unix/random-sysctl-linux.c - src/unix/sysinfo-loadavg.c) -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") - list(APPEND uv_sources src/unix/netbsd.c) - list(APPEND uv_libraries kvm) -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") - list(APPEND uv_sources src/unix/openbsd.c) -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "OS/390") - list(APPEND uv_defines PATH_MAX=255) - list(APPEND uv_defines _AE_BIMODAL) - list(APPEND uv_defines _ALL_SOURCE) - list(APPEND uv_defines _LARGE_TIME_API) - list(APPEND uv_defines _OPEN_MSGQ_EXT) - list(APPEND uv_defines _OPEN_SYS_FILE_EXT) - list(APPEND uv_defines _OPEN_SYS_IF_EXT) - list(APPEND uv_defines _OPEN_SYS_SOCK_EXT3) - list(APPEND uv_defines _OPEN_SYS_SOCK_IPV6) - list(APPEND uv_defines _UNIX03_SOURCE) - list(APPEND uv_defines _UNIX03_THREADS) - list(APPEND uv_defines _UNIX03_WITHDRAWN) - list(APPEND uv_defines _XOPEN_SOURCE_EXTENDED) - list(APPEND uv_sources - src/unix/pthread-fixes.c - src/unix/pthread-barrier.c - src/unix/os390.c - src/unix/os390-syscalls.c) -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "SunOS") - list(APPEND uv_defines __EXTENSIONS__ _XOPEN_SOURCE=500) - list(APPEND uv_libraries kstat nsl sendfile socket) - list(APPEND uv_sources src/unix/no-proctitle.c src/unix/sunos.c) -endif() - -if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|Linux|NetBSD|OpenBSD") - list(APPEND uv_test_libraries util) -endif() - -set(uv_sources_tmp "") -foreach(file ${uv_sources}) - list(APPEND uv_sources_tmp "${LIBUV_ROOT_DIR}/${file}") -endforeach(file) -set(uv_sources "${uv_sources_tmp}") - -list(APPEND uv_defines CLICKHOUSE_GLIBC_COMPATIBILITY) - -add_library(uv SHARED ${uv_sources}) -target_compile_definitions(uv - INTERFACE USING_UV_SHARED=1 - PRIVATE ${uv_defines} BUILDING_UV_SHARED=1) -target_compile_options(uv PRIVATE ${uv_cflags}) -target_include_directories(uv PUBLIC ${LIBUV_ROOT_DIR}/include PRIVATE ${LIBUV_ROOT_DIR}/src) -target_link_libraries(uv ${uv_libraries}) - -add_library(uv_a STATIC ${uv_sources}) -target_compile_definitions(uv_a PRIVATE ${uv_defines}) -target_compile_options(uv_a PRIVATE ${uv_cflags}) -target_include_directories(uv_a PUBLIC ${LIBUV_ROOT_DIR}/include PRIVATE ${LIBUV_ROOT_DIR}/src) -target_link_libraries(uv_a ${uv_libraries}) - -#if(LIBUV_BUILD_TESTS) -# add_executable(uv_run_tests ${uv_test_sources}) -# target_compile_definitions(uv_run_tests -# PRIVATE ${uv_defines} USING_UV_SHARED=1) -# target_compile_options(uv_run_tests PRIVATE ${uv_cflags}) -# target_link_libraries(uv_run_tests uv ${uv_test_libraries}) -# add_test(NAME uv_test -# COMMAND uv_run_tests -# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -# add_executable(uv_run_tests_a ${uv_test_sources}) -# target_compile_definitions(uv_run_tests_a PRIVATE ${uv_defines}) -# target_compile_options(uv_run_tests_a PRIVATE ${uv_cflags}) -# target_link_libraries(uv_run_tests_a uv_a ${uv_test_libraries}) -# add_test(NAME uv_test_a -# COMMAND uv_run_tests_a -# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -#endif() - -if(UNIX) - # Now for some gibbering horrors from beyond the stars... - foreach(x ${uv_libraries}) - set(LIBS "${LIBS} -l${x}") - endforeach(x) - file(STRINGS ${LIBUV_ROOT_DIR}/configure.ac configure_ac REGEX ^AC_INIT) - string(REGEX MATCH [0-9]+[.][0-9]+[.][0-9]+ PACKAGE_VERSION "${configure_ac}") - string(REGEX MATCH ^[0-9]+ UV_VERSION_MAJOR "${PACKAGE_VERSION}") - # The version in the filename is mirroring the behaviour of autotools. - set_target_properties(uv PROPERTIES VERSION ${UV_VERSION_MAJOR}.0.0 - SOVERSION ${UV_VERSION_MAJOR}) - set(includedir ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) - set(libdir ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}) - set(prefix ${CMAKE_INSTALL_PREFIX}) - configure_file(${LIBUV_ROOT_DIR}/libuv.pc.in ${LIBUV_ROOT_DIR}/libuv.pc @ONLY) - - install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DOCDIR}) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libuv.pc - DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) - install(TARGETS uv LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) - install(TARGETS uv_a ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) -endif() - -#if(WIN32) -# install(DIRECTORY include/ DESTINATION include) -# install(FILES LICENSE DESTINATION .) -# install(TARGETS uv uv_a -# RUNTIME DESTINATION lib/$ -# ARCHIVE DESTINATION lib/$) -#endif() diff --git a/tests/integration/helpers/docker_compose_cassandra.yml b/docker/test/integration/compose/docker_compose_cassandra.yml similarity index 100% rename from tests/integration/helpers/docker_compose_cassandra.yml rename to docker/test/integration/compose/docker_compose_cassandra.yml diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 650630701be..71b719ce996 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -633,9 +633,35 @@ Example of settings: localhost - 6349 + 9042 + username + qwerty123 + database_name + table_name + 1 + 1 + One + "SomeColumn" = 42 + 8 ``` +Setting fields: +- `host` – The Cassandra host or comma-separated list of hosts. +- `port` – The port on the Cassandra servers. If not specified, default port is used. +- `user` – Name of the Cassandra user. +- `password` – Password of the Cassandra user. +- `keyspace` – Name of the keyspace (database). +- `column_family` – Name of the column family (table). +- `allow_filering` – Flag to allow or not potentially expensive conditions on clustering key columns. Default value is 1. +- `partition_key_prefix` – Number of partition key columns in primary key of the Cassandra table. + Required for compose key dictionaries. Order of key columns in the dictionary definition must be the same as in Cassandra. + Default value is 1 (the first key column is a partition key and other key columns are clustering key). +- `consistency` – Consistency level. Possible values: `One`, `Two`, `Three`, + `All`, `EachQuorum`, `Quorum`, `LocalQuorum`, `LocalOne`, `Serial`, `LocalSerial`. Default is `One`. +- `where` – Optional selection criteria. +- `max_threads` – The maximum number of threads to use for loading data from multiple partitions in compose key dictionaries. + + [Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_sources/) diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index e00fd5ec3e9..8fb9eb3f93c 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -16,189 +16,261 @@ namespace DB { + namespace ErrorCodes { - extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; + extern const int TYPE_MISMATCH; extern const int CASSANDRA_INTERNAL_ERROR; } CassandraBlockInputStream::CassandraBlockInputStream( - const CassClusterPtr & cluster, + const CassSessionShared & session_, const String & query_str, const Block & sample_block, - const size_t max_block_size_) - : statement(query_str.c_str(), /*parameters count*/ 0) + size_t max_block_size_) + : session(session_) + , statement(query_str.c_str(), /*parameters count*/ 0) , max_block_size(max_block_size_) , has_more_pages(cass_true) { description.init(sample_block); cassandraCheck(cass_statement_set_paging_size(statement, max_block_size)); - cassandraWaitAndCheck(cass_session_connect(session, cluster)); } -namespace +void CassandraBlockInputStream::insertValue(IColumn & column, ValueType type, const CassValue * cass_value) const { - using ValueType = ExternalResultDescription::ValueType; - - void insertValue(IColumn & column, const ValueType type, const CassValue * cass_value) + switch (type) { - /// Cassandra does not support unsigned integers (cass_uint32_t is for Date) - switch (type) + case ValueType::vtUInt8: { - case ValueType::vtUInt8: - { - cass_int8_t value; - cass_value_get_int8(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtUInt16: - { - cass_int16_t value; - cass_value_get_int16(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtUInt32: - { - cass_int32_t value; - cass_value_get_int32(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtUInt64: - { - cass_int64_t value; - cass_value_get_int64(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtInt8: - { - cass_int8_t value; - cass_value_get_int8(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtInt16: - { - cass_int16_t value; - cass_value_get_int16(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtInt32: - { - cass_int32_t value; - cass_value_get_int32(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtInt64: - { - cass_int64_t value; - cass_value_get_int64(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtFloat32: - { - cass_float_t value; - cass_value_get_float(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtFloat64: - { - cass_double_t value; - cass_value_get_double(cass_value, &value); - assert_cast(column).insertValue(value); - break; - } - case ValueType::vtString: - { - const char * value; - size_t value_length; - cass_value_get_string(cass_value, &value, &value_length); - assert_cast(column).insertData(value, value_length); - break; - } - case ValueType::vtDate: - { - cass_uint32_t value; - cass_value_get_uint32(cass_value, &value); - assert_cast(column).insertValue(static_cast(value)); - break; - } - case ValueType::vtDateTime: - { - cass_int64_t value; - cass_value_get_int64(cass_value, &value); - assert_cast(column).insertValue(static_cast(value / 1000)); - break; - } - case ValueType::vtUUID: - { - CassUuid value; - cass_value_get_uuid(cass_value, &value); - std::array uuid_str; - cass_uuid_string(value, uuid_str.data()); - assert_cast(column).insert(parse(uuid_str.data(), uuid_str.size())); - break; - } + cass_int8_t value; + cass_value_get_int8(cass_value, &value); + assert_cast(column).insertValue(static_cast(value)); + break; + } + case ValueType::vtUInt16: + { + cass_int16_t value; + cass_value_get_int16(cass_value, &value); + assert_cast(column).insertValue(static_cast(value)); + break; + } + case ValueType::vtUInt32: + { + cass_int32_t value; + cass_value_get_int32(cass_value, &value); + assert_cast(column).insertValue(static_cast(value)); + break; + } + case ValueType::vtUInt64: + { + cass_int64_t value; + cass_value_get_int64(cass_value, &value); + assert_cast(column).insertValue(static_cast(value)); + break; + } + case ValueType::vtInt8: + { + cass_int8_t value; + cass_value_get_int8(cass_value, &value); + assert_cast(column).insertValue(value); + break; + } + case ValueType::vtInt16: + { + cass_int16_t value; + cass_value_get_int16(cass_value, &value); + assert_cast(column).insertValue(value); + break; + } + case ValueType::vtInt32: + { + cass_int32_t value; + cass_value_get_int32(cass_value, &value); + assert_cast(column).insertValue(value); + break; + } + case ValueType::vtInt64: + { + cass_int64_t value; + cass_value_get_int64(cass_value, &value); + assert_cast(column).insertValue(value); + break; + } + case ValueType::vtFloat32: + { + cass_float_t value; + cass_value_get_float(cass_value, &value); + assert_cast(column).insertValue(value); + break; + } + case ValueType::vtFloat64: + { + cass_double_t value; + cass_value_get_double(cass_value, &value); + assert_cast(column).insertValue(value); + break; + } + case ValueType::vtString: + { + const char * value = nullptr; + size_t value_length; + cass_value_get_string(cass_value, &value, &value_length); + assert_cast(column).insertData(value, value_length); + break; + } + case ValueType::vtDate: + { + cass_uint32_t value; + cass_value_get_uint32(cass_value, &value); + assert_cast(column).insertValue(static_cast(value)); + break; + } + case ValueType::vtDateTime: + { + cass_int64_t value; + cass_value_get_int64(cass_value, &value); + assert_cast(column).insertValue(static_cast(value / 1000)); + break; + } + case ValueType::vtUUID: + { + CassUuid value; + cass_value_get_uuid(cass_value, &value); + std::array uuid_str; + cass_uuid_string(value, uuid_str.data()); + assert_cast(column).insert(parse(uuid_str.data(), uuid_str.size())); + break; } } } - Block CassandraBlockInputStream::readImpl() +void CassandraBlockInputStream::readPrefix() +{ + result_future = cass_session_execute(*session, statement); +} + +Block CassandraBlockInputStream::readImpl() +{ + if (!has_more_pages) + return {}; + + MutableColumns columns = description.sample_block.cloneEmptyColumns(); + + cassandraWaitAndCheck(result_future); + CassResultPtr result = cass_future_get_result(result_future); + + assert(cass_result_column_count(result) == columns.size()); + + assertTypes(result); + + has_more_pages = cass_result_has_more_pages(result); + if (has_more_pages) { - if (!has_more_pages) - return {}; - - MutableColumns columns = description.sample_block.cloneEmptyColumns(); - CassFuturePtr query_future = cass_session_execute(session, statement); - - CassResultPtr result = cass_future_get_result(query_future); - - if (!result) { - const char* error_message; - size_t error_message_length; - cass_future_error_message(query_future, &error_message, &error_message_length); - - throw Exception{error_message, ErrorCodes::CASSANDRA_INTERNAL_ERROR}; - } - - [[maybe_unused]] size_t row_count = 0; - assert(cass_result_column_count(result) == columns.size()); - CassIteratorPtr rows_iter = cass_iterator_from_result(result); /// Points to rows[-1] - while (cass_iterator_next(rows_iter)) - { - const CassRow * row = cass_iterator_get_row(rows_iter); - for (size_t col_idx = 0; col_idx < columns.size(); ++col_idx) - { - const CassValue * val = cass_row_get_column(row, col_idx); - if (cass_value_is_null(val)) - columns[col_idx]->insertDefault(); - else if (description.types[col_idx].second) - { - ColumnNullable & column_nullable = assert_cast(*columns[col_idx]); - insertValue(column_nullable.getNestedColumn(), description.types[col_idx].first, val); - column_nullable.getNullMapData().emplace_back(0); - } - else - insertValue(*columns[col_idx], description.types[col_idx].first, val); - } - ++row_count; - } - assert(cass_result_row_count(result) == row_count); - - has_more_pages = cass_result_has_more_pages(result); - - if (has_more_pages) - cassandraCheck(cass_statement_set_paging_state(statement, result)); - - return description.sample_block.cloneWithColumns(std::move(columns)); + cassandraCheck(cass_statement_set_paging_state(statement, result)); + result_future = cass_session_execute(*session, statement); } + CassIteratorPtr rows_iter = cass_iterator_from_result(result); /// Points to rows[-1] + while (cass_iterator_next(rows_iter)) + { + const CassRow * row = cass_iterator_get_row(rows_iter); + for (size_t col_idx = 0; col_idx < columns.size(); ++col_idx) + { + const CassValue * val = cass_row_get_column(row, col_idx); + if (cass_value_is_null(val)) + columns[col_idx]->insertDefault(); + else if (description.types[col_idx].second) + { + ColumnNullable & column_nullable = assert_cast(*columns[col_idx]); + insertValue(column_nullable.getNestedColumn(), description.types[col_idx].first, val); + column_nullable.getNullMapData().emplace_back(0); + } + else + insertValue(*columns[col_idx], description.types[col_idx].first, val); + } + } + + assert(cass_result_row_count(result) == columns.front()->size()); + + return description.sample_block.cloneWithColumns(std::move(columns)); +} + +void CassandraBlockInputStream::assertTypes(const CassResultPtr & result) +{ + if (!assert_types) + return; + + size_t column_count = cass_result_column_count(result); + for (size_t i = 0; i < column_count; ++i) + { + CassValueType expected; + String expected_text; + + /// Cassandra does not support unsigned integers (cass_uint32_t is for Date) + switch (description.types[i].first) + { + case ExternalResultDescription::ValueType::vtInt8: + case ExternalResultDescription::ValueType::vtUInt8: + expected = CASS_VALUE_TYPE_TINY_INT; + expected_text = "tinyint"; + break; + case ExternalResultDescription::ValueType::vtInt16: + case ExternalResultDescription::ValueType::vtUInt16: + expected = CASS_VALUE_TYPE_SMALL_INT; + expected_text = "smallint"; + break; + case ExternalResultDescription::ValueType::vtUInt32: + case ExternalResultDescription::ValueType::vtInt32: + expected = CASS_VALUE_TYPE_INT; + expected_text = "int"; + break; + case ExternalResultDescription::ValueType::vtInt64: + case ExternalResultDescription::ValueType::vtUInt64: + expected = CASS_VALUE_TYPE_BIGINT; + expected_text = "bigint"; + break; + case ExternalResultDescription::ValueType::vtFloat32: + expected = CASS_VALUE_TYPE_FLOAT; + expected_text = "float"; + break; + case ExternalResultDescription::ValueType::vtFloat64: + expected = CASS_VALUE_TYPE_DOUBLE; + expected_text = "double"; + break; + case ExternalResultDescription::ValueType::vtString: + expected = CASS_VALUE_TYPE_TEXT; + expected_text = "text, ascii or varchar"; + break; + case ExternalResultDescription::ValueType::vtDate: + expected = CASS_VALUE_TYPE_DATE; + expected_text = "date"; + break; + case ExternalResultDescription::ValueType::vtDateTime: + expected = CASS_VALUE_TYPE_TIMESTAMP; + expected_text = "timestamp"; + break; + case ExternalResultDescription::ValueType::vtUUID: + expected = CASS_VALUE_TYPE_UUID; + expected_text = "uuid"; + break; + } + + CassValueType got = cass_result_column_type(result, i); + + if (got != expected) + { + if (expected == CASS_VALUE_TYPE_TEXT && (got == CASS_VALUE_TYPE_ASCII || got == CASS_VALUE_TYPE_VARCHAR)) + continue; + + const auto & column_name = description.sample_block.getColumnsWithTypeAndName()[i].name; + throw Exception("Type mismatch for column " + column_name + ": expected Cassandra type " + expected_text, + ErrorCodes::TYPE_MISMATCH); + } + } + + assert_types = false; +} + } #endif diff --git a/src/Dictionaries/CassandraBlockInputStream.h b/src/Dictionaries/CassandraBlockInputStream.h index 700211ebb3e..5208e516a0e 100644 --- a/src/Dictionaries/CassandraBlockInputStream.h +++ b/src/Dictionaries/CassandraBlockInputStream.h @@ -9,30 +9,35 @@ namespace DB { +class CassandraBlockInputStream final : public IBlockInputStream +{ +public: + CassandraBlockInputStream( + const CassSessionShared & session_, + const String & query_str, + const Block & sample_block, + size_t max_block_size); + String getName() const override { return "Cassandra"; } -/// Allows processing results of a Cassandra query as a sequence of Blocks, simplifies chaining - class CassandraBlockInputStream final : public IBlockInputStream - { - public: - CassandraBlockInputStream( - const CassClusterPtr & cluster, - const String & query_str, - const Block & sample_block, - const size_t max_block_size); + Block getHeader() const override { return description.sample_block.cloneEmpty(); } - String getName() const override { return "Cassandra"; } + void readPrefix() override; - Block getHeader() const override { return description.sample_block.cloneEmpty(); } +private: + using ValueType = ExternalResultDescription::ValueType; - private: - Block readImpl() override; + Block readImpl() override; + void insertValue(IColumn & column, ValueType type, const CassValue * cass_value) const; + void assertTypes(const CassResultPtr & result); - CassSessionPtr session; - CassStatementPtr statement; - const size_t max_block_size; - ExternalResultDescription description; - cass_bool_t has_more_pages; - }; + CassSessionShared session; + CassStatementPtr statement; + CassFuturePtr result_future; + const size_t max_block_size; + ExternalResultDescription description; + cass_bool_t has_more_pages; + bool assert_types = true; +}; } diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index fec60fe3d83..c41f528db91 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -1,37 +1,35 @@ #include "CassandraDictionarySource.h" #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" -#include -#include -#include -#include namespace DB { - namespace ErrorCodes - { - extern const int SUPPORT_IS_DISABLED; - } - void registerDictionarySourceCassandra(DictionarySourceFactory & factory) +namespace ErrorCodes +{ + extern const int SUPPORT_IS_DISABLED; + extern const int NOT_IMPLEMENTED; +} + +void registerDictionarySourceCassandra(DictionarySourceFactory & factory) +{ + auto create_table_source = [=]([[maybe_unused]] const DictionaryStructure & dict_struct, + [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, + [[maybe_unused]] const std::string & config_prefix, + [[maybe_unused]] Block & sample_block, + const Context & /* context */, + bool /*check_config*/) -> DictionarySourcePtr { - auto create_table_source = [=]([[maybe_unused]] const DictionaryStructure & dict_struct, - [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, - [[maybe_unused]] const std::string & config_prefix, - [[maybe_unused]] Block & sample_block, - const Context & /* context */, - bool /*check_config*/) -> DictionarySourcePtr - { #if USE_CASSANDRA - setupCassandraDriverLibraryLogging(CASS_LOG_TRACE); - return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); + setupCassandraDriverLibraryLogging(CASS_LOG_INFO); + return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); #else - throw Exception{"Dictionary source of type `cassandra` is disabled because library was built without cassandra support.", - ErrorCodes::SUPPORT_IS_DISABLED}; + throw Exception{"Dictionary source of type `cassandra` is disabled because ClickHouse was built without cassandra support.", + ErrorCodes::SUPPORT_IS_DISABLED}; #endif - }; - factory.registerSource("cassandra", create_table_source); - } + }; + factory.registerSource("cassandra", create_table_source); +} } @@ -39,8 +37,9 @@ namespace DB #include #include -#include #include "CassandraBlockInputStream.h" +#include +#include namespace DB { @@ -57,7 +56,7 @@ CassandraSettings::CassandraSettings( , port(config.getUInt(config_prefix + ".port", 0)) , user(config.getString(config_prefix + ".user", "")) , password(config.getString(config_prefix + ".password", "")) - , db(config.getString(config_prefix + ".keyspace", "")) + , db(config.getString(config_prefix + ".keyspace")) , table(config.getString(config_prefix + ".column_family")) , allow_filtering(config.getBool(config_prefix + ".allow_filtering", false)) , partition_key_prefix(config.getUInt(config_prefix + ".partition_key_prefix", 1)) @@ -124,7 +123,7 @@ CassandraDictionarySource::CassandraDictionarySource( { } -void CassandraDictionarySource::maybeAllowFiltering(String & query) +void CassandraDictionarySource::maybeAllowFiltering(String & query) const { if (!settings.allow_filtering) return; @@ -137,10 +136,11 @@ BlockInputStreamPtr CassandraDictionarySource::loadAll() String query = query_builder.composeLoadAllQuery(); maybeAllowFiltering(query); LOG_INFO(log, "Loading all using query: {}", query); - return std::make_shared(cluster, query, sample_block, max_block_size); + return std::make_shared(getSession(), query, sample_block, max_block_size); } -std::string CassandraDictionarySource::toString() const { +std::string CassandraDictionarySource::toString() const +{ return "Cassandra: " + settings.db + '.' + settings.table; } @@ -149,7 +149,7 @@ BlockInputStreamPtr CassandraDictionarySource::loadIds(const std::vector String query = query_builder.composeLoadIdsQuery(ids); maybeAllowFiltering(query); LOG_INFO(log, "Loading ids using query: {}", query); - return std::make_shared(cluster, query, sample_block, max_block_size); + return std::make_shared(getSession(), query, sample_block, max_block_size); } BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) @@ -162,7 +162,7 @@ BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_colu for (const auto & row : requested_rows) { SipHash partition_key; - for (const auto i : ext::range(0, settings.partition_key_prefix)) + for (size_t i = 0; i < settings.partition_key_prefix; ++i) key_columns[i]->updateHashWithValue(row, partition_key); partitions[partition_key.get64()].push_back(row); } @@ -173,7 +173,7 @@ BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_colu String query = query_builder.composeLoadKeysQuery(key_columns, partition.second, ExternalQueryBuilder::CASSANDRA_SEPARATE_PARTITION_KEY, settings.partition_key_prefix); maybeAllowFiltering(query); LOG_INFO(log, "Loading keys for partition hash {} using query: {}", partition.first, query); - streams.push_back(std::make_shared(cluster, query, sample_block, max_block_size)); + streams.push_back(std::make_shared(getSession(), query, sample_block, max_block_size)); } if (streams.size() == 1) @@ -182,6 +182,30 @@ BlockInputStreamPtr CassandraDictionarySource::loadKeys(const Columns & key_colu return std::make_shared(streams, nullptr, settings.max_threads); } +BlockInputStreamPtr CassandraDictionarySource::loadUpdatedAll() +{ + throw Exception("Method loadUpdatedAll is unsupported for CassandraDictionarySource", ErrorCodes::NOT_IMPLEMENTED); +} + +CassSessionShared CassandraDictionarySource::getSession() +{ + /// Reuse connection if exists, create new one if not + auto session = maybe_session.lock(); + if (session) + return session; + + std::lock_guard lock(connect_mutex); + session = maybe_session.lock(); + if (session) + return session; + + session = std::make_shared(); + CassFuturePtr future = cass_session_connect(*session, cluster); + cassandraWaitAndCheck(future); + maybe_session = session; + return session; +} + } #endif diff --git a/src/Dictionaries/CassandraDictionarySource.h b/src/Dictionaries/CassandraDictionarySource.h index dff93fcd029..18db66b94c2 100644 --- a/src/Dictionaries/CassandraDictionarySource.h +++ b/src/Dictionaries/CassandraDictionarySource.h @@ -34,7 +34,8 @@ struct CassandraSettings void setConsistency(const String & config_str); }; -class CassandraDictionarySource final : public IDictionarySource { +class CassandraDictionarySource final : public IDictionarySource +{ public: CassandraDictionarySource( const DictionaryStructure & dict_struct, @@ -64,15 +65,13 @@ public: BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector & requested_rows) override; - BlockInputStreamPtr loadUpdatedAll() override - { - throw Exception{"Method loadUpdatedAll is unsupported for CassandraDictionarySource", ErrorCodes::NOT_IMPLEMENTED}; - } + BlockInputStreamPtr loadUpdatedAll() override; String toString() const override; private: - void maybeAllowFiltering(String & query); + void maybeAllowFiltering(String & query) const; + CassSessionShared getSession(); Poco::Logger * log; const DictionaryStructure dict_struct; @@ -80,7 +79,9 @@ private: Block sample_block; ExternalQueryBuilder query_builder; + std::mutex connect_mutex; CassClusterPtr cluster; + CassSessionWeak maybe_session; }; } diff --git a/src/Dictionaries/CassandraHelpers.cpp b/src/Dictionaries/CassandraHelpers.cpp index 4f92a75a1f3..6de80a455c7 100644 --- a/src/Dictionaries/CassandraHelpers.cpp +++ b/src/Dictionaries/CassandraHelpers.cpp @@ -21,7 +21,7 @@ void cassandraCheck(CassError code) } -void cassandraWaitAndCheck(CassFuturePtr && future) +void cassandraWaitAndCheck(CassFuturePtr & future) { auto code = cass_future_error_code(future); /// Waits if not ready if (code == CASS_OK) diff --git a/src/Dictionaries/CassandraHelpers.h b/src/Dictionaries/CassandraHelpers.h index 2a91815e37d..70b38acf15c 100644 --- a/src/Dictionaries/CassandraHelpers.h +++ b/src/Dictionaries/CassandraHelpers.h @@ -7,6 +7,7 @@ #if USE_CASSANDRA #include #include +#include namespace DB { @@ -37,6 +38,7 @@ public: Dtor(ptr); ptr = rhs.ptr; rhs.ptr = nullptr; + return *this; } ~ObjectHolder() @@ -54,8 +56,12 @@ public: /// These object are created on pointer construction using CassClusterPtr = Cassandra::ObjectHolder; -using CassSessionPtr = Cassandra::ObjectHolder; using CassStatementPtr = Cassandra::ObjectHolder; +using CassSessionPtr = Cassandra::ObjectHolder; + +/// Share connections between streams. Executing statements in one session object is thread-safe +using CassSessionShared = std::shared_ptr; +using CassSessionWeak = std::weak_ptr; /// The following objects are created inside Cassandra driver library, /// but must be freed by user code @@ -65,7 +71,7 @@ using CassIteratorPtr = Cassandra::ObjectHoldertest {table} 1 + "Int64_" < 1000000000000000000 '''.format( host=self.docker_hostname, diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index 98ba191c948..8cd6940d587 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -3,12 +3,10 @@ import os from helpers.cluster import ClickHouseCluster from dictionary import Field, Row, Dictionary, DictionaryStructure, Layout - from external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, SourceExecutableHashed -from external_sources import SourceMongo, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra from external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra import math -import time + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries') @@ -212,7 +210,6 @@ def get_dictionaries(fold, total_folds, all_dicts): return all_dicts[fold * chunk_len : (fold + 1) * chunk_len] -#@pytest.mark.timeout(3000) @pytest.mark.parametrize("fold", list(range(10))) def test_simple_dictionaries(started_cluster, fold): fields = FIELDS["simple"] From 64c013610a852211148e5a9ef1694a7fa13c7a28 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 4 Jun 2020 01:03:49 +0300 Subject: [PATCH 16/29] fix --- src/Dictionaries/CassandraBlockInputStream.cpp | 2 -- src/Dictionaries/CassandraBlockInputStream.h | 4 ++++ .../test_dictionaries_all_layouts_and_sources/test.py | 2 -- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index 8fb9eb3f93c..8a14add868e 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include "CassandraBlockInputStream.h" @@ -20,7 +19,6 @@ namespace DB namespace ErrorCodes { extern const int TYPE_MISMATCH; - extern const int CASSANDRA_INTERNAL_ERROR; } CassandraBlockInputStream::CassandraBlockInputStream( diff --git a/src/Dictionaries/CassandraBlockInputStream.h b/src/Dictionaries/CassandraBlockInputStream.h index 5208e516a0e..667d686fd31 100644 --- a/src/Dictionaries/CassandraBlockInputStream.h +++ b/src/Dictionaries/CassandraBlockInputStream.h @@ -1,6 +1,8 @@ #pragma once #include + +#if USE_CASSANDRA #include #include #include @@ -41,3 +43,5 @@ private: }; } + +#endif diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index 8cd6940d587..0a812ea2a8b 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -225,8 +225,6 @@ def test_simple_dictionaries(started_cluster, fold): node.query("system reload dictionaries") - #time.sleep(3000) - queries_with_answers = [] for dct in simple_dicts: for row in data: From 4dae169216c7721739bf843780d777af4c8bae16 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 4 Jun 2020 16:03:06 +0300 Subject: [PATCH 17/29] fix gcc warnings --- src/Dictionaries/CassandraBlockInputStream.cpp | 4 ++-- src/Dictionaries/CassandraBlockInputStream.h | 2 +- src/Dictionaries/CassandraDictionarySource.h | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/CassandraBlockInputStream.cpp b/src/Dictionaries/CassandraBlockInputStream.cpp index 8a14add868e..4f6a62a0eea 100644 --- a/src/Dictionaries/CassandraBlockInputStream.cpp +++ b/src/Dictionaries/CassandraBlockInputStream.cpp @@ -35,7 +35,7 @@ CassandraBlockInputStream::CassandraBlockInputStream( cassandraCheck(cass_statement_set_paging_size(statement, max_block_size)); } -void CassandraBlockInputStream::insertValue(IColumn & column, ValueType type, const CassValue * cass_value) const +void CassandraBlockInputStream::insertValue(IColumn & column, ValueType type, const CassValue * cass_value) { switch (type) { @@ -202,7 +202,7 @@ void CassandraBlockInputStream::assertTypes(const CassResultPtr & result) size_t column_count = cass_result_column_count(result); for (size_t i = 0; i < column_count; ++i) { - CassValueType expected; + CassValueType expected = CASS_VALUE_TYPE_UNKNOWN; String expected_text; /// Cassandra does not support unsigned integers (cass_uint32_t is for Date) diff --git a/src/Dictionaries/CassandraBlockInputStream.h b/src/Dictionaries/CassandraBlockInputStream.h index 667d686fd31..3b0e583e3ad 100644 --- a/src/Dictionaries/CassandraBlockInputStream.h +++ b/src/Dictionaries/CassandraBlockInputStream.h @@ -30,7 +30,7 @@ private: using ValueType = ExternalResultDescription::ValueType; Block readImpl() override; - void insertValue(IColumn & column, ValueType type, const CassValue * cass_value) const; + static void insertValue(IColumn & column, ValueType type, const CassValue * cass_value); void assertTypes(const CassResultPtr & result); CassSessionShared session; diff --git a/src/Dictionaries/CassandraDictionarySource.h b/src/Dictionaries/CassandraDictionarySource.h index 18db66b94c2..c0a4e774d23 100644 --- a/src/Dictionaries/CassandraDictionarySource.h +++ b/src/Dictionaries/CassandraDictionarySource.h @@ -9,6 +9,7 @@ #include "ExternalQueryBuilder.h" #include #include +#include namespace DB { From 1c982d00e38cc04d3cf83a612fb7c9bae7a662b3 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 4 Jun 2020 16:59:12 +0300 Subject: [PATCH 18/29] try fix sync --- src/Dictionaries/CassandraHelpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/CassandraHelpers.h b/src/Dictionaries/CassandraHelpers.h index 70b38acf15c..8a00e372c96 100644 --- a/src/Dictionaries/CassandraHelpers.h +++ b/src/Dictionaries/CassandraHelpers.h @@ -5,7 +5,7 @@ #endif #if USE_CASSANDRA -#include +#include // Y_IGNORE #include #include From e2e5e1d5f857abe5e21a61c1566e404bced4024a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 8 Jun 2020 16:57:33 +0300 Subject: [PATCH 19/29] More stable perf tests --- docker/test/performance-comparison/compare.sh | 8 +- docker/test/performance-comparison/report.py | 9 +- .../performance/agg_functions_min_max_any.xml | 178 +++++++++--------- tests/performance/array_element.xml | 9 +- .../bit_operations_fixed_string.xml | 13 +- .../bit_operations_fixed_string_numbers.xml | 6 +- tests/performance/codecs_int_insert.xml | 2 +- .../performance/number_formatting_formats.xml | 22 ++- tests/performance/point_in_polygon.xml | 2 +- tests/performance/sum_map.xml | 1 - 10 files changed, 125 insertions(+), 125 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index e63ba6122c8..4993e381bd9 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -409,11 +409,11 @@ create table all_query_runs_json engine File(JSON, 'report/all-query-runs.json') ; create table changed_perf_tsv engine File(TSV, 'report/changed-perf.tsv') as - select left, right, diff, stat_threshold, changed_fail, test, query_display_name + select left, right, diff, stat_threshold, changed_fail, test, query_index, query_display_name from queries where changed_show order by abs(diff) desc; create table unstable_queries_tsv engine File(TSV, 'report/unstable-queries.tsv') as - select left, right, diff, stat_threshold, unstable_fail, test, query_display_name + select left, right, diff, stat_threshold, unstable_fail, test, query_index, query_display_name from queries where unstable_show order by stat_threshold desc; create table queries_for_flamegraph engine File(TSVWithNamesAndTypes, @@ -464,8 +464,8 @@ create table all_tests_tsv engine File(TSV, 'report/all-queries.tsv') as select changed_fail, unstable_fail, left, right, diff, floor(left > right ? left / right : right / left, 3), - stat_threshold, test, query_display_name - from queries order by test, query_display_name; + stat_threshold, test, query_index, query_display_name + from queries order by test, query_index; -- new report for all queries with all metrics (no page yet) create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.tsv') as diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 866e78da098..5d4d30e0098 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -207,7 +207,8 @@ if args.report == 'main': 'p < 0.001 threshold', # 3 # Failed # 4 'Test', # 5 - 'Query', # 6 + '#', # 6 + 'Query', # 7 ] print(tableHeader(columns)) @@ -248,7 +249,8 @@ if args.report == 'main': 'p < 0.001 threshold', #3 # Failed #4 'Test', #5 - 'Query' #6 + '#', #6 + 'Query' #7 ] print(tableStart('Unstable queries')) @@ -391,7 +393,8 @@ elif args.report == 'all-queries': 'Times speedup / slowdown', #5 'p < 0.001 threshold', #6 'Test', #7 - 'Query', #8 + '#', #8 + 'Query', #9 ] print(tableStart('All query times')) diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index 9c16cb88970..d40d19b6e38 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -1,95 +1,95 @@ - test.hits + hits_100m_single -select min(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(Title) from test.hits where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(URL) from test.hits where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(Referer) from test.hits where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(FlashMinor2) from test.hits where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(MobilePhoneModel) from test.hits where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(Params) from test.hits where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(SearchPhrase) from test.hits where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(PageCharset) from test.hits where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(SocialNetwork) from test.hits where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(SocialAction) from test.hits where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(SocialSourcePage) from test.hits where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(ParamOrderID) from test.hits where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(OpenstatServiceName) from test.hits where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(OpenstatCampaignID) from test.hits where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(OpenstatAdID) from test.hits where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(OpenstatSourceID) from test.hits where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(UTMSource) from test.hits where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(UTMMedium) from test.hits where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(UTMCampaign) from test.hits where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(UTMContent) from test.hits where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(UTMTerm) from test.hits where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(FromTag) from test.hits where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(Title) from hits_100m_single where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(Title) from hits_100m_single where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(Title) from hits_100m_single where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(Title) from hits_100m_single where Title != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(URL) from hits_100m_single where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(URL) from hits_100m_single where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(URL) from hits_100m_single where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(URL) from hits_100m_single where URL != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(Referer) from hits_100m_single where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(Referer) from hits_100m_single where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(Referer) from hits_100m_single where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(Referer) from hits_100m_single where Referer != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(FlashMinor2) from hits_100m_single where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(FlashMinor2) from hits_100m_single where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(FlashMinor2) from hits_100m_single where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(FlashMinor2) from hits_100m_single where FlashMinor2 != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(MobilePhoneModel) from hits_100m_single where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(MobilePhoneModel) from hits_100m_single where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(MobilePhoneModel) from hits_100m_single where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(MobilePhoneModel) from hits_100m_single where MobilePhoneModel != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(Params) from hits_100m_single where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(Params) from hits_100m_single where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(Params) from hits_100m_single where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(Params) from hits_100m_single where Params != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(SearchPhrase) from hits_100m_single where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(SearchPhrase) from hits_100m_single where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(SearchPhrase) from hits_100m_single where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(SearchPhrase) from hits_100m_single where SearchPhrase != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(PageCharset) from hits_100m_single where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(PageCharset) from hits_100m_single where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(PageCharset) from hits_100m_single where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(PageCharset) from hits_100m_single where PageCharset != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(ParamOrderID) from hits_100m_single where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(ParamOrderID) from hits_100m_single where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(ParamOrderID) from hits_100m_single where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(ParamOrderID) from hits_100m_single where ParamOrderID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OpenstatServiceName) from hits_100m_single where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OpenstatServiceName) from hits_100m_single where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OpenstatServiceName) from hits_100m_single where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OpenstatServiceName) from hits_100m_single where OpenstatServiceName != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OpenstatCampaignID) from hits_100m_single where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OpenstatCampaignID) from hits_100m_single where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OpenstatCampaignID) from hits_100m_single where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OpenstatCampaignID) from hits_100m_single where OpenstatCampaignID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OpenstatAdID) from hits_100m_single where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OpenstatAdID) from hits_100m_single where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OpenstatAdID) from hits_100m_single where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OpenstatAdID) from hits_100m_single where OpenstatAdID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(OpenstatSourceID) from hits_100m_single where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(OpenstatSourceID) from hits_100m_single where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(OpenstatSourceID) from hits_100m_single where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(OpenstatSourceID) from hits_100m_single where OpenstatSourceID != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMSource) from hits_100m_single where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMSource) from hits_100m_single where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMSource) from hits_100m_single where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMSource) from hits_100m_single where UTMSource != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMMedium) from hits_100m_single where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMMedium) from hits_100m_single where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMMedium) from hits_100m_single where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMMedium) from hits_100m_single where UTMMedium != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMCampaign) from hits_100m_single where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMCampaign) from hits_100m_single where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMCampaign) from hits_100m_single where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMCampaign) from hits_100m_single where UTMCampaign != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMContent) from hits_100m_single where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMContent) from hits_100m_single where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMContent) from hits_100m_single where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMContent) from hits_100m_single where UTMContent != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(UTMTerm) from hits_100m_single where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(UTMTerm) from hits_100m_single where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(UTMTerm) from hits_100m_single where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(UTMTerm) from hits_100m_single where UTMTerm != '' group by intHash32(UserID) % 1000000 FORMAT Null +select min(FromTag) from hits_100m_single where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null +select max(FromTag) from hits_100m_single where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null +select any(FromTag) from hits_100m_single where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null +select anyHeavy(FromTag) from hits_100m_single where FromTag != '' group by intHash32(UserID) % 1000000 FORMAT Null diff --git a/tests/performance/array_element.xml b/tests/performance/array_element.xml index 456ce55cbe1..1f82b833380 100644 --- a/tests/performance/array_element.xml +++ b/tests/performance/array_element.xml @@ -1,8 +1,5 @@ - - - - SELECT count() FROM numbers(10000000) WHERE NOT ignore([[1], [2]][number % 2 + 2]) - SELECT count() FROM numbers(10000000) WHERE NOT ignore([[], [2]][number % 2 + 2]) - SELECT count() FROM numbers(10000000) WHERE NOT ignore([[], []][number % 2 + 2]) + SELECT count() FROM numbers(100000000) WHERE NOT ignore([[1], [2]][number % 2 + 2]) + SELECT count() FROM numbers(100000000) WHERE NOT ignore([[], [2]][number % 2 + 2]) + SELECT count() FROM numbers(100000000) WHERE NOT ignore([[], []][number % 2 + 2]) diff --git a/tests/performance/bit_operations_fixed_string.xml b/tests/performance/bit_operations_fixed_string.xml index c08761ba8fc..19972535fd9 100644 --- a/tests/performance/bit_operations_fixed_string.xml +++ b/tests/performance/bit_operations_fixed_string.xml @@ -1,13 +1,10 @@ - - - - test.hits + hits_100m_single - SELECT count() FROM test.hits WHERE NOT ignore(bitAnd(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) - SELECT count() FROM test.hits WHERE NOT ignore(bitOr(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) - SELECT count() FROM test.hits WHERE NOT ignore(bitXor(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) - SELECT count() FROM test.hits WHERE NOT ignore(bitNot(toFixedString(ClientIP6, 16))) + SELECT count() FROM hits_100m_single WHERE NOT ignore(bitAnd(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) + SELECT count() FROM hits_100m_single WHERE NOT ignore(bitOr(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) + SELECT count() FROM hits_100m_single WHERE NOT ignore(bitXor(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) + SELECT count() FROM hits_100m_single WHERE NOT ignore(bitNot(toFixedString(ClientIP6, 16))) diff --git a/tests/performance/bit_operations_fixed_string_numbers.xml b/tests/performance/bit_operations_fixed_string_numbers.xml index e10e665ac81..5d3d9aa804d 100644 --- a/tests/performance/bit_operations_fixed_string_numbers.xml +++ b/tests/performance/bit_operations_fixed_string_numbers.xml @@ -1,6 +1,4 @@ - - - SELECT count() FROM numbers(10000000) WHERE NOT ignore(bitXor(reinterpretAsFixedString(number), reinterpretAsFixedString(number + 1))) - SELECT count() FROM numbers(10000000) WHERE NOT ignore(bitXor(reinterpretAsFixedString(number), reinterpretAsFixedString(0xabcd0123cdef4567))) + SELECT count() FROM numbers(100000000) WHERE NOT ignore(bitXor(reinterpretAsFixedString(number), reinterpretAsFixedString(number + 1))) + SELECT count() FROM numbers(100000000) WHERE NOT ignore(bitXor(reinterpretAsFixedString(number), reinterpretAsFixedString(0xabcd0123cdef4567))) diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index 0ad04f00f78..662df80ae70 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -33,7 +33,7 @@ num_rows - 10000000 + 20000000 diff --git a/tests/performance/number_formatting_formats.xml b/tests/performance/number_formatting_formats.xml index e6b5a8344dc..a9faf9369ec 100644 --- a/tests/performance/number_formatting_formats.xml +++ b/tests/performance/number_formatting_formats.xml @@ -1,6 +1,4 @@ - CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File(`{format}`) - format @@ -13,22 +11,30 @@ JSONEachRow TSKV RowBinary - Native XML Parquet ODBCDriver2 - Null MySQLWire - - test.hits - + + + format_fast + + Native + Null + + + + + CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File(`{format}`) + CREATE TABLE IF NOT EXISTS table_{format_fast} (x UInt64) ENGINE = File(`{format}`) INSERT INTO table_{format} SELECT number FROM numbers(10000000) + INSERT INTO table_{format_fast} SELECT number FROM numbers(100000000) DROP TABLE IF EXISTS table_{format} - + DROP TABLE IF EXISTS table_{format_fast} diff --git a/tests/performance/point_in_polygon.xml b/tests/performance/point_in_polygon.xml index b15fecbbfb0..c1325720429 100644 --- a/tests/performance/point_in_polygon.xml +++ b/tests/performance/point_in_polygon.xml @@ -4,7 +4,7 @@ INSERT INTO polygons WITH number + 1 AS radius SELECT [arrayMap(x -> (cos(x / 90. * pi()) * radius, sin(x / 90. * pi()) * radius), range(180))] - FROM numbers(100000) + FROM numbers(1000000) SELECT pointInPolygon((100, 100), polygon) FROM polygons diff --git a/tests/performance/sum_map.xml b/tests/performance/sum_map.xml index 9cc03035cce..cb1a4cb5bc6 100644 --- a/tests/performance/sum_map.xml +++ b/tests/performance/sum_map.xml @@ -7,7 +7,6 @@ scale - 100000 1000000 From 6a3df071a2204a63d767f3eb1cf9cc39e5d79e2b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 9 Jun 2020 01:13:08 +0300 Subject: [PATCH 20/29] fixup --- docker/test/performance-comparison/compare.sh | 2 +- .../performance/agg_functions_min_max_any.xml | 7 +- .../arithmetic_operations_in_aggr_func.xml | 20 +- tests/performance/base64_hits.xml | 5 +- tests/performance/basename.xml | 12 +- .../bit_operations_fixed_string.xml | 15 +- tests/performance/cpu_synthetic.xml | 5 +- tests/performance/int_parsing.xml | 171 +++++++++--------- .../performance/number_formatting_formats.xml | 4 +- tests/performance/point_in_polygon.xml | 2 +- tests/performance/set_index.xml | 2 +- tests/performance/website.xml | 2 +- 12 files changed, 118 insertions(+), 129 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 4993e381bd9..835e1424fd9 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -582,7 +582,7 @@ create table metric_devation engine File(TSVWithNamesAndTypes, union all select * from unstable_run_traces union all select * from unstable_run_metrics_2) mm group by test, query_index, metric - having d > 0.5 + having d > 0.5 and q[3] > 5 ) metrics left join query_display_names using (test, query_index) order by test, query_index, d desc diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index d40d19b6e38..f212ab227b8 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -1,4 +1,4 @@ - + hits_100m_single @@ -39,10 +39,7 @@ select max(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null select any(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null select anyHeavy(SocialNetwork) from hits_100m_single where SocialNetwork != '' group by intHash32(UserID) % 1000000 FORMAT Null -select min(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null -select max(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null -select any(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null -select anyHeavy(SocialAction) from hits_100m_single where SocialAction != '' group by intHash32(UserID) % 1000000 FORMAT Null + select min(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null select max(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null select any(SocialSourcePage) from hits_100m_single where SocialSourcePage != '' group by intHash32(UserID) % 1000000 FORMAT Null diff --git a/tests/performance/arithmetic_operations_in_aggr_func.xml b/tests/performance/arithmetic_operations_in_aggr_func.xml index 28f13823731..c91033bf9e4 100644 --- a/tests/performance/arithmetic_operations_in_aggr_func.xml +++ b/tests/performance/arithmetic_operations_in_aggr_func.xml @@ -1,22 +1,14 @@ + SELECT max(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(500000000) - - - 10 - - + SELECT min(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(500000000) + SELECT sum(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(500000000) - SELECT max(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(120000000) + SELECT min(-1 + (((-2 + (number + -3)) + -4) + -5)) FROM numbers(500000000) - SELECT min(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(120000000) + SELECT max(-1 + (((-2 + (number + -3)) + -4) + -5)) FROM numbers(500000000) - SELECT sum(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(120000000) - - SELECT min(-1 + (((-2 + (number + -3)) + -4) + -5)) FROM numbers(120000000) - - SELECT max(-1 + (((-2 + (number + -3)) + -4) + -5)) FROM numbers(120000000) - - SELECT max(((((number) * 10) * -2) * 3) * 2) + min(((((number) * 10) * -2) * 3) * 2) FROM numbers(120000000) + SELECT max(((((number) * 10) * -2) * 3) * 2) + min(((((number) * 10) * -2) * 3) * 2) FROM numbers(500000000) diff --git a/tests/performance/base64_hits.xml b/tests/performance/base64_hits.xml index b07212bd598..65828d6ca1f 100644 --- a/tests/performance/base64_hits.xml +++ b/tests/performance/base64_hits.xml @@ -1,10 +1,11 @@ - hits_100m_single - + + 1 + diff --git a/tests/performance/basename.xml b/tests/performance/basename.xml index 7d040da5998..4ccc7ddf3a0 100644 --- a/tests/performance/basename.xml +++ b/tests/performance/basename.xml @@ -1,12 +1,12 @@ - - - - test.hits + hits_100m_single -SELECT count() FROM test.hits WHERE NOT ignore(basename(URL)) -SELECT count() FROM test.hits WHERE NOT ignore(basename(Referer)) + + 1 + + SELECT count() FROM hits_100m_single WHERE NOT ignore(basename(URL)) + SELECT count() FROM hits_100m_single WHERE NOT ignore(basename(Referer)) diff --git a/tests/performance/bit_operations_fixed_string.xml b/tests/performance/bit_operations_fixed_string.xml index 19972535fd9..56b6ff804f7 100644 --- a/tests/performance/bit_operations_fixed_string.xml +++ b/tests/performance/bit_operations_fixed_string.xml @@ -1,10 +1,15 @@ - hits_100m_single + test.hits - SELECT count() FROM hits_100m_single WHERE NOT ignore(bitAnd(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) - SELECT count() FROM hits_100m_single WHERE NOT ignore(bitOr(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) - SELECT count() FROM hits_100m_single WHERE NOT ignore(bitXor(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) - SELECT count() FROM hits_100m_single WHERE NOT ignore(bitNot(toFixedString(ClientIP6, 16))) + + 1 + + + + SELECT count() FROM test.hits WHERE NOT ignore(bitAnd(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) + SELECT count() FROM test.hits WHERE NOT ignore(bitOr(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) + SELECT count() FROM test.hits WHERE NOT ignore(bitXor(toFixedString(ClientIP6, 16), IPv6StringToNum('ffff:ffff:ffff:0000:0000:0000:0000:0000'))) + SELECT count() FROM test.hits WHERE NOT ignore(bitNot(toFixedString(ClientIP6, 16))) diff --git a/tests/performance/cpu_synthetic.xml b/tests/performance/cpu_synthetic.xml index 1076e051207..2888f7bbbd6 100644 --- a/tests/performance/cpu_synthetic.xml +++ b/tests/performance/cpu_synthetic.xml @@ -1,7 +1,4 @@ - - - - + hits_100m_single hits_10m_single diff --git a/tests/performance/int_parsing.xml b/tests/performance/int_parsing.xml index a9258875b5e..1a242c28393 100644 --- a/tests/performance/int_parsing.xml +++ b/tests/performance/int_parsing.xml @@ -1,92 +1,89 @@ - - - - test.hits + hits_100m_single - SELECT count() FROM test.hits WHERE NOT ignore(toString(WatchID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(JavaEnable)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(GoodEvent)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(CounterID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ClientIP)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(RegionID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(UserID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(CounterClass)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(OS)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(UserAgent)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(Refresh)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsRobot)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ResolutionWidth)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ResolutionHeight)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ResolutionDepth)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(FlashMajor)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(FlashMinor)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(NetMajor)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(NetMinor)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(UserAgentMajor)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(CookieEnable)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(JavascriptEnable)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsMobile)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(MobilePhone)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IPNetworkID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(TraficSourceID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(SearchEngineID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(SearchPhrase)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(AdvEngineID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsArtifical)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(WindowClientWidth)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(WindowClientHeight)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ClientTimeZone)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(SilverlightVersion1)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(SilverlightVersion2)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(SilverlightVersion3)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(SilverlightVersion4)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(CodeVersion)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsLink)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsDownload)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsNotBounce)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(FUniqID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(HID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsOldCounter)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsEvent)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(IsParameter)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(DontCountHits)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(WithHash)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(Age)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(Sex)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(Income)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(Interests)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(Robotness)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(RemoteIP)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(WindowName)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(OpenerName)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(HistoryLength)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(HTTPError)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(SendTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(DNSTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ConnectTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ResponseStartTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ResponseEndTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(FetchTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(RedirectTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(DOMInteractiveTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(DOMContentLoadedTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(DOMCompleteTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(LoadEventStartTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(LoadEventEndTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(NSToDOMContentLoadedTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(FirstPaintTiming)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(RedirectCount)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(SocialSourceNetworkID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ParamPrice)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(ParamCurrencyID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(HasGCLID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(RefererHash)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(URLHash)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(CLID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(YCLID)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(RequestNum)) SETTINGS max_threads = 1 - SELECT count() FROM test.hits WHERE NOT ignore(toString(RequestTry)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(WatchID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(JavaEnable)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(GoodEvent)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(CounterID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ClientIP)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RegionID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(UserID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(CounterClass)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(OS)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(UserAgent)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(Refresh)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsRobot)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionWidth)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionHeight)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionDepth)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FlashMajor)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FlashMinor)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(NetMajor)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(NetMinor)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(UserAgentMajor)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(CookieEnable)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(JavascriptEnable)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsMobile)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(MobilePhone)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IPNetworkID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(TraficSourceID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SearchEngineID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SearchPhrase)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(AdvEngineID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsArtifical)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(WindowClientWidth)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(WindowClientHeight)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ClientTimeZone)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SilverlightVersion1)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SilverlightVersion2)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SilverlightVersion3)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SilverlightVersion4)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(CodeVersion)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsLink)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsDownload)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsNotBounce)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FUniqID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(HID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsOldCounter)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsEvent)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsParameter)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DontCountHits)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(WithHash)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(Age)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(Sex)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(Income)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(Interests)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(Robotness)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RemoteIP)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(WindowName)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(OpenerName)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(HistoryLength)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(HTTPError)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SendTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DNSTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ConnectTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResponseStartTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResponseEndTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FetchTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RedirectTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMInteractiveTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMContentLoadedTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMCompleteTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(LoadEventStartTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(LoadEventEndTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(NSToDOMContentLoadedTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FirstPaintTiming)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RedirectCount)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SocialSourceNetworkID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ParamPrice)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ParamCurrencyID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(HasGCLID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RefererHash)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(URLHash)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(CLID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(YCLID)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RequestNum)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RequestTry)) SETTINGS max_threads = 1 diff --git a/tests/performance/number_formatting_formats.xml b/tests/performance/number_formatting_formats.xml index a9faf9369ec..c4a17b1f133 100644 --- a/tests/performance/number_formatting_formats.xml +++ b/tests/performance/number_formatting_formats.xml @@ -32,8 +32,8 @@ CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File(`{format}`) CREATE TABLE IF NOT EXISTS table_{format_fast} (x UInt64) ENGINE = File(`{format}`) - INSERT INTO table_{format} SELECT number FROM numbers(10000000) - INSERT INTO table_{format_fast} SELECT number FROM numbers(100000000) + INSERT INTO table_{format} SELECT number FROM numbers(10000000) + INSERT INTO table_{format_fast} SELECT number FROM numbers(20000000) DROP TABLE IF EXISTS table_{format} DROP TABLE IF EXISTS table_{format_fast} diff --git a/tests/performance/point_in_polygon.xml b/tests/performance/point_in_polygon.xml index c1325720429..a1ef4891577 100644 --- a/tests/performance/point_in_polygon.xml +++ b/tests/performance/point_in_polygon.xml @@ -7,7 +7,7 @@ FROM numbers(1000000) - SELECT pointInPolygon((100, 100), polygon) FROM polygons + SELECT pointInPolygon((100, 100), polygon) FROM polygons FORMAT Null DROP TABLE IF EXISTS polygons diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index d9b263159b5..46fe32e5486 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -15,7 +15,7 @@ SELECT count() FROM test_in WHERE -toInt64(a) NOT IN (SELECT toInt64(rand(1)) FROM numbers(100000)) settings max_rows_to_read=1, read_overflow_mode='break' - SELECT count() FROM numbers(1000) WHERE toString(number) IN ('41577', '83972', '51697', '50014', '37553', '93459', '87438', '95971', '83186', '74326', '67871', '50406', '83678', '29655', '18580', '83905', '61518', '29059', '56700', '82787', '98672', '30884', '81822', '39850', '80852', '57627', '91346', '64522', '17781', '49467', '41099', '41929', '85618', '91389', '68564', '91769', '81219', '52218', '37220', '97097', '2129', '9886', '52049', '34847', '25364', '36429', '76897', '71868', '58121', '71199', '84819', '69991', '34046', '64507', '34892', '24228', '36986', '28588', '51159', '53444', '80531', '9941', '20256', '48103', '32565', '62890', '5379', '60302', '46434', '3205', '18821', '31030', '19794', '71557', '71703', '15024', '14004', '82164', '95659', '40227', '83358', '24395', '9610', '19814', '48491', '66412', '16012', '71586', '42143', '51103', '24463', '89949', '35694', '39193', '63904', '40489', '77144', '94014', '84836', '9980', '46554', '43905', '25588', '25205', '72624', '10249', '35888', '98478', '99030', '26834', '31', '81499', '14847', '82997', '92357', '92893', '17426', '56630', '22252', '68119', '62710', '8740', '82144', '79916', '23391', '30192', '99271', '96435', '44237', '98327', '69481', '16691', '13643', '84554', '38571', '70926', '99283', '79000', '20926', '86495', '4834', '1222', '39486', '57697', '58002', '40790', '15623', '3999', '31515', '12694', '26143', '35951', '54085', '97534', '35329', '73535', '88715', '29572', '75799', '45166', '32066', '48023', '69523', '93150', '8740', '96790', '15534', '63252', '5142', '67045', '93992', '16663', '292', '63924', '6588', '12190', '31506', '69590', '35394', '55168', '65223', '79183', '32600', '69676', '28316', '72111', '53531', '15073', '41127', '73451', '24725', '61647', '65315', '41143', '26493', '95608', '34407', '76098', '53105', '83691', '48755', '35696', '62587', '81826', '3963', '45766', '82751', '12430', '97685', '29919', '78155', '71636', '50215', '89734', '9892', '47151', '54855', '3428', '9712', '52592', '2403', '79602', '81243', '79859', '57361', '82000', '42107', '28860', '99591', '28296', '57337', '64969', '32332', '25535', '30924', '21313', '32554', '17342', '87311', '19825', '24898', '61323', '83209', '79322', '79009', '50746', '33396', '62033', '16548', '17427', '24073', '34640', '52368', '4724', '80408', '40', '33787', '16666', '19665', '86751', '27264', '2241', '88134', '53566', '10589', '79711', '92823', '58972', '91767', '60885', '51659', '7867', '96849', '30360', '20914', '9584', '1250', '22871', '23282', '99312', '4683', '33429', '68361', '82614', '81440', '47863', '69790', '11968', '75210', '66854', '37002', '61142', '71514', '1588', '42336', '11069', '26291', '2261', '71056', '13492', '9133', '91216', '72207', '71586', '86535', '83898', '24392', '45384', '48545', '61972', '503', '80180', '35834', '97025', '70411', '55039', '35430', '27631', '82533', '96831', '74077', '42533', '14451', '26943', '53783', '69489', '71969', '8432', '37230', '61348', '19472', '59115', '9886', '50951', '57109', '7141', '1902', '84130', '4323', '55889', '47784', '2220', '75988', '66988', '63721', '8131', '95601', '95207', '2311', '26541', '50991', '6717', '2969', '71857', '51034', '65958', '94716', '90275', '21012', '46859', '7984', '31131', '46457', '69578', '44540', '7294', '80117', '9925', '60155', '90608', '82684', '32193', '87071', '28006', '87604', '24501', '79087', '2848', '29237', '11221', '81319', '40966', '87641', '35325', '78705', '88636', '78717', '62831', '56390', '99271', '43821', '14453', '17923', '62695', '77322', '21038', '67677', '41271', '4376', '65426', '46091', '19887', '97251', '55583', '58763', '3826', '35037', '73533', '64267', '82319', '9836', '42622', '96829', '16363', '10455', '49290', '99992', '98229', '66356', '59087', '73998', '25986', '4279', '56790', '69540', '588', '36620', '60358', '45056', '89297', '42740', '8323', '19245', '82417', '41431', '699', '11554', '73910', '44491', '56019', '68901', '45816', '68126', '89379', '23885', '13263', '56395', '73130', '19089', '23771', '10335', '48547', '16903', '6453', '33560', '89668', '38159', '43177', '90655', '49712', '62', '66920', '34180', '12150', '48564', '39538', '85026', '87195', '14928', '8956', '71157', '53287', '39161', '67583', '83309', '92054', '86977', '56188', '15229', '88170', '60894', '58497', '89254', '40082', '86890', '60161', '97291', '45878', '23368', '14577', '92870', '37017', '97356', '99426', '76061', '89186', '99751', '85153', '61580', '39360', '90107', '25603', '26798', '76224', '6469', '7912', '69838', '16404', '67497', '28965', '80836', '80365', '91249', '48713', '17113', '33090', '40793', '70450', '66689', '83698', '17802', '43869', '13355', '18959', '79411', '87930', '9265', '37504', '44876', '97234', '94149', '35040', '22049', '49248', '6535', '36080', '28346', '94437', '78319', '17961', '89056', '56161', '35810', '41632', '45494', '53351', '89729', '99510', '51584', '59688', '6193', '70809', '51093', '92589', '90247', '34910', '78235', '17362', '49423', '63324', '525', '37638', '72325', '89356', '15298', '59116', '17848', '65429', '27029', '84781', '70247', '8825', '35082', '70451', '22522', '58125', '91879', '90531', '2478', '463', '37902', '54405', '87267', '72688', '22803', '33134', '35177', '84551', '44974', '88375', '76407', '27774', '33849', '19915', '82014', '80434', '26380', '48777', '53811', '14838', '26829', '56441', '99869', '49574', '85476', '19723', '16907', '4018', '37338', '78510', '47912', '13030', '65277', '95716', '67363', '21393', '89887', '78842', '81650', '903', '17436', '30704', '49223', '27198', '25500', '52214', '54258', '70082', '53950', '49312', '43615', '99473', '94348', '53661', '96213', '96346', '62010', '38268', '32861', '75660', '10392', '89491', '68335', '29817', '88706', '24184', '36298', '43440', '21626', '26535', '44560', '46363', '12534', '99070', '95606', '33714', '73070', '8303', '29853', '23014', '99982', '4530', '14955', '45803', '50', '90750', '30394', '81276', '95563', '47314', '58520', '91299', '88944', '54402', '67405', '29253', '47079', '71734', '99728', '17652', '13307', '35556', '18962', '26780', '17771', '53712', '60055', '37628', '35830', '90739', '61151', '41309', '27652', '3051', '53167', '98417', '19382', '36833', '75085', '65374', '87732', '30352', '31776', '32765', '97565', '92199', '49050', '29503', '51024', '18834', '8515', '24069', '96216', '10777', '90680', '18974', '68884', '85305', '36007', '56707', '4212', '47352', '34426', '13185', '92939', '95782', '70577', '58080', '98279', '3906', '5065', '56896', '16382', '31273', '17117', '98602', '12786', '24086', '63970', '72756', '35798', '82367', '7356', '53398', '68503', '2962', '16425', '67334', '68461', '65439', '15620', '70906', '29649', '46461', '74602', '38012', '71714', '16825', '89480', '53386', '88532', '35104', '28556', '82120', '23155', '23347', '24797', '60061', '54962', '99427', '82248', '82447', '39968', '63727', '27431', '81511', '91168', '71425', '80740', '84127', '40717', '15503', '15419', '46594', '61263', '19212', '53175', '70724', '74445', '23034', '71818', '40246', '18886', '53066', '4880', '83701', '86107', '87862', '44751', '392', '73440', '90291', '93395', '20894', '38463', '32664', '55158', '20090', '50004', '79070', '98471', '85478', '96615', '68149', '78334', '97752', '73207', '71678', '91238', '96757', '82598', '194', '35797', '45120', '60782', '28721', '17676', '78066', '60957', '11826', '51563', '50516', '16485', '47053', '31738', '48923', '23554', '96850', '42033', '73701', '78607', '45979', '54571', '12415', '31693', '15356', '36902', '9126', '3767', '3295', '90402', '24005', '95350', '67033', '49137', '72606', '51899', '17522', '31957', '44641', '53982', '23767', '68257', '15766', '19995', '2107', '48788', '11765', '91055', '46576', '54651', '50381', '62827', '73636', '46606', '98753', '37631', '70441', '87916', '66983', '33870', '31125', '12904', '57040', '4874', '58632', '42037', '18782', '5998', '18974', '57949', '81010', '90407', '99874', '20462', '89949', '10952', '71454', '95130', '46115', '3518', '13384', '69039', '79482', '22076', '59782', '32042', '40930', '60243', '29298', '6790', '46985', '44398', '85631', '14380', '66179', '2629', '32126', '49833', '14118', '58492', '31493', '81172', '96638', '8745', '89663', '76842', '78633', '41373', '83721', '42886', '11123', '32739', '11051', '1303', '92314', '83324', '85600', '44276', '69064', '56125', '84650', '31028', '12628', '14502', '64764', '39405', '44855', '79046', '51716', '46824', '83389', '1941', '1257', '9280', '73176', '84729', '2579', '63366', '22606', '35541', '51096', '13447', '18355', '68037', '28436', '94116', '81070', '78355', '67897', '5296', '32742', '77645', '91853', '18767', '67949', '40963', '5792', '17278', '25597', '41884', '80829', '7099', '18645', '60295', '12082', '81800', '78415', '18082', '38789', '16295', '72377', '74949', '55583', '66853', '15402', '72977', '15123', '99434', '34999', '21687', '76049', '42987', '83748', '88256', '66688', '21766', '20304', '29271', '10069', '19822', '11792', '42526', '74143', '17289', '30253', '6367', '20888', '12975', '94073', '98639', '30134', '26320', '65507', '69002', '53120', '4550', '38893', '18954', '38283', '54863', '17698', '99670', '10521', '92467', '60994', '18052', '48673', '35811', '87282', '62706', '16061', '53112', '22652', '37780', '55662', '26331', '49410', '79074', '10623', '69577', '79613', '9491', '31229', '43922', '84231', '58409', '36386', '46875', '74431', '76735', '38776', '23350', '7314', '9079', '51519', '98544', '70216', '63380', '90381', '1295', '46901', '58225', '55339', '89918', '75522', '35431', '89460', '49552', '89302', '23068', '28493', '3042', '25194', '59520', '9810', '95706', '81297', '89638', '54794', '94527', '45262', '97932', '78685', '6947', '22818', '48700', '9153', '12289', '22011', '58825', '93854', '65438', '4509', '33741', '28208', '69061', '48578', '40247', '77725', '31837', '39003', '69363', '78113', '76398', '97262', '67795', + SELECT count() FROM numbers(10000) WHERE toString(number) IN ('41577', '83972', '51697', '50014', '37553', '93459', '87438', '95971', '83186', '74326', '67871', '50406', '83678', '29655', '18580', '83905', '61518', '29059', '56700', '82787', '98672', '30884', '81822', '39850', '80852', '57627', '91346', '64522', '17781', '49467', '41099', '41929', '85618', '91389', '68564', '91769', '81219', '52218', '37220', '97097', '2129', '9886', '52049', '34847', '25364', '36429', '76897', '71868', '58121', '71199', '84819', '69991', '34046', '64507', '34892', '24228', '36986', '28588', '51159', '53444', '80531', '9941', '20256', '48103', '32565', '62890', '5379', '60302', '46434', '3205', '18821', '31030', '19794', '71557', '71703', '15024', '14004', '82164', '95659', '40227', '83358', '24395', '9610', '19814', '48491', '66412', '16012', '71586', '42143', '51103', '24463', '89949', '35694', '39193', '63904', '40489', '77144', '94014', '84836', '9980', '46554', '43905', '25588', '25205', '72624', '10249', '35888', '98478', '99030', '26834', '31', '81499', '14847', '82997', '92357', '92893', '17426', '56630', '22252', '68119', '62710', '8740', '82144', '79916', '23391', '30192', '99271', '96435', '44237', '98327', '69481', '16691', '13643', '84554', '38571', '70926', '99283', '79000', '20926', '86495', '4834', '1222', '39486', '57697', '58002', '40790', '15623', '3999', '31515', '12694', '26143', '35951', '54085', '97534', '35329', '73535', '88715', '29572', '75799', '45166', '32066', '48023', '69523', '93150', '8740', '96790', '15534', '63252', '5142', '67045', '93992', '16663', '292', '63924', '6588', '12190', '31506', '69590', '35394', '55168', '65223', '79183', '32600', '69676', '28316', '72111', '53531', '15073', '41127', '73451', '24725', '61647', '65315', '41143', '26493', '95608', '34407', '76098', '53105', '83691', '48755', '35696', '62587', '81826', '3963', '45766', '82751', '12430', '97685', '29919', '78155', '71636', '50215', '89734', '9892', '47151', '54855', '3428', '9712', '52592', '2403', '79602', '81243', '79859', '57361', '82000', '42107', '28860', '99591', '28296', '57337', '64969', '32332', '25535', '30924', '21313', '32554', '17342', '87311', '19825', '24898', '61323', '83209', '79322', '79009', '50746', '33396', '62033', '16548', '17427', '24073', '34640', '52368', '4724', '80408', '40', '33787', '16666', '19665', '86751', '27264', '2241', '88134', '53566', '10589', '79711', '92823', '58972', '91767', '60885', '51659', '7867', '96849', '30360', '20914', '9584', '1250', '22871', '23282', '99312', '4683', '33429', '68361', '82614', '81440', '47863', '69790', '11968', '75210', '66854', '37002', '61142', '71514', '1588', '42336', '11069', '26291', '2261', '71056', '13492', '9133', '91216', '72207', '71586', '86535', '83898', '24392', '45384', '48545', '61972', '503', '80180', '35834', '97025', '70411', '55039', '35430', '27631', '82533', '96831', '74077', '42533', '14451', '26943', '53783', '69489', '71969', '8432', '37230', '61348', '19472', '59115', '9886', '50951', '57109', '7141', '1902', '84130', '4323', '55889', '47784', '2220', '75988', '66988', '63721', '8131', '95601', '95207', '2311', '26541', '50991', '6717', '2969', '71857', '51034', '65958', '94716', '90275', '21012', '46859', '7984', '31131', '46457', '69578', '44540', '7294', '80117', '9925', '60155', '90608', '82684', '32193', '87071', '28006', '87604', '24501', '79087', '2848', '29237', '11221', '81319', '40966', '87641', '35325', '78705', '88636', '78717', '62831', '56390', '99271', '43821', '14453', '17923', '62695', '77322', '21038', '67677', '41271', '4376', '65426', '46091', '19887', '97251', '55583', '58763', '3826', '35037', '73533', '64267', '82319', '9836', '42622', '96829', '16363', '10455', '49290', '99992', '98229', '66356', '59087', '73998', '25986', '4279', '56790', '69540', '588', '36620', '60358', '45056', '89297', '42740', '8323', '19245', '82417', '41431', '699', '11554', '73910', '44491', '56019', '68901', '45816', '68126', '89379', '23885', '13263', '56395', '73130', '19089', '23771', '10335', '48547', '16903', '6453', '33560', '89668', '38159', '43177', '90655', '49712', '62', '66920', '34180', '12150', '48564', '39538', '85026', '87195', '14928', '8956', '71157', '53287', '39161', '67583', '83309', '92054', '86977', '56188', '15229', '88170', '60894', '58497', '89254', '40082', '86890', '60161', '97291', '45878', '23368', '14577', '92870', '37017', '97356', '99426', '76061', '89186', '99751', '85153', '61580', '39360', '90107', '25603', '26798', '76224', '6469', '7912', '69838', '16404', '67497', '28965', '80836', '80365', '91249', '48713', '17113', '33090', '40793', '70450', '66689', '83698', '17802', '43869', '13355', '18959', '79411', '87930', '9265', '37504', '44876', '97234', '94149', '35040', '22049', '49248', '6535', '36080', '28346', '94437', '78319', '17961', '89056', '56161', '35810', '41632', '45494', '53351', '89729', '99510', '51584', '59688', '6193', '70809', '51093', '92589', '90247', '34910', '78235', '17362', '49423', '63324', '525', '37638', '72325', '89356', '15298', '59116', '17848', '65429', '27029', '84781', '70247', '8825', '35082', '70451', '22522', '58125', '91879', '90531', '2478', '463', '37902', '54405', '87267', '72688', '22803', '33134', '35177', '84551', '44974', '88375', '76407', '27774', '33849', '19915', '82014', '80434', '26380', '48777', '53811', '14838', '26829', '56441', '99869', '49574', '85476', '19723', '16907', '4018', '37338', '78510', '47912', '13030', '65277', '95716', '67363', '21393', '89887', '78842', '81650', '903', '17436', '30704', '49223', '27198', '25500', '52214', '54258', '70082', '53950', '49312', '43615', '99473', '94348', '53661', '96213', '96346', '62010', '38268', '32861', '75660', '10392', '89491', '68335', '29817', '88706', '24184', '36298', '43440', '21626', '26535', '44560', '46363', '12534', '99070', '95606', '33714', '73070', '8303', '29853', '23014', '99982', '4530', '14955', '45803', '50', '90750', '30394', '81276', '95563', '47314', '58520', '91299', '88944', '54402', '67405', '29253', '47079', '71734', '99728', '17652', '13307', '35556', '18962', '26780', '17771', '53712', '60055', '37628', '35830', '90739', '61151', '41309', '27652', '3051', '53167', '98417', '19382', '36833', '75085', '65374', '87732', '30352', '31776', '32765', '97565', '92199', '49050', '29503', '51024', '18834', '8515', '24069', '96216', '10777', '90680', '18974', '68884', '85305', '36007', '56707', '4212', '47352', '34426', '13185', '92939', '95782', '70577', '58080', '98279', '3906', '5065', '56896', '16382', '31273', '17117', '98602', '12786', '24086', '63970', '72756', '35798', '82367', '7356', '53398', '68503', '2962', '16425', '67334', '68461', '65439', '15620', '70906', '29649', '46461', '74602', '38012', '71714', '16825', '89480', '53386', '88532', '35104', '28556', '82120', '23155', '23347', '24797', '60061', '54962', '99427', '82248', '82447', '39968', '63727', '27431', '81511', '91168', '71425', '80740', '84127', '40717', '15503', '15419', '46594', '61263', '19212', '53175', '70724', '74445', '23034', '71818', '40246', '18886', '53066', '4880', '83701', '86107', '87862', '44751', '392', '73440', '90291', '93395', '20894', '38463', '32664', '55158', '20090', '50004', '79070', '98471', '85478', '96615', '68149', '78334', '97752', '73207', '71678', '91238', '96757', '82598', '194', '35797', '45120', '60782', '28721', '17676', '78066', '60957', '11826', '51563', '50516', '16485', '47053', '31738', '48923', '23554', '96850', '42033', '73701', '78607', '45979', '54571', '12415', '31693', '15356', '36902', '9126', '3767', '3295', '90402', '24005', '95350', '67033', '49137', '72606', '51899', '17522', '31957', '44641', '53982', '23767', '68257', '15766', '19995', '2107', '48788', '11765', '91055', '46576', '54651', '50381', '62827', '73636', '46606', '98753', '37631', '70441', '87916', '66983', '33870', '31125', '12904', '57040', '4874', '58632', '42037', '18782', '5998', '18974', '57949', '81010', '90407', '99874', '20462', '89949', '10952', '71454', '95130', '46115', '3518', '13384', '69039', '79482', '22076', '59782', '32042', '40930', '60243', '29298', '6790', '46985', '44398', '85631', '14380', '66179', '2629', '32126', '49833', '14118', '58492', '31493', '81172', '96638', '8745', '89663', '76842', '78633', '41373', '83721', '42886', '11123', '32739', '11051', '1303', '92314', '83324', '85600', '44276', '69064', '56125', '84650', '31028', '12628', '14502', '64764', '39405', '44855', '79046', '51716', '46824', '83389', '1941', '1257', '9280', '73176', '84729', '2579', '63366', '22606', '35541', '51096', '13447', '18355', '68037', '28436', '94116', '81070', '78355', '67897', '5296', '32742', '77645', '91853', '18767', '67949', '40963', '5792', '17278', '25597', '41884', '80829', '7099', '18645', '60295', '12082', '81800', '78415', '18082', '38789', '16295', '72377', '74949', '55583', '66853', '15402', '72977', '15123', '99434', '34999', '21687', '76049', '42987', '83748', '88256', '66688', '21766', '20304', '29271', '10069', '19822', '11792', '42526', '74143', '17289', '30253', '6367', '20888', '12975', '94073', '98639', '30134', '26320', '65507', '69002', '53120', '4550', '38893', '18954', '38283', '54863', '17698', '99670', '10521', '92467', '60994', '18052', '48673', '35811', '87282', '62706', '16061', '53112', '22652', '37780', '55662', '26331', '49410', '79074', '10623', '69577', '79613', '9491', '31229', '43922', '84231', '58409', '36386', '46875', '74431', '76735', '38776', '23350', '7314', '9079', '51519', '98544', '70216', '63380', '90381', '1295', '46901', '58225', '55339', '89918', '75522', '35431', '89460', '49552', '89302', '23068', '28493', '3042', '25194', '59520', '9810', '95706', '81297', '89638', '54794', '94527', '45262', '97932', '78685', '6947', '22818', '48700', '9153', '12289', '22011', '58825', '93854', '65438', '4509', '33741', '28208', '69061', '48578', '40247', '77725', '31837', '39003', '69363', '78113', '76398', '97262', '67795', '68446', '58896', '60969', '19849', '6722', '91854', '49519', '13949', '67109', '48824', '31723', '75554', '69575', '94986', '75350', '18628', '15284', '41943', '15433', '52607', '41', '22340', '29528', '24059', '34145', '72517', '46316', '10667', '54510', '19882', '47764', '69124', '41963', '84350', '48420', '4646', '24958', '69020', '97121', '26178', '62664', '50622', '32554', '49655', '45398', '11267', '72222', '73363', '89554', '89046', '57384', '29259', '37888', '24850', '74353', '57343', '34762', '2900', '11393', '42154', '94306', '70552', '75265', '921', '26003', '64352', '89857', '83171', '58249', '48940', '53512', '66335', '44865', '68729', '19299', '58003', '39854', '99122', '3860', '80173', '52242', '90966', '53183', '71982', '82325', '87842', '15000', '55627', '71132', '6354', '42402', '91719', '91644', '94533', '74925', '66278', '66911', '85576', '40495', '70919', '71797', '87835', '29845', '71832', '3390', '7994', '33499', '70715', '54897', '82710', '63077', '78105', '24758', '89585', '84607', '46477', '78618', '10989', '39222', '98749', '51685', '94664', '31008', '32823', '89521', '72160', '26952', '4001', '21421', '5737', '74027', '88179', '45553', '83743', '19913', '49435', '65616', '82641', '5149', '76959', '40681', '73023', '2670', '30845', '18863', '35094', '88400', '80963', '9154', '16571', '64192', '59694', '41317', '59942', '58856', '99281', '67260', '66971', '22716', '76089', '58047', '67071', '53707', '462', '52518', '72277', '10681', '69', '98855', '12593', '88842', '67242', '73543', '37439', '18413', '67211', '93495', '45576', '70614', '27988', '53210', '18618', '21318', '68059', '25518', '55917', '56522', '16548', '2404', '93538', '61452', '66358', '3709', '23914', '92426', '81439', '38070', '28988', '29939', '2948', '85720', '45628', '51101', '89431', '86365', '17571', '50987', '83849', '11015', '83812', '66187', '26362', '66786', '22024', '93866', '36161', '90080', '64874', '37294', '83860', '73821', '80279', '36766', '73117', '44620', '84556', '42070', '90383', '27862', '20665', '67576', '34997', '57958', '80638', '84351', '63961', '1362', '14338', '80377', '24192', '41294', '57368', '51189', '27287', '45764', '86289', '65600', '708', '84090', '96005', '55676', '84855', '72385', '70018', '9336', '82701', '3710', '52083', '74045', '96454', '30956', '67369', '78941', '81810', '71906', '23194', '33042', '50794', '61256', '24449', '48639', '22916', '78303', '13666', '40762', '43942', '51075', '89783', '95786', '90462', '6181', '36482', '40675', '4970', '6388', '91849', '72579', '94983', '86084', '20140', '68427', '48123', '43122', '98066', '37560', '6927', '72803', '5546', '62259', '98439', '6457', '98568', '70499', '33022', '28226', '29675', '20917', '75365', '20900', '8190', '56736', '99153', '77779', '49333', '50293', '97650', '4067', '47278', '42761', '71875', '13966', '11223', '46783', '18059', '61355', '29638', '75681', '24466', '89634', '20759', '83252', '37780', '15931', '74893', '6703', '64524', '80656', '85990', '78427', '18411', '20696', '86432', '93176', '69889', '15072', '15180', '9935', '10467', '60248', '42430', '62590', '89596', '27743', '26398', '79912', '60048', '50943', '38870', '69383', '72261', '98059', '55242', '74905', '5667', '54321', '70415', '39903', '49711', '85318', '79979', '59262', '82321', '15263', '17416', '74554', '94733', '72112', '49872', '54849', '73883', '78250', '74935', '68559', '57564', '50541', '45730', '41595', '5588', '83723', '42891', '11898', '14348', '99732', '14481', '85233', '21277', '94508', '52551', '74187', '7634', '42912', '25100', '43536', '35798', '48190', '86477', '22680', '48148', '59501', '56563', '16802', '81496', '97568', '68657', '51462', '67953', '99660', '39002', '54170', '57190', '68086', '52700', '6487', '55709', '70418', '62629', '70420', '35695', '36152', '45360', '53503', '46623', '76000', '50648', '97876', '44815', '29163', '1356', '64123', '71388', '17658', '99084', '58727', '59437', '38773', '71254', '81286', '97545', '18786', '56834', '20346', '36401', '62316', '58082', '67959', '99876', '69895', '80099', '62747', '20517', '99777', '6472', '49189', '31321', '39992', '68073', '13378', '51806', '21776', '52060', '96983', '25754', '93709', '96627', '8644', '93726', '14002', '37716', '87620', '34507', '76339', '24491', '5849', '44110', '522', '66521', '12776', '44887', '80535', '14548', '75248', '671', '73071', '35715', '59474', '7061', '82243', '56170', '20179', '59717', '1725', '24634', '11270', '77023', '63840', '46608', '44667', '22422', '59771', '94768', '73033', '82905', '16463', '40971', '22204', '58366', '28721', '14907', '76468', '81872', '38418', '36989', '61439', '10610', '131', '44296', '35453', '10117', '75856', '94603', '99602', '68075', '35949', '13599', '50030', '69633', '55956', '85465', '16429', '86081', '11145', '6195', '82207', '90598', '92814', '23725', '83204', '80346', '71542', '46634', '15820', '54123', '45397', '15322', '61743', '9273', '71347', '6835', '64006', '91718', '43677', '32923', '21486', '17098', '61694', '43347', '40019', '4071', '52443', '42386', '56839', '83514', '27633', '40780', '51749', '92101', '62384', '92206', '56044', '66174', '11137', '73966', '78471', '30468', '31643', '33197', '6888', '8066', '86603', '74383', '6098', '54411', '98819', '89862', '88639', '94422', '89371', '80526', '91747', '91220', '64944', '76658', '42046', '58518', '27249', '6646', '3028', '1346', '33763', '9734', '31737', '65527', '5892', '60813', '3410', '35464', '43009', '98382', '70580', '93898', '56404', '32995', '62771', '71556', '40538', '55612', '45656', '10758', '20268', '33603', '38310', '14242', '74397', '10722', '71575', '22590', '49043', '91439', '9055', '23668', '9101', '5268', '64133', '77501', '64684', '11337', '47575', '50732', '88680', '93730', '46785', '17589', '3520', '57595', '71241', '34994', '8753', '36147', '88844', '41914', '11250', '94632', '71927', '4624', '86279', '7664', '2659', '94853', '65386', '30438', '86005', '92883', '84629', '59910', '44484', '1306', '8404', '56962', '29990', '38445', '96191', '73013', '66590', '40951', '24712', '18825', '37268', '87843', '18972', '12154', '7779', '52149', '76152', '65799', '86011', '35475', '78083', '88232', '91551', '65532', '93516', '73827', '24227', '44687', '55759', '83819', '45088', '10856', '60488', '39051', '14103', '76650', '81181', '46731', '737', '58788', '78945', '42096', '66731', '66740', '72273', '88969', '5655', '86590', '41096', '80038', '32430', '51877', '23970', '91900', '13082', '45880', '94367', '19739', '61998', '71665', '16083', '57035', '26916', '10166', '18834', '46798', '66881', '28444', '68840', '10459', '81087', '4728', '76224', '39257', '23470', '93524', '37345', '30074', '49856', '22022', '55279', '5159', '5193', '58030', '57539', '12514', '49759', '96222', '52597', '67192', '88187', '53614', '16084', '79915', '28212', '79334', '85283', '32306', '31058', '43113', '74707', '74869', '2213', '32134', '6379', '85426', '87098', '35984', '51105', '69287', '16803', '83337', '14913', '62531', '58098', '7914', '20105', '28850', '1384', '43173', '62983', '87113', '76066', '86320', '77684', '45191', '95225', '41503', '36713', '48404', '91228', '53865', '98981', '59161', '61237', '84561', '17455', '14379', '57789', '80895', '99260', '84595', '72942', '53220', '84448', '81332', '49437', '83086', '93414', '54519', '52288', '74772', '22460', '49324', '11168', '96071', '61985', '38284', '6405', '54698', '71727', '60093', '37340', '87884', '83403', '4542', '94949', '19636', '15855', '39105', '10424', '67418', '91022', '69254', '8481', '38411', '3832', '44354', '93548', '57172', '28481', '372', '81497', '52179', '41060', '72141', '41396', '65590', '70432', '82819', '93814', '26118', '84780', '88485', '70821', '8222', '83000', '47067', '38516', '33347', '47681', '48202', '60749', '52112', '7937', '28105', '11394', '45746', '43252', '34494', '2979', '69715', '42486', '82315', '71760', '97413', '66137', '94487', '7429', '74434', '22964', '55251', '3448', '53534', '2574', '9693', '96157', '2955', '4348', '19566', '56930', '83319', '31310', '53905', '1148', '41726', '22233', '76045', '37351', '10545', '17581', '28047', '30199', '4741', '58111', '33497', '67796', '67730', '31247', '43772', '29461', '45970', '73353', '22534', '53962', '32147', '71392', '62579', '66345', '58246', '33442', '9581', '29705', '14058', '86471', '76125', '59363', '94982', '74810', '89149', '20066', '3366', '3568', '25752', '80036', '64119', '27270', '40061', '91052', '69022', '9852', '77112', '83075', '43924', '61661', '56133', '96652', '57944', '72576', '82170', '79236', '55745', '15309', '88878', '72761', '37647', '67465', '12777', '97309', '93202', '41470', '8787', '64920', '48514', '18917', '35157', '59151', '4640', '5317', '38134', '76548', '82788', '9214', '58418', '73185', '90554', '10543', '47182', '62936', '91765', '89751', '68931', '48865', '64607', '7150', '77862', '14297', '14828', '33013', '91698', '67593', '98096', '16595', '51639', '86531', '24719', '1703', '78788', '43810', '38918', '95491', '99903', '82671', '8291', '68288', '31224', '39863', '4265', '77798', '7698', '33804', '92286', '4744', '37038', '44203', '98212', '17369', '77442', '62879', '4145', '96881', '15646', '36824', '19959', '45451', '76049', '54272', '97577', '95298', '81115', '30204', '82041', '8037', '10052', '8756', '76833', '82851', '24276', '75574', '36037', '78079', '92807', '29064', '90000', '84150', '17102', '75092', '49424', '35597', '4693', '82853', '42511', '16119', '23478', '65240', '55585', '91762', '71671', '46682', '72479', '97696', '24615', '12579', '30274', '48255', '2336', '90202', '5808', '45426', '76308', '74639', '31245', '99894', '89638', '6233', '33893', '71899', '85273', '89429', '29761', '50231', '57249', '99347', '22642', '66972', '86221', '47514', '88274', '10819', '73150', '53754', '13304', '20478', '38099', '619', '14669', '8011', '97657', '26569', '65430', '13467', '38180', '23675', '72350', '42257', '39875', '23529', '53407', '11833', '29599', '95621', '7727', '59527', '86846', '22860', '5358', '3730', '87555', '362', '95755', '54565', '29935', '68950', '52349', '98344', '86576', '7420', '12236', '15844', '48099', '97535', '97081', '50261', '31187', '60496', '24123', '24042', '6376', '6679', '99806', '20306', '60676', '36881', '77309', '5247', '96569', '53417', '73252', '64179', '35318', '75732', '65119', '32621', '40464', '22887', '96152', '65161', '83381', '8915', '68142', '7328', '85031', '15688', '72519', '93992', '86927', '75538', '38205', '50877', '70039', '97538', '94822', '52131', '49643', '85206', '1347', '14574', '88736', '53442', '49991', '64925', '72283', '82213', '60905', '36118', '62963', '16983', '79185', '15111', '26059', '17792', '98218', '33214', '1094', '41754', '77275', '65173', '13190', '91004', '90422', '44387', '92672', '98641', '54609', '83295', '37395', '70104', '32986', '72524', '82478', '5837', '83916', '52736', '57112', '55985', '42642', '42136', '89642', '35712', '49489', '19726', '65824', '24384', '48112', '15366', '99206', '68384', '51389', '529', '21475', '75749', '95182', '60110', '70571', '74174', '38105', '78107', '4101', '8982', '11215', '23987', '3303', '28706', '54629', '98000', '67510', '30036', '99140', '48896', '40971', '7735', '79984', '50134', '94928', '57023', '52880', '83067', '41940', '62994', '89213', '38593', '19283', '68206', '22234', '19245', '26266', '32403', '65889', '17022', '64280', '42797', '27161', '57675', '42313', '93606', '93082', '20659', '90824', '1226', '66266', '12503', '57104', '15247', '51160', '92398', '71967', '59476', '44465', '35765', '10787', '47737', '45792', '2292', '47599', '89612', '8162', '87622', '69410', '45727', '31158', '99791', '89544', '27214', '99588', '40516', '75616', '36505', '46079', '95448', '97999', '47462', '47799', '82729', '34038', '60789', '96938', '22682', '79062', '93307', '36038', '49016', '90983', '48219', '50889', '32517', '72219', '71229', '82643', '1195', '70543', '17', '22178', '23544', '72371', '1163', '28527', '7336', '39846', '31956', '80963', '41804', '59791', '41831', '1940', '52377', '79494', '12531', '81112', '44320', '18746', '5774', '63869', '4085', '59922', '12751', '99443', '13530', '23872', '36026', '83360', '32711', '92980', '11140', '99323', '57263', '98149', '29265', '25548', '65995', '4818', '15593', '8535', '37863', '12217', '14474', '66584', '89272', '86690', '58777', '39666', '44756', '18442', '52586', '98030', '40850', '38708', '49304', '68923', '65008', '84388', '83639', '29866', '63675', '26793', '49227', '82099', '24090', '57535', '24201', '65776', '74054', '89833', '62979', '26613', '5851', '99766', '63484', '66605', '37179', '90760', '59336', '58390', '93239', '84578', '11396', '93994', '73818', '23972', '37720', '72369', '25063', '32952', '71036', '76612', '31285', '34090', '19136', '53783', '66436', '61478', '96749', '43658', '7399', '31574', '67073', '40480', '20727', '70993', '65549', '30800', '21507', '53785', '89574', '86381', '56492', '62603', '44856', '68687', '63794', '70996', '7475', '84238', '71939', '86886', '94792', '15036', '36936', '95722', '17771', '67850', '33371', '49314', '40744', '5432', '81057', '41201', '75986', '22961', '15323', '1570', '18657', '95219', '19130', '53127', '15867', '81135', '73206', '76668', '36386', '48828', '31417', '56916', '70891', '60534', '95777', '10022', '94053', '2928', '56326', '16559', '79656', '6414', '81247', '78270', '55687', '19151', '61597', '99857', '81142', '27725', '53493', '12185', '1455', '48501', '59425', '20591', '24900', '66079', '84889', '32024', '18919', '2043', '7076', '71201', '88258', '86521', '93348', '26395', '39646', '44145', '33911', '46231', '67054', '39979', '11630', '23020', '76278', '88056', '11480', '4723', '78612', '70211', '60622', '84687', '59092', '65675', '38479', '64399', '64699', '95964', '42764', '69060', '28189', '4193', '95805', '75462', '17245', '59640', '94773', '84292', '53092', '98507', '61353', '32483', '53027', '48912', '87221', '47788', '59263', '65196', '35567', '17494', '64253', '50223', '7057', '87467', '62414', '2523', '50910', '72353', '78986', '78104', '47719', '29108', '12957', '5114', '64435', '66707', '37449', '70399', '45334', '71606', '55338', '55072', '58765', '12151', '22012', '16954', '87366', '14240', '98041', '72296', '47408', '56879', '99584', '63172', '92316', '28071', '29880', '19608', '13839', '87484', '56541', '88662', '87098', '72124', '78282', '27653', '38993', '31870', '67239', '99445', '7376', '78487', '98880', '12180', '86773', '67773', '15416', '58172', '13075', '67559', '97510', '29705', '86985', '57024', '11827', '31236', '91920', '26116', '94614', '14486', '46252', '78847', '43786', '70048', '96739', '35240', '39933', '58209', '27852', '65669', '47323', '58150', '84444', '44344', '95882', '41258', '31314', '69060', '19916', '6979', '19436', '45572', '16259', '74566', '6306', '24705', '53422', '593', '97031', '22308', '26875', '23042', '78035', '34229', '61976', '23175', '50072', '90896', '50810', '71730', '86468', '94807', '8218', '36032', '58628', '60560', '51206', '37943', '27987', '15014', '49905', '70018', '66799', '80851', '23594', '29982', '6438', '97381', '47715', '96294', '17985', '48545', '12672', '5250', '9988', '24601', '3736', '97815', '54363', '64703', '44167', '68376', '16595', '38073', '29630', '59630', '1858', '71823', '75580', '70083', '14493', '93821', '93394', '85369', '3818', '8435', '59988', '43966', '13961', '15855', '83332', '80312', '27299', '88840', '76964', '56173', '62794', '79389', '82642', '85843', '47116', '43064', '16061', '28905', '54415', '72832', '91252', '93488', '79457', '99336', '70744', '80432', '6487', '880', '87701', '154', '86574', '86677', '17892', '81488', '95260', '12515', '43189', '9211', '55403', '41417', '60046', '54785', '83655', '28274', '65745', '63062', '44549', '36391', '48051', '7328', '3572', '33226', '49177', '25123', '59065', '19691', '15109', '10172', '95578', '29497', '48152', '20276', '36270', '78866', '48309', '53209', '55475', '30073', '19717', '16004', '45692', '83430', '9291', '45935', '57030', '92613', '91656', '67697', '34915', '28156', '56594', '3273', '11194', '98270', '34370', '2621', '66679', '97451', '97717', '87923', '48310', '37725', '69743', '75103', '84956', '75163', '16069', '65304', '19397', '18071', '27273', '49823', '57595', '98324', '82174', '10293', '80943', '64184', '19472', '4198', '9410', '25927', '65961', '33155', '95168', '33692', '61712', '69877', '13308', '17415', '10022', '2491', '67310', '96140', '68050', '76272', '17143', '76805', '57176', '7539', '22690', '95483', '87592', '27221', '90821', '51154', '99828', '68998', '54581', '74222', '10269', '65057', '45467', '96089', '55058', '89779', '60837', '74122', '52886', '58055', '14880', '93208', '66652', '68830', '24121', '62407', '87257', '18802', '14925', '45423', '98624', '55195', '59072', '41414', '77840', '66075', '62705', '26549', '19063', '57552', '2507', '52069', '57620', '66688', '14833', '33700', '90666', '98052', '5367', '2268', '43093', '69063', '22030', '85564', '92258', '1847', '24446', '65835', '38660', '91899', '87732', '52396', '31952', '36000', '86944', '16109', '80729', '53757', '60226', '59103', '84187', '36674', '72823', '29884', '4654', '69139', '20440', '57413', '3651', '39639', '44564', '57492', '84159', '751', '99748', '9659', '72661', '39220', '99742', '74734', '75729', '38071', '69934', '73640', '65294', '54524', '64372', '37927', '17187', '7863', '12732', '40296', '36197', '15821', '76831', '4400', '71933', '4040', '22072', '33064', '25702', '13324', '91275', '27388', '97729', '14620', '45989', '80737', '17934', '4219', '3032', '43457', '31051', '24469', '67041', '29328', '75499', '80951', '88212', '92595', '49969', '24612', '58732', '2718', '3805', '50918', '99426', '8614', '35580', '93273', '989', '24385', '41185', '25687', '47146', '25227', '95839', '56355', '98536', '79824', '31725', '46447', '26690', '68418', '47783', '33725', '21729', '70797', '59038', '60376', '25087', '68332', '67950', '12411', '95918', '64736', '65336', '74947', '64605', '4106', '42712', '96640', '28492', '28648', '42429', '821', '24333', '69677', '38959', '23484', '92005', '29352', '29159', '52873', '99947', '21834', '85347', '93479', '28298', '55608', '3226', '69714', '80283', '6577', '18849', '44605', '75286', '28139', '26541', '12867', '57500', '86617', '33005', '57498', '60223', '74954', '51401', '55246', '5648', '16513', '40930', '43821', '32090', '66002', '65530', '76083', '6047', '6879', '94987', '80787', '11688', '77161', '92670', '6696', '400', '28572', '47234', '51375', '88518', '762', '92617', '54260', '7560', '60180', '43331', '64059', '27616', '75839', '21392', '47756', '46254', '19486', '88533', '30130', '93694', '8557', '66534', '94447', '16910', '6480', '77440', '24366', '6195', '48946', '28597', '44429', '50300', '73556', '40638', '98709', '94413', '15987', '43860', '64871', '93953', '34506', '7296', '31753', '30626', '77510', '39829', '25696', '39776', '69185', '36540', '65413', '31528', '43446', '73532', '49776', '30282', '30004', '26725', '15200', '33958', '90320', '71836', '48051', '31970', '5326', '96194', '69695', '60898', '60945', '18271', '50868', '61468', '23593', '68985', '20628', '58044', '8942', '34849', '7384', '50500', '62895', '78780', '48946', '65278', '4067', '973', '34761', '15512', '73739', '23138', '47322', '55568', '32259', '71816', '49277', '75218', '76104', '19579', '68312', '67904', '33886', '53888', '26421', '43859', '40291', '39068', '31711', '36542', '10195', '39781', '72352', '13188', '34113', '9428', '60443', '4987', '13783', '80744', '63483', '18266', '11961', '87167', '46987', '28480', '74214', '39191', '8146', '38090', '75727', '79245', '47720', '52547', '45321', '4972', '49701', '74354', '69672', '63455', '41902', '5667', '54166', '4962', '25873', '44509', '73332', '73383', '29438', '21455', '12320', '11997', '16921', '49379', '63027', '86175', '8110', '76149', '2520', '11256', '25863', '50518', '69001', '79113', '9447', '91840', '5242', '10998', '46496', '2448', '56058', '20970', '10517', '17783', '25723', '97137', '62840', '1264', '78691', '81020', '55335', '48524', '2088', '90413', '76651', '26855', '16177', '14954', '62914', '21344', '5708', '75560', '39311', '95865', '28783', '64902', '95657', '46276', '33426', '4799', '11588', '57513', '73689', '77677', '63011', '97795', '34954', '76866', '32043', '32697', '26643', '36890', '53476', '3011', '13963', '49551', '87671', '67761', '17488', '94770', '50599', '33272', '23091', '38079', '41177', '22395', '91656', '79679', '38687', '57384', '80118', '42507', '4098', '78949', '45669', '48802', '83915', '78292', '4369', '57657', '49146', '45192', '98491', '72457', '46331', '207', '81601', '7409', '70856', '91605', '70295', '9171', '72293', '32997', '78025', '16795', '73534', '68780', '21284', '31767', '94381', '86439', '12420', '53285', '99563', '60502', '67954', '55012', '99809', '5431', '69978', '99712', '14401', '79498', '4495', '3045', '528', '72542', '91604', '72725', '39378', '80378', '41996', '20138', '54545', '59730', '36951', '45157', '37964', '97690', '12184', '4944', '53803', '93605', '60851', '68938', '46285', '89663', '90309', '6907', '87239', '81791', '83292', '90013', '68927', '14725', '81840', '63836', '52068', '43830', '4794', '931', '59255', '8263', '99057', '94401', '69033', '7437', '20364', '92884', '28193', '43932', '37629', '59426', '18891', '8583', '79551', '87242', '1483', '6725', '65786', '16844', '12650', '99305', '42841', '9811', '18800', '39313', '51373', '31874', '84558', '27831', '48614', '48975', '55509', '83363', '31854', '64001', '94028', '76125', '79314', '24893', '81132', '9441', '86015', '28356', '40358', '10160', '23328', '7330', '76538', '37611', '89351', '84132', '97047', '26109', '95222', '35130', '75600', '88602', '15073', '87835', '71649', '28948', '81615', '37498', '28674', '59776', '44095', '65924', '64368', '94536', '12518', '61711', '55619', '82949', '4114', '21540', '70544', '28022', '79983', '28781', '7749', '97873', '4951', '50076', '47611', '99522', '56820', '38653', '49047', '36283', '83908', '72452', '85625', '10811', '36998', '44083', '34864', '44975', '39057', '4551', '68450', '24781', '1503', '9871', '46885', '11424', '21259', '54900', '97669', '85669', '6015', '2521', '37661', '14915', '57423', '91903', '94789', '32059', '64972', '4600', '61465', '27118', '79785', '13547', '49766', '38410', '68860', '63756', '23621', '64387', '46255', '63408', '11297', '41081', '56326', '58349', '98703', '72268', '73574', '32098', '42534', '91502', '38083', '11241', '56828', '12098', '25377', '37054', '56328', '30034', '26922', '68401', '93478', '63275', '62650', '81407', '773', '79499', '14970', '47217', '1187', '57428', '69980', '77764', '74791', '22107', '54363', '39247', '56028', '56982', '84244', '21464', '18716', '25533', '94589', '94768', '21537', '18436', '81135', '27654', '79713', '56630', '61571', '58453', '26758', '68450', '68449', '2994', '15347', '83954', '71823', '6428', '44210', '79597', '95144', '32871', '1991', '320', '77157', '63607', '31154', '48846', '71125', '61750', '59608', '33038', '35733', '68915', '94127', '50383', '64242', '49708', '57270', '65019', '8581', '12111', '18487', '50013', '58664', '22214', '19033', '33681', '44754', '28830', '10381', '52318', '34959', '20682', '55453', '53800', '65774', '99164', '72102', '36986', '44157', '56716', '7974', '81475', '25926', '39402', '33688', '99671', '95312', '42268', '26536', '14482', '67377', '57993', '89147', '15834', '64995', '4700', '18714', '30221', '39095', '32749', '69257', '55204', '30497', '31839', '63045', '30009', '62683', '31232', '77680', '93551', '63589', '6989', '77246', '42169', '46117', '73226', '37427', '1858', '83649', '37410', '86369', '4641', '74481', '66168', '48041', '22597', '14670', '27464', '57165', '20939', '36282', '76940', '73358', '50521', '69603', '8895', '81793', '57743', '81903', '64025', '91641', '25276', '34040', '62642', '64015', '57657', '84890', '73832', '782', '60160', '16998', '40023', '24590', '88613', '76640', '53091', '67600', '80183', '45674', '64464', '25163', '42384', '66972', '13953', '41966', '66048', '15135', '73745', '19466', '53657', '34619', '13462', '15905', '48257', '73297', '238', '93525', '80556', '5942', '5411', '66169', '9090', '95130', '74316', '57321', '48083', '62355', '68113', '15239', '36644', '80326', '65817', '54428', '61955', '58849', '77206', '16073', '98261', '92091', '39178', '35464', '85109', '85452', '21128', '25665', '81860', '44664', '24024', '56960', '95124', '39786', '18836', '11121', '44163', '81074', '79064', '46219', '94694', '44233', '81469', '24642', '15030', '21995', '13587', '40755', '6669', '81093', '74305', '1881', '55649', '37273', '80827', '98643', '46694', '59281', '79231', '42813', '84984', '7052', '98113', '17296', '84434', '31205', '46894', '71219', '74530', '44686', '70744', '91388', '20692', '96853', '73803', '15836', '18126', '49686', '4179', '47588', '87892', '65425', '68012', '97468', '92510', '99271', '58694', '11918', '37051', '18644', '57228', '14265', '57572', '57022', '52186', '30193', '93570', '87872', '5257', '26784', '6476', '61746', '68559', '1720', '26202', '16519', '27688', '10645', '87174', '60845', '73385', '82075', '6933', '98828', '56895', '17344', '84253', '36561', '51648', '24939', '63470', '31034', '95052', '51090', '51465', '87979', '68650', '30181', '29598', '19137', '43221', '81353', '90170', '96985', '61115', '17385', '92314', '80650', '55821', '17874', '84333', '93272', '48260', '87272', '22764', '59957', '51870', '85988', '39222', '77241', '62535', '28344', '6011', '80831', '64551', '46299', '75195', '71177', '8660', '58943', '57003', '3306', '74413', '74068', '15073', '89016', '93140', '13911', '57170', '19880', '41870', '9131', '57495', '73032', '86979', '60094', '87026', '30880', '4736', '86301', '92707', '21689', '83565', '71275', '47665', '65687', '71184', '89897', '32490', '97577', '38723', '79113', '37531', '97500', '94450', '15699', '58019', '84423', '27057', '56017', '97148', '47365', '30669', '33818', '80406', '99690', '33012', '95178', '46809', '48448', '79350', '9146', '99701', '98976', '71197', '44161', '75069', '36602', '79650', '97301', '12020', '56658', '25701', '46392', '78609', '63073', '69419', '57736', '20102', '42415', '79044', '20277', '56280', '47903', '94311', '25558', '40336', '91305', '90505', '66769', '64562', '83737', '62892', '10375', '71024', '19988', '56946', '76110', '21847', '43162', '50578', '46086', '54167', '61722', '53463', '63134', '69288', '12838', '14116', '71687', '50846', '59810', '24826', '84138', '82885', '91496', '98600', '82769', '40049', '4125', '50694', '1294', '2805', '29691', '82321', '76462', '85945', '115', '29188', '66918', '71340', '31585', '61638', '95472', '52978', '50622', '81990', '60955', '70519', '22270', '35610', '95871', '89222', '41038', '52546', '1163', '67943', '1793', '92010', '35755', '74509', '66665', '95759', '8568', '44299', '67822', '5806', '85839', '13895', '87675', '31357', '88014', '40026', '53050', '28951', '31992', '42495', '82892', '51567', '2869', '45808', '20238', '20781', '56098', '66307', '95701', '614', '60833', '3091', '81339', '24195', '65639', '85976', '28116', '66224', '51502', '73637', '13207', '88302', '36488', '65518', '98187', '26', '74367', '64706', '53943', '86760', '25783', '82112', '34958', '86621', '20848', '63459', '14049', '84943', '91873', '50238', '77773', '64109', '8602', '87934', '47583', '66053', '30287', '5507', '80312', '37464', '57457', '86200', '17806', '16522', '38843', '94334', '59958', '63864', '53427', '74506', '33980', '90449', '30842', '53616', '36738', '52', '13595', '53051', '13174', '60163', '71420', '73835', '67119', '79018', '42782', '45059', '952', '46360', '85879', '71552', '84741', '29746', '32577', '10041', '7208', '97528', '51256', '916', '55973', '17684', '99046', '38782', '58660', '97798', '66032', '48339', '51329', '12532', '97904', '95454', '42737', '62541', '96702', '82953', '94610', '26645', '86813', '25480', '99713', '26078', '23028', '93056', '21445', '73209', '89318', '69987', '34705', '30064', '17094', '51135', '54141', '26625', '1086', '13082', '30843', '98672', '56864', '42605', '5833', '60850', '69366', '27351', '16456', '92609', '48030', '54322', '69891', '46502', '34578', '77918', '63276', '75958', '42519', '60266', '85576', '4855', '14258', '67017', '10545', '35078', '53012', '71922', '85784', '73402', '74363', '58457', '94102', '23510', '51559', '39482', '87057', '9377', '10106', '82985', '33931', '16523', '6484', '97749', '83172', '53753', '27466', '23073', '96083', '67302', '57465', '21877', '18013', '99804', '32873', '43123', '72365', '53197', '80578', '69770', '97471', '86954', '67183', '98497', '78474', '28450', '63183', '98699', '42738', '61433', '3491', '27304', '49311', '94980', '92740', '43272', '86549', '11406', '79636', '85582', '38086', '657', '2354', '26567', '77450', '42086', '21600', '49011', '44059', '47872', '75761', '96577', '11642', '83471', '79616', '23749', '77082', '96876', '65302', '84027', '48955', '59887', '20657', '75090', '9058', '50347', '66088', '70745', '76342', '58026', '95568', '61504', '93473', '84590', '47089', '74717', '93090', '46334', '68273', '59500', '54345', '72608', '54048', '86156', '40296', '74046', '6813', '36369', '74543', '18305', '85236', '31316', '37061', '96893', '23112', '5529', '10166', '19037', '1467', '70810', '30932', '18410', '92837', '81324', '12268', '54705', '25207', '90366', '56528', '3392', '88747', '39951', '97957', '99404', '23685', '13533', '15640', '11434', '66516', '71025', '65770', '88000', '52232', '32360', '10787', '37438', '2264', '94460', '80214', '42288', '59062', '29010', '64093', '21225', '22297', '36935', '19202', '5925', '85373', '27414', '28991', '9191', '42273', '56587', '89719', '77191', '64334', '61542', '28763', '28978', '79184', '59815', '95200', '30246', '54022', '287', '91808', '66347', '50833', '15356', '78614', diff --git a/tests/performance/website.xml b/tests/performance/website.xml index 0011d225d6c..66357352f3e 100644 --- a/tests/performance/website.xml +++ b/tests/performance/website.xml @@ -55,7 +55,7 @@ SELECT URL, count() AS c FROM {table} GROUP BY URL ORDER BY c DESC LIMIT 10 SELECT 1, URL, count() AS c FROM {table} GROUP BY 1, URL ORDER BY c DESC LIMIT 10 SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM hits_100m_single GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10 -SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10 +SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10 SETTINGS max_threads = 1 SELECT Title, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10 SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000 SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000 From 32772073ea8655298394fef45f2b5d16679c1f88 Mon Sep 17 00:00:00 2001 From: Ivan Starkov Date: Tue, 9 Jun 2020 13:52:21 +0300 Subject: [PATCH 21/29] Fix docker-entrypoint-initdb.d wait Fixes 'Address family not supported by protocol' for any docker-entrypoint-initdb.d script. wget uses 'localhost' which resolves for both ipv4 and ipv6 with current config (/etc/hosts) and so the wget fails _(Address family not supported by protocol)_ and does not retry. Forcing it to use IPv4 fixes the issue --- docker/server/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 2af8a377b92..6111b0057ed 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -94,7 +94,7 @@ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then # check if clickhouse is ready to accept connections # will try to send ping clickhouse via http_port (max 12 retries, with 1 sec delay) - if ! wget --spider --quiet --tries=12 --waitretry=1 --retry-connrefused "http://localhost:$HTTP_PORT/ping" ; then + if ! wget --spider --quiet -4 --tries=12 --waitretry=1 --retry-connrefused "http://localhost:$HTTP_PORT/ping" ; then echo >&2 'ClickHouse init process failed.' exit 1 fi From 5719549ee169bccf6c0b19000c96895057c224e6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 9 Jun 2020 16:29:07 +0300 Subject: [PATCH 22/29] fixup --- docker/test/performance-comparison/compare.sh | 36 +++++++++++++++++-- docker/test/performance-comparison/report.py | 6 ++-- tests/performance/base64_hits.xml | 8 ++--- tests/performance/basename.xml | 6 ++-- tests/performance/int_parsing.xml | 13 ------- tests/performance/set_index.xml | 9 +++-- 6 files changed, 50 insertions(+), 28 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 835e1424fd9..983db608b47 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -421,9 +421,39 @@ create table queries_for_flamegraph engine File(TSVWithNamesAndTypes, select test, query_index from queries where unstable_show or changed_show ; -create table unstable_tests_tsv engine File(TSV, 'report/bad-tests.tsv') as - select test, sum(unstable_fail) u, sum(changed_fail) c, u + c s from queries - group by test having s > 0 order by s desc; +create table test_time_changes_tsv engine File(TSV, 'report/test-time-changes.tsv') as + select test, queries, average_time_change from ( + select test, count(*) queries, + sum(left) as left, sum(right) as right, + (right - left) / right average_time_change + from queries + group by test + order by abs(average_time_change) desc + ) + ; + +create table unstable_tests_tsv engine File(TSV, 'report/unstable-tests.tsv') as + select test, sum(unstable_show) total_unstable, sum(changed_show) total_changed + from queries + group by test + order by total_unstable + total_changed desc + ; + +create table test_perf_changes_tsv engine File(TSV, 'report/test-perf-changes.tsv') as + select test, + queries, + coalesce(total_unstable, 0) total_unstable, + coalesce(total_changed, 0) total_changed, + total_unstable + total_changed total_bad, + coalesce(toString(floor(average_time_change, 3)), '??') average_time_change_str + from test_time_changes_tsv + full join unstable_tests_tsv + using test + where (abs(average_time_change) > 0.05 and queries > 5) + or (total_bad > 0) + order by total_bad desc, average_time_change desc + settings join_use_nulls = 1 + ; create table query_time engine Memory as select * from file('analyze/client-times.tsv', TSV, diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 5d4d30e0098..98602bac546 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -274,9 +274,9 @@ if args.report == 'main': skipped_tests_rows = tsvRows('analyze/skipped-tests.tsv') printSimpleTable('Skipped tests', ['Test', 'Reason'], skipped_tests_rows) - printSimpleTable('Tests with most unstable queries', - ['Test', 'Unstable', 'Changed perf', 'Total not OK'], - tsvRows('report/bad-tests.tsv')) + printSimpleTable('Test performance changes', + ['Test', 'Queries', 'Unstable', 'Changed perf', 'Total not OK', 'Avg relative time diff'], + tsvRows('report/test-perf-changes.tsv')) def print_test_times(): global slow_average_tests diff --git a/tests/performance/base64_hits.xml b/tests/performance/base64_hits.xml index 65828d6ca1f..7e7ffc6bd84 100644 --- a/tests/performance/base64_hits.xml +++ b/tests/performance/base64_hits.xml @@ -1,6 +1,6 @@ - hits_100m_single + hits_10m_single @@ -19,7 +19,7 @@ - SELECT count() FROM hits_100m_single WHERE NOT ignore(base64Encode({string})) - SELECT count() FROM hits_100m_single WHERE base64Decode(base64Encode({string})) != {string} - SELECT count() FROM hits_100m_single WHERE tryBase64Decode(base64Encode({string})) != {string} + SELECT count() FROM hits_10m_single WHERE NOT ignore(base64Encode({string})) + SELECT count() FROM hits_10m_single WHERE base64Decode(base64Encode({string})) != {string} + SELECT count() FROM hits_10m_single WHERE tryBase64Decode(base64Encode({string})) != {string} diff --git a/tests/performance/basename.xml b/tests/performance/basename.xml index 4ccc7ddf3a0..daa0e9605b3 100644 --- a/tests/performance/basename.xml +++ b/tests/performance/basename.xml @@ -1,12 +1,12 @@ - hits_100m_single + hits_10m_single 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(basename(URL)) - SELECT count() FROM hits_100m_single WHERE NOT ignore(basename(Referer)) + SELECT count() FROM hits_10m_single WHERE NOT ignore(basename(URL)) + SELECT count() FROM hits_10m_single WHERE NOT ignore(basename(Referer)) diff --git a/tests/performance/int_parsing.xml b/tests/performance/int_parsing.xml index 1a242c28393..3008ed06684 100644 --- a/tests/performance/int_parsing.xml +++ b/tests/performance/int_parsing.xml @@ -14,7 +14,6 @@ SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(OS)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(UserAgent)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(Refresh)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(IsRobot)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionWidth)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionHeight)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResolutionDepth)) SETTINGS max_threads = 1 @@ -67,15 +66,6 @@ SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResponseStartTiming)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ResponseEndTiming)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FetchTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RedirectTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMInteractiveTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMContentLoadedTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(DOMCompleteTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(LoadEventStartTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(LoadEventEndTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(NSToDOMContentLoadedTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(FirstPaintTiming)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RedirectCount)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(SocialSourceNetworkID)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ParamPrice)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(ParamCurrencyID)) SETTINGS max_threads = 1 @@ -83,7 +73,4 @@ SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RefererHash)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(URLHash)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(CLID)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(YCLID)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RequestNum)) SETTINGS max_threads = 1 - SELECT count() FROM hits_100m_single WHERE NOT ignore(toString(RequestTry)) SETTINGS max_threads = 1 diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index 46fe32e5486..e85ded1d8fa 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -14,8 +14,13 @@ SELECT count() FROM test_in WHERE -toInt64(a) NOT IN (SELECT toInt64(rand(1)) FROM numbers(100000)) settings max_rows_to_read=1, read_overflow_mode='break' - - SELECT count() FROM numbers(10000) WHERE toString(number) IN ('41577', '83972', '51697', '50014', '37553', '93459', '87438', '95971', '83186', '74326', '67871', '50406', '83678', '29655', '18580', '83905', '61518', '29059', '56700', '82787', '98672', '30884', '81822', '39850', '80852', '57627', '91346', '64522', '17781', '49467', '41099', '41929', '85618', '91389', '68564', '91769', '81219', '52218', '37220', '97097', '2129', '9886', '52049', '34847', '25364', '36429', '76897', '71868', '58121', '71199', '84819', '69991', '34046', '64507', '34892', '24228', '36986', '28588', '51159', '53444', '80531', '9941', '20256', '48103', '32565', '62890', '5379', '60302', '46434', '3205', '18821', '31030', '19794', '71557', '71703', '15024', '14004', '82164', '95659', '40227', '83358', '24395', '9610', '19814', '48491', '66412', '16012', '71586', '42143', '51103', '24463', '89949', '35694', '39193', '63904', '40489', '77144', '94014', '84836', '9980', '46554', '43905', '25588', '25205', '72624', '10249', '35888', '98478', '99030', '26834', '31', '81499', '14847', '82997', '92357', '92893', '17426', '56630', '22252', '68119', '62710', '8740', '82144', '79916', '23391', '30192', '99271', '96435', '44237', '98327', '69481', '16691', '13643', '84554', '38571', '70926', '99283', '79000', '20926', '86495', '4834', '1222', '39486', '57697', '58002', '40790', '15623', '3999', '31515', '12694', '26143', '35951', '54085', '97534', '35329', '73535', '88715', '29572', '75799', '45166', '32066', '48023', '69523', '93150', '8740', '96790', '15534', '63252', '5142', '67045', '93992', '16663', '292', '63924', '6588', '12190', '31506', '69590', '35394', '55168', '65223', '79183', '32600', '69676', '28316', '72111', '53531', '15073', '41127', '73451', '24725', '61647', '65315', '41143', '26493', '95608', '34407', '76098', '53105', '83691', '48755', '35696', '62587', '81826', '3963', '45766', '82751', '12430', '97685', '29919', '78155', '71636', '50215', '89734', '9892', '47151', '54855', '3428', '9712', '52592', '2403', '79602', '81243', '79859', '57361', '82000', '42107', '28860', '99591', '28296', '57337', '64969', '32332', '25535', '30924', '21313', '32554', '17342', '87311', '19825', '24898', '61323', '83209', '79322', '79009', '50746', '33396', '62033', '16548', '17427', '24073', '34640', '52368', '4724', '80408', '40', '33787', '16666', '19665', '86751', '27264', '2241', '88134', '53566', '10589', '79711', '92823', '58972', '91767', '60885', '51659', '7867', '96849', '30360', '20914', '9584', '1250', '22871', '23282', '99312', '4683', '33429', '68361', '82614', '81440', '47863', '69790', '11968', '75210', '66854', '37002', '61142', '71514', '1588', '42336', '11069', '26291', '2261', '71056', '13492', '9133', '91216', '72207', '71586', '86535', '83898', '24392', '45384', '48545', '61972', '503', '80180', '35834', '97025', '70411', '55039', '35430', '27631', '82533', '96831', '74077', '42533', '14451', '26943', '53783', '69489', '71969', '8432', '37230', '61348', '19472', '59115', '9886', '50951', '57109', '7141', '1902', '84130', '4323', '55889', '47784', '2220', '75988', '66988', '63721', '8131', '95601', '95207', '2311', '26541', '50991', '6717', '2969', '71857', '51034', '65958', '94716', '90275', '21012', '46859', '7984', '31131', '46457', '69578', '44540', '7294', '80117', '9925', '60155', '90608', '82684', '32193', '87071', '28006', '87604', '24501', '79087', '2848', '29237', '11221', '81319', '40966', '87641', '35325', '78705', '88636', '78717', '62831', '56390', '99271', '43821', '14453', '17923', '62695', '77322', '21038', '67677', '41271', '4376', '65426', '46091', '19887', '97251', '55583', '58763', '3826', '35037', '73533', '64267', '82319', '9836', '42622', '96829', '16363', '10455', '49290', '99992', '98229', '66356', '59087', '73998', '25986', '4279', '56790', '69540', '588', '36620', '60358', '45056', '89297', '42740', '8323', '19245', '82417', '41431', '699', '11554', '73910', '44491', '56019', '68901', '45816', '68126', '89379', '23885', '13263', '56395', '73130', '19089', '23771', '10335', '48547', '16903', '6453', '33560', '89668', '38159', '43177', '90655', '49712', '62', '66920', '34180', '12150', '48564', '39538', '85026', '87195', '14928', '8956', '71157', '53287', '39161', '67583', '83309', '92054', '86977', '56188', '15229', '88170', '60894', '58497', '89254', '40082', '86890', '60161', '97291', '45878', '23368', '14577', '92870', '37017', '97356', '99426', '76061', '89186', '99751', '85153', '61580', '39360', '90107', '25603', '26798', '76224', '6469', '7912', '69838', '16404', '67497', '28965', '80836', '80365', '91249', '48713', '17113', '33090', '40793', '70450', '66689', '83698', '17802', '43869', '13355', '18959', '79411', '87930', '9265', '37504', '44876', '97234', '94149', '35040', '22049', '49248', '6535', '36080', '28346', '94437', '78319', '17961', '89056', '56161', '35810', '41632', '45494', '53351', '89729', '99510', '51584', '59688', '6193', '70809', '51093', '92589', '90247', '34910', '78235', '17362', '49423', '63324', '525', '37638', '72325', '89356', '15298', '59116', '17848', '65429', '27029', '84781', '70247', '8825', '35082', '70451', '22522', '58125', '91879', '90531', '2478', '463', '37902', '54405', '87267', '72688', '22803', '33134', '35177', '84551', '44974', '88375', '76407', '27774', '33849', '19915', '82014', '80434', '26380', '48777', '53811', '14838', '26829', '56441', '99869', '49574', '85476', '19723', '16907', '4018', '37338', '78510', '47912', '13030', '65277', '95716', '67363', '21393', '89887', '78842', '81650', '903', '17436', '30704', '49223', '27198', '25500', '52214', '54258', '70082', '53950', '49312', '43615', '99473', '94348', '53661', '96213', '96346', '62010', '38268', '32861', '75660', '10392', '89491', '68335', '29817', '88706', '24184', '36298', '43440', '21626', '26535', '44560', '46363', '12534', '99070', '95606', '33714', '73070', '8303', '29853', '23014', '99982', '4530', '14955', '45803', '50', '90750', '30394', '81276', '95563', '47314', '58520', '91299', '88944', '54402', '67405', '29253', '47079', '71734', '99728', '17652', '13307', '35556', '18962', '26780', '17771', '53712', '60055', '37628', '35830', '90739', '61151', '41309', '27652', '3051', '53167', '98417', '19382', '36833', '75085', '65374', '87732', '30352', '31776', '32765', '97565', '92199', '49050', '29503', '51024', '18834', '8515', '24069', '96216', '10777', '90680', '18974', '68884', '85305', '36007', '56707', '4212', '47352', '34426', '13185', '92939', '95782', '70577', '58080', '98279', '3906', '5065', '56896', '16382', '31273', '17117', '98602', '12786', '24086', '63970', '72756', '35798', '82367', '7356', '53398', '68503', '2962', '16425', '67334', '68461', '65439', '15620', '70906', '29649', '46461', '74602', '38012', '71714', '16825', '89480', '53386', '88532', '35104', '28556', '82120', '23155', '23347', '24797', '60061', '54962', '99427', '82248', '82447', '39968', '63727', '27431', '81511', '91168', '71425', '80740', '84127', '40717', '15503', '15419', '46594', '61263', '19212', '53175', '70724', '74445', '23034', '71818', '40246', '18886', '53066', '4880', '83701', '86107', '87862', '44751', '392', '73440', '90291', '93395', '20894', '38463', '32664', '55158', '20090', '50004', '79070', '98471', '85478', '96615', '68149', '78334', '97752', '73207', '71678', '91238', '96757', '82598', '194', '35797', '45120', '60782', '28721', '17676', '78066', '60957', '11826', '51563', '50516', '16485', '47053', '31738', '48923', '23554', '96850', '42033', '73701', '78607', '45979', '54571', '12415', '31693', '15356', '36902', '9126', '3767', '3295', '90402', '24005', '95350', '67033', '49137', '72606', '51899', '17522', '31957', '44641', '53982', '23767', '68257', '15766', '19995', '2107', '48788', '11765', '91055', '46576', '54651', '50381', '62827', '73636', '46606', '98753', '37631', '70441', '87916', '66983', '33870', '31125', '12904', '57040', '4874', '58632', '42037', '18782', '5998', '18974', '57949', '81010', '90407', '99874', '20462', '89949', '10952', '71454', '95130', '46115', '3518', '13384', '69039', '79482', '22076', '59782', '32042', '40930', '60243', '29298', '6790', '46985', '44398', '85631', '14380', '66179', '2629', '32126', '49833', '14118', '58492', '31493', '81172', '96638', '8745', '89663', '76842', '78633', '41373', '83721', '42886', '11123', '32739', '11051', '1303', '92314', '83324', '85600', '44276', '69064', '56125', '84650', '31028', '12628', '14502', '64764', '39405', '44855', '79046', '51716', '46824', '83389', '1941', '1257', '9280', '73176', '84729', '2579', '63366', '22606', '35541', '51096', '13447', '18355', '68037', '28436', '94116', '81070', '78355', '67897', '5296', '32742', '77645', '91853', '18767', '67949', '40963', '5792', '17278', '25597', '41884', '80829', '7099', '18645', '60295', '12082', '81800', '78415', '18082', '38789', '16295', '72377', '74949', '55583', '66853', '15402', '72977', '15123', '99434', '34999', '21687', '76049', '42987', '83748', '88256', '66688', '21766', '20304', '29271', '10069', '19822', '11792', '42526', '74143', '17289', '30253', '6367', '20888', '12975', '94073', '98639', '30134', '26320', '65507', '69002', '53120', '4550', '38893', '18954', '38283', '54863', '17698', '99670', '10521', '92467', '60994', '18052', '48673', '35811', '87282', '62706', '16061', '53112', '22652', '37780', '55662', '26331', '49410', '79074', '10623', '69577', '79613', '9491', '31229', '43922', '84231', '58409', '36386', '46875', '74431', '76735', '38776', '23350', '7314', '9079', '51519', '98544', '70216', '63380', '90381', '1295', '46901', '58225', '55339', '89918', '75522', '35431', '89460', '49552', '89302', '23068', '28493', '3042', '25194', '59520', '9810', '95706', '81297', '89638', '54794', '94527', '45262', '97932', '78685', '6947', '22818', '48700', '9153', '12289', '22011', '58825', '93854', '65438', '4509', '33741', '28208', '69061', '48578', '40247', '77725', '31837', '39003', '69363', '78113', '76398', '97262', '67795', + + SELECT count() FROM numbers(100) WHERE toString(number) IN ('41577', '83972', '51697', '50014', '37553', '93459', '87438', '95971', '83186', '74326', '67871', '50406', '83678', '29655', '18580', '83905', '61518', '29059', '56700', '82787', '98672', '30884', '81822', '39850', '80852', '57627', '91346', '64522', '17781', '49467', '41099', '41929', '85618', '91389', '68564', '91769', '81219', '52218', '37220', '97097', '2129', '9886', '52049', '34847', '25364', '36429', '76897', '71868', '58121', '71199', '84819', '69991', '34046', '64507', '34892', '24228', '36986', '28588', '51159', '53444', '80531', '9941', '20256', '48103', '32565', '62890', '5379', '60302', '46434', '3205', '18821', '31030', '19794', '71557', '71703', '15024', '14004', '82164', '95659', '40227', '83358', '24395', '9610', '19814', '48491', '66412', '16012', '71586', '42143', '51103', '24463', '89949', '35694', '39193', '63904', '40489', '77144', '94014', '84836', '9980', '46554', '43905', '25588', '25205', '72624', '10249', '35888', '98478', '99030', '26834', '31', '81499', '14847', '82997', '92357', '92893', '17426', '56630', '22252', '68119', '62710', '8740', '82144', '79916', '23391', '30192', '99271', '96435', '44237', '98327', '69481', '16691', '13643', '84554', '38571', '70926', '99283', '79000', '20926', '86495', '4834', '1222', '39486', '57697', '58002', '40790', '15623', '3999', '31515', '12694', '26143', '35951', '54085', '97534', '35329', '73535', '88715', '29572', '75799', '45166', '32066', '48023', '69523', '93150', '8740', '96790', '15534', '63252', '5142', '67045', '93992', '16663', '292', '63924', '6588', '12190', '31506', '69590', '35394', '55168', '65223', '79183', '32600', '69676', '28316', '72111', '53531', '15073', '41127', '73451', '24725', '61647', '65315', '41143', '26493', '95608', '34407', '76098', '53105', '83691', '48755', '35696', '62587', '81826', '3963', '45766', '82751', '12430', '97685', '29919', '78155', '71636', '50215', '89734', '9892', '47151', '54855', '3428', '9712', '52592', '2403', '79602', '81243', '79859', '57361', '82000', '42107', '28860', '99591', '28296', '57337', '64969', '32332', '25535', '30924', '21313', '32554', '17342', '87311', '19825', '24898', '61323', '83209', '79322', '79009', '50746', '33396', '62033', '16548', '17427', '24073', '34640', '52368', '4724', '80408', '40', '33787', '16666', '19665', '86751', '27264', '2241', '88134', '53566', '10589', '79711', '92823', '58972', '91767', '60885', '51659', '7867', '96849', '30360', '20914', '9584', '1250', '22871', '23282', '99312', '4683', '33429', '68361', '82614', '81440', '47863', '69790', '11968', '75210', '66854', '37002', '61142', '71514', '1588', '42336', '11069', '26291', '2261', '71056', '13492', '9133', '91216', '72207', '71586', '86535', '83898', '24392', '45384', '48545', '61972', '503', '80180', '35834', '97025', '70411', '55039', '35430', '27631', '82533', '96831', '74077', '42533', '14451', '26943', '53783', '69489', '71969', '8432', '37230', '61348', '19472', '59115', '9886', '50951', '57109', '7141', '1902', '84130', '4323', '55889', '47784', '2220', '75988', '66988', '63721', '8131', '95601', '95207', '2311', '26541', '50991', '6717', '2969', '71857', '51034', '65958', '94716', '90275', '21012', '46859', '7984', '31131', '46457', '69578', '44540', '7294', '80117', '9925', '60155', '90608', '82684', '32193', '87071', '28006', '87604', '24501', '79087', '2848', '29237', '11221', '81319', '40966', '87641', '35325', '78705', '88636', '78717', '62831', '56390', '99271', '43821', '14453', '17923', '62695', '77322', '21038', '67677', '41271', '4376', '65426', '46091', '19887', '97251', '55583', '58763', '3826', '35037', '73533', '64267', '82319', '9836', '42622', '96829', '16363', '10455', '49290', '99992', '98229', '66356', '59087', '73998', '25986', '4279', '56790', '69540', '588', '36620', '60358', '45056', '89297', '42740', '8323', '19245', '82417', '41431', '699', '11554', '73910', '44491', '56019', '68901', '45816', '68126', '89379', '23885', '13263', '56395', '73130', '19089', '23771', '10335', '48547', '16903', '6453', '33560', '89668', '38159', '43177', '90655', '49712', '62', '66920', '34180', '12150', '48564', '39538', '85026', '87195', '14928', '8956', '71157', '53287', '39161', '67583', '83309', '92054', '86977', '56188', '15229', '88170', '60894', '58497', '89254', '40082', '86890', '60161', '97291', '45878', '23368', '14577', '92870', '37017', '97356', '99426', '76061', '89186', '99751', '85153', '61580', '39360', '90107', '25603', '26798', '76224', '6469', '7912', '69838', '16404', '67497', '28965', '80836', '80365', '91249', '48713', '17113', '33090', '40793', '70450', '66689', '83698', '17802', '43869', '13355', '18959', '79411', '87930', '9265', '37504', '44876', '97234', '94149', '35040', '22049', '49248', '6535', '36080', '28346', '94437', '78319', '17961', '89056', '56161', '35810', '41632', '45494', '53351', '89729', '99510', '51584', '59688', '6193', '70809', '51093', '92589', '90247', '34910', '78235', '17362', '49423', '63324', '525', '37638', '72325', '89356', '15298', '59116', '17848', '65429', '27029', '84781', '70247', '8825', '35082', '70451', '22522', '58125', '91879', '90531', '2478', '463', '37902', '54405', '87267', '72688', '22803', '33134', '35177', '84551', '44974', '88375', '76407', '27774', '33849', '19915', '82014', '80434', '26380', '48777', '53811', '14838', '26829', '56441', '99869', '49574', '85476', '19723', '16907', '4018', '37338', '78510', '47912', '13030', '65277', '95716', '67363', '21393', '89887', '78842', '81650', '903', '17436', '30704', '49223', '27198', '25500', '52214', '54258', '70082', '53950', '49312', '43615', '99473', '94348', '53661', '96213', '96346', '62010', '38268', '32861', '75660', '10392', '89491', '68335', '29817', '88706', '24184', '36298', '43440', '21626', '26535', '44560', '46363', '12534', '99070', '95606', '33714', '73070', '8303', '29853', '23014', '99982', '4530', '14955', '45803', '50', '90750', '30394', '81276', '95563', '47314', '58520', '91299', '88944', '54402', '67405', '29253', '47079', '71734', '99728', '17652', '13307', '35556', '18962', '26780', '17771', '53712', '60055', '37628', '35830', '90739', '61151', '41309', '27652', '3051', '53167', '98417', '19382', '36833', '75085', '65374', '87732', '30352', '31776', '32765', '97565', '92199', '49050', '29503', '51024', '18834', '8515', '24069', '96216', '10777', '90680', '18974', '68884', '85305', '36007', '56707', '4212', '47352', '34426', '13185', '92939', '95782', '70577', '58080', '98279', '3906', '5065', '56896', '16382', '31273', '17117', '98602', '12786', '24086', '63970', '72756', '35798', '82367', '7356', '53398', '68503', '2962', '16425', '67334', '68461', '65439', '15620', '70906', '29649', '46461', '74602', '38012', '71714', '16825', '89480', '53386', '88532', '35104', '28556', '82120', '23155', '23347', '24797', '60061', '54962', '99427', '82248', '82447', '39968', '63727', '27431', '81511', '91168', '71425', '80740', '84127', '40717', '15503', '15419', '46594', '61263', '19212', '53175', '70724', '74445', '23034', '71818', '40246', '18886', '53066', '4880', '83701', '86107', '87862', '44751', '392', '73440', '90291', '93395', '20894', '38463', '32664', '55158', '20090', '50004', '79070', '98471', '85478', '96615', '68149', '78334', '97752', '73207', '71678', '91238', '96757', '82598', '194', '35797', '45120', '60782', '28721', '17676', '78066', '60957', '11826', '51563', '50516', '16485', '47053', '31738', '48923', '23554', '96850', '42033', '73701', '78607', '45979', '54571', '12415', '31693', '15356', '36902', '9126', '3767', '3295', '90402', '24005', '95350', '67033', '49137', '72606', '51899', '17522', '31957', '44641', '53982', '23767', '68257', '15766', '19995', '2107', '48788', '11765', '91055', '46576', '54651', '50381', '62827', '73636', '46606', '98753', '37631', '70441', '87916', '66983', '33870', '31125', '12904', '57040', '4874', '58632', '42037', '18782', '5998', '18974', '57949', '81010', '90407', '99874', '20462', '89949', '10952', '71454', '95130', '46115', '3518', '13384', '69039', '79482', '22076', '59782', '32042', '40930', '60243', '29298', '6790', '46985', '44398', '85631', '14380', '66179', '2629', '32126', '49833', '14118', '58492', '31493', '81172', '96638', '8745', '89663', '76842', '78633', '41373', '83721', '42886', '11123', '32739', '11051', '1303', '92314', '83324', '85600', '44276', '69064', '56125', '84650', '31028', '12628', '14502', '64764', '39405', '44855', '79046', '51716', '46824', '83389', '1941', '1257', '9280', '73176', '84729', '2579', '63366', '22606', '35541', '51096', '13447', '18355', '68037', '28436', '94116', '81070', '78355', '67897', '5296', '32742', '77645', '91853', '18767', '67949', '40963', '5792', '17278', '25597', '41884', '80829', '7099', '18645', '60295', '12082', '81800', '78415', '18082', '38789', '16295', '72377', '74949', '55583', '66853', '15402', '72977', '15123', '99434', '34999', '21687', '76049', '42987', '83748', '88256', '66688', '21766', '20304', '29271', '10069', '19822', '11792', '42526', '74143', '17289', '30253', '6367', '20888', '12975', '94073', '98639', '30134', '26320', '65507', '69002', '53120', '4550', '38893', '18954', '38283', '54863', '17698', '99670', '10521', '92467', '60994', '18052', '48673', '35811', '87282', '62706', '16061', '53112', '22652', '37780', '55662', '26331', '49410', '79074', '10623', '69577', '79613', '9491', '31229', '43922', '84231', '58409', '36386', '46875', '74431', '76735', '38776', '23350', '7314', '9079', '51519', '98544', '70216', '63380', '90381', '1295', '46901', '58225', '55339', '89918', '75522', '35431', '89460', '49552', '89302', '23068', '28493', '3042', '25194', '59520', '9810', '95706', '81297', '89638', '54794', '94527', '45262', '97932', '78685', '6947', '22818', '48700', '9153', '12289', '22011', '58825', '93854', '65438', '4509', '33741', '28208', '69061', '48578', '40247', '77725', '31837', '39003', '69363', '78113', '76398', '97262', '67795', '68446', '58896', '60969', '19849', '6722', '91854', '49519', '13949', '67109', '48824', '31723', '75554', '69575', '94986', '75350', '18628', '15284', '41943', '15433', '52607', '41', '22340', '29528', '24059', '34145', '72517', '46316', '10667', '54510', '19882', '47764', '69124', '41963', '84350', '48420', '4646', '24958', '69020', '97121', '26178', '62664', '50622', '32554', '49655', '45398', '11267', '72222', '73363', '89554', '89046', '57384', '29259', '37888', '24850', '74353', '57343', '34762', '2900', '11393', '42154', '94306', '70552', '75265', '921', '26003', '64352', '89857', '83171', '58249', '48940', '53512', '66335', '44865', '68729', '19299', '58003', '39854', '99122', '3860', '80173', '52242', '90966', '53183', '71982', '82325', '87842', '15000', '55627', '71132', '6354', '42402', '91719', '91644', '94533', '74925', '66278', '66911', '85576', '40495', '70919', '71797', '87835', '29845', '71832', '3390', '7994', '33499', '70715', '54897', '82710', '63077', '78105', '24758', '89585', '84607', '46477', '78618', '10989', '39222', '98749', '51685', '94664', '31008', '32823', '89521', '72160', '26952', '4001', '21421', '5737', '74027', '88179', '45553', '83743', '19913', '49435', '65616', '82641', '5149', '76959', '40681', '73023', '2670', '30845', '18863', '35094', '88400', '80963', '9154', '16571', '64192', '59694', '41317', '59942', '58856', '99281', '67260', '66971', '22716', '76089', '58047', '67071', '53707', '462', '52518', '72277', '10681', '69', '98855', '12593', '88842', '67242', '73543', '37439', '18413', '67211', '93495', '45576', '70614', '27988', '53210', '18618', '21318', '68059', '25518', '55917', '56522', '16548', '2404', '93538', '61452', '66358', '3709', '23914', '92426', '81439', '38070', '28988', '29939', '2948', '85720', '45628', '51101', '89431', '86365', '17571', '50987', '83849', '11015', '83812', '66187', '26362', '66786', '22024', '93866', '36161', '90080', '64874', '37294', '83860', '73821', '80279', '36766', '73117', '44620', '84556', '42070', '90383', '27862', '20665', '67576', '34997', '57958', '80638', '84351', '63961', '1362', '14338', '80377', '24192', '41294', '57368', '51189', '27287', '45764', '86289', '65600', '708', '84090', '96005', '55676', '84855', '72385', '70018', '9336', '82701', '3710', '52083', '74045', '96454', '30956', '67369', '78941', '81810', '71906', '23194', '33042', '50794', '61256', '24449', '48639', '22916', '78303', '13666', '40762', '43942', '51075', '89783', '95786', '90462', '6181', '36482', '40675', '4970', '6388', '91849', '72579', '94983', '86084', '20140', '68427', '48123', '43122', '98066', '37560', '6927', '72803', '5546', '62259', '98439', '6457', '98568', '70499', '33022', '28226', '29675', '20917', '75365', '20900', '8190', '56736', '99153', '77779', '49333', '50293', '97650', '4067', '47278', '42761', '71875', '13966', '11223', '46783', '18059', '61355', '29638', '75681', '24466', '89634', '20759', '83252', '37780', '15931', '74893', '6703', '64524', '80656', '85990', '78427', '18411', '20696', '86432', '93176', '69889', '15072', '15180', '9935', '10467', '60248', '42430', '62590', '89596', '27743', '26398', '79912', '60048', '50943', '38870', '69383', '72261', '98059', '55242', '74905', '5667', '54321', '70415', '39903', '49711', '85318', '79979', '59262', '82321', '15263', '17416', '74554', '94733', '72112', '49872', '54849', '73883', '78250', '74935', '68559', '57564', '50541', '45730', '41595', '5588', '83723', '42891', '11898', '14348', '99732', '14481', '85233', '21277', '94508', '52551', '74187', '7634', '42912', '25100', '43536', '35798', '48190', '86477', '22680', '48148', '59501', '56563', '16802', '81496', '97568', '68657', '51462', '67953', '99660', '39002', '54170', '57190', '68086', '52700', '6487', '55709', '70418', '62629', '70420', '35695', '36152', '45360', '53503', '46623', '76000', '50648', '97876', '44815', '29163', '1356', '64123', '71388', '17658', '99084', '58727', '59437', '38773', '71254', '81286', '97545', '18786', '56834', '20346', '36401', '62316', '58082', '67959', '99876', '69895', '80099', '62747', '20517', '99777', '6472', '49189', '31321', '39992', '68073', '13378', '51806', '21776', '52060', '96983', '25754', '93709', '96627', '8644', '93726', '14002', '37716', '87620', '34507', '76339', '24491', '5849', '44110', '522', '66521', '12776', '44887', '80535', '14548', '75248', '671', '73071', '35715', '59474', '7061', '82243', '56170', '20179', '59717', '1725', '24634', '11270', '77023', '63840', '46608', '44667', '22422', '59771', '94768', '73033', '82905', '16463', '40971', '22204', '58366', '28721', '14907', '76468', '81872', '38418', '36989', '61439', '10610', '131', '44296', '35453', '10117', '75856', '94603', '99602', '68075', '35949', '13599', '50030', '69633', '55956', '85465', '16429', '86081', '11145', '6195', '82207', '90598', '92814', '23725', '83204', '80346', '71542', '46634', '15820', '54123', '45397', '15322', '61743', '9273', '71347', '6835', '64006', '91718', '43677', '32923', '21486', '17098', '61694', '43347', '40019', '4071', '52443', '42386', '56839', '83514', '27633', '40780', '51749', '92101', '62384', '92206', '56044', '66174', '11137', '73966', '78471', '30468', '31643', '33197', '6888', '8066', '86603', '74383', '6098', '54411', '98819', '89862', '88639', '94422', '89371', '80526', '91747', '91220', '64944', '76658', '42046', '58518', '27249', '6646', '3028', '1346', '33763', '9734', '31737', '65527', '5892', '60813', '3410', '35464', '43009', '98382', '70580', '93898', '56404', '32995', '62771', '71556', '40538', '55612', '45656', '10758', '20268', '33603', '38310', '14242', '74397', '10722', '71575', '22590', '49043', '91439', '9055', '23668', '9101', '5268', '64133', '77501', '64684', '11337', '47575', '50732', '88680', '93730', '46785', '17589', '3520', '57595', '71241', '34994', '8753', '36147', '88844', '41914', '11250', '94632', '71927', '4624', '86279', '7664', '2659', '94853', '65386', '30438', '86005', '92883', '84629', '59910', '44484', '1306', '8404', '56962', '29990', '38445', '96191', '73013', '66590', '40951', '24712', '18825', '37268', '87843', '18972', '12154', '7779', '52149', '76152', '65799', '86011', '35475', '78083', '88232', '91551', '65532', '93516', '73827', '24227', '44687', '55759', '83819', '45088', '10856', '60488', '39051', '14103', '76650', '81181', '46731', '737', '58788', '78945', '42096', '66731', '66740', '72273', '88969', '5655', '86590', '41096', '80038', '32430', '51877', '23970', '91900', '13082', '45880', '94367', '19739', '61998', '71665', '16083', '57035', '26916', '10166', '18834', '46798', '66881', '28444', '68840', '10459', '81087', '4728', '76224', '39257', '23470', '93524', '37345', '30074', '49856', '22022', '55279', '5159', '5193', '58030', '57539', '12514', '49759', '96222', '52597', '67192', '88187', '53614', '16084', '79915', '28212', '79334', '85283', '32306', '31058', '43113', '74707', '74869', '2213', '32134', '6379', '85426', '87098', '35984', '51105', '69287', '16803', '83337', '14913', '62531', '58098', '7914', '20105', '28850', '1384', '43173', '62983', '87113', '76066', '86320', '77684', '45191', '95225', '41503', '36713', '48404', '91228', '53865', '98981', '59161', '61237', '84561', '17455', '14379', '57789', '80895', '99260', '84595', '72942', '53220', '84448', '81332', '49437', '83086', '93414', '54519', '52288', '74772', '22460', '49324', '11168', '96071', '61985', '38284', '6405', '54698', '71727', '60093', '37340', '87884', '83403', '4542', '94949', '19636', '15855', '39105', '10424', '67418', '91022', '69254', '8481', '38411', '3832', '44354', '93548', '57172', '28481', '372', '81497', '52179', '41060', '72141', '41396', '65590', '70432', '82819', '93814', '26118', '84780', '88485', '70821', '8222', '83000', '47067', '38516', '33347', '47681', '48202', '60749', '52112', '7937', '28105', '11394', '45746', '43252', '34494', '2979', '69715', '42486', '82315', '71760', '97413', '66137', '94487', '7429', '74434', '22964', '55251', '3448', '53534', '2574', '9693', '96157', '2955', '4348', '19566', '56930', '83319', '31310', '53905', '1148', '41726', '22233', '76045', '37351', '10545', '17581', '28047', '30199', '4741', '58111', '33497', '67796', '67730', '31247', '43772', '29461', '45970', '73353', '22534', '53962', '32147', '71392', '62579', '66345', '58246', '33442', '9581', '29705', '14058', '86471', '76125', '59363', '94982', '74810', '89149', '20066', '3366', '3568', '25752', '80036', '64119', '27270', '40061', '91052', '69022', '9852', '77112', '83075', '43924', '61661', '56133', '96652', '57944', '72576', '82170', '79236', '55745', '15309', '88878', '72761', '37647', '67465', '12777', '97309', '93202', '41470', '8787', '64920', '48514', '18917', '35157', '59151', '4640', '5317', '38134', '76548', '82788', '9214', '58418', '73185', '90554', '10543', '47182', '62936', '91765', '89751', '68931', '48865', '64607', '7150', '77862', '14297', '14828', '33013', '91698', '67593', '98096', '16595', '51639', '86531', '24719', '1703', '78788', '43810', '38918', '95491', '99903', '82671', '8291', '68288', '31224', '39863', '4265', '77798', '7698', '33804', '92286', '4744', '37038', '44203', '98212', '17369', '77442', '62879', '4145', '96881', '15646', '36824', '19959', '45451', '76049', '54272', '97577', '95298', '81115', '30204', '82041', '8037', '10052', '8756', '76833', '82851', '24276', '75574', '36037', '78079', '92807', '29064', '90000', '84150', '17102', '75092', '49424', '35597', '4693', '82853', '42511', '16119', '23478', '65240', '55585', '91762', '71671', '46682', '72479', '97696', '24615', '12579', '30274', '48255', '2336', '90202', '5808', '45426', '76308', '74639', '31245', '99894', '89638', '6233', '33893', '71899', '85273', '89429', '29761', '50231', '57249', '99347', '22642', '66972', '86221', '47514', '88274', '10819', '73150', '53754', '13304', '20478', '38099', '619', '14669', '8011', '97657', '26569', '65430', '13467', '38180', '23675', '72350', '42257', '39875', '23529', '53407', '11833', '29599', '95621', '7727', '59527', '86846', '22860', '5358', '3730', '87555', '362', '95755', '54565', '29935', '68950', '52349', '98344', '86576', '7420', '12236', '15844', '48099', '97535', '97081', '50261', '31187', '60496', '24123', '24042', '6376', '6679', '99806', '20306', '60676', '36881', '77309', '5247', '96569', '53417', '73252', '64179', '35318', '75732', '65119', '32621', '40464', '22887', '96152', '65161', '83381', '8915', '68142', '7328', '85031', '15688', '72519', '93992', '86927', '75538', '38205', '50877', '70039', '97538', '94822', '52131', '49643', '85206', '1347', '14574', '88736', '53442', '49991', '64925', '72283', '82213', '60905', '36118', '62963', '16983', '79185', '15111', '26059', '17792', '98218', '33214', '1094', '41754', '77275', '65173', '13190', '91004', '90422', '44387', '92672', '98641', '54609', '83295', '37395', '70104', '32986', '72524', '82478', '5837', '83916', '52736', '57112', '55985', '42642', '42136', '89642', '35712', '49489', '19726', '65824', '24384', '48112', '15366', '99206', '68384', '51389', '529', '21475', '75749', '95182', '60110', '70571', '74174', '38105', '78107', '4101', '8982', '11215', '23987', '3303', '28706', '54629', '98000', '67510', '30036', '99140', '48896', '40971', '7735', '79984', '50134', '94928', '57023', '52880', '83067', '41940', '62994', '89213', '38593', '19283', '68206', '22234', '19245', '26266', '32403', '65889', '17022', '64280', '42797', '27161', '57675', '42313', '93606', '93082', '20659', '90824', '1226', '66266', '12503', '57104', '15247', '51160', '92398', '71967', '59476', '44465', '35765', '10787', '47737', '45792', '2292', '47599', '89612', '8162', '87622', '69410', '45727', '31158', '99791', '89544', '27214', '99588', '40516', '75616', '36505', '46079', '95448', '97999', '47462', '47799', '82729', '34038', '60789', '96938', '22682', '79062', '93307', '36038', '49016', '90983', '48219', '50889', '32517', '72219', '71229', '82643', '1195', '70543', '17', '22178', '23544', '72371', '1163', '28527', '7336', '39846', '31956', '80963', '41804', '59791', '41831', '1940', '52377', '79494', '12531', '81112', '44320', '18746', '5774', '63869', '4085', '59922', '12751', '99443', '13530', '23872', '36026', '83360', '32711', '92980', '11140', '99323', '57263', '98149', '29265', '25548', '65995', '4818', '15593', '8535', '37863', '12217', '14474', '66584', '89272', '86690', '58777', '39666', '44756', '18442', '52586', '98030', '40850', '38708', '49304', '68923', '65008', '84388', '83639', '29866', '63675', '26793', '49227', '82099', '24090', '57535', '24201', '65776', '74054', '89833', '62979', '26613', '5851', '99766', '63484', '66605', '37179', '90760', '59336', '58390', '93239', '84578', '11396', '93994', '73818', '23972', '37720', '72369', '25063', '32952', '71036', '76612', '31285', '34090', '19136', '53783', '66436', '61478', '96749', '43658', '7399', '31574', '67073', '40480', '20727', '70993', '65549', '30800', '21507', '53785', '89574', '86381', '56492', '62603', '44856', '68687', '63794', '70996', '7475', '84238', '71939', '86886', '94792', '15036', '36936', '95722', '17771', '67850', '33371', '49314', '40744', '5432', '81057', '41201', '75986', '22961', '15323', '1570', '18657', '95219', '19130', '53127', '15867', '81135', '73206', '76668', '36386', '48828', '31417', '56916', '70891', '60534', '95777', '10022', '94053', '2928', '56326', '16559', '79656', '6414', '81247', '78270', '55687', '19151', '61597', '99857', '81142', '27725', '53493', '12185', '1455', '48501', '59425', '20591', '24900', '66079', '84889', '32024', '18919', '2043', '7076', '71201', '88258', '86521', '93348', '26395', '39646', '44145', '33911', '46231', '67054', '39979', '11630', '23020', '76278', '88056', '11480', '4723', '78612', '70211', '60622', '84687', '59092', '65675', '38479', '64399', '64699', '95964', '42764', '69060', '28189', '4193', '95805', '75462', '17245', '59640', '94773', '84292', '53092', '98507', '61353', '32483', '53027', '48912', '87221', '47788', '59263', '65196', '35567', '17494', '64253', '50223', '7057', '87467', '62414', '2523', '50910', '72353', '78986', '78104', '47719', '29108', '12957', '5114', '64435', '66707', '37449', '70399', '45334', '71606', '55338', '55072', '58765', '12151', '22012', '16954', '87366', '14240', '98041', '72296', '47408', '56879', '99584', '63172', '92316', '28071', '29880', '19608', '13839', '87484', '56541', '88662', '87098', '72124', '78282', '27653', '38993', '31870', '67239', '99445', '7376', '78487', '98880', '12180', '86773', '67773', '15416', '58172', '13075', '67559', '97510', '29705', '86985', '57024', '11827', '31236', '91920', '26116', '94614', '14486', '46252', '78847', '43786', '70048', '96739', '35240', '39933', '58209', '27852', '65669', '47323', '58150', '84444', '44344', '95882', '41258', '31314', '69060', '19916', '6979', '19436', '45572', '16259', '74566', '6306', '24705', '53422', '593', '97031', '22308', '26875', '23042', '78035', '34229', '61976', '23175', '50072', '90896', '50810', '71730', '86468', '94807', '8218', '36032', '58628', '60560', '51206', '37943', '27987', '15014', '49905', '70018', '66799', '80851', '23594', '29982', '6438', '97381', '47715', '96294', '17985', '48545', '12672', '5250', '9988', '24601', '3736', '97815', '54363', '64703', '44167', '68376', '16595', '38073', '29630', '59630', '1858', '71823', '75580', '70083', '14493', '93821', '93394', '85369', '3818', '8435', '59988', '43966', '13961', '15855', '83332', '80312', '27299', '88840', '76964', '56173', '62794', '79389', '82642', '85843', '47116', '43064', '16061', '28905', '54415', '72832', '91252', '93488', '79457', '99336', '70744', '80432', '6487', '880', '87701', '154', '86574', '86677', '17892', '81488', '95260', '12515', '43189', '9211', '55403', '41417', '60046', '54785', '83655', '28274', '65745', '63062', '44549', '36391', '48051', '7328', '3572', '33226', '49177', '25123', '59065', '19691', '15109', '10172', '95578', '29497', '48152', '20276', '36270', '78866', '48309', '53209', '55475', '30073', '19717', '16004', '45692', '83430', '9291', '45935', '57030', '92613', '91656', '67697', '34915', '28156', '56594', '3273', '11194', '98270', '34370', '2621', '66679', '97451', '97717', '87923', '48310', '37725', '69743', '75103', '84956', '75163', '16069', '65304', '19397', '18071', '27273', '49823', '57595', '98324', '82174', '10293', '80943', '64184', '19472', '4198', '9410', '25927', '65961', '33155', '95168', '33692', '61712', '69877', '13308', '17415', '10022', '2491', '67310', '96140', '68050', '76272', '17143', '76805', '57176', '7539', '22690', '95483', '87592', '27221', '90821', '51154', '99828', '68998', '54581', '74222', '10269', '65057', '45467', '96089', '55058', '89779', '60837', '74122', '52886', '58055', '14880', '93208', '66652', '68830', '24121', '62407', '87257', '18802', '14925', '45423', '98624', '55195', '59072', '41414', '77840', '66075', '62705', '26549', '19063', '57552', '2507', '52069', '57620', '66688', '14833', '33700', '90666', '98052', '5367', '2268', '43093', '69063', '22030', '85564', '92258', '1847', '24446', '65835', '38660', '91899', '87732', '52396', '31952', '36000', '86944', '16109', '80729', '53757', '60226', '59103', '84187', '36674', '72823', '29884', '4654', '69139', '20440', '57413', '3651', '39639', '44564', '57492', '84159', '751', '99748', '9659', '72661', '39220', '99742', '74734', '75729', '38071', '69934', '73640', '65294', '54524', '64372', '37927', '17187', '7863', '12732', '40296', '36197', '15821', '76831', '4400', '71933', '4040', '22072', '33064', '25702', '13324', '91275', '27388', '97729', '14620', '45989', '80737', '17934', '4219', '3032', '43457', '31051', '24469', '67041', '29328', '75499', '80951', '88212', '92595', '49969', '24612', '58732', '2718', '3805', '50918', '99426', '8614', '35580', '93273', '989', '24385', '41185', '25687', '47146', '25227', '95839', '56355', '98536', '79824', '31725', '46447', '26690', '68418', '47783', '33725', '21729', '70797', '59038', '60376', '25087', '68332', '67950', '12411', '95918', '64736', '65336', '74947', '64605', '4106', '42712', '96640', '28492', '28648', '42429', '821', '24333', '69677', '38959', '23484', '92005', '29352', '29159', '52873', '99947', '21834', '85347', '93479', '28298', '55608', '3226', '69714', '80283', '6577', '18849', '44605', '75286', '28139', '26541', '12867', '57500', '86617', '33005', '57498', '60223', '74954', '51401', '55246', '5648', '16513', '40930', '43821', '32090', '66002', '65530', '76083', '6047', '6879', '94987', '80787', '11688', '77161', '92670', '6696', '400', '28572', '47234', '51375', '88518', '762', '92617', '54260', '7560', '60180', '43331', '64059', '27616', '75839', '21392', '47756', '46254', '19486', '88533', '30130', '93694', '8557', '66534', '94447', '16910', '6480', '77440', '24366', '6195', '48946', '28597', '44429', '50300', '73556', '40638', '98709', '94413', '15987', '43860', '64871', '93953', '34506', '7296', '31753', '30626', '77510', '39829', '25696', '39776', '69185', '36540', '65413', '31528', '43446', '73532', '49776', '30282', '30004', '26725', '15200', '33958', '90320', '71836', '48051', '31970', '5326', '96194', '69695', '60898', '60945', '18271', '50868', '61468', '23593', '68985', '20628', '58044', '8942', '34849', '7384', '50500', '62895', '78780', '48946', '65278', '4067', '973', '34761', '15512', '73739', '23138', '47322', '55568', '32259', '71816', '49277', '75218', '76104', '19579', '68312', '67904', '33886', '53888', '26421', '43859', '40291', '39068', '31711', '36542', '10195', '39781', '72352', '13188', '34113', '9428', '60443', '4987', '13783', '80744', '63483', '18266', '11961', '87167', '46987', '28480', '74214', '39191', '8146', '38090', '75727', '79245', '47720', '52547', '45321', '4972', '49701', '74354', '69672', '63455', '41902', '5667', '54166', '4962', '25873', '44509', '73332', '73383', '29438', '21455', '12320', '11997', '16921', '49379', '63027', '86175', '8110', '76149', '2520', '11256', '25863', '50518', '69001', '79113', '9447', '91840', '5242', '10998', '46496', '2448', '56058', '20970', '10517', '17783', '25723', '97137', '62840', '1264', '78691', '81020', '55335', '48524', '2088', '90413', '76651', '26855', '16177', '14954', '62914', '21344', '5708', '75560', '39311', '95865', '28783', '64902', '95657', '46276', '33426', '4799', '11588', '57513', '73689', '77677', '63011', '97795', '34954', '76866', '32043', '32697', '26643', '36890', '53476', '3011', '13963', '49551', '87671', '67761', '17488', '94770', '50599', '33272', '23091', '38079', '41177', '22395', '91656', '79679', '38687', '57384', '80118', '42507', '4098', '78949', '45669', '48802', '83915', '78292', '4369', '57657', '49146', '45192', '98491', '72457', '46331', '207', '81601', '7409', '70856', '91605', '70295', '9171', '72293', '32997', '78025', '16795', '73534', '68780', '21284', '31767', '94381', '86439', '12420', '53285', '99563', '60502', '67954', '55012', '99809', '5431', '69978', '99712', '14401', '79498', '4495', '3045', '528', '72542', '91604', '72725', '39378', '80378', '41996', '20138', '54545', '59730', '36951', '45157', '37964', '97690', '12184', '4944', '53803', '93605', '60851', '68938', '46285', '89663', '90309', '6907', '87239', '81791', '83292', '90013', '68927', '14725', '81840', '63836', '52068', '43830', '4794', '931', '59255', '8263', '99057', '94401', '69033', '7437', '20364', '92884', '28193', '43932', '37629', '59426', '18891', '8583', '79551', '87242', '1483', '6725', '65786', '16844', '12650', '99305', '42841', '9811', '18800', '39313', '51373', '31874', '84558', '27831', '48614', '48975', '55509', '83363', '31854', '64001', '94028', '76125', '79314', '24893', '81132', '9441', '86015', '28356', '40358', '10160', '23328', '7330', '76538', '37611', '89351', '84132', '97047', '26109', '95222', '35130', '75600', '88602', '15073', '87835', '71649', '28948', '81615', '37498', '28674', '59776', '44095', '65924', '64368', '94536', '12518', '61711', '55619', '82949', '4114', '21540', '70544', '28022', '79983', '28781', '7749', '97873', '4951', '50076', '47611', '99522', '56820', '38653', '49047', '36283', '83908', '72452', '85625', '10811', '36998', '44083', '34864', '44975', '39057', '4551', '68450', '24781', '1503', '9871', '46885', '11424', '21259', '54900', '97669', '85669', '6015', '2521', '37661', '14915', '57423', '91903', '94789', '32059', '64972', '4600', '61465', '27118', '79785', '13547', '49766', '38410', '68860', '63756', '23621', '64387', '46255', '63408', '11297', '41081', '56326', '58349', '98703', '72268', '73574', '32098', '42534', '91502', '38083', '11241', '56828', '12098', '25377', '37054', '56328', '30034', '26922', '68401', '93478', '63275', '62650', '81407', '773', '79499', '14970', '47217', '1187', '57428', '69980', '77764', '74791', '22107', '54363', '39247', '56028', '56982', '84244', '21464', '18716', '25533', '94589', '94768', '21537', '18436', '81135', '27654', '79713', '56630', '61571', '58453', '26758', '68450', '68449', '2994', '15347', '83954', '71823', '6428', '44210', '79597', '95144', '32871', '1991', '320', '77157', '63607', '31154', '48846', '71125', '61750', '59608', '33038', '35733', '68915', '94127', '50383', '64242', '49708', '57270', '65019', '8581', '12111', '18487', '50013', '58664', '22214', '19033', '33681', '44754', '28830', '10381', '52318', '34959', '20682', '55453', '53800', '65774', '99164', '72102', '36986', '44157', '56716', '7974', '81475', '25926', '39402', '33688', '99671', '95312', '42268', '26536', '14482', '67377', '57993', '89147', '15834', '64995', '4700', '18714', '30221', '39095', '32749', '69257', '55204', '30497', '31839', '63045', '30009', '62683', '31232', '77680', '93551', '63589', '6989', '77246', '42169', '46117', '73226', '37427', '1858', '83649', '37410', '86369', '4641', '74481', '66168', '48041', '22597', '14670', '27464', '57165', '20939', '36282', '76940', '73358', '50521', '69603', '8895', '81793', '57743', '81903', '64025', '91641', '25276', '34040', '62642', '64015', '57657', '84890', '73832', '782', '60160', '16998', '40023', '24590', '88613', '76640', '53091', '67600', '80183', '45674', '64464', '25163', '42384', '66972', '13953', '41966', '66048', '15135', '73745', '19466', '53657', '34619', '13462', '15905', '48257', '73297', '238', '93525', '80556', '5942', '5411', '66169', '9090', '95130', '74316', '57321', '48083', '62355', '68113', '15239', '36644', '80326', '65817', '54428', '61955', '58849', '77206', '16073', '98261', '92091', '39178', '35464', '85109', '85452', '21128', '25665', '81860', '44664', '24024', '56960', '95124', '39786', '18836', '11121', '44163', '81074', '79064', '46219', '94694', '44233', '81469', '24642', '15030', '21995', '13587', '40755', '6669', '81093', '74305', '1881', '55649', '37273', '80827', '98643', '46694', '59281', '79231', '42813', '84984', '7052', '98113', '17296', '84434', '31205', '46894', '71219', '74530', '44686', '70744', '91388', '20692', '96853', '73803', '15836', '18126', '49686', '4179', '47588', '87892', '65425', '68012', '97468', '92510', '99271', '58694', '11918', '37051', '18644', '57228', '14265', '57572', '57022', '52186', '30193', '93570', '87872', '5257', '26784', '6476', '61746', '68559', '1720', '26202', '16519', '27688', '10645', '87174', '60845', '73385', '82075', '6933', '98828', '56895', '17344', '84253', '36561', '51648', '24939', '63470', '31034', '95052', '51090', '51465', '87979', '68650', '30181', '29598', '19137', '43221', '81353', '90170', '96985', '61115', '17385', '92314', '80650', '55821', '17874', '84333', '93272', '48260', '87272', '22764', '59957', '51870', '85988', '39222', '77241', '62535', '28344', '6011', '80831', '64551', '46299', '75195', '71177', '8660', '58943', '57003', '3306', '74413', '74068', '15073', '89016', '93140', '13911', '57170', '19880', '41870', '9131', '57495', '73032', '86979', '60094', '87026', '30880', '4736', '86301', '92707', '21689', '83565', '71275', '47665', '65687', '71184', '89897', '32490', '97577', '38723', '79113', '37531', '97500', '94450', '15699', '58019', '84423', '27057', '56017', '97148', '47365', '30669', '33818', '80406', '99690', '33012', '95178', '46809', '48448', '79350', '9146', '99701', '98976', '71197', '44161', '75069', '36602', '79650', '97301', '12020', '56658', '25701', '46392', '78609', '63073', '69419', '57736', '20102', '42415', '79044', '20277', '56280', '47903', '94311', '25558', '40336', '91305', '90505', '66769', '64562', '83737', '62892', '10375', '71024', '19988', '56946', '76110', '21847', '43162', '50578', '46086', '54167', '61722', '53463', '63134', '69288', '12838', '14116', '71687', '50846', '59810', '24826', '84138', '82885', '91496', '98600', '82769', '40049', '4125', '50694', '1294', '2805', '29691', '82321', '76462', '85945', '115', '29188', '66918', '71340', '31585', '61638', '95472', '52978', '50622', '81990', '60955', '70519', '22270', '35610', '95871', '89222', '41038', '52546', '1163', '67943', '1793', '92010', '35755', '74509', '66665', '95759', '8568', '44299', '67822', '5806', '85839', '13895', '87675', '31357', '88014', '40026', '53050', '28951', '31992', '42495', '82892', '51567', '2869', '45808', '20238', '20781', '56098', '66307', '95701', '614', '60833', '3091', '81339', '24195', '65639', '85976', '28116', '66224', '51502', '73637', '13207', '88302', '36488', '65518', '98187', '26', '74367', '64706', '53943', '86760', '25783', '82112', '34958', '86621', '20848', '63459', '14049', '84943', '91873', '50238', '77773', '64109', '8602', '87934', '47583', '66053', '30287', '5507', '80312', '37464', '57457', '86200', '17806', '16522', '38843', '94334', '59958', '63864', '53427', '74506', '33980', '90449', '30842', '53616', '36738', '52', '13595', '53051', '13174', '60163', '71420', '73835', '67119', '79018', '42782', '45059', '952', '46360', '85879', '71552', '84741', '29746', '32577', '10041', '7208', '97528', '51256', '916', '55973', '17684', '99046', '38782', '58660', '97798', '66032', '48339', '51329', '12532', '97904', '95454', '42737', '62541', '96702', '82953', '94610', '26645', '86813', '25480', '99713', '26078', '23028', '93056', '21445', '73209', '89318', '69987', '34705', '30064', '17094', '51135', '54141', '26625', '1086', '13082', '30843', '98672', '56864', '42605', '5833', '60850', '69366', '27351', '16456', '92609', '48030', '54322', '69891', '46502', '34578', '77918', '63276', '75958', '42519', '60266', '85576', '4855', '14258', '67017', '10545', '35078', '53012', '71922', '85784', '73402', '74363', '58457', '94102', '23510', '51559', '39482', '87057', '9377', '10106', '82985', '33931', '16523', '6484', '97749', '83172', '53753', '27466', '23073', '96083', '67302', '57465', '21877', '18013', '99804', '32873', '43123', '72365', '53197', '80578', '69770', '97471', '86954', '67183', '98497', '78474', '28450', '63183', '98699', '42738', '61433', '3491', '27304', '49311', '94980', '92740', '43272', '86549', '11406', '79636', '85582', '38086', '657', '2354', '26567', '77450', '42086', '21600', '49011', '44059', '47872', '75761', '96577', '11642', '83471', '79616', '23749', '77082', '96876', '65302', '84027', '48955', '59887', '20657', '75090', '9058', '50347', '66088', '70745', '76342', '58026', '95568', '61504', '93473', '84590', '47089', '74717', '93090', '46334', '68273', '59500', '54345', '72608', '54048', '86156', '40296', '74046', '6813', '36369', '74543', '18305', '85236', '31316', '37061', '96893', '23112', '5529', '10166', '19037', '1467', '70810', '30932', '18410', '92837', '81324', '12268', '54705', '25207', '90366', '56528', '3392', '88747', '39951', '97957', '99404', '23685', '13533', '15640', '11434', '66516', '71025', '65770', '88000', '52232', '32360', '10787', '37438', '2264', '94460', '80214', '42288', '59062', '29010', '64093', '21225', '22297', '36935', '19202', '5925', '85373', '27414', '28991', '9191', '42273', '56587', '89719', '77191', '64334', '61542', '28763', '28978', '79184', '59815', '95200', '30246', '54022', '287', '91808', '66347', '50833', '15356', '78614', From bdcf1170ced2f334d9a273815b9ac8064f4046ba Mon Sep 17 00:00:00 2001 From: Ivan Starkov Date: Tue, 9 Jun 2020 16:32:49 +0300 Subject: [PATCH 23/29] Instead of forcing IPv4 protocol force wget to use protocols in order --- docker/server/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 6111b0057ed..059f3cb631b 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -94,7 +94,7 @@ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then # check if clickhouse is ready to accept connections # will try to send ping clickhouse via http_port (max 12 retries, with 1 sec delay) - if ! wget --spider --quiet -4 --tries=12 --waitretry=1 --retry-connrefused "http://localhost:$HTTP_PORT/ping" ; then + if ! wget --spider --quiet --prefer-family=IPv6 --tries=12 --waitretry=1 --retry-connrefused "http://localhost:$HTTP_PORT/ping" ; then echo >&2 'ClickHouse init process failed.' exit 1 fi From f623ef635b3be124d218e1c480df837f1d539abe Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 9 Jun 2020 18:26:51 +0300 Subject: [PATCH 24/29] Avoid copying whole response stream into memory in S3 HTTP client. --- contrib/aws | 2 +- src/IO/S3/PocoHTTPClient.cpp | 5 +++-- src/IO/S3/PocoHTTPClientFactory.cpp | 6 ++++-- src/IO/S3/PocoHTTPClientFactory.h | 11 +++++++---- src/IO/S3/PocoHTTPResponseStream.cpp | 12 ++++++++++++ src/IO/S3/PocoHTTPResponseStream.h | 21 +++++++++++++++++++++ 6 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 src/IO/S3/PocoHTTPResponseStream.cpp create mode 100644 src/IO/S3/PocoHTTPResponseStream.h diff --git a/contrib/aws b/contrib/aws index f7d9ce39f41..17e10c0fc77 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit f7d9ce39f41323300044567be007c233338bb94a +Subproject commit 17e10c0fc77f22afe890fa6d1b283760e5edaa56 diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index b8de483a5a8..0dfa80ca107 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include #include @@ -149,8 +151,7 @@ void PocoHTTPClient::MakeRequestInternal( response->SetClientErrorMessage(error_message); } else - /// TODO: Do not copy whole stream. - Poco::StreamCopier::copyStream(response_body_stream, response->GetResponseBody()); + response->GetResponseStream().SetUnderlyingStream(std::make_shared(session, response_body_stream)); break; } diff --git a/src/IO/S3/PocoHTTPClientFactory.cpp b/src/IO/S3/PocoHTTPClientFactory.cpp index 033ad4af37c..e4b86593ec1 100644 --- a/src/IO/S3/PocoHTTPClientFactory.cpp +++ b/src/IO/S3/PocoHTTPClientFactory.cpp @@ -21,10 +21,12 @@ std::shared_ptr PocoHTTPClientFactory::CreateHttpRequest } std::shared_ptr PocoHTTPClientFactory::CreateHttpRequest( - const Aws::Http::URI & uri, Aws::Http::HttpMethod method, const Aws::IOStreamFactory & streamFactory) const + const Aws::Http::URI & uri, Aws::Http::HttpMethod method, const Aws::IOStreamFactory &) const { auto request = Aws::MakeShared("PocoHTTPClientFactory", uri, method); - request->SetResponseStreamFactory(streamFactory); + + /// Don't create default response stream. Actual response stream will be set later in PocoHTTPClient. + request->SetResponseStreamFactory(null_factory); return request; } diff --git a/src/IO/S3/PocoHTTPClientFactory.h b/src/IO/S3/PocoHTTPClientFactory.h index ac586289113..4e555f05502 100644 --- a/src/IO/S3/PocoHTTPClientFactory.h +++ b/src/IO/S3/PocoHTTPClientFactory.h @@ -4,22 +4,25 @@ namespace Aws::Http { - class HttpClient; - class HttpRequest; +class HttpClient; +class HttpRequest; } namespace DB::S3 { - class PocoHTTPClientFactory : public Aws::Http::HttpClientFactory { public: ~PocoHTTPClientFactory() override = default; - [[nodiscard]] std::shared_ptr CreateHttpClient(const Aws::Client::ClientConfiguration & clientConfiguration) const override; + [[nodiscard]] std::shared_ptr + CreateHttpClient(const Aws::Client::ClientConfiguration & clientConfiguration) const override; [[nodiscard]] std::shared_ptr CreateHttpRequest(const Aws::String & uri, Aws::Http::HttpMethod method, const Aws::IOStreamFactory & streamFactory) const override; [[nodiscard]] std::shared_ptr CreateHttpRequest(const Aws::Http::URI & uri, Aws::Http::HttpMethod method, const Aws::IOStreamFactory & streamFactory) const override; + +private: + const Aws::IOStreamFactory null_factory = []() { return nullptr; }; }; } diff --git a/src/IO/S3/PocoHTTPResponseStream.cpp b/src/IO/S3/PocoHTTPResponseStream.cpp new file mode 100644 index 00000000000..0a198268f2e --- /dev/null +++ b/src/IO/S3/PocoHTTPResponseStream.cpp @@ -0,0 +1,12 @@ +#include "PocoHTTPResponseStream.h" + +#include + +namespace DB::S3 +{ +PocoHTTPResponseStream::PocoHTTPResponseStream(std::shared_ptr session_, std::istream & response_stream_) + : Aws::IStream(response_stream_.rdbuf()), session(std::move(session_)) +{ +} + +} diff --git a/src/IO/S3/PocoHTTPResponseStream.h b/src/IO/S3/PocoHTTPResponseStream.h new file mode 100644 index 00000000000..8167ddc4346 --- /dev/null +++ b/src/IO/S3/PocoHTTPResponseStream.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +namespace DB::S3 +{ +/** + * Wrapper of IStream to store response stream and corresponding HTTP session. + */ +class PocoHTTPResponseStream : public Aws::IStream +{ +public: + PocoHTTPResponseStream(std::shared_ptr session_, std::istream & response_stream_); + +private: + /// Poco HTTP session is holder of response stream. + std::shared_ptr session; +}; + +} From c95d5786fdda6cb8057e2eff885b750f07839315 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Jun 2020 01:52:25 +0300 Subject: [PATCH 25/29] Fixup --- tests/performance/generate_table_function.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/generate_table_function.xml b/tests/performance/generate_table_function.xml index 74d4ad4d014..f09753c963a 100644 --- a/tests/performance/generate_table_function.xml +++ b/tests/performance/generate_table_function.xml @@ -3,7 +3,7 @@ SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 0, 10, 10) LIMIT 10000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 0, 10, 10) LIMIT 10000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 0, 10, 10) LIMIT 10000000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200))', 0, 10, 10) LIMIT 10000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 0, 10, 10) LIMIT 10000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 0, 10, 10) LIMIT 10000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('f32 Float32, f64 Float64', 0, 10, 10) LIMIT 10000000); From 898d83491dcf3592ceba4bf66eacf975588f26a6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 10 Jun 2020 02:37:20 +0300 Subject: [PATCH 26/29] fixup --- tests/performance/set_hits.xml | 12 ++++++------ tests/performance/set_index.xml | 4 +--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/performance/set_hits.xml b/tests/performance/set_hits.xml index 8b9ae1da83b..f788bb43196 100644 --- a/tests/performance/set_hits.xml +++ b/tests/performance/set_hits.xml @@ -1,16 +1,16 @@ - + hits_10m_single hits_100m_single - - - SELECT count() FROM hits_100m_single WHERE UserID IN (SELECT UserID FROM hits_100m_single WHERE AdvEngineID != 0) + + SELECT count() FROM hits_100m_single WHERE UserID IN (SELECT UserID FROM hits_100m_single WHERE AdvEngineID != 0) SETTINGS max_threads = 1 SELECT count() FROM hits_10m_single WHERE UserID IN (SELECT UserID FROM hits_10m_single) SELECT count() FROM hits_10m_single WHERE SearchPhrase IN (SELECT SearchPhrase FROM hits_10m_single) - SELECT count() FROM hits_10m_single WHERE URL IN (SELECT URL FROM hits_10m_single WHERE AdvEngineID != 0) + SELECT count() FROM hits_100m_single WHERE URL IN (SELECT URL FROM hits_100m_single WHERE AdvEngineID != 0) SELECT count() FROM hits_10m_single WHERE URL IN (SELECT URL FROM hits_10m_single WHERE SearchEngineID != 0) - SELECT count() FROM hits_10m_single WHERE RegionID IN (SELECT RegionID FROM hits_10m_single) + SELECT count() FROM hits_100m_single WHERE RegionID IN (SELECT RegionID FROM hits_100m_single) diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index e85ded1d8fa..021f6dc60f3 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -1,9 +1,7 @@ - CREATE TABLE test_in (`a` UInt32) ENGINE = MergeTree() ORDER BY a INSERT INTO test_in SELECT number FROM numbers(500000000) - SELECT count() FROM test_in WHERE a IN (SELECT rand(1) FROM numbers(100000)) SETTINGS max_rows_to_read = 1, read_overflow_mode = 'break' @@ -20,7 +18,7 @@ comparison), because it tests that the big set is parsed fast enough, not the lookups we do in this set. --> - SELECT count() FROM numbers(100) WHERE toString(number) IN ('41577', '83972', '51697', '50014', '37553', '93459', '87438', '95971', '83186', '74326', '67871', '50406', '83678', '29655', '18580', '83905', '61518', '29059', '56700', '82787', '98672', '30884', '81822', '39850', '80852', '57627', '91346', '64522', '17781', '49467', '41099', '41929', '85618', '91389', '68564', '91769', '81219', '52218', '37220', '97097', '2129', '9886', '52049', '34847', '25364', '36429', '76897', '71868', '58121', '71199', '84819', '69991', '34046', '64507', '34892', '24228', '36986', '28588', '51159', '53444', '80531', '9941', '20256', '48103', '32565', '62890', '5379', '60302', '46434', '3205', '18821', '31030', '19794', '71557', '71703', '15024', '14004', '82164', '95659', '40227', '83358', '24395', '9610', '19814', '48491', '66412', '16012', '71586', '42143', '51103', '24463', '89949', '35694', '39193', '63904', '40489', '77144', '94014', '84836', '9980', '46554', '43905', '25588', '25205', '72624', '10249', '35888', '98478', '99030', '26834', '31', '81499', '14847', '82997', '92357', '92893', '17426', '56630', '22252', '68119', '62710', '8740', '82144', '79916', '23391', '30192', '99271', '96435', '44237', '98327', '69481', '16691', '13643', '84554', '38571', '70926', '99283', '79000', '20926', '86495', '4834', '1222', '39486', '57697', '58002', '40790', '15623', '3999', '31515', '12694', '26143', '35951', '54085', '97534', '35329', '73535', '88715', '29572', '75799', '45166', '32066', '48023', '69523', '93150', '8740', '96790', '15534', '63252', '5142', '67045', '93992', '16663', '292', '63924', '6588', '12190', '31506', '69590', '35394', '55168', '65223', '79183', '32600', '69676', '28316', '72111', '53531', '15073', '41127', '73451', '24725', '61647', '65315', '41143', '26493', '95608', '34407', '76098', '53105', '83691', '48755', '35696', '62587', '81826', '3963', '45766', '82751', '12430', '97685', '29919', '78155', '71636', '50215', '89734', '9892', '47151', '54855', '3428', '9712', '52592', '2403', '79602', '81243', '79859', '57361', '82000', '42107', '28860', '99591', '28296', '57337', '64969', '32332', '25535', '30924', '21313', '32554', '17342', '87311', '19825', '24898', '61323', '83209', '79322', '79009', '50746', '33396', '62033', '16548', '17427', '24073', '34640', '52368', '4724', '80408', '40', '33787', '16666', '19665', '86751', '27264', '2241', '88134', '53566', '10589', '79711', '92823', '58972', '91767', '60885', '51659', '7867', '96849', '30360', '20914', '9584', '1250', '22871', '23282', '99312', '4683', '33429', '68361', '82614', '81440', '47863', '69790', '11968', '75210', '66854', '37002', '61142', '71514', '1588', '42336', '11069', '26291', '2261', '71056', '13492', '9133', '91216', '72207', '71586', '86535', '83898', '24392', '45384', '48545', '61972', '503', '80180', '35834', '97025', '70411', '55039', '35430', '27631', '82533', '96831', '74077', '42533', '14451', '26943', '53783', '69489', '71969', '8432', '37230', '61348', '19472', '59115', '9886', '50951', '57109', '7141', '1902', '84130', '4323', '55889', '47784', '2220', '75988', '66988', '63721', '8131', '95601', '95207', '2311', '26541', '50991', '6717', '2969', '71857', '51034', '65958', '94716', '90275', '21012', '46859', '7984', '31131', '46457', '69578', '44540', '7294', '80117', '9925', '60155', '90608', '82684', '32193', '87071', '28006', '87604', '24501', '79087', '2848', '29237', '11221', '81319', '40966', '87641', '35325', '78705', '88636', '78717', '62831', '56390', '99271', '43821', '14453', '17923', '62695', '77322', '21038', '67677', '41271', '4376', '65426', '46091', '19887', '97251', '55583', '58763', '3826', '35037', '73533', '64267', '82319', '9836', '42622', '96829', '16363', '10455', '49290', '99992', '98229', '66356', '59087', '73998', '25986', '4279', '56790', '69540', '588', '36620', '60358', '45056', '89297', '42740', '8323', '19245', '82417', '41431', '699', '11554', '73910', '44491', '56019', '68901', '45816', '68126', '89379', '23885', '13263', '56395', '73130', '19089', '23771', '10335', '48547', '16903', '6453', '33560', '89668', '38159', '43177', '90655', '49712', '62', '66920', '34180', '12150', '48564', '39538', '85026', '87195', '14928', '8956', '71157', '53287', '39161', '67583', '83309', '92054', '86977', '56188', '15229', '88170', '60894', '58497', '89254', '40082', '86890', '60161', '97291', '45878', '23368', '14577', '92870', '37017', '97356', '99426', '76061', '89186', '99751', '85153', '61580', '39360', '90107', '25603', '26798', '76224', '6469', '7912', '69838', '16404', '67497', '28965', '80836', '80365', '91249', '48713', '17113', '33090', '40793', '70450', '66689', '83698', '17802', '43869', '13355', '18959', '79411', '87930', '9265', '37504', '44876', '97234', '94149', '35040', '22049', '49248', '6535', '36080', '28346', '94437', '78319', '17961', '89056', '56161', '35810', '41632', '45494', '53351', '89729', '99510', '51584', '59688', '6193', '70809', '51093', '92589', '90247', '34910', '78235', '17362', '49423', '63324', '525', '37638', '72325', '89356', '15298', '59116', '17848', '65429', '27029', '84781', '70247', '8825', '35082', '70451', '22522', '58125', '91879', '90531', '2478', '463', '37902', '54405', '87267', '72688', '22803', '33134', '35177', '84551', '44974', '88375', '76407', '27774', '33849', '19915', '82014', '80434', '26380', '48777', '53811', '14838', '26829', '56441', '99869', '49574', '85476', '19723', '16907', '4018', '37338', '78510', '47912', '13030', '65277', '95716', '67363', '21393', '89887', '78842', '81650', '903', '17436', '30704', '49223', '27198', '25500', '52214', '54258', '70082', '53950', '49312', '43615', '99473', '94348', '53661', '96213', '96346', '62010', '38268', '32861', '75660', '10392', '89491', '68335', '29817', '88706', '24184', '36298', '43440', '21626', '26535', '44560', '46363', '12534', '99070', '95606', '33714', '73070', '8303', '29853', '23014', '99982', '4530', '14955', '45803', '50', '90750', '30394', '81276', '95563', '47314', '58520', '91299', '88944', '54402', '67405', '29253', '47079', '71734', '99728', '17652', '13307', '35556', '18962', '26780', '17771', '53712', '60055', '37628', '35830', '90739', '61151', '41309', '27652', '3051', '53167', '98417', '19382', '36833', '75085', '65374', '87732', '30352', '31776', '32765', '97565', '92199', '49050', '29503', '51024', '18834', '8515', '24069', '96216', '10777', '90680', '18974', '68884', '85305', '36007', '56707', '4212', '47352', '34426', '13185', '92939', '95782', '70577', '58080', '98279', '3906', '5065', '56896', '16382', '31273', '17117', '98602', '12786', '24086', '63970', '72756', '35798', '82367', '7356', '53398', '68503', '2962', '16425', '67334', '68461', '65439', '15620', '70906', '29649', '46461', '74602', '38012', '71714', '16825', '89480', '53386', '88532', '35104', '28556', '82120', '23155', '23347', '24797', '60061', '54962', '99427', '82248', '82447', '39968', '63727', '27431', '81511', '91168', '71425', '80740', '84127', '40717', '15503', '15419', '46594', '61263', '19212', '53175', '70724', '74445', '23034', '71818', '40246', '18886', '53066', '4880', '83701', '86107', '87862', '44751', '392', '73440', '90291', '93395', '20894', '38463', '32664', '55158', '20090', '50004', '79070', '98471', '85478', '96615', '68149', '78334', '97752', '73207', '71678', '91238', '96757', '82598', '194', '35797', '45120', '60782', '28721', '17676', '78066', '60957', '11826', '51563', '50516', '16485', '47053', '31738', '48923', '23554', '96850', '42033', '73701', '78607', '45979', '54571', '12415', '31693', '15356', '36902', '9126', '3767', '3295', '90402', '24005', '95350', '67033', '49137', '72606', '51899', '17522', '31957', '44641', '53982', '23767', '68257', '15766', '19995', '2107', '48788', '11765', '91055', '46576', '54651', '50381', '62827', '73636', '46606', '98753', '37631', '70441', '87916', '66983', '33870', '31125', '12904', '57040', '4874', '58632', '42037', '18782', '5998', '18974', '57949', '81010', '90407', '99874', '20462', '89949', '10952', '71454', '95130', '46115', '3518', '13384', '69039', '79482', '22076', '59782', '32042', '40930', '60243', '29298', '6790', '46985', '44398', '85631', '14380', '66179', '2629', '32126', '49833', '14118', '58492', '31493', '81172', '96638', '8745', '89663', '76842', '78633', '41373', '83721', '42886', '11123', '32739', '11051', '1303', '92314', '83324', '85600', '44276', '69064', '56125', '84650', '31028', '12628', '14502', '64764', '39405', '44855', '79046', '51716', '46824', '83389', '1941', '1257', '9280', '73176', '84729', '2579', '63366', '22606', '35541', '51096', '13447', '18355', '68037', '28436', '94116', '81070', '78355', '67897', '5296', '32742', '77645', '91853', '18767', '67949', '40963', '5792', '17278', '25597', '41884', '80829', '7099', '18645', '60295', '12082', '81800', '78415', '18082', '38789', '16295', '72377', '74949', '55583', '66853', '15402', '72977', '15123', '99434', '34999', '21687', '76049', '42987', '83748', '88256', '66688', '21766', '20304', '29271', '10069', '19822', '11792', '42526', '74143', '17289', '30253', '6367', '20888', '12975', '94073', '98639', '30134', '26320', '65507', '69002', '53120', '4550', '38893', '18954', '38283', '54863', '17698', '99670', '10521', '92467', '60994', '18052', '48673', '35811', '87282', '62706', '16061', '53112', '22652', '37780', '55662', '26331', '49410', '79074', '10623', '69577', '79613', '9491', '31229', '43922', '84231', '58409', '36386', '46875', '74431', '76735', '38776', '23350', '7314', '9079', '51519', '98544', '70216', '63380', '90381', '1295', '46901', '58225', '55339', '89918', '75522', '35431', '89460', '49552', '89302', '23068', '28493', '3042', '25194', '59520', '9810', '95706', '81297', '89638', '54794', '94527', '45262', '97932', '78685', '6947', '22818', '48700', '9153', '12289', '22011', '58825', '93854', '65438', '4509', '33741', '28208', '69061', '48578', '40247', '77725', '31837', '39003', '69363', '78113', '76398', '97262', '67795', + SELECT toString(rand()) IN ('41577', '83972', '51697', '50014', '37553', '93459', '87438', '95971', '83186', '74326', '67871', '50406', '83678', '29655', '18580', '83905', '61518', '29059', '56700', '82787', '98672', '30884', '81822', '39850', '80852', '57627', '91346', '64522', '17781', '49467', '41099', '41929', '85618', '91389', '68564', '91769', '81219', '52218', '37220', '97097', '2129', '9886', '52049', '34847', '25364', '36429', '76897', '71868', '58121', '71199', '84819', '69991', '34046', '64507', '34892', '24228', '36986', '28588', '51159', '53444', '80531', '9941', '20256', '48103', '32565', '62890', '5379', '60302', '46434', '3205', '18821', '31030', '19794', '71557', '71703', '15024', '14004', '82164', '95659', '40227', '83358', '24395', '9610', '19814', '48491', '66412', '16012', '71586', '42143', '51103', '24463', '89949', '35694', '39193', '63904', '40489', '77144', '94014', '84836', '9980', '46554', '43905', '25588', '25205', '72624', '10249', '35888', '98478', '99030', '26834', '31', '81499', '14847', '82997', '92357', '92893', '17426', '56630', '22252', '68119', '62710', '8740', '82144', '79916', '23391', '30192', '99271', '96435', '44237', '98327', '69481', '16691', '13643', '84554', '38571', '70926', '99283', '79000', '20926', '86495', '4834', '1222', '39486', '57697', '58002', '40790', '15623', '3999', '31515', '12694', '26143', '35951', '54085', '97534', '35329', '73535', '88715', '29572', '75799', '45166', '32066', '48023', '69523', '93150', '8740', '96790', '15534', '63252', '5142', '67045', '93992', '16663', '292', '63924', '6588', '12190', '31506', '69590', '35394', '55168', '65223', '79183', '32600', '69676', '28316', '72111', '53531', '15073', '41127', '73451', '24725', '61647', '65315', '41143', '26493', '95608', '34407', '76098', '53105', '83691', '48755', '35696', '62587', '81826', '3963', '45766', '82751', '12430', '97685', '29919', '78155', '71636', '50215', '89734', '9892', '47151', '54855', '3428', '9712', '52592', '2403', '79602', '81243', '79859', '57361', '82000', '42107', '28860', '99591', '28296', '57337', '64969', '32332', '25535', '30924', '21313', '32554', '17342', '87311', '19825', '24898', '61323', '83209', '79322', '79009', '50746', '33396', '62033', '16548', '17427', '24073', '34640', '52368', '4724', '80408', '40', '33787', '16666', '19665', '86751', '27264', '2241', '88134', '53566', '10589', '79711', '92823', '58972', '91767', '60885', '51659', '7867', '96849', '30360', '20914', '9584', '1250', '22871', '23282', '99312', '4683', '33429', '68361', '82614', '81440', '47863', '69790', '11968', '75210', '66854', '37002', '61142', '71514', '1588', '42336', '11069', '26291', '2261', '71056', '13492', '9133', '91216', '72207', '71586', '86535', '83898', '24392', '45384', '48545', '61972', '503', '80180', '35834', '97025', '70411', '55039', '35430', '27631', '82533', '96831', '74077', '42533', '14451', '26943', '53783', '69489', '71969', '8432', '37230', '61348', '19472', '59115', '9886', '50951', '57109', '7141', '1902', '84130', '4323', '55889', '47784', '2220', '75988', '66988', '63721', '8131', '95601', '95207', '2311', '26541', '50991', '6717', '2969', '71857', '51034', '65958', '94716', '90275', '21012', '46859', '7984', '31131', '46457', '69578', '44540', '7294', '80117', '9925', '60155', '90608', '82684', '32193', '87071', '28006', '87604', '24501', '79087', '2848', '29237', '11221', '81319', '40966', '87641', '35325', '78705', '88636', '78717', '62831', '56390', '99271', '43821', '14453', '17923', '62695', '77322', '21038', '67677', '41271', '4376', '65426', '46091', '19887', '97251', '55583', '58763', '3826', '35037', '73533', '64267', '82319', '9836', '42622', '96829', '16363', '10455', '49290', '99992', '98229', '66356', '59087', '73998', '25986', '4279', '56790', '69540', '588', '36620', '60358', '45056', '89297', '42740', '8323', '19245', '82417', '41431', '699', '11554', '73910', '44491', '56019', '68901', '45816', '68126', '89379', '23885', '13263', '56395', '73130', '19089', '23771', '10335', '48547', '16903', '6453', '33560', '89668', '38159', '43177', '90655', '49712', '62', '66920', '34180', '12150', '48564', '39538', '85026', '87195', '14928', '8956', '71157', '53287', '39161', '67583', '83309', '92054', '86977', '56188', '15229', '88170', '60894', '58497', '89254', '40082', '86890', '60161', '97291', '45878', '23368', '14577', '92870', '37017', '97356', '99426', '76061', '89186', '99751', '85153', '61580', '39360', '90107', '25603', '26798', '76224', '6469', '7912', '69838', '16404', '67497', '28965', '80836', '80365', '91249', '48713', '17113', '33090', '40793', '70450', '66689', '83698', '17802', '43869', '13355', '18959', '79411', '87930', '9265', '37504', '44876', '97234', '94149', '35040', '22049', '49248', '6535', '36080', '28346', '94437', '78319', '17961', '89056', '56161', '35810', '41632', '45494', '53351', '89729', '99510', '51584', '59688', '6193', '70809', '51093', '92589', '90247', '34910', '78235', '17362', '49423', '63324', '525', '37638', '72325', '89356', '15298', '59116', '17848', '65429', '27029', '84781', '70247', '8825', '35082', '70451', '22522', '58125', '91879', '90531', '2478', '463', '37902', '54405', '87267', '72688', '22803', '33134', '35177', '84551', '44974', '88375', '76407', '27774', '33849', '19915', '82014', '80434', '26380', '48777', '53811', '14838', '26829', '56441', '99869', '49574', '85476', '19723', '16907', '4018', '37338', '78510', '47912', '13030', '65277', '95716', '67363', '21393', '89887', '78842', '81650', '903', '17436', '30704', '49223', '27198', '25500', '52214', '54258', '70082', '53950', '49312', '43615', '99473', '94348', '53661', '96213', '96346', '62010', '38268', '32861', '75660', '10392', '89491', '68335', '29817', '88706', '24184', '36298', '43440', '21626', '26535', '44560', '46363', '12534', '99070', '95606', '33714', '73070', '8303', '29853', '23014', '99982', '4530', '14955', '45803', '50', '90750', '30394', '81276', '95563', '47314', '58520', '91299', '88944', '54402', '67405', '29253', '47079', '71734', '99728', '17652', '13307', '35556', '18962', '26780', '17771', '53712', '60055', '37628', '35830', '90739', '61151', '41309', '27652', '3051', '53167', '98417', '19382', '36833', '75085', '65374', '87732', '30352', '31776', '32765', '97565', '92199', '49050', '29503', '51024', '18834', '8515', '24069', '96216', '10777', '90680', '18974', '68884', '85305', '36007', '56707', '4212', '47352', '34426', '13185', '92939', '95782', '70577', '58080', '98279', '3906', '5065', '56896', '16382', '31273', '17117', '98602', '12786', '24086', '63970', '72756', '35798', '82367', '7356', '53398', '68503', '2962', '16425', '67334', '68461', '65439', '15620', '70906', '29649', '46461', '74602', '38012', '71714', '16825', '89480', '53386', '88532', '35104', '28556', '82120', '23155', '23347', '24797', '60061', '54962', '99427', '82248', '82447', '39968', '63727', '27431', '81511', '91168', '71425', '80740', '84127', '40717', '15503', '15419', '46594', '61263', '19212', '53175', '70724', '74445', '23034', '71818', '40246', '18886', '53066', '4880', '83701', '86107', '87862', '44751', '392', '73440', '90291', '93395', '20894', '38463', '32664', '55158', '20090', '50004', '79070', '98471', '85478', '96615', '68149', '78334', '97752', '73207', '71678', '91238', '96757', '82598', '194', '35797', '45120', '60782', '28721', '17676', '78066', '60957', '11826', '51563', '50516', '16485', '47053', '31738', '48923', '23554', '96850', '42033', '73701', '78607', '45979', '54571', '12415', '31693', '15356', '36902', '9126', '3767', '3295', '90402', '24005', '95350', '67033', '49137', '72606', '51899', '17522', '31957', '44641', '53982', '23767', '68257', '15766', '19995', '2107', '48788', '11765', '91055', '46576', '54651', '50381', '62827', '73636', '46606', '98753', '37631', '70441', '87916', '66983', '33870', '31125', '12904', '57040', '4874', '58632', '42037', '18782', '5998', '18974', '57949', '81010', '90407', '99874', '20462', '89949', '10952', '71454', '95130', '46115', '3518', '13384', '69039', '79482', '22076', '59782', '32042', '40930', '60243', '29298', '6790', '46985', '44398', '85631', '14380', '66179', '2629', '32126', '49833', '14118', '58492', '31493', '81172', '96638', '8745', '89663', '76842', '78633', '41373', '83721', '42886', '11123', '32739', '11051', '1303', '92314', '83324', '85600', '44276', '69064', '56125', '84650', '31028', '12628', '14502', '64764', '39405', '44855', '79046', '51716', '46824', '83389', '1941', '1257', '9280', '73176', '84729', '2579', '63366', '22606', '35541', '51096', '13447', '18355', '68037', '28436', '94116', '81070', '78355', '67897', '5296', '32742', '77645', '91853', '18767', '67949', '40963', '5792', '17278', '25597', '41884', '80829', '7099', '18645', '60295', '12082', '81800', '78415', '18082', '38789', '16295', '72377', '74949', '55583', '66853', '15402', '72977', '15123', '99434', '34999', '21687', '76049', '42987', '83748', '88256', '66688', '21766', '20304', '29271', '10069', '19822', '11792', '42526', '74143', '17289', '30253', '6367', '20888', '12975', '94073', '98639', '30134', '26320', '65507', '69002', '53120', '4550', '38893', '18954', '38283', '54863', '17698', '99670', '10521', '92467', '60994', '18052', '48673', '35811', '87282', '62706', '16061', '53112', '22652', '37780', '55662', '26331', '49410', '79074', '10623', '69577', '79613', '9491', '31229', '43922', '84231', '58409', '36386', '46875', '74431', '76735', '38776', '23350', '7314', '9079', '51519', '98544', '70216', '63380', '90381', '1295', '46901', '58225', '55339', '89918', '75522', '35431', '89460', '49552', '89302', '23068', '28493', '3042', '25194', '59520', '9810', '95706', '81297', '89638', '54794', '94527', '45262', '97932', '78685', '6947', '22818', '48700', '9153', '12289', '22011', '58825', '93854', '65438', '4509', '33741', '28208', '69061', '48578', '40247', '77725', '31837', '39003', '69363', '78113', '76398', '97262', '67795', '68446', '58896', '60969', '19849', '6722', '91854', '49519', '13949', '67109', '48824', '31723', '75554', '69575', '94986', '75350', '18628', '15284', '41943', '15433', '52607', '41', '22340', '29528', '24059', '34145', '72517', '46316', '10667', '54510', '19882', '47764', '69124', '41963', '84350', '48420', '4646', '24958', '69020', '97121', '26178', '62664', '50622', '32554', '49655', '45398', '11267', '72222', '73363', '89554', '89046', '57384', '29259', '37888', '24850', '74353', '57343', '34762', '2900', '11393', '42154', '94306', '70552', '75265', '921', '26003', '64352', '89857', '83171', '58249', '48940', '53512', '66335', '44865', '68729', '19299', '58003', '39854', '99122', '3860', '80173', '52242', '90966', '53183', '71982', '82325', '87842', '15000', '55627', '71132', '6354', '42402', '91719', '91644', '94533', '74925', '66278', '66911', '85576', '40495', '70919', '71797', '87835', '29845', '71832', '3390', '7994', '33499', '70715', '54897', '82710', '63077', '78105', '24758', '89585', '84607', '46477', '78618', '10989', '39222', '98749', '51685', '94664', '31008', '32823', '89521', '72160', '26952', '4001', '21421', '5737', '74027', '88179', '45553', '83743', '19913', '49435', '65616', '82641', '5149', '76959', '40681', '73023', '2670', '30845', '18863', '35094', '88400', '80963', '9154', '16571', '64192', '59694', '41317', '59942', '58856', '99281', '67260', '66971', '22716', '76089', '58047', '67071', '53707', '462', '52518', '72277', '10681', '69', '98855', '12593', '88842', '67242', '73543', '37439', '18413', '67211', '93495', '45576', '70614', '27988', '53210', '18618', '21318', '68059', '25518', '55917', '56522', '16548', '2404', '93538', '61452', '66358', '3709', '23914', '92426', '81439', '38070', '28988', '29939', '2948', '85720', '45628', '51101', '89431', '86365', '17571', '50987', '83849', '11015', '83812', '66187', '26362', '66786', '22024', '93866', '36161', '90080', '64874', '37294', '83860', '73821', '80279', '36766', '73117', '44620', '84556', '42070', '90383', '27862', '20665', '67576', '34997', '57958', '80638', '84351', '63961', '1362', '14338', '80377', '24192', '41294', '57368', '51189', '27287', '45764', '86289', '65600', '708', '84090', '96005', '55676', '84855', '72385', '70018', '9336', '82701', '3710', '52083', '74045', '96454', '30956', '67369', '78941', '81810', '71906', '23194', '33042', '50794', '61256', '24449', '48639', '22916', '78303', '13666', '40762', '43942', '51075', '89783', '95786', '90462', '6181', '36482', '40675', '4970', '6388', '91849', '72579', '94983', '86084', '20140', '68427', '48123', '43122', '98066', '37560', '6927', '72803', '5546', '62259', '98439', '6457', '98568', '70499', '33022', '28226', '29675', '20917', '75365', '20900', '8190', '56736', '99153', '77779', '49333', '50293', '97650', '4067', '47278', '42761', '71875', '13966', '11223', '46783', '18059', '61355', '29638', '75681', '24466', '89634', '20759', '83252', '37780', '15931', '74893', '6703', '64524', '80656', '85990', '78427', '18411', '20696', '86432', '93176', '69889', '15072', '15180', '9935', '10467', '60248', '42430', '62590', '89596', '27743', '26398', '79912', '60048', '50943', '38870', '69383', '72261', '98059', '55242', '74905', '5667', '54321', '70415', '39903', '49711', '85318', '79979', '59262', '82321', '15263', '17416', '74554', '94733', '72112', '49872', '54849', '73883', '78250', '74935', '68559', '57564', '50541', '45730', '41595', '5588', '83723', '42891', '11898', '14348', '99732', '14481', '85233', '21277', '94508', '52551', '74187', '7634', '42912', '25100', '43536', '35798', '48190', '86477', '22680', '48148', '59501', '56563', '16802', '81496', '97568', '68657', '51462', '67953', '99660', '39002', '54170', '57190', '68086', '52700', '6487', '55709', '70418', '62629', '70420', '35695', '36152', '45360', '53503', '46623', '76000', '50648', '97876', '44815', '29163', '1356', '64123', '71388', '17658', '99084', '58727', '59437', '38773', '71254', '81286', '97545', '18786', '56834', '20346', '36401', '62316', '58082', '67959', '99876', '69895', '80099', '62747', '20517', '99777', '6472', '49189', '31321', '39992', '68073', '13378', '51806', '21776', '52060', '96983', '25754', '93709', '96627', '8644', '93726', '14002', '37716', '87620', '34507', '76339', '24491', '5849', '44110', '522', '66521', '12776', '44887', '80535', '14548', '75248', '671', '73071', '35715', '59474', '7061', '82243', '56170', '20179', '59717', '1725', '24634', '11270', '77023', '63840', '46608', '44667', '22422', '59771', '94768', '73033', '82905', '16463', '40971', '22204', '58366', '28721', '14907', '76468', '81872', '38418', '36989', '61439', '10610', '131', '44296', '35453', '10117', '75856', '94603', '99602', '68075', '35949', '13599', '50030', '69633', '55956', '85465', '16429', '86081', '11145', '6195', '82207', '90598', '92814', '23725', '83204', '80346', '71542', '46634', '15820', '54123', '45397', '15322', '61743', '9273', '71347', '6835', '64006', '91718', '43677', '32923', '21486', '17098', '61694', '43347', '40019', '4071', '52443', '42386', '56839', '83514', '27633', '40780', '51749', '92101', '62384', '92206', '56044', '66174', '11137', '73966', '78471', '30468', '31643', '33197', '6888', '8066', '86603', '74383', '6098', '54411', '98819', '89862', '88639', '94422', '89371', '80526', '91747', '91220', '64944', '76658', '42046', '58518', '27249', '6646', '3028', '1346', '33763', '9734', '31737', '65527', '5892', '60813', '3410', '35464', '43009', '98382', '70580', '93898', '56404', '32995', '62771', '71556', '40538', '55612', '45656', '10758', '20268', '33603', '38310', '14242', '74397', '10722', '71575', '22590', '49043', '91439', '9055', '23668', '9101', '5268', '64133', '77501', '64684', '11337', '47575', '50732', '88680', '93730', '46785', '17589', '3520', '57595', '71241', '34994', '8753', '36147', '88844', '41914', '11250', '94632', '71927', '4624', '86279', '7664', '2659', '94853', '65386', '30438', '86005', '92883', '84629', '59910', '44484', '1306', '8404', '56962', '29990', '38445', '96191', '73013', '66590', '40951', '24712', '18825', '37268', '87843', '18972', '12154', '7779', '52149', '76152', '65799', '86011', '35475', '78083', '88232', '91551', '65532', '93516', '73827', '24227', '44687', '55759', '83819', '45088', '10856', '60488', '39051', '14103', '76650', '81181', '46731', '737', '58788', '78945', '42096', '66731', '66740', '72273', '88969', '5655', '86590', '41096', '80038', '32430', '51877', '23970', '91900', '13082', '45880', '94367', '19739', '61998', '71665', '16083', '57035', '26916', '10166', '18834', '46798', '66881', '28444', '68840', '10459', '81087', '4728', '76224', '39257', '23470', '93524', '37345', '30074', '49856', '22022', '55279', '5159', '5193', '58030', '57539', '12514', '49759', '96222', '52597', '67192', '88187', '53614', '16084', '79915', '28212', '79334', '85283', '32306', '31058', '43113', '74707', '74869', '2213', '32134', '6379', '85426', '87098', '35984', '51105', '69287', '16803', '83337', '14913', '62531', '58098', '7914', '20105', '28850', '1384', '43173', '62983', '87113', '76066', '86320', '77684', '45191', '95225', '41503', '36713', '48404', '91228', '53865', '98981', '59161', '61237', '84561', '17455', '14379', '57789', '80895', '99260', '84595', '72942', '53220', '84448', '81332', '49437', '83086', '93414', '54519', '52288', '74772', '22460', '49324', '11168', '96071', '61985', '38284', '6405', '54698', '71727', '60093', '37340', '87884', '83403', '4542', '94949', '19636', '15855', '39105', '10424', '67418', '91022', '69254', '8481', '38411', '3832', '44354', '93548', '57172', '28481', '372', '81497', '52179', '41060', '72141', '41396', '65590', '70432', '82819', '93814', '26118', '84780', '88485', '70821', '8222', '83000', '47067', '38516', '33347', '47681', '48202', '60749', '52112', '7937', '28105', '11394', '45746', '43252', '34494', '2979', '69715', '42486', '82315', '71760', '97413', '66137', '94487', '7429', '74434', '22964', '55251', '3448', '53534', '2574', '9693', '96157', '2955', '4348', '19566', '56930', '83319', '31310', '53905', '1148', '41726', '22233', '76045', '37351', '10545', '17581', '28047', '30199', '4741', '58111', '33497', '67796', '67730', '31247', '43772', '29461', '45970', '73353', '22534', '53962', '32147', '71392', '62579', '66345', '58246', '33442', '9581', '29705', '14058', '86471', '76125', '59363', '94982', '74810', '89149', '20066', '3366', '3568', '25752', '80036', '64119', '27270', '40061', '91052', '69022', '9852', '77112', '83075', '43924', '61661', '56133', '96652', '57944', '72576', '82170', '79236', '55745', '15309', '88878', '72761', '37647', '67465', '12777', '97309', '93202', '41470', '8787', '64920', '48514', '18917', '35157', '59151', '4640', '5317', '38134', '76548', '82788', '9214', '58418', '73185', '90554', '10543', '47182', '62936', '91765', '89751', '68931', '48865', '64607', '7150', '77862', '14297', '14828', '33013', '91698', '67593', '98096', '16595', '51639', '86531', '24719', '1703', '78788', '43810', '38918', '95491', '99903', '82671', '8291', '68288', '31224', '39863', '4265', '77798', '7698', '33804', '92286', '4744', '37038', '44203', '98212', '17369', '77442', '62879', '4145', '96881', '15646', '36824', '19959', '45451', '76049', '54272', '97577', '95298', '81115', '30204', '82041', '8037', '10052', '8756', '76833', '82851', '24276', '75574', '36037', '78079', '92807', '29064', '90000', '84150', '17102', '75092', '49424', '35597', '4693', '82853', '42511', '16119', '23478', '65240', '55585', '91762', '71671', '46682', '72479', '97696', '24615', '12579', '30274', '48255', '2336', '90202', '5808', '45426', '76308', '74639', '31245', '99894', '89638', '6233', '33893', '71899', '85273', '89429', '29761', '50231', '57249', '99347', '22642', '66972', '86221', '47514', '88274', '10819', '73150', '53754', '13304', '20478', '38099', '619', '14669', '8011', '97657', '26569', '65430', '13467', '38180', '23675', '72350', '42257', '39875', '23529', '53407', '11833', '29599', '95621', '7727', '59527', '86846', '22860', '5358', '3730', '87555', '362', '95755', '54565', '29935', '68950', '52349', '98344', '86576', '7420', '12236', '15844', '48099', '97535', '97081', '50261', '31187', '60496', '24123', '24042', '6376', '6679', '99806', '20306', '60676', '36881', '77309', '5247', '96569', '53417', '73252', '64179', '35318', '75732', '65119', '32621', '40464', '22887', '96152', '65161', '83381', '8915', '68142', '7328', '85031', '15688', '72519', '93992', '86927', '75538', '38205', '50877', '70039', '97538', '94822', '52131', '49643', '85206', '1347', '14574', '88736', '53442', '49991', '64925', '72283', '82213', '60905', '36118', '62963', '16983', '79185', '15111', '26059', '17792', '98218', '33214', '1094', '41754', '77275', '65173', '13190', '91004', '90422', '44387', '92672', '98641', '54609', '83295', '37395', '70104', '32986', '72524', '82478', '5837', '83916', '52736', '57112', '55985', '42642', '42136', '89642', '35712', '49489', '19726', '65824', '24384', '48112', '15366', '99206', '68384', '51389', '529', '21475', '75749', '95182', '60110', '70571', '74174', '38105', '78107', '4101', '8982', '11215', '23987', '3303', '28706', '54629', '98000', '67510', '30036', '99140', '48896', '40971', '7735', '79984', '50134', '94928', '57023', '52880', '83067', '41940', '62994', '89213', '38593', '19283', '68206', '22234', '19245', '26266', '32403', '65889', '17022', '64280', '42797', '27161', '57675', '42313', '93606', '93082', '20659', '90824', '1226', '66266', '12503', '57104', '15247', '51160', '92398', '71967', '59476', '44465', '35765', '10787', '47737', '45792', '2292', '47599', '89612', '8162', '87622', '69410', '45727', '31158', '99791', '89544', '27214', '99588', '40516', '75616', '36505', '46079', '95448', '97999', '47462', '47799', '82729', '34038', '60789', '96938', '22682', '79062', '93307', '36038', '49016', '90983', '48219', '50889', '32517', '72219', '71229', '82643', '1195', '70543', '17', '22178', '23544', '72371', '1163', '28527', '7336', '39846', '31956', '80963', '41804', '59791', '41831', '1940', '52377', '79494', '12531', '81112', '44320', '18746', '5774', '63869', '4085', '59922', '12751', '99443', '13530', '23872', '36026', '83360', '32711', '92980', '11140', '99323', '57263', '98149', '29265', '25548', '65995', '4818', '15593', '8535', '37863', '12217', '14474', '66584', '89272', '86690', '58777', '39666', '44756', '18442', '52586', '98030', '40850', '38708', '49304', '68923', '65008', '84388', '83639', '29866', '63675', '26793', '49227', '82099', '24090', '57535', '24201', '65776', '74054', '89833', '62979', '26613', '5851', '99766', '63484', '66605', '37179', '90760', '59336', '58390', '93239', '84578', '11396', '93994', '73818', '23972', '37720', '72369', '25063', '32952', '71036', '76612', '31285', '34090', '19136', '53783', '66436', '61478', '96749', '43658', '7399', '31574', '67073', '40480', '20727', '70993', '65549', '30800', '21507', '53785', '89574', '86381', '56492', '62603', '44856', '68687', '63794', '70996', '7475', '84238', '71939', '86886', '94792', '15036', '36936', '95722', '17771', '67850', '33371', '49314', '40744', '5432', '81057', '41201', '75986', '22961', '15323', '1570', '18657', '95219', '19130', '53127', '15867', '81135', '73206', '76668', '36386', '48828', '31417', '56916', '70891', '60534', '95777', '10022', '94053', '2928', '56326', '16559', '79656', '6414', '81247', '78270', '55687', '19151', '61597', '99857', '81142', '27725', '53493', '12185', '1455', '48501', '59425', '20591', '24900', '66079', '84889', '32024', '18919', '2043', '7076', '71201', '88258', '86521', '93348', '26395', '39646', '44145', '33911', '46231', '67054', '39979', '11630', '23020', '76278', '88056', '11480', '4723', '78612', '70211', '60622', '84687', '59092', '65675', '38479', '64399', '64699', '95964', '42764', '69060', '28189', '4193', '95805', '75462', '17245', '59640', '94773', '84292', '53092', '98507', '61353', '32483', '53027', '48912', '87221', '47788', '59263', '65196', '35567', '17494', '64253', '50223', '7057', '87467', '62414', '2523', '50910', '72353', '78986', '78104', '47719', '29108', '12957', '5114', '64435', '66707', '37449', '70399', '45334', '71606', '55338', '55072', '58765', '12151', '22012', '16954', '87366', '14240', '98041', '72296', '47408', '56879', '99584', '63172', '92316', '28071', '29880', '19608', '13839', '87484', '56541', '88662', '87098', '72124', '78282', '27653', '38993', '31870', '67239', '99445', '7376', '78487', '98880', '12180', '86773', '67773', '15416', '58172', '13075', '67559', '97510', '29705', '86985', '57024', '11827', '31236', '91920', '26116', '94614', '14486', '46252', '78847', '43786', '70048', '96739', '35240', '39933', '58209', '27852', '65669', '47323', '58150', '84444', '44344', '95882', '41258', '31314', '69060', '19916', '6979', '19436', '45572', '16259', '74566', '6306', '24705', '53422', '593', '97031', '22308', '26875', '23042', '78035', '34229', '61976', '23175', '50072', '90896', '50810', '71730', '86468', '94807', '8218', '36032', '58628', '60560', '51206', '37943', '27987', '15014', '49905', '70018', '66799', '80851', '23594', '29982', '6438', '97381', '47715', '96294', '17985', '48545', '12672', '5250', '9988', '24601', '3736', '97815', '54363', '64703', '44167', '68376', '16595', '38073', '29630', '59630', '1858', '71823', '75580', '70083', '14493', '93821', '93394', '85369', '3818', '8435', '59988', '43966', '13961', '15855', '83332', '80312', '27299', '88840', '76964', '56173', '62794', '79389', '82642', '85843', '47116', '43064', '16061', '28905', '54415', '72832', '91252', '93488', '79457', '99336', '70744', '80432', '6487', '880', '87701', '154', '86574', '86677', '17892', '81488', '95260', '12515', '43189', '9211', '55403', '41417', '60046', '54785', '83655', '28274', '65745', '63062', '44549', '36391', '48051', '7328', '3572', '33226', '49177', '25123', '59065', '19691', '15109', '10172', '95578', '29497', '48152', '20276', '36270', '78866', '48309', '53209', '55475', '30073', '19717', '16004', '45692', '83430', '9291', '45935', '57030', '92613', '91656', '67697', '34915', '28156', '56594', '3273', '11194', '98270', '34370', '2621', '66679', '97451', '97717', '87923', '48310', '37725', '69743', '75103', '84956', '75163', '16069', '65304', '19397', '18071', '27273', '49823', '57595', '98324', '82174', '10293', '80943', '64184', '19472', '4198', '9410', '25927', '65961', '33155', '95168', '33692', '61712', '69877', '13308', '17415', '10022', '2491', '67310', '96140', '68050', '76272', '17143', '76805', '57176', '7539', '22690', '95483', '87592', '27221', '90821', '51154', '99828', '68998', '54581', '74222', '10269', '65057', '45467', '96089', '55058', '89779', '60837', '74122', '52886', '58055', '14880', '93208', '66652', '68830', '24121', '62407', '87257', '18802', '14925', '45423', '98624', '55195', '59072', '41414', '77840', '66075', '62705', '26549', '19063', '57552', '2507', '52069', '57620', '66688', '14833', '33700', '90666', '98052', '5367', '2268', '43093', '69063', '22030', '85564', '92258', '1847', '24446', '65835', '38660', '91899', '87732', '52396', '31952', '36000', '86944', '16109', '80729', '53757', '60226', '59103', '84187', '36674', '72823', '29884', '4654', '69139', '20440', '57413', '3651', '39639', '44564', '57492', '84159', '751', '99748', '9659', '72661', '39220', '99742', '74734', '75729', '38071', '69934', '73640', '65294', '54524', '64372', '37927', '17187', '7863', '12732', '40296', '36197', '15821', '76831', '4400', '71933', '4040', '22072', '33064', '25702', '13324', '91275', '27388', '97729', '14620', '45989', '80737', '17934', '4219', '3032', '43457', '31051', '24469', '67041', '29328', '75499', '80951', '88212', '92595', '49969', '24612', '58732', '2718', '3805', '50918', '99426', '8614', '35580', '93273', '989', '24385', '41185', '25687', '47146', '25227', '95839', '56355', '98536', '79824', '31725', '46447', '26690', '68418', '47783', '33725', '21729', '70797', '59038', '60376', '25087', '68332', '67950', '12411', '95918', '64736', '65336', '74947', '64605', '4106', '42712', '96640', '28492', '28648', '42429', '821', '24333', '69677', '38959', '23484', '92005', '29352', '29159', '52873', '99947', '21834', '85347', '93479', '28298', '55608', '3226', '69714', '80283', '6577', '18849', '44605', '75286', '28139', '26541', '12867', '57500', '86617', '33005', '57498', '60223', '74954', '51401', '55246', '5648', '16513', '40930', '43821', '32090', '66002', '65530', '76083', '6047', '6879', '94987', '80787', '11688', '77161', '92670', '6696', '400', '28572', '47234', '51375', '88518', '762', '92617', '54260', '7560', '60180', '43331', '64059', '27616', '75839', '21392', '47756', '46254', '19486', '88533', '30130', '93694', '8557', '66534', '94447', '16910', '6480', '77440', '24366', '6195', '48946', '28597', '44429', '50300', '73556', '40638', '98709', '94413', '15987', '43860', '64871', '93953', '34506', '7296', '31753', '30626', '77510', '39829', '25696', '39776', '69185', '36540', '65413', '31528', '43446', '73532', '49776', '30282', '30004', '26725', '15200', '33958', '90320', '71836', '48051', '31970', '5326', '96194', '69695', '60898', '60945', '18271', '50868', '61468', '23593', '68985', '20628', '58044', '8942', '34849', '7384', '50500', '62895', '78780', '48946', '65278', '4067', '973', '34761', '15512', '73739', '23138', '47322', '55568', '32259', '71816', '49277', '75218', '76104', '19579', '68312', '67904', '33886', '53888', '26421', '43859', '40291', '39068', '31711', '36542', '10195', '39781', '72352', '13188', '34113', '9428', '60443', '4987', '13783', '80744', '63483', '18266', '11961', '87167', '46987', '28480', '74214', '39191', '8146', '38090', '75727', '79245', '47720', '52547', '45321', '4972', '49701', '74354', '69672', '63455', '41902', '5667', '54166', '4962', '25873', '44509', '73332', '73383', '29438', '21455', '12320', '11997', '16921', '49379', '63027', '86175', '8110', '76149', '2520', '11256', '25863', '50518', '69001', '79113', '9447', '91840', '5242', '10998', '46496', '2448', '56058', '20970', '10517', '17783', '25723', '97137', '62840', '1264', '78691', '81020', '55335', '48524', '2088', '90413', '76651', '26855', '16177', '14954', '62914', '21344', '5708', '75560', '39311', '95865', '28783', '64902', '95657', '46276', '33426', '4799', '11588', '57513', '73689', '77677', '63011', '97795', '34954', '76866', '32043', '32697', '26643', '36890', '53476', '3011', '13963', '49551', '87671', '67761', '17488', '94770', '50599', '33272', '23091', '38079', '41177', '22395', '91656', '79679', '38687', '57384', '80118', '42507', '4098', '78949', '45669', '48802', '83915', '78292', '4369', '57657', '49146', '45192', '98491', '72457', '46331', '207', '81601', '7409', '70856', '91605', '70295', '9171', '72293', '32997', '78025', '16795', '73534', '68780', '21284', '31767', '94381', '86439', '12420', '53285', '99563', '60502', '67954', '55012', '99809', '5431', '69978', '99712', '14401', '79498', '4495', '3045', '528', '72542', '91604', '72725', '39378', '80378', '41996', '20138', '54545', '59730', '36951', '45157', '37964', '97690', '12184', '4944', '53803', '93605', '60851', '68938', '46285', '89663', '90309', '6907', '87239', '81791', '83292', '90013', '68927', '14725', '81840', '63836', '52068', '43830', '4794', '931', '59255', '8263', '99057', '94401', '69033', '7437', '20364', '92884', '28193', '43932', '37629', '59426', '18891', '8583', '79551', '87242', '1483', '6725', '65786', '16844', '12650', '99305', '42841', '9811', '18800', '39313', '51373', '31874', '84558', '27831', '48614', '48975', '55509', '83363', '31854', '64001', '94028', '76125', '79314', '24893', '81132', '9441', '86015', '28356', '40358', '10160', '23328', '7330', '76538', '37611', '89351', '84132', '97047', '26109', '95222', '35130', '75600', '88602', '15073', '87835', '71649', '28948', '81615', '37498', '28674', '59776', '44095', '65924', '64368', '94536', '12518', '61711', '55619', '82949', '4114', '21540', '70544', '28022', '79983', '28781', '7749', '97873', '4951', '50076', '47611', '99522', '56820', '38653', '49047', '36283', '83908', '72452', '85625', '10811', '36998', '44083', '34864', '44975', '39057', '4551', '68450', '24781', '1503', '9871', '46885', '11424', '21259', '54900', '97669', '85669', '6015', '2521', '37661', '14915', '57423', '91903', '94789', '32059', '64972', '4600', '61465', '27118', '79785', '13547', '49766', '38410', '68860', '63756', '23621', '64387', '46255', '63408', '11297', '41081', '56326', '58349', '98703', '72268', '73574', '32098', '42534', '91502', '38083', '11241', '56828', '12098', '25377', '37054', '56328', '30034', '26922', '68401', '93478', '63275', '62650', '81407', '773', '79499', '14970', '47217', '1187', '57428', '69980', '77764', '74791', '22107', '54363', '39247', '56028', '56982', '84244', '21464', '18716', '25533', '94589', '94768', '21537', '18436', '81135', '27654', '79713', '56630', '61571', '58453', '26758', '68450', '68449', '2994', '15347', '83954', '71823', '6428', '44210', '79597', '95144', '32871', '1991', '320', '77157', '63607', '31154', '48846', '71125', '61750', '59608', '33038', '35733', '68915', '94127', '50383', '64242', '49708', '57270', '65019', '8581', '12111', '18487', '50013', '58664', '22214', '19033', '33681', '44754', '28830', '10381', '52318', '34959', '20682', '55453', '53800', '65774', '99164', '72102', '36986', '44157', '56716', '7974', '81475', '25926', '39402', '33688', '99671', '95312', '42268', '26536', '14482', '67377', '57993', '89147', '15834', '64995', '4700', '18714', '30221', '39095', '32749', '69257', '55204', '30497', '31839', '63045', '30009', '62683', '31232', '77680', '93551', '63589', '6989', '77246', '42169', '46117', '73226', '37427', '1858', '83649', '37410', '86369', '4641', '74481', '66168', '48041', '22597', '14670', '27464', '57165', '20939', '36282', '76940', '73358', '50521', '69603', '8895', '81793', '57743', '81903', '64025', '91641', '25276', '34040', '62642', '64015', '57657', '84890', '73832', '782', '60160', '16998', '40023', '24590', '88613', '76640', '53091', '67600', '80183', '45674', '64464', '25163', '42384', '66972', '13953', '41966', '66048', '15135', '73745', '19466', '53657', '34619', '13462', '15905', '48257', '73297', '238', '93525', '80556', '5942', '5411', '66169', '9090', '95130', '74316', '57321', '48083', '62355', '68113', '15239', '36644', '80326', '65817', '54428', '61955', '58849', '77206', '16073', '98261', '92091', '39178', '35464', '85109', '85452', '21128', '25665', '81860', '44664', '24024', '56960', '95124', '39786', '18836', '11121', '44163', '81074', '79064', '46219', '94694', '44233', '81469', '24642', '15030', '21995', '13587', '40755', '6669', '81093', '74305', '1881', '55649', '37273', '80827', '98643', '46694', '59281', '79231', '42813', '84984', '7052', '98113', '17296', '84434', '31205', '46894', '71219', '74530', '44686', '70744', '91388', '20692', '96853', '73803', '15836', '18126', '49686', '4179', '47588', '87892', '65425', '68012', '97468', '92510', '99271', '58694', '11918', '37051', '18644', '57228', '14265', '57572', '57022', '52186', '30193', '93570', '87872', '5257', '26784', '6476', '61746', '68559', '1720', '26202', '16519', '27688', '10645', '87174', '60845', '73385', '82075', '6933', '98828', '56895', '17344', '84253', '36561', '51648', '24939', '63470', '31034', '95052', '51090', '51465', '87979', '68650', '30181', '29598', '19137', '43221', '81353', '90170', '96985', '61115', '17385', '92314', '80650', '55821', '17874', '84333', '93272', '48260', '87272', '22764', '59957', '51870', '85988', '39222', '77241', '62535', '28344', '6011', '80831', '64551', '46299', '75195', '71177', '8660', '58943', '57003', '3306', '74413', '74068', '15073', '89016', '93140', '13911', '57170', '19880', '41870', '9131', '57495', '73032', '86979', '60094', '87026', '30880', '4736', '86301', '92707', '21689', '83565', '71275', '47665', '65687', '71184', '89897', '32490', '97577', '38723', '79113', '37531', '97500', '94450', '15699', '58019', '84423', '27057', '56017', '97148', '47365', '30669', '33818', '80406', '99690', '33012', '95178', '46809', '48448', '79350', '9146', '99701', '98976', '71197', '44161', '75069', '36602', '79650', '97301', '12020', '56658', '25701', '46392', '78609', '63073', '69419', '57736', '20102', '42415', '79044', '20277', '56280', '47903', '94311', '25558', '40336', '91305', '90505', '66769', '64562', '83737', '62892', '10375', '71024', '19988', '56946', '76110', '21847', '43162', '50578', '46086', '54167', '61722', '53463', '63134', '69288', '12838', '14116', '71687', '50846', '59810', '24826', '84138', '82885', '91496', '98600', '82769', '40049', '4125', '50694', '1294', '2805', '29691', '82321', '76462', '85945', '115', '29188', '66918', '71340', '31585', '61638', '95472', '52978', '50622', '81990', '60955', '70519', '22270', '35610', '95871', '89222', '41038', '52546', '1163', '67943', '1793', '92010', '35755', '74509', '66665', '95759', '8568', '44299', '67822', '5806', '85839', '13895', '87675', '31357', '88014', '40026', '53050', '28951', '31992', '42495', '82892', '51567', '2869', '45808', '20238', '20781', '56098', '66307', '95701', '614', '60833', '3091', '81339', '24195', '65639', '85976', '28116', '66224', '51502', '73637', '13207', '88302', '36488', '65518', '98187', '26', '74367', '64706', '53943', '86760', '25783', '82112', '34958', '86621', '20848', '63459', '14049', '84943', '91873', '50238', '77773', '64109', '8602', '87934', '47583', '66053', '30287', '5507', '80312', '37464', '57457', '86200', '17806', '16522', '38843', '94334', '59958', '63864', '53427', '74506', '33980', '90449', '30842', '53616', '36738', '52', '13595', '53051', '13174', '60163', '71420', '73835', '67119', '79018', '42782', '45059', '952', '46360', '85879', '71552', '84741', '29746', '32577', '10041', '7208', '97528', '51256', '916', '55973', '17684', '99046', '38782', '58660', '97798', '66032', '48339', '51329', '12532', '97904', '95454', '42737', '62541', '96702', '82953', '94610', '26645', '86813', '25480', '99713', '26078', '23028', '93056', '21445', '73209', '89318', '69987', '34705', '30064', '17094', '51135', '54141', '26625', '1086', '13082', '30843', '98672', '56864', '42605', '5833', '60850', '69366', '27351', '16456', '92609', '48030', '54322', '69891', '46502', '34578', '77918', '63276', '75958', '42519', '60266', '85576', '4855', '14258', '67017', '10545', '35078', '53012', '71922', '85784', '73402', '74363', '58457', '94102', '23510', '51559', '39482', '87057', '9377', '10106', '82985', '33931', '16523', '6484', '97749', '83172', '53753', '27466', '23073', '96083', '67302', '57465', '21877', '18013', '99804', '32873', '43123', '72365', '53197', '80578', '69770', '97471', '86954', '67183', '98497', '78474', '28450', '63183', '98699', '42738', '61433', '3491', '27304', '49311', '94980', '92740', '43272', '86549', '11406', '79636', '85582', '38086', '657', '2354', '26567', '77450', '42086', '21600', '49011', '44059', '47872', '75761', '96577', '11642', '83471', '79616', '23749', '77082', '96876', '65302', '84027', '48955', '59887', '20657', '75090', '9058', '50347', '66088', '70745', '76342', '58026', '95568', '61504', '93473', '84590', '47089', '74717', '93090', '46334', '68273', '59500', '54345', '72608', '54048', '86156', '40296', '74046', '6813', '36369', '74543', '18305', '85236', '31316', '37061', '96893', '23112', '5529', '10166', '19037', '1467', '70810', '30932', '18410', '92837', '81324', '12268', '54705', '25207', '90366', '56528', '3392', '88747', '39951', '97957', '99404', '23685', '13533', '15640', '11434', '66516', '71025', '65770', '88000', '52232', '32360', '10787', '37438', '2264', '94460', '80214', '42288', '59062', '29010', '64093', '21225', '22297', '36935', '19202', '5925', '85373', '27414', '28991', '9191', '42273', '56587', '89719', '77191', '64334', '61542', '28763', '28978', '79184', '59815', '95200', '30246', '54022', '287', '91808', '66347', '50833', '15356', '78614', From af1c8f8a5f1dcbc9bf12cef3dc55b02f87edb212 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 10 Jun 2020 02:57:31 +0300 Subject: [PATCH 27/29] fixup --- docker/test/performance-comparison/compare.sh | 8 +++++--- tests/performance/set_index.xml | 13 ++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 983db608b47..f7986689020 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -347,9 +347,11 @@ create table query_metric_stats engine File(TSVWithNamesAndTypes, create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') as select -- FIXME Comparison mode doesn't make sense for queries that complete - -- immediately, so for now we pretend they don't exist. We don't want to - -- remove them altogether because we want to be able to detect regressions, - -- but the right way to do this is not yet clear. + -- immediately (on the same order of time as noise). We compute average + -- run time between old and new version, and if it is below a threshold, + -- we just skip the query. If there is a significant regression, the + -- average will be above threshold, we'll process it normally and will + -- detect the regression. (left + right) / 2 < 0.02 as short, not short and abs(diff) > report_threshold and abs(diff) > stat_threshold as changed_fail, diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index 021f6dc60f3..88dbf457cba 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -14,9 +14,9 @@ SELECT toString(rand()) IN ('41577', '83972', '51697', '50014', '37553', '93459', '87438', '95971', '83186', '74326', '67871', '50406', '83678', '29655', '18580', '83905', '61518', '29059', '56700', '82787', '98672', '30884', '81822', '39850', '80852', '57627', '91346', '64522', '17781', '49467', '41099', '41929', '85618', '91389', '68564', '91769', '81219', '52218', '37220', '97097', '2129', '9886', '52049', '34847', '25364', '36429', '76897', '71868', '58121', '71199', '84819', '69991', '34046', '64507', '34892', '24228', '36986', '28588', '51159', '53444', '80531', '9941', '20256', '48103', '32565', '62890', '5379', '60302', '46434', '3205', '18821', '31030', '19794', '71557', '71703', '15024', '14004', '82164', '95659', '40227', '83358', '24395', '9610', '19814', '48491', '66412', '16012', '71586', '42143', '51103', '24463', '89949', '35694', '39193', '63904', '40489', '77144', '94014', '84836', '9980', '46554', '43905', '25588', '25205', '72624', '10249', '35888', '98478', '99030', '26834', '31', '81499', '14847', '82997', '92357', '92893', '17426', '56630', '22252', '68119', '62710', '8740', '82144', '79916', '23391', '30192', '99271', '96435', '44237', '98327', '69481', '16691', '13643', '84554', '38571', '70926', '99283', '79000', '20926', '86495', '4834', '1222', '39486', '57697', '58002', '40790', '15623', '3999', '31515', '12694', '26143', '35951', '54085', '97534', '35329', '73535', '88715', '29572', '75799', '45166', '32066', '48023', '69523', '93150', '8740', '96790', '15534', '63252', '5142', '67045', '93992', '16663', '292', '63924', '6588', '12190', '31506', '69590', '35394', '55168', '65223', '79183', '32600', '69676', '28316', '72111', '53531', '15073', '41127', '73451', '24725', '61647', '65315', '41143', '26493', '95608', '34407', '76098', '53105', '83691', '48755', '35696', '62587', '81826', '3963', '45766', '82751', '12430', '97685', '29919', '78155', '71636', '50215', '89734', '9892', '47151', '54855', '3428', '9712', '52592', '2403', '79602', '81243', '79859', '57361', '82000', '42107', '28860', '99591', '28296', '57337', '64969', '32332', '25535', '30924', '21313', '32554', '17342', '87311', '19825', '24898', '61323', '83209', '79322', '79009', '50746', '33396', '62033', '16548', '17427', '24073', '34640', '52368', '4724', '80408', '40', '33787', '16666', '19665', '86751', '27264', '2241', '88134', '53566', '10589', '79711', '92823', '58972', '91767', '60885', '51659', '7867', '96849', '30360', '20914', '9584', '1250', '22871', '23282', '99312', '4683', '33429', '68361', '82614', '81440', '47863', '69790', '11968', '75210', '66854', '37002', '61142', '71514', '1588', '42336', '11069', '26291', '2261', '71056', '13492', '9133', '91216', '72207', '71586', '86535', '83898', '24392', '45384', '48545', '61972', '503', '80180', '35834', '97025', '70411', '55039', '35430', '27631', '82533', '96831', '74077', '42533', '14451', '26943', '53783', '69489', '71969', '8432', '37230', '61348', '19472', '59115', '9886', '50951', '57109', '7141', '1902', '84130', '4323', '55889', '47784', '2220', '75988', '66988', '63721', '8131', '95601', '95207', '2311', '26541', '50991', '6717', '2969', '71857', '51034', '65958', '94716', '90275', '21012', '46859', '7984', '31131', '46457', '69578', '44540', '7294', '80117', '9925', '60155', '90608', '82684', '32193', '87071', '28006', '87604', '24501', '79087', '2848', '29237', '11221', '81319', '40966', '87641', '35325', '78705', '88636', '78717', '62831', '56390', '99271', '43821', '14453', '17923', '62695', '77322', '21038', '67677', '41271', '4376', '65426', '46091', '19887', '97251', '55583', '58763', '3826', '35037', '73533', '64267', '82319', '9836', '42622', '96829', '16363', '10455', '49290', '99992', '98229', '66356', '59087', '73998', '25986', '4279', '56790', '69540', '588', '36620', '60358', '45056', '89297', '42740', '8323', '19245', '82417', '41431', '699', '11554', '73910', '44491', '56019', '68901', '45816', '68126', '89379', '23885', '13263', '56395', '73130', '19089', '23771', '10335', '48547', '16903', '6453', '33560', '89668', '38159', '43177', '90655', '49712', '62', '66920', '34180', '12150', '48564', '39538', '85026', '87195', '14928', '8956', '71157', '53287', '39161', '67583', '83309', '92054', '86977', '56188', '15229', '88170', '60894', '58497', '89254', '40082', '86890', '60161', '97291', '45878', '23368', '14577', '92870', '37017', '97356', '99426', '76061', '89186', '99751', '85153', '61580', '39360', '90107', '25603', '26798', '76224', '6469', '7912', '69838', '16404', '67497', '28965', '80836', '80365', '91249', '48713', '17113', '33090', '40793', '70450', '66689', '83698', '17802', '43869', '13355', '18959', '79411', '87930', '9265', '37504', '44876', '97234', '94149', '35040', '22049', '49248', '6535', '36080', '28346', '94437', '78319', '17961', '89056', '56161', '35810', '41632', '45494', '53351', '89729', '99510', '51584', '59688', '6193', '70809', '51093', '92589', '90247', '34910', '78235', '17362', '49423', '63324', '525', '37638', '72325', '89356', '15298', '59116', '17848', '65429', '27029', '84781', '70247', '8825', '35082', '70451', '22522', '58125', '91879', '90531', '2478', '463', '37902', '54405', '87267', '72688', '22803', '33134', '35177', '84551', '44974', '88375', '76407', '27774', '33849', '19915', '82014', '80434', '26380', '48777', '53811', '14838', '26829', '56441', '99869', '49574', '85476', '19723', '16907', '4018', '37338', '78510', '47912', '13030', '65277', '95716', '67363', '21393', '89887', '78842', '81650', '903', '17436', '30704', '49223', '27198', '25500', '52214', '54258', '70082', '53950', '49312', '43615', '99473', '94348', '53661', '96213', '96346', '62010', '38268', '32861', '75660', '10392', '89491', '68335', '29817', '88706', '24184', '36298', '43440', '21626', '26535', '44560', '46363', '12534', '99070', '95606', '33714', '73070', '8303', '29853', '23014', '99982', '4530', '14955', '45803', '50', '90750', '30394', '81276', '95563', '47314', '58520', '91299', '88944', '54402', '67405', '29253', '47079', '71734', '99728', '17652', '13307', '35556', '18962', '26780', '17771', '53712', '60055', '37628', '35830', '90739', '61151', '41309', '27652', '3051', '53167', '98417', '19382', '36833', '75085', '65374', '87732', '30352', '31776', '32765', '97565', '92199', '49050', '29503', '51024', '18834', '8515', '24069', '96216', '10777', '90680', '18974', '68884', '85305', '36007', '56707', '4212', '47352', '34426', '13185', '92939', '95782', '70577', '58080', '98279', '3906', '5065', '56896', '16382', '31273', '17117', '98602', '12786', '24086', '63970', '72756', '35798', '82367', '7356', '53398', '68503', '2962', '16425', '67334', '68461', '65439', '15620', '70906', '29649', '46461', '74602', '38012', '71714', '16825', '89480', '53386', '88532', '35104', '28556', '82120', '23155', '23347', '24797', '60061', '54962', '99427', '82248', '82447', '39968', '63727', '27431', '81511', '91168', '71425', '80740', '84127', '40717', '15503', '15419', '46594', '61263', '19212', '53175', '70724', '74445', '23034', '71818', '40246', '18886', '53066', '4880', '83701', '86107', '87862', '44751', '392', '73440', '90291', '93395', '20894', '38463', '32664', '55158', '20090', '50004', '79070', '98471', '85478', '96615', '68149', '78334', '97752', '73207', '71678', '91238', '96757', '82598', '194', '35797', '45120', '60782', '28721', '17676', '78066', '60957', '11826', '51563', '50516', '16485', '47053', '31738', '48923', '23554', '96850', '42033', '73701', '78607', '45979', '54571', '12415', '31693', '15356', '36902', '9126', '3767', '3295', '90402', '24005', '95350', '67033', '49137', '72606', '51899', '17522', '31957', '44641', '53982', '23767', '68257', '15766', '19995', '2107', '48788', '11765', '91055', '46576', '54651', '50381', '62827', '73636', '46606', '98753', '37631', '70441', '87916', '66983', '33870', '31125', '12904', '57040', '4874', '58632', '42037', '18782', '5998', '18974', '57949', '81010', '90407', '99874', '20462', '89949', '10952', '71454', '95130', '46115', '3518', '13384', '69039', '79482', '22076', '59782', '32042', '40930', '60243', '29298', '6790', '46985', '44398', '85631', '14380', '66179', '2629', '32126', '49833', '14118', '58492', '31493', '81172', '96638', '8745', '89663', '76842', '78633', '41373', '83721', '42886', '11123', '32739', '11051', '1303', '92314', '83324', '85600', '44276', '69064', '56125', '84650', '31028', '12628', '14502', '64764', '39405', '44855', '79046', '51716', '46824', '83389', '1941', '1257', '9280', '73176', '84729', '2579', '63366', '22606', '35541', '51096', '13447', '18355', '68037', '28436', '94116', '81070', '78355', '67897', '5296', '32742', '77645', '91853', '18767', '67949', '40963', '5792', '17278', '25597', '41884', '80829', '7099', '18645', '60295', '12082', '81800', '78415', '18082', '38789', '16295', '72377', '74949', '55583', '66853', '15402', '72977', '15123', '99434', '34999', '21687', '76049', '42987', '83748', '88256', '66688', '21766', '20304', '29271', '10069', '19822', '11792', '42526', '74143', '17289', '30253', '6367', '20888', '12975', '94073', '98639', '30134', '26320', '65507', '69002', '53120', '4550', '38893', '18954', '38283', '54863', '17698', '99670', '10521', '92467', '60994', '18052', '48673', '35811', '87282', '62706', '16061', '53112', '22652', '37780', '55662', '26331', '49410', '79074', '10623', '69577', '79613', '9491', '31229', '43922', '84231', '58409', '36386', '46875', '74431', '76735', '38776', '23350', '7314', '9079', '51519', '98544', '70216', '63380', '90381', '1295', '46901', '58225', '55339', '89918', '75522', '35431', '89460', '49552', '89302', '23068', '28493', '3042', '25194', '59520', '9810', '95706', '81297', '89638', '54794', '94527', '45262', '97932', '78685', '6947', '22818', '48700', '9153', '12289', '22011', '58825', '93854', '65438', '4509', '33741', '28208', '69061', '48578', '40247', '77725', '31837', '39003', '69363', '78113', '76398', '97262', '67795', '68446', '58896', '60969', '19849', '6722', '91854', '49519', '13949', '67109', '48824', '31723', '75554', '69575', '94986', '75350', '18628', '15284', '41943', '15433', '52607', '41', '22340', '29528', '24059', '34145', '72517', '46316', '10667', '54510', '19882', '47764', '69124', '41963', '84350', '48420', '4646', '24958', '69020', '97121', '26178', '62664', '50622', '32554', '49655', '45398', '11267', '72222', '73363', '89554', '89046', '57384', '29259', '37888', '24850', '74353', '57343', '34762', '2900', '11393', '42154', '94306', '70552', '75265', '921', '26003', '64352', '89857', '83171', '58249', '48940', '53512', '66335', '44865', '68729', '19299', '58003', '39854', '99122', '3860', '80173', '52242', '90966', '53183', '71982', '82325', '87842', '15000', '55627', '71132', '6354', '42402', '91719', '91644', '94533', '74925', '66278', '66911', '85576', '40495', '70919', '71797', '87835', '29845', '71832', '3390', '7994', '33499', '70715', '54897', '82710', '63077', '78105', '24758', '89585', '84607', '46477', '78618', '10989', '39222', '98749', '51685', '94664', '31008', '32823', '89521', '72160', '26952', '4001', '21421', '5737', '74027', '88179', '45553', '83743', '19913', '49435', '65616', '82641', '5149', '76959', '40681', '73023', '2670', '30845', '18863', '35094', '88400', '80963', '9154', '16571', '64192', '59694', '41317', '59942', '58856', '99281', '67260', '66971', '22716', '76089', '58047', '67071', '53707', '462', '52518', '72277', '10681', '69', '98855', '12593', '88842', '67242', '73543', '37439', '18413', '67211', '93495', '45576', '70614', '27988', '53210', '18618', '21318', '68059', '25518', '55917', '56522', '16548', '2404', '93538', '61452', '66358', '3709', '23914', '92426', '81439', '38070', '28988', '29939', '2948', '85720', '45628', '51101', '89431', '86365', '17571', '50987', '83849', '11015', '83812', '66187', '26362', '66786', '22024', '93866', '36161', '90080', '64874', '37294', '83860', '73821', '80279', '36766', '73117', '44620', '84556', '42070', '90383', '27862', '20665', '67576', '34997', '57958', '80638', '84351', '63961', '1362', '14338', '80377', '24192', '41294', '57368', '51189', '27287', '45764', '86289', '65600', '708', '84090', '96005', '55676', '84855', '72385', '70018', '9336', '82701', '3710', '52083', '74045', '96454', '30956', '67369', '78941', '81810', '71906', '23194', '33042', '50794', '61256', '24449', '48639', '22916', '78303', '13666', '40762', '43942', '51075', '89783', '95786', '90462', '6181', '36482', '40675', '4970', '6388', '91849', '72579', '94983', '86084', '20140', '68427', '48123', '43122', '98066', '37560', '6927', '72803', '5546', '62259', '98439', '6457', '98568', '70499', '33022', '28226', '29675', '20917', '75365', '20900', '8190', '56736', '99153', '77779', '49333', '50293', '97650', '4067', '47278', '42761', '71875', '13966', '11223', '46783', '18059', '61355', '29638', '75681', '24466', '89634', '20759', '83252', '37780', '15931', '74893', '6703', '64524', '80656', '85990', '78427', '18411', '20696', '86432', '93176', '69889', '15072', '15180', '9935', '10467', '60248', '42430', '62590', '89596', '27743', '26398', '79912', '60048', '50943', '38870', '69383', '72261', '98059', '55242', '74905', '5667', '54321', '70415', '39903', '49711', '85318', '79979', '59262', '82321', '15263', '17416', '74554', '94733', '72112', '49872', '54849', '73883', '78250', '74935', '68559', '57564', '50541', '45730', '41595', '5588', '83723', '42891', '11898', '14348', '99732', '14481', '85233', '21277', '94508', '52551', '74187', '7634', '42912', '25100', '43536', '35798', '48190', '86477', '22680', '48148', '59501', '56563', '16802', '81496', '97568', '68657', '51462', '67953', '99660', '39002', '54170', '57190', '68086', '52700', '6487', '55709', '70418', '62629', '70420', '35695', '36152', '45360', '53503', '46623', '76000', '50648', '97876', '44815', '29163', '1356', '64123', '71388', '17658', '99084', '58727', '59437', '38773', '71254', '81286', '97545', '18786', '56834', '20346', '36401', '62316', '58082', '67959', '99876', '69895', '80099', '62747', '20517', '99777', '6472', '49189', '31321', '39992', '68073', '13378', '51806', '21776', '52060', '96983', '25754', '93709', '96627', '8644', '93726', '14002', '37716', '87620', '34507', '76339', '24491', '5849', '44110', '522', '66521', '12776', '44887', '80535', '14548', '75248', '671', '73071', '35715', '59474', '7061', '82243', '56170', '20179', '59717', '1725', '24634', '11270', '77023', '63840', '46608', '44667', '22422', '59771', '94768', '73033', '82905', '16463', '40971', '22204', '58366', '28721', '14907', '76468', '81872', '38418', '36989', '61439', '10610', '131', '44296', '35453', '10117', '75856', '94603', '99602', '68075', '35949', '13599', '50030', '69633', '55956', '85465', '16429', '86081', '11145', '6195', '82207', '90598', '92814', '23725', '83204', '80346', '71542', '46634', '15820', '54123', '45397', '15322', '61743', '9273', '71347', '6835', '64006', '91718', '43677', '32923', '21486', '17098', '61694', '43347', '40019', '4071', '52443', '42386', '56839', '83514', '27633', '40780', '51749', '92101', '62384', '92206', '56044', '66174', '11137', '73966', '78471', '30468', '31643', '33197', '6888', '8066', '86603', '74383', '6098', '54411', '98819', '89862', '88639', '94422', '89371', '80526', '91747', '91220', '64944', '76658', '42046', '58518', '27249', '6646', '3028', '1346', '33763', '9734', '31737', '65527', '5892', '60813', '3410', '35464', '43009', '98382', '70580', '93898', '56404', '32995', '62771', '71556', '40538', '55612', '45656', '10758', '20268', '33603', '38310', '14242', '74397', '10722', '71575', '22590', '49043', '91439', '9055', '23668', '9101', '5268', '64133', '77501', '64684', '11337', '47575', '50732', '88680', '93730', '46785', '17589', '3520', '57595', '71241', '34994', '8753', '36147', '88844', '41914', '11250', '94632', '71927', '4624', '86279', '7664', '2659', '94853', '65386', '30438', '86005', '92883', '84629', '59910', '44484', '1306', '8404', '56962', '29990', '38445', '96191', '73013', '66590', '40951', '24712', '18825', '37268', '87843', '18972', '12154', '7779', '52149', '76152', '65799', '86011', '35475', '78083', '88232', '91551', '65532', '93516', '73827', '24227', '44687', '55759', '83819', '45088', '10856', '60488', '39051', '14103', '76650', '81181', '46731', '737', '58788', '78945', '42096', '66731', '66740', '72273', '88969', '5655', '86590', '41096', '80038', '32430', '51877', '23970', '91900', '13082', '45880', '94367', '19739', '61998', '71665', '16083', '57035', '26916', '10166', '18834', '46798', '66881', '28444', '68840', '10459', '81087', '4728', '76224', '39257', '23470', '93524', '37345', '30074', '49856', '22022', '55279', '5159', '5193', '58030', '57539', '12514', '49759', '96222', '52597', '67192', '88187', '53614', '16084', '79915', '28212', '79334', '85283', '32306', '31058', '43113', '74707', '74869', '2213', '32134', '6379', '85426', '87098', '35984', '51105', '69287', '16803', '83337', '14913', '62531', '58098', '7914', '20105', '28850', '1384', '43173', '62983', '87113', '76066', '86320', '77684', '45191', '95225', '41503', '36713', '48404', '91228', '53865', '98981', '59161', '61237', '84561', '17455', '14379', '57789', '80895', '99260', '84595', '72942', '53220', '84448', '81332', '49437', '83086', '93414', '54519', '52288', '74772', '22460', '49324', '11168', '96071', '61985', '38284', '6405', '54698', '71727', '60093', '37340', '87884', '83403', '4542', '94949', '19636', '15855', '39105', '10424', '67418', '91022', '69254', '8481', '38411', '3832', '44354', '93548', '57172', '28481', '372', '81497', '52179', '41060', '72141', '41396', '65590', '70432', '82819', '93814', '26118', '84780', '88485', '70821', '8222', '83000', '47067', '38516', '33347', '47681', '48202', '60749', '52112', '7937', '28105', '11394', '45746', '43252', '34494', '2979', '69715', '42486', '82315', '71760', '97413', '66137', '94487', '7429', '74434', '22964', '55251', '3448', '53534', '2574', '9693', '96157', '2955', '4348', '19566', '56930', '83319', '31310', '53905', '1148', '41726', '22233', '76045', '37351', '10545', '17581', '28047', '30199', '4741', '58111', '33497', '67796', '67730', '31247', '43772', '29461', '45970', '73353', '22534', '53962', '32147', '71392', '62579', '66345', '58246', '33442', '9581', '29705', '14058', '86471', '76125', '59363', '94982', '74810', '89149', '20066', '3366', '3568', '25752', '80036', '64119', '27270', '40061', '91052', '69022', '9852', '77112', '83075', '43924', '61661', '56133', '96652', '57944', '72576', '82170', '79236', '55745', '15309', '88878', '72761', '37647', '67465', '12777', '97309', '93202', '41470', '8787', '64920', '48514', '18917', '35157', '59151', '4640', '5317', '38134', '76548', '82788', '9214', '58418', '73185', '90554', '10543', '47182', '62936', '91765', '89751', '68931', '48865', '64607', '7150', '77862', '14297', '14828', '33013', '91698', '67593', '98096', '16595', '51639', '86531', '24719', '1703', '78788', '43810', '38918', '95491', '99903', '82671', '8291', '68288', '31224', '39863', '4265', '77798', '7698', '33804', '92286', '4744', '37038', '44203', '98212', '17369', '77442', '62879', '4145', '96881', '15646', '36824', '19959', '45451', '76049', '54272', '97577', '95298', '81115', '30204', '82041', '8037', '10052', '8756', '76833', '82851', '24276', '75574', '36037', '78079', '92807', '29064', '90000', '84150', '17102', '75092', '49424', '35597', '4693', '82853', '42511', '16119', '23478', '65240', '55585', '91762', '71671', '46682', '72479', '97696', '24615', '12579', '30274', '48255', '2336', '90202', '5808', '45426', '76308', '74639', '31245', '99894', '89638', '6233', '33893', '71899', '85273', '89429', '29761', '50231', '57249', '99347', '22642', '66972', '86221', '47514', '88274', '10819', '73150', '53754', '13304', '20478', '38099', '619', '14669', '8011', '97657', '26569', '65430', '13467', '38180', '23675', '72350', '42257', '39875', '23529', '53407', '11833', @@ -28,8 +28,11 @@ '37674', '44977', '54370', '97381', '60218', '2423', '99591', '69913', '26507', '19708', '6279', '58955', '20126', '1495', '57894', '7638', '38700', '77148', '36844', '7539', '91452', '6914', '74349', '66850', '49104', '6516', '58535', '20851', '27859', '32881', '72919', '28203', '32882', '2419', '77583', '63822', '37703', '66793', '65784', '62281', '55867', '70703', '89344', '1498', '33770', '87176', '95636', '64891', '90736', '95521', '10989', '5237', '99010', '21106', '11422', '1831', '67239', '52557', '36468', '71713', '39637', '49574', '50455', '14953', '96900', '70852', '96982', '4341', '44585', '95651', '79669', '29652', '87294', '74692', '16221', '768', '35380', '21352', '50907', '27259', '11718', '5017', '55964', '94137', '52347', '10595', '12968', '85602', '97965', '18836', '90511', '70960', '97336', '44575', '23791', '42195', '64776', '29363', '42379', '1805', '28919', '6772', '78143', '54797', '27362', '56149', '59048', '38567', '6339', '27787', '42167', '45990', '95532', '54839', '26572', '38496', '89797', '6634', '16468', '24898', '66814', '98126', '31762', '36133', '64539', '43167', '87022', '61295', '30364', '89249', '25756', '63570', '91484', '10564', '79648', '5756', '41376', '61897', '40388', '88927', '62891', '79708', '25495', '22204', '33892', '36871', '19879', '58646', '57061', '73100', '75831', '20029', '67462', '54675', '7766', '2409', '24506', '7877', '11720', '86252', '9897', '8080', '70684', '74497', '2242', '24604', '31969', '83999', '56635', '5283', '64971', '79152', '27470', '89042', '22835', '21476', '50292', '56081', '96342', '32763', '84487', '64856', '79152', '64656', '72169', '69971', '93094', '52804', '80917', '53152', '56016', '28496', '79110', '17133', '12581', '91742', '78929', '2676', '46700', '59528', '93808', '4535', '54035', '40161', '62796', '3598', '97088', '13599', '36337', '73395', '17494', '86275', '62058', '61937', '87747', '94883', '90677', '88544', '72553', '50210', '75481', '64378', '74464', '21659', '30970', '71989', '84846', '72289', '88716', '39143', '8487', '4912', '91013', '18623', '19122', '36507', '76438', '7516', '67970', '72350', '69873', '33635', '55983', '69008', '49545', '3134', '60056', '52509', '63304', '15560', '23651', '81090', '7027', '8317', '33060', '37295', '51961', '53037', '97431', '40512', '23536', '25168', '78455', '85613', '12304', '40733', '99890', '51238', '55439', '96201', '73559', '92533', '90173', '16721', '6078', '29854', '38894', '31117', '63040', '86795', '81786', '21149', '38998', '61811', '48622', '73019', '59296', '13576', '92559', '36300', '77294', '26794', '50912', '98380', '13176', '57746', '75286', '15330', '40921', '7337', '4664', '20384', '4674', '44516', '27633', '31950', '88210', '54536', '9839', '80137', '77491', '18434', '45152', '96942', '41005', '76103', '34825', '86869', '14772', '13384', '21051', '37348', '34434', '97210', '54960', '26598', '60981', '41889', '6446', '64492', '95310', '86236', '81885', '35684', '16539', '98476', '32028', '96470', '6318', '99576', '93935', '48609', '86090', '2476', '65576', '80636', '44817', '99646', '98963', '20486', '26261', '27334', '72946', '82023', '33506', '80193', '13762', '98133', '21134', '33268', '63477', '74609', '30454', '51477', '93391', '96805', '68653', '2714', '63642', '51520', '22972', '13305', '96058', '42336', '74461', '31597', '12050', '81712', '37977', '25718', '4834', '56608', '75731', '406', '28585', '63924', '23702', '29849', '16941', '91921', '65842', '76525', '68534', '50902', '17609', '23852', '53703', '31286', '58526', '9633', '87596', '10654', '2085', '52766', '22135', '76524', '32295', '90072', '70078', '77786', '93741', '87320', '70309', '44024', '95286', '12361', '29682', '59766', '26685', '90686', '81691', '49704', '23431', '53955', '39023', '47261', '1530', '58265', '80065', '95620', '90621', '63760', '90676', '81653', '36397', '20252', '81754', '20256', '67098', '7838', '49408', '88400', '87941', '84533', '6570', '22567', '18850', '55472', '40129', '48425', '23497', '39308', '34698', '53092', '89480', '47785', '57282', '25508', '19006', '50604', '86917', '9436', '88921', '3168', '70537', '3185', '34988', '5462', '69482', '45768', '91955', '56898', '15307', '99731', '89292', '19356', '20646', '66712', '7281', '12856', '31174', '19577', '8726', '62971', '33008', '37118', '59055', '84101', '68445', '91957', '47526', '15627', '79914', '20013', '26147', '80821', '56372', '74205', '28531', '25352', '51775', '93948', '55212', '17863', '91521', '74911', '88160', '2360', '98260', '18294', '62402', '84268', '9580', '42668', '1467', '40059', '5221', '4216', '9917', '35420', '16496', '34369', '50253', '95234', '95114', '84193', '28322', '37031', '81284', '88628', '36782', '42572', '73347', '66188', '43342', '77285', '16513', '89064', '63066', '72645', '67075', '48208', '18181', '77898', '65795', '53707', '39856', '92883', '92567', '49733', '30236', '10273', '53029', '69773', '78379', '72108', '47696', '97557', '95184', '14688', '29853', '62694', '70431', '88435', '58799', '21883', '99866', '69178', '55870', '14414', '85274', '27321', '55555', '613', '15067', '88217', '73655', '99548', '13631', '78789', '36690', '7952', '60830', '77438', '40059', '95602', '43097', '3429', '93731', '90537', '2932', '35702', '16125', '6652', '39632', '39349', '9910', '38103', '78608', '73565', '48556', '28978', '7128', '82326', '53980', '28059', '28212', '87101', '77752', '99170', '56753', '30484', '71470', '32607', '24674', '32687', '25098', '94712', '64024', '48239', '90408', '17316', '99243', '3656', '67402', '48009', '98427', '52800', '56024', '4417', '89747', '93338', '18758', '56411', '44810', '82456', '30808', '75470', '67115', '66876', '53906', '78403', '56059', '34383', '60056', '89136', '7237', '11129', '21351', '78662', '43606', '37454', '45465', '9292', '38099', '81699', '50195', '49368', '47503', '44605', '6523', '81478', '37910', '397', '20256', '6835', '2787', '80383', '4241', '65986', '83870', '21205', '10879', '26593', '44357', '72604', '56131', '43423', '80206', '26240', '87198', '99445', '53504', '10632', '2465', '31793', '89575', '64184', '39988', '60049', '87100', '37151', '61585', '82180', '52065', '72519', '72935', '3201', '5862', '20560', '95339', '21661', '17533', '17182', '71189', '91564', '57999', '35490', '94773', '95056', '51583', '59394', '10727', '8655', '48123', '10701', '25314', '20100', '6533', '46435', '43188', '23001', '23018', '76637', '32018', '36603', '18701', '9550', '61550', '47541', '36500', '67507', '81574', '95490', '69169', '32584', '30045', '64699', '83539', '89396', '42517', '61979', '41528', '8271', '88377', '61423', '1158', '89724', '70789', '14886', '64823', '56675', '97747', '23990', '58495', '82064', '17062', '90258', '86854', '93304', '12925', '49975', '45074', '87155', '72223', '67344', '42733', '42516', '40110', '15444', '88285', '39371', '23198', '61544', '90205', '6192', '15718', '19803', '92712', '20081', '31397', '5555', '70463', '19521', '80401', '74097', '32060', '26495', '20507', '40473', '1449', '57215', '46142', '39303', '50359', '35898', '46908', '90752', '7823', '27416', '73770', '98790', '17907', '29999', '76417', '49926', '76752', '21608', '26524', '88209', '6000', '88897', '19541', '41451', '59538', '56560', '1456', '67828', '82407', '45722', '93344', '54279', '78594', '38354', '93807', '10929', '91560', '60681', '70615', '32527', '10108', '48303', '63134', '28500', '18257', '57081', '24801', '99077', '52197', '15390', '52300', '57116', '417', '7503', '20054', '75315', '81359', '69091', '18853', '2465', '25600', '13522', '74575', '12661', '83071', '15191', '27543', '21730', '60853', '18961', '14773', '89185', '33694', '51143', '1449', '68831', '78062', '65173', '32697', '41674', '9429', '22156', '96022', '46305', '97534', '5685', '48870', '89988', '20686', '66705', '6865', '94250', '16872', '13178', '7420', '73531', '92723', '60620', '48843', '74207', '60016', '50943', '62699', '63507', '76537', '87066', '76922', '24711', '34809', '5021', '31293', '53854', '77607', '52322', '10934', '50284', '87804', '36730', '86946', '80749', '43325', '97958', '7362', '39582', '10042', '42053', '66236', '69931', '23463', '87996', '33563', '4468', '32905', '50815', '79478', '28658', '46018', '23186', '26080', '13494', '6237', '42762', '86440', '77407', '10426', '62902', '73251', '36861', '92357', '98754', '1839', '46391', '11420', '27132', '93028', '39609', '42015', '68218', '54228', '5456', '38705', '64307', '49483', '878', '54360', '54480', '66684', '55089', '4537', '82073', '72602', '96238', '56708', '58625', '32991', '74205', '72868', '79086', '64250', '56376', '10621', '76607', '47706', '72760', '70303', '60715', '14644', '44186', '36264', '29489', '14184', '62699', '30567', '16700', '31222', '15650', '1500', '22950', '54628', '41004', '96094', '70028', '74178', '65328', '26605', '63076', '75271', '79285', '8151', '42101', '56362', '25961', '87864', '972', '29510', '2747', '8877', '9780', '61052', '84105', '15573', '27475', '44570', '25334', '18517', '44237', '84094', '67524', '76761', '65678', '79284', '2462', '42631', '22696', '19223', '29728', '67742', '11883', '59027', '12377', '80538', '2165', '17377', '15030', '49838', '23920', '26025', '68179', '75894', '43783', '97106', '75558', '35528', '52081', '16951', '68855', '402', '21459', '97550', '16948', '5369', '4641', '2663', '15233', '79974', '71093', '15234', '42690', '22322', '54282', '95845', '90010', '40530', '88298', '41885', '7079', '6098', '72786', '36603', '77378', '48393', '45723', '41996', '96025', '89297', '75586', '8422', '24360', '170', '46036', '46725', '67944', '74029', '73069', '45371', '99916', '71085', '42608', '89904', '6393', '51274', '42729', '58924', '82497', '64143', '88622', '18818', '89041', '56090', '21369', '78224', '90450', '45488', '58830', '4133', '98062', '81113', '11285', '51457', '3183', '38800', '65278', '42169', '28602', '52648', '44683', '75647', '11778', '32151', '33528', '23773', '68268', '23367', '70964', '23548', '35575', '67570', '77681', '74158', '25374', '62714', '43100', '4977', '51678', '83460', '29755', '15890', '64626', '54044', '14793', '64339', '94008', '97126', '49202', '33889', '12601', '12275', '56123', '94557', '68226', '67200', '9374', '70687', '29211', '8039', '14598', '74548', '37433', '98991', '29933', '37203', '23973', '96482', '64774', '58350', '61781', '31824', '57193', '26476', '21814', '32297', '32627', '44277', '33876', '55468', '81715', '82505', '61462', '20324', '84293', '40116', '51087', '43594', '6854', '59077', '39841', '26023', '22777', '66859', '82460', '89515', '41712', '33711', '71875', '10685', '12655', '50138', '31063', '37040', '95819', '38919', '27391', '29833', '34350', '65646', '7697', '2688', '41146', '13241', '50305', '86568', '24487', '78741', '96370', '21015', '31719', '39750', '25014', '72415', '8486', '90668', '51143', '49488', '21057', '92803', '53528', '39550', '76039', '44185', '32404', '30217', '19796', '38084', '49161', '80140', '20241', '39357', '68908', '93083', '77231', '6952', '36322', '50790', '623', '29730', '13616', '57546', '17434', '93811', '35148', '81419', '40250', '40329', '89126', '72402', '16053', '27107', '28919', '16829', '96582', '65057', '28416', '30801', '77742', '27420', '73118', '89352', '54706', '23035', '88413', '64608', '61930', '15037', '47327', '59596', '18700', '57576', '63628', '56823', '60091', '68209', '21001', '14962', '72257', '83802', '33721', '86343', '11133', '65737', '68477', '90725', '86869', '98403', '47393', '25356', '61372', '8873', '19888', '48836', '66005', '23531', '72520', '26461', '78508', '28213', '96394', '22983', '37856', '71814', '27425', '72753', '27511', '65471', '38592', '3683', '24652', '64505', '92543', '53201', '40639', '99542', '53425', '35321', '47669', '14134', '47727', '48202', '71931', '32119', '50086', '50266', '67159', '89317', '81905', '30315', '49154', '8690', '69365', '56881', '46473', '64100', '38365', '59377', '65630', '54871', '52745', '91536', '16106', '70066', '62063', '84530', '88103', '33599', '51063', '87299', '41880', '25335', '51252', '42788', '13568', '1721', '62424', '83308', '36787', '91536', '92555', '27600', '24030', '12267', '66336', '30242', '7183', '67624', '28471', '48593', '79766', '31178', '47818', '94522', '88855', '45262', '43670', '18065', '25062', '44558', '37189', '69225', '35216', '42683', '26289', '72816', '31947', '65871', '45715', '59452', '22014', '56669', '60331', '33450', '60601', '95047', '30789', '90107', '81565', '32266', '3252', '5446', '58756', '55370', '34034', '81071', '2560', '39054', '39564', '15010', '5389', '60002', '53320', '49545', '48444', '31415', '39278', '79879', '30148', '10186', '60358', '29011', '14419', '95159', '94815', '55251', '90910', '80582', '92304', '11697', '60061', '38577', '84439', '76196', '34542', '50963', '36294', '11123', '59763', '29873', '47383', '12979', '22119', '21723', '64725', '48377', '77132', '9817', '79920', '47653', '60069', '12924', '53808', '55962', '66969', '13757', '60615', '10994', '9138', '34119', '58436', '64407', '75170', '73524', '51864', '94183', '86847', '15585', '57616', '96267', '5340', '52929', '49096', '50291', '5559', '32382', '84077', '6598', '87921', '59719', '31726', '44772', '63373', '75420', '66829', '47275', '98264', '61387', '94945', '44540', '50098', '13078', '44729', '95332', '63555', '30782', '63203', '15071', '60996', '72812', '17418', '80215', '37610', '30670', '44674', '74822', '15471', '25236', '16266', '76213', '35820', '19567', '8715', '72003', '90606', '1434', '53545', '88170', '75014', '62287', '35436', '38669', '12927', '83877', '38622', '28313', '82884', '73969', '38671', '10450', '24158', '22941', '73162', '86548', '42482', '95315', '92016', '96156', '44012', '35962', '6366', '3881', '74300', '26248', '30182', '19164', '67105', '66771', '52587', '69894', '61820', '16551', '50743', '10096', '69030', '24451', '89165', '23929', '96291', '30685', '64413', '19913', '9049', '71383', '61684', '45384', '45927', '81840', '49521', '89594', '30055', '83430', '14930', '60316', '86585', '99375', '80170', '14207', '19584', '20067', '82874', '30159', '46647', '6942', '66777', '32638', '55662', '75470', '77622', '26893', '96149', '14373', '33252', '50574', '7945', '20696', '56662', '94348', '3384', '20956', '89668', '99052', '65131', '56847', '17589', '16419', '2670', '10705', '59587', '92902', '92424', '48570', '11034', '69149', '35733', '17315', '84966', '69353', '69590', '52834', '32561', '6049', '50156', '71676', '76423', '32361', '61509', '8845', '75709', '35956', '21912', '31188', '59083', '43459', '38614', '92206', '55645', '38737', '34193', '6451', '94163', '24326', '49976', '71600', '58024', '67160', '4365', '38270', '59558', '80834', '60739', '54318', '19738', '42196', '43191', '13463', '88914', '99239', '66869', '75691', '33085', '4323', '7170', '46184', '41423', '89835', '46877', '20349', '14365', '32727', '35322', '841', '23597', '43370', '57527', '73250', '32553', '71489', '44617', '98323', '37672', '59549', '96023', '63176', '13524', '15621', '30448', '28136', '45549', '3513', '64153', '19839', '24219', '41987', '51083', '90268', '52052', '31430', '4727', '99409', '43595', '82374', '61251', '51470', '66562', '98724', '23529', '53895', '67562', '87573', '89964', '30821', '15733', '33062', '86963', '33450', '75338', '32570', '14453', '38080', '36335', '84226', '52790', '42883', '61156', '42789', '57846', '60096', '29946', '80178', '15882', '1971', '60722', '62458', '8754', '59991', '89321', '584', '70565', '36458', '21226', '23561', '9837', '39364', '23065', '30675', '9306', '40085', '52082', '89976', '73283', '77851', '36174', '54470', '63250', '72111', '70853', '26723', '42590', '91230', '47512', '13983', '70898', '70927', '40721', '30642', '41628', '90010', '27306', '1933', '43304', '44499', '87890', '22201', '89249', '63935', '48438', '58588', '1061', '70061', '63075', '9676', '65820', '82156', '82668', '111', '54350', '10328', '23466', '98936', '18285', '53919', '32422', '84859', '58387', '24022', '32423', '6010', '56417', '49452', '69999', '14885', '47102', '59577', '24999', '75984', '96464', '59088', '85987', '71442', '88789', '4753', '8229', '76883', '15284', '90610', '40507', '78882', '55575', '25315', '7214', '70602', '4796', '35767', '54657', '42153', '16050', '93607', '99249', '77236', '59949', '52871', '47837', '33534', '30023', '89137', '99938', '35824', '50775', '30282', '82798', '53312', '65277', '68375', '91445', '58166', '43344', '6589', '82515', '34632', '78588', '152', '67554', '15877', '74334', '32783', '45147', '39483', '92067', '59029', '38298', '55229', '28268', '85140', '33451', '15424', '46695', '23201', '83329', '28372', '19518', '89198', '33305', '43892', '470', '37662', '9407', '14376', '80310', '21459', '72381', '80414', '88305', '69073', '63101', '91054', '47190', '48595', '24696', '41426', '35133', '94399', '21790', '55040', '73279', '20809', '67805', '94115', '58633', '78053', '89444', '4112', '8', '34517', '22106', '85934', '86814', '53333', '93437', '85062', '32791', '72744', '99843', '51161', '22730', '34908', '82918', '92566', '22467', '41226', '98518', '29235', '94042', '84371', '79100', '25214', '7764', '59427', '47891', '61092', '23775', '13641', '30837', '77377', '43032', '38441', '29462', '20300', '19070', '20982', '73987', '87836', '68062', '6419', '51563', '40084', '85694', '86677', '47142', '27222', '17844', '19158', '45120', '88524', '74724', '73229', '42470', '38751', '1132', '28603', '61188', '55021', '88825', '58005', '62411', '8843', '94852', '93664', '39253', '27473', '247', '43824', '1804', '8905', '11509', '95659', '7811', '80691', '15779', '49794', '8991', '76099', '29223', '36060', '85399', '41369', '22885', '38473', '22376', '50446', '89578', '25818', '61333', '78787', '47605', '83654', '99068', '52120', '48367', '86381', '19803', '72600', '31998', '37755', '88031', '83969', '42319', '27974', '35780', '93662', '46808', '60529', '15491', '10447', '48829', '33886', '68333', '44855', '86554', '64794', '66376', '58222', '14021', '52043', '56375', '1300', '38105', '89159', '97456', '26800', '93124', '3673', '32279', '30658', '84475', '3708', '93952', '39245', '91980', '55333', '79440', '64407', '46559', '60759', '10688', '49872', '45810', '87405', '66932', '56530', '57751', '9619', '27361', '6356', '65848', '7524', '20273', '22362', '20504', '28042', '39475', '51677', '85733', '32426', '54558', '17222', '56485', '34928', '90917', '70528', '51732', '61014', '98420', '67265', '41383', '3883', '47642', '53324', '93679', '93088', '57534', '44449', '46779', '81482', '54279', '80135', '11216', '92545', '18426', '96005', '57801', '21898', '5104', '83467', '72015', '43783', '89674', '57468', '96686', '95167', '38507', '95187', '64923', '71214', '42834', '93219', '47342', '24476', '84834', '29080', '86533', '30687', '68400', '26933', '37396', '65169', '89767', '20642', '53843', '85167', '77306', '46723', '68501', '4243', '35044', '15950', '40388', '53630', '76125', '10816', '83285', '4120', '11402', '91344', '95169') - - SELECT count() FROM numbers(10000) WHERE (number, number) IN ((17258, 93148), (4508, 52749), (68660, 70017), (77797, 23528), (1136, 37393), (53237, 15379), (68370, 73211), (15782, 54962), (59432, 45415), (68396, 920), (96154, 21016), (12700, 26887), (88016, 43191), (68153, 51575), (91315, 40005), (18070, 73178), (86, 631), (77717, 20324), (3227, 76188), (74960, 43147), (77538, 19628), (82292, 6525), (24293, 12566), (85244, 96287), (93982, 1329), (38064, 54723), (83999, 45810), (71921, 53673), (88638, 9669), (1959, 39535), (82235, 95796), (27907, 90975), (42383, 91015), (9948, 91514), (81712, 47309), (400, 25808), (31791, 46948), (39740, 36098), (25943, 84598), (99598, 52939), (77134, 15845), (40313, 72174), (85017, 94036), (36595, 14303), (83961, 68078), (55792, 72759), (73574, 43606), (9853, 63560), (28580, 56721), (74804, 41025), (32095, 55657), (52881, 63416), (91368, 90310), (23922, 38883), (30592, 10758), (66448, 61183), (31880, 96697), (11362, 20633), (75331, 2015), (71129, 8785), (1115, 70955), (7886, 83698), (18961, 84556), (16677, 43028), (37347, 70220), (31699, 71244), (10578, 96159), (67600, 39041), (78791, 86687), (21545, 54174), (68774, 37637), (46132, 81768), (98413, 20605), (2960, 23665), (31507, 35719), (96209, 18368), (60558, 38035), (21952, 3264), (11834, 86458), (21651, 17650), (86276, 36087), (18818, 24849), (61951, 3390), (59637, 62545), (30346, 72253), (36281, 2992), (78340, 49872), (94326, 93723), (3416, 94405), (12272, 8741), (22600, 22095), (57636, 37106), (38702, 14889), (70238, 11276), (17325, 60648), (16492, 41271), (52100, 1304), (93416, 7795), (57209, 71008), (48010, 36078), (20384, 74420), (77440, 34439), (69224, 45099), (30374, 33884), (49038, 90140), (1154, 84725), (64926, 86985), (91746, 73472), (59757, 75755), (45860, 71557), (45833, 36526), (74618, 73598), (91360, 65168), (58029, 30793), (56332, 14973), (99943, 96877), (97454, 6450), (64502, 77301), (73182, 31853), (76809, 83964), (82916, 86188), (78736, 65427), (36495, 7422), (76196, 2804), (96117, 61093), (9177, 26099), (52942, 63007), (48578, 47876), (50638, 89903), (7113, 97316), (35301, 12750), (47807, 7254), (38217, 55418), (56970, 41687), (20527, 62886), (358, 14021), (64018, 18582), (91740, 21683), (81967, 53589), (45437, 38450), (45476, 67752), (76851, 72072), (7304, 60091), (40097, 12897), (39906, 29247), (84262, 58734), (30857, 43791), (56087, 78929), (20498, 45954), (48726, 500), (62723, 43763), (28368, 30756), (74048, 52403), (15045, 95926), (75542, 55384), (52543, 22525), (56001, 6935), (11431, 46745), (77731, 7310), (36718, 59909), (32235, 91254), (92417, 25917), (21782, 79277), (46378, 87536), (35324, 26075), (6310, 76915), (1551, 69473), (50642, 68865), (55190, 72934), (49780, 21873), (99466, 29686), (90761, 13179), (72959, 57033), (20020, 90200), (46186, 79105), (73871, 52382), (59559, 38801), (59916, 16082), (33610, 94966), (46001, 45225), (86679, 26469), (77245, 91929), (32887, 36623), (11179, 46898), (87881, 68087), (45438, 47991), (24950, 94525), (91664, 51656), (43914, 47805), (15736, 96156), (56346, 20283), (85053, 48931), (17790, 26179), (96195, 55728), (43765, 54807), (44988, 89269), (55911, 99411), (52446, 47397), (28346, 65442), (96669, 68226), (66194, 26848), (37276, 55864), (14116, 41583), (18058, 16317), (93136, 85318), (35616, 86252), (29222, 29969), (33386, 85372), (71094, 44238), (27733, 31838), (64626, 16692), (52904, 97899), (97619, 12663), (50165, 4688), (67557, 44053), (69184, 66269), (73164, 89705), (39822, 15169), (65499, 72808), (30068, 63697), (30154, 64235), (97016, 58716), (94366, 36592), (1592, 16261), (87985, 52102), (12554, 23652), (15909, 25292), (2527, 91531), (92139, 36031), (28986, 30032), (3038, 56314), (32239, 26707), (15973, 34901), (70246, 39680), (82529, 38132), (45827, 74783), (53665, 64111), (55218, 84170), (20466, 16130), (55734, 71203), (31438, 96906), (66338, 85858), (35988, 68511), (78391, 15191), (80747, 59213), (5357, 11546), (16822, 16607), (36607, 41106), (74949, 30739), (45726, 64887), (1524, 54847), (37371, 89195), (28726, 27788), (22600, 44777), (53999, 63625), (84304, 98338), (49260, 76480), (74564, 53907), (89867, 97096), (60157, 61299), (17165, 10146), (56334, 36268), (62114, 49222), (22715, 23620), (42830, 11539), (41091, 69151), (75471, 68364), (18681, 43249), (42738, 63219), (35474, 98454), (76815, 46024), (66310, 36521), (86095, 77013), (63693, 77319), (80731, 63031), (95478, 92387), (23787, 63724), (46299, 68994), (4800, 2460), (9663, 80639), (77231, 85814), (81615, 11311), (35638, 27340), (13598, 14322), (30657, 17238), (90957, 96846), (69962, 52140), (41681, 65962), (96836, 58177), (36190, 11623), (4231, 40500), (43049, 41949), (71177, 98492), (30193, 39750), (19744, 33204), (63358, 30210), (45638, 58918), (43641, 38741), (35598, 40932), (33238, 36236), (50835, 20968), (25099, 34071), (84986, 88456), (35333, 1529), (79771, 23985), (647, 61658), (9424, 11743), (77766, 31528), (77811, 86973), (76403, 74377), (55568, 79251), (68858, 20762), (68520, 66773), (93598, 89823), (8080, 82539), (87760, 52247), (25191, 16905), (17837, 8339), (85177, 59050), (51680, 77374), (3287, 43018), (43479, 62141), (34909, 46322), (11869, 5885), (96193, 58417), (101, 47460), (34937, 88582), (83216, 88388), (28571, 15292), (66683, 62613), (34478, 8924), (2680, 89973), (62438, 44460), (11724, 4791), (5383, 72888), (88206, 67586), (8124, 21690), (28779, 75789), (66791, 4757), (6176, 47760), (6403, 78084), (78122, 35446), (99494, 73608), (39691, 89098), (59182, 19484), (25389, 98963), (96487, 3692), (76222, 67381), (21199, 50358), (95998, 58137), (28777, 43913), (14176, 60117), (52257, 81703), (14604, 13438), (71301, 14401), (19758, 66914), (15506, 29873), (87205, 29449), (93295, 15930), (63651, 11287), (19785, 15966), (30795, 75112), (69462, 37655), (18793, 85764), (36240, 31236), (98153, 73724), (72491, 4223), (66930, 35048), (25686, 13269), (13940, 13259), (69163, 11235), (1183, 86961), (54323, 67315), (85044, 60872), (48875, 3683), (43052, 92861), (87574, 32969), (92552, 80564), (94832, 47682), (72011, 80994), (60182, 917), (97788, 34169), (66432, 47940), (87468, 80954), (35385, 68758), (50555, 63710), (55311, 44337), (87065, 26514), (84581, 98736), (23212, 56499), (75120, 72447), (56087, 38285), (58171, 45629), (28401, 44319), (70432, 27883), (18891, 14646), (26206, 49924), (79957, 44914), (56064, 27529), (99090, 29197), (49435, 340), (53525, 65601), (76998, 88349), (50416, 70860), (42506, 75290), (34024, 13295), (86663, 46523), (88814, 231), (57809, 21), (84914, 84771), (43042, 66892), (17288, 33908), (4934, 63195), (50590, 1516), (97843, 80208), (20091, 86717), (71566, 15929), (19531, 23634), (41646, 45549), (89226, 82902), (96683, 63386), (31072, 53788), (51135, 41099), (78912, 65609), (36094, 23603), (88403, 51455), (73795, 47066), (26448, 82852), (22829, 2894), (30041, 92548), (27733, 20608), (70180, 19892), (51650, 63440), (76328, 13666), (40514, 6677), (2786, 51059), (40809, 16499), (10857, 82541), (78221, 61067), (17982, 51969), (85369, 66965), (47153, 47149), (43965, 75796), (82725, 60767), (42407, 97249), (51475, 81224), (60957, 89414), (33065, 21663), (36601, 5290), (95842, 67301), (64630, 60398), (55212, 35638), (41750, 44235), (75260, 82400), (91291, 25843), (6477, 8311), (14919, 52306), (66220, 33180), (45736, 2313), (37450, 64444), (98614, 61344), (75007, 50946), (56701, 28117), (66632, 5174), (92323, 76613), (6796, 73695), (33696, 76280), (86876, 5614), (50863, 67993), (36068, 17049), (91912, 34271), (70706, 1904), (97798, 41117), (68154, 72483), (83862, 25578), (61643, 17204), (69974, 64232), (77926, 19637), (64901, 88988), (71424, 91703), (91655, 17147), (46872, 56530), (44189, 98087), (95939, 54420), (72651, 68785), (67624, 84875), (92587, 87663), (65275, 81256), (53798, 2506), (14702, 3638), (71291, 50452), (14909, 13903), (66965, 26606), (14127, 60345), (35306, 1738), (77234, 10468), (53521, 41218), (80681, 82583), (44227, 26521), (32263, 21482), (82270, 56963), (50580, 80567), (11593, 22346), (20074, 26867), (73126, 28667), (62996, 24317), (20295, 57163), (1506, 57668), (69567, 45236), (43366, 26001), (88052, 40181), (1599, 89349), (36789, 1579), (39895, 46673), (30381, 3206), (31723, 5625), (19252, 31317), (16932, 77149), (48794, 34409), (55986, 30328), (47551, 75088), (57363, 78365), (95221, 63385), (26449, 5733), (96588, 53077), (52980, 41140), (8187, 85947), (36723, 26520), (23579, 38909), (33350, 19275), (63930, 19357), (43536, 59941), (31117, 77322), (44638, 94812), (44730, 99097), (95108, 48170), (57813, 49503), (79959, 89436), (86980, 62031), (8275, 44009), (36666, 94645), (22064, 38882), (40471, 16939), (31156, 11337), (13101, 96977), (17906, 26835), (89861, 51405), (73369, 67946), (99141, 58572), (27131, 98703), (15900, 43412), (51768, 93125), (78579, 46689), (23029, 13895), (60870, 55830), (22553, 8236), (76449, 96207), (83766, 51024), (27630, 50614), (53484, 90104), (77626, 21944), (46755, 41583), (53616, 34240), (94159, 44415), (13914, 90059), (44387, 89012), (27499, 64579), (83415, 30809), (77558, 82619), (88880, 9814), (8466, 4424), (43598, 91921), (24695, 3349), (46295, 65208), (51256, 82461), (49126, 93012), (16186, 96585), (43284, 22655), (93130, 90393), (77495, 34372), (85509, 65856), (86662, 61906), (50988, 44393), (29828, 17737), (91651, 35308), (29796, 49716), (14019, 87751), (29688, 71207), (82845, 19100), (11989, 50132), (21158, 99905), (54732, 42547), (32314, 12851), (46405, 43794), (87849, 45643), (53524, 21212), (61925, 75491), (12498, 21937), (30185, 69475), (48421, 52487), (15112, 90935), (33187, 17801), (61704, 25514), (17889, 23917), (18758, 57197), (7693, 47232), (47905, 24618), (11494, 78950), (95662, 54561), (8075, 33909), (90427, 46065), (73962, 19821), (50691, 79400), (58218, 4881), (94106, 2509), (60633, 55169), (49600, 83054), (23339, 13270), (70262, 58946), (48417, 97266), (27629, 46905), (74465, 75514), (41687, 2564), (12814, 19492), (78899, 30168), (17745, 35206), (37972, 35296), (22288, 80001), + + SELECT (rand(), rand()) IN ((17258, 93148), (4508, 52749), (68660, 70017), (77797, 23528), (1136, 37393), (53237, 15379), (68370, 73211), (15782, 54962), (59432, 45415), (68396, 920), (96154, 21016), (12700, 26887), (88016, 43191), (68153, 51575), (91315, 40005), (18070, 73178), (86, 631), (77717, 20324), (3227, 76188), (74960, 43147), (77538, 19628), (82292, 6525), (24293, 12566), (85244, 96287), (93982, 1329), (38064, 54723), (83999, 45810), (71921, 53673), (88638, 9669), (1959, 39535), (82235, 95796), (27907, 90975), (42383, 91015), (9948, 91514), (81712, 47309), (400, 25808), (31791, 46948), (39740, 36098), (25943, 84598), (99598, 52939), (77134, 15845), (40313, 72174), (85017, 94036), (36595, 14303), (83961, 68078), (55792, 72759), (73574, 43606), (9853, 63560), (28580, 56721), (74804, 41025), (32095, 55657), (52881, 63416), (91368, 90310), (23922, 38883), (30592, 10758), (66448, 61183), (31880, 96697), (11362, 20633), (75331, 2015), (71129, 8785), (1115, 70955), (7886, 83698), (18961, 84556), (16677, 43028), (37347, 70220), (31699, 71244), (10578, 96159), (67600, 39041), (78791, 86687), (21545, 54174), (68774, 37637), (46132, 81768), (98413, 20605), (2960, 23665), (31507, 35719), (96209, 18368), (60558, 38035), (21952, 3264), (11834, 86458), (21651, 17650), (86276, 36087), (18818, 24849), (61951, 3390), (59637, 62545), (30346, 72253), (36281, 2992), (78340, 49872), (94326, 93723), (3416, 94405), (12272, 8741), (22600, 22095), (57636, 37106), (38702, 14889), (70238, 11276), (17325, 60648), (16492, 41271), (52100, 1304), (93416, 7795), (57209, 71008), (48010, 36078), (20384, 74420), (77440, 34439), (69224, 45099), (30374, 33884), (49038, 90140), (1154, 84725), (64926, 86985), (91746, 73472), (59757, 75755), (45860, 71557), (45833, 36526), (74618, 73598), (91360, 65168), (58029, 30793), (56332, 14973), (99943, 96877), (97454, 6450), (64502, 77301), (73182, 31853), (76809, 83964), (82916, 86188), (78736, 65427), (36495, 7422), (76196, 2804), (96117, 61093), (9177, 26099), (52942, 63007), (48578, 47876), (50638, 89903), (7113, 97316), (35301, 12750), (47807, 7254), (38217, 55418), (56970, 41687), (20527, 62886), (358, 14021), (64018, 18582), (91740, 21683), (81967, 53589), (45437, 38450), (45476, 67752), (76851, 72072), (7304, 60091), (40097, 12897), (39906, 29247), (84262, 58734), (30857, 43791), (56087, 78929), (20498, 45954), (48726, 500), (62723, 43763), (28368, 30756), (74048, 52403), (15045, 95926), (75542, 55384), (52543, 22525), (56001, 6935), (11431, 46745), (77731, 7310), (36718, 59909), (32235, 91254), (92417, 25917), (21782, 79277), (46378, 87536), (35324, 26075), (6310, 76915), (1551, 69473), (50642, 68865), (55190, 72934), (49780, 21873), (99466, 29686), (90761, 13179), (72959, 57033), (20020, 90200), (46186, 79105), (73871, 52382), (59559, 38801), (59916, 16082), (33610, 94966), (46001, 45225), (86679, 26469), (77245, 91929), (32887, 36623), (11179, 46898), (87881, 68087), (45438, 47991), (24950, 94525), (91664, 51656), (43914, 47805), (15736, 96156), (56346, 20283), (85053, 48931), (17790, 26179), (96195, 55728), (43765, 54807), (44988, 89269), (55911, 99411), (52446, 47397), (28346, 65442), (96669, 68226), (66194, 26848), (37276, 55864), (14116, 41583), (18058, 16317), (93136, 85318), (35616, 86252), (29222, 29969), (33386, 85372), (71094, 44238), (27733, 31838), (64626, 16692), (52904, 97899), (97619, 12663), (50165, 4688), (67557, 44053), (69184, 66269), (73164, 89705), (39822, 15169), (65499, 72808), (30068, 63697), (30154, 64235), (97016, 58716), (94366, 36592), (1592, 16261), (87985, 52102), (12554, 23652), (15909, 25292), (2527, 91531), (92139, 36031), (28986, 30032), (3038, 56314), (32239, 26707), (15973, 34901), (70246, 39680), (82529, 38132), (45827, 74783), (53665, 64111), (55218, 84170), (20466, 16130), (55734, 71203), (31438, 96906), (66338, 85858), (35988, 68511), (78391, 15191), (80747, 59213), (5357, 11546), (16822, 16607), (36607, 41106), (74949, 30739), (45726, 64887), (1524, 54847), (37371, 89195), (28726, 27788), (22600, 44777), (53999, 63625), (84304, 98338), (49260, 76480), (74564, 53907), (89867, 97096), (60157, 61299), (17165, 10146), (56334, 36268), (62114, 49222), (22715, 23620), (42830, 11539), (41091, 69151), (75471, 68364), (18681, 43249), (42738, 63219), (35474, 98454), (76815, 46024), (66310, 36521), (86095, 77013), (63693, 77319), (80731, 63031), (95478, 92387), (23787, 63724), (46299, 68994), (4800, 2460), (9663, 80639), (77231, 85814), (81615, 11311), (35638, 27340), (13598, 14322), (30657, 17238), (90957, 96846), (69962, 52140), (41681, 65962), (96836, 58177), (36190, 11623), (4231, 40500), (43049, 41949), (71177, 98492), (30193, 39750), (19744, 33204), (63358, 30210), (45638, 58918), (43641, 38741), (35598, 40932), (33238, 36236), (50835, 20968), (25099, 34071), (84986, 88456), (35333, 1529), (79771, 23985), (647, 61658), (9424, 11743), (77766, 31528), (77811, 86973), (76403, 74377), (55568, 79251), (68858, 20762), (68520, 66773), (93598, 89823), (8080, 82539), (87760, 52247), (25191, 16905), (17837, 8339), (85177, 59050), (51680, 77374), (3287, 43018), (43479, 62141), (34909, 46322), (11869, 5885), (96193, 58417), (101, 47460), (34937, 88582), (83216, 88388), (28571, 15292), (66683, 62613), (34478, 8924), (2680, 89973), (62438, 44460), (11724, 4791), (5383, 72888), (88206, 67586), (8124, 21690), (28779, 75789), (66791, 4757), (6176, 47760), (6403, 78084), (78122, 35446), (99494, 73608), (39691, 89098), (59182, 19484), (25389, 98963), (96487, 3692), (76222, 67381), (21199, 50358), (95998, 58137), (28777, 43913), (14176, 60117), (52257, 81703), (14604, 13438), (71301, 14401), (19758, 66914), (15506, 29873), (87205, 29449), (93295, 15930), (63651, 11287), (19785, 15966), (30795, 75112), (69462, 37655), (18793, 85764), (36240, 31236), (98153, 73724), (72491, 4223), (66930, 35048), (25686, 13269), (13940, 13259), (69163, 11235), (1183, 86961), (54323, 67315), (85044, 60872), (48875, 3683), (43052, 92861), (87574, 32969), (92552, 80564), (94832, 47682), (72011, 80994), (60182, 917), (97788, 34169), (66432, 47940), (87468, 80954), (35385, 68758), (50555, 63710), (55311, 44337), (87065, 26514), (84581, 98736), (23212, 56499), (75120, 72447), (56087, 38285), (58171, 45629), (28401, 44319), (70432, 27883), (18891, 14646), (26206, 49924), (79957, 44914), (56064, 27529), (99090, 29197), (49435, 340), (53525, 65601), (76998, 88349), (50416, 70860), (42506, 75290), (34024, 13295), (86663, 46523), (88814, 231), (57809, 21), (84914, 84771), (43042, 66892), (17288, 33908), (4934, 63195), (50590, 1516), (97843, 80208), (20091, 86717), (71566, 15929), (19531, 23634), (41646, 45549), (89226, 82902), (96683, 63386), (31072, 53788), (51135, 41099), (78912, 65609), (36094, 23603), (88403, 51455), (73795, 47066), (26448, 82852), (22829, 2894), (30041, 92548), (27733, 20608), (70180, 19892), (51650, 63440), (76328, 13666), (40514, 6677), (2786, 51059), (40809, 16499), (10857, 82541), (78221, 61067), (17982, 51969), (85369, 66965), (47153, 47149), (43965, 75796), (82725, 60767), (42407, 97249), (51475, 81224), (60957, 89414), (33065, 21663), (36601, 5290), (95842, 67301), (64630, 60398), (55212, 35638), (41750, 44235), (75260, 82400), (91291, 25843), (6477, 8311), (14919, 52306), (66220, 33180), (45736, 2313), (37450, 64444), (98614, 61344), (75007, 50946), (56701, 28117), (66632, 5174), (92323, 76613), (6796, 73695), (33696, 76280), (86876, 5614), (50863, 67993), (36068, 17049), (91912, 34271), (70706, 1904), (97798, 41117), (68154, 72483), (83862, 25578), (61643, 17204), (69974, 64232), (77926, 19637), (64901, 88988), (71424, 91703), (91655, 17147), (46872, 56530), (44189, 98087), (95939, 54420), (72651, 68785), (67624, 84875), (92587, 87663), (65275, 81256), (53798, 2506), (14702, 3638), (71291, 50452), (14909, 13903), (66965, 26606), (14127, 60345), (35306, 1738), (77234, 10468), (53521, 41218), (80681, 82583), (44227, 26521), (32263, 21482), (82270, 56963), (50580, 80567), (11593, 22346), (20074, 26867), (73126, 28667), (62996, 24317), (20295, 57163), (1506, 57668), (69567, 45236), (43366, 26001), (88052, 40181), (1599, 89349), (36789, 1579), (39895, 46673), (30381, 3206), (31723, 5625), (19252, 31317), (16932, 77149), (48794, 34409), (55986, 30328), (47551, 75088), (57363, 78365), (95221, 63385), (26449, 5733), (96588, 53077), (52980, 41140), (8187, 85947), (36723, 26520), (23579, 38909), (33350, 19275), (63930, 19357), (43536, 59941), (31117, 77322), (44638, 94812), (44730, 99097), (95108, 48170), (57813, 49503), (79959, 89436), (86980, 62031), (8275, 44009), (36666, 94645), (22064, 38882), (40471, 16939), (31156, 11337), (13101, 96977), (17906, 26835), (89861, 51405), (73369, 67946), (99141, 58572), (27131, 98703), (15900, 43412), (51768, 93125), (78579, 46689), (23029, 13895), (60870, 55830), (22553, 8236), (76449, 96207), (83766, 51024), (27630, 50614), (53484, 90104), (77626, 21944), (46755, 41583), (53616, 34240), (94159, 44415), (13914, 90059), (44387, 89012), (27499, 64579), (83415, 30809), (77558, 82619), (88880, 9814), (8466, 4424), (43598, 91921), (24695, 3349), (46295, 65208), (51256, 82461), (49126, 93012), (16186, 96585), (43284, 22655), (93130, 90393), (77495, 34372), (85509, 65856), (86662, 61906), (50988, 44393), (29828, 17737), (91651, 35308), (29796, 49716), (14019, 87751), (29688, 71207), (82845, 19100), (11989, 50132), (21158, 99905), (54732, 42547), (32314, 12851), (46405, 43794), (87849, 45643), (53524, 21212), (61925, 75491), (12498, 21937), (30185, 69475), (48421, 52487), (15112, 90935), (33187, 17801), (61704, 25514), (17889, 23917), (18758, 57197), (7693, 47232), (47905, 24618), (11494, 78950), (95662, 54561), (8075, 33909), (90427, 46065), (73962, 19821), (50691, 79400), (58218, 4881), (94106, 2509), (60633, 55169), (49600, 83054), (23339, 13270), (70262, 58946), (48417, 97266), (27629, 46905), (74465, 75514), (41687, 2564), (12814, 19492), (78899, 30168), (17745, 35206), (37972, 35296), (22288, 80001), (68026, 36558), (40187, 12234), (92380, 22866), (56488, 64402), (41404, 62562), (47802, 45287), (83302, 85215), (58999, 85776), (35158, 16804), (13416, 94146), (62953, 28243), (83290, 19103), (4564, 21789), (64468, 20927), (25582, 47206), (57810, 18693), (28938, 97986), (61704, 14838), (19214, 3232), (12911, 25438), (85802, 28837), (56506, 89458), (66392, 47773), (68190, 43841), (43044, 52214), (57886, 32830), (15943, 59771), (37081, 89294), (4032, 32960), (46931, 85790), (69656, 72737), (28217, 39872), (86170, 42776), (55116, 51495), (90485, 45274), (60773, 36788), (2193, 2636), (70222, 62086), (75720, 70712), (17549, 51460), (23609, 31515), (70254, 39825), (63762, 11061), (13107, 15394), (45916, 72130), (91558, 86662), (99524, 69106), (93073, 29881), (31724, 3007), (69051, 59452), (59701, 86760), (4967, 82028), (57404, 48226), (71829, 79910), (23714, 62439), (73881, 67618), (63269, 40085), (6164, 23415), (48156, 93907), (18627, 16570), (6676, 22991), (36916, 41488), (99079, 13264), (32533, 99243), (55505, 63339), (89564, 3290), (24886, 34916), (91310, 9343), (49779, 12740), (26320, 3406), (57661, 5702), (10765, 57881), (5518, 47638), (93148, 27438), (73451, 24477), (84075, 96822), (58883, 58883), (96812, 82388), (30659, 59654), (24498, 95808), (25591, 21834), (13090, 87704), (76495, 17249), (75975, 84318), (55459, 70426), (84256, 88604), (79438, 43104), (45331, 7495), (63619, 11123), (24772, 2601), (63343, 14138), (39957, 98339), (55595, 17823), (97676, 53933), (91867, 25023), (64677, 67859), (43737, 34315), (24800, 53968), (93157, 17507), (24264, 35273), (33889, 507), (10207, 40542), (40213, 57800), (38321, 74160), (42391, 7651), (80267, 94736), (52473, 79634), (17075, 2531), (8595, 75890), (31496, 50367), (16069, 79896), (70067, 200), (23420, 49517), (1628, 45646), (8916, 36794), (72294, 88976), (40603, 86008), (91871, 71098), (5447, 70998), (24152, 17561), (65046, 34951), (56950, 9292), (19244, 31385), (74693, 31813), (97343, 21572), (38834, 135), (79717, 62486), (38, 10308), (58035, 71344), (85802, 81079), (5943, 156), (38735, 38867), (3803, 99366), (15853, 19408), (62988, 62008), (8316, 44684), (17035, 71012), (48584, 2117), (75425, 37336), (2405, 50420), (43653, 28836), (12394, 69430), (54522, 4954), (33359, 148), (41018, 82851), (79995, 55417), (65008, 32342), (36547, 88185), (8131, 7054), (38980, 20146), (27976, 63039), (53119, 67009), (40043, 98393), (29333, 51980), (85818, 98405), (77956, 20099), (99747, 16916), (11597, 50181), (40961, 8262), (75103, 13912), (62339, 69155), (3869, 85481), (7053, 30956), (33563, 53272), (96178, 81751), (99365, 88728), (34447, 11164), (62856, 30939), (92486, 3357), (56605, 35330), (42180, 15137), (83946, 62984), (61869, 55711), (52880, 49871), (44588, 27387), (16332, 24496), (1781, 13508), (56674, 95773), (21328, 19628), (96455, 24155), (14302, 74435), (54053, 24590), (86642, 22177), (24089, 16186), (70281, 4601), (18552, 70708), (95442, 5895), (96714, 6293), (43803, 45857), (93257, 18497), (90032, 85086), (40566, 87233), (32674, 73822), (95599, 49334), (62745, 51898), (8245, 93882), (14093, 40977), (47215, 53001), (59737, 68452), (90937, 25354), (43805, 82571), (81953, 68572), (37298, 96262), (94899, 65066), (34772, 80762), (55469, 1186), (8734, 91665), (18622, 51150), (85200, 39575), (65381, 15979), (89734, 89656), (64712, 53691), (87187, 58256), (8476, 89694), (49935, 35239), (63730, 34982), (27687, 91571), (87543, 15350), (85208, 18781), (14783, 2574), (44699, 666), (56440, 87617), (32732, 49301), (76725, 3895), (10419, 90580), (34725, 69476), (14831, 81588), (93924, 38057), (38528, 99060), (57136, 44206), (74685, 99559), (43083, 87511), (43105, 35474), (35582, 17560), (5578, 98727), (78947, 53865), (32013, 95029), (61552, 42674), (52191, 49975), (71566, 16403), (78534, 16350), (18520, 80501), (29114, 46547), (11488, 5069), (89591, 82384), (13741, 42318), (74385, 58849), (49739, 63421), (83821, 6676), (51997, 93321), (36677, 81768), (37915, 73495), (47175, 6086), (39989, 83110), (6489, 48112), (88822, 20370), (12846, 13952), (28930, 20879), (25139, 84552), (76434, 2665), (55145, 31523), (21177, 18630), (81077, 96275), (61006, 30845), (77722, 62651), (61181, 72545), (93838, 84287), (59300, 19014), (75076, 97980), (76979, 1473), (48409, 13097), (51718, 5325), (36522, 72119), (60917, 18995), (61469, 42853), (34387, 37322), (38684, 28120), (64136, 8559), (15368, 99424), (97824, 7864), (33833, 72029), (7024, 9961), (49400, 66220), (63025, 97179), (6135, 98878), (19873, 8438), (3963, 35670), (65186, 89423), (26653, 65943), (83132, 67000), (82578, 35007), (42680, 60479), (71102, 98589), (74842, 94010), (22931, 33725), (46537, 42629), (75793, 48115), (21630, 92454), (97993, 81332), (25747, 31814), (91231, 65953), (91981, 12219), (64719, 16254), (60914, 8334), (15887, 96432), (42110, 28837), (7295, 83147), (50334, 7053), (3949, 33594), (1524, 98230), (17265, 98024), (75969, 36232), (89538, 5212), (13444, 55946), (69823, 81848), (32578, 74024), (52018, 98290), (59118, 40186), (61002, 16977), (69537, 44780), (92, 13937), (33715, 42663), (46347, 8312), (86196, 59301), (17128, 85014), (26429, 57682), (45888, 99588), (22750, 96110), (46809, 49251), (24521, 40071), (287, 22115), (11741, 36315), (22742, 17581), (35808, 3110), (98904, 30407), (4584, 13383), (28585, 69669), (94823, 29715), (9551, 36389), (77997, 45746), (49894, 55722), (23415, 69459), (58246, 85144), (74136, 18102), (97366, 85724), (34271, 51601), (47535, 70883), (59443, 90103), (45213, 45811), (62741, 86898), (17324, 50034), (62080, 25193), (89524, 4421), (13476, 51456), (69198, 56718), (58024, 22969), (65210, 67941), (32561, 44881), (62295, 67448), (66135, 95453), (9417, 20443), (82486, 23745), (19185, 99041), (40662, 91714), (3423, 58624), (4512, 74502), (67772, 98023), (69575, 75779), (69107, 62805), (517, 33801), (47406, 7581), (81108, 10546), (12976, 47001), (16742, 83811), (44593, 82124), (52731, 34642), (81725, 20555), (94126, 91919), (24800, 59302), (97253, 39249), (71692, 10769), (88721, 56321), (7019, 69771), (31464, 61774), (29597, 19263), (65557, 31875), (28653, 69636), (58074, 76848), (15906, 80620), (18259, 40193), (99991, 4769), (98935, 99269), (12123, 60124), (20787, 47346), (13526, 33592), (95370, 40350), (17479, 42884), (58368, 83218), (63290, 74406), (97030, 35102), (45298, 27660), (64593, 21262), (76268, 82641), (1107, 44044), (21427, 79959), (85180, 62412), (7359, 1318), (83618, 9762), (1425, 55804), (32874, 97943), (68191, 38742), (41715, 17902), (3771, 15032), (7848, 74950), (33881, 40904), (75295, 26151), (75775, 13760), (90262, 89822), (88169, 18679), (57506, 32356), (94983, 44281), (37385, 37432), (18248, 48162), (45573, 66278), (25277, 72788), (26977, 36778), (26254, 61758), (12860, 48026), (96819, 3339), (13134, 1173), (26822, 53374), (15989, 29698), (11258, 54515), (37866, 34928), (22996, 26577), (39952, 42732), (6754, 70595), (86245, 44669), (47044, 34170), (6789, 45220), (31706, 2090), (42582, 40023), (35147, 46591), (88210, 11307), (53644, 7680), (11280, 91075), (42961, 65122), (40066, 52185), (20050, 6154), (98440, 20393), (88992, 75432), (32386, 66731), (36952, 34149), (18453, 32715), (84413, 10378), (59440, 2374), (45354, 85009), (50382, 66510), (64428, 95401), (9336, 41760), (26317, 91416), (81941, 99504), (26600, 53522), (81069, 40236), (51126, 27911), (97144, 14243), (62738, 50287), (37372, 28962), (12053, 9090), (69492, 95524), (68141, 52931), (17276, 16487), (69227, 25949), (14143, 70193), (7077, 53032), (65463, 74082), (94997, 66496), (80443, 55832), (66796, 5970), (15852, 95662), (81559, 97272), (55851, 18977), (91142, 48976), (91143, 950), (79225, 31004), (61310, 20760), (74541, 90842), (80322, 11630), (84631, 544), (66785, 86591), (25650, 63252), (59635, 18586), (2964, 6741), (37091, 71148), (11984, 43077), (87505, 62049), (61925, 92290), (18808, 3937), (8300, 33268), (70850, 50661), (86024, 73730), (85161, 47116), (50193, 89155), (37773, 40845), (9251, 41688), (6940, 65399), (42479, 95630), (19401, 43102), (48069, 36040), (62760, 95013), (394, 2641), (32567, 29306), (13870, 58835), (98248, 47291), (49803, 4523), (40222, 12883), (53576, 73105), (88265, 23629), (67865, 67875), (33473, 27144), (80219, 53893), (74878, 47341), (78070, 84803), (30003, 5600), (41103, 6145), (83490, 81076), (55059, 66736), (45015, 10239), (79555, 85819), (81808, 34970), (19235, 85480), (91807, 52177), (40887, 87009), (5003, 2687), (64964, 88122), (765, 94893), (93573, 20504), (28854, 38438), (94244, 93475), (72996, 84801), (75427, 81692), (63161, 98637), (18814, 61343), (22863, 60110), (8949, 12694), (19675, 94313), (43857, 74073), (15737, 58218), (48895, 68474), (22220, 92926), (69055, 50282), (40532, 74934), (59062, 66405), (85784, 87704), (58494, 88222), (2260, 20401), (73112, 99666), (46739, 95433), (21179, 85119), (11545, 38801), (59993, 50866), (10086, 4709), (70560, 29611), (27095, 89017), (6896, 2279), (92506, 5013), (48600, 90491), (18782, 54638), (54337, 82734), (52054, 13481), (38297, 56559), (15998, 30591), (89789, 7522), (18149, 28725), (3532, 28625), (70934, 49617), (84599, 55664), (74229, 52269), (55431, 11893), (32807, 72543), (83882, 53025), (11490, 83442), (14844, 88612), (12526, 45953), (906, 2231), (68240, 95612), (18818, 31535), (57774, 91290), (67250, 67400), (77332, 23550), (42332, 57775), (28792, 11539), (19108, 34608), (12399, 38591), (7329, 10740), (84288, 50928), (29461, 17629), (63884, 88489), (47479, 61085), (75357, 57255), (60107, 94046), (32934, 66312), (28615, 42600), (55553, 85213), (57838, 91426), (9783, 11513), (73677, 28821), (75408, 75561), (22995, 59224), (74874, 54145), (18513, 75901), (46440, 69414), (36072, 22263), (60560, 73325), (69967, 93358), (75949, 98634), (3688, 57991), (43482, 94541), (40922, 31011), (57763, 74497), (93576, 96392), (83038, 80656), (47757, 87045), (14061, 53465), (65619, 33775), (11341, 6702), (6249, 87358), (15766, 85937), (13135, 93945), (24495, 95900), (80359, 1739), (15468, 73426), (49240, 44999), (82839, 90808), (87438, 75613), (348, 73144), (99523, 85853), (21557, 70210), (64933, 1672), (38154, 17477), (97136, 67363), (96491, 8038), (97981, 3434), (54372, 27038), (88480, 86675), (21028, 21083), (43197, 4440), (31702, 78290), (66631, 24438), (11482, 17922), (90351, 39503), (46186, 32439), (73828, 6640), (56916, 26029), (62840, 1815), (20281, 28488), (18211, 30043), (65211, 93012), (43614, 58012), (90322, 77343), (64293, 94525), (59489, 39760), (93219, 78440), (74613, 9732), (38085, 19191), (58029, 48186), (88762, 1764), (28627, 21993), (49975, 41225), (70486, 43480), (82764, 96425), (27218, 78327), (17844, 73333), (70463, 37629), (10500, 33826), (97343, 66575), (82833, 51210), (77353, 45073), (27163, 39728), (78076, 46691), (80302, 39342), (77142, 1319), (87403, 80110), (53805, 27786), (50558, 74264), (83146, 31358), (11567, 4438), (30041, 54287), (91731, 18496), (57591, 93894), (72534, 59009), (98064, 59148), (69626, 66615), (20951, 43949), (61960, 68060), (48892, 67918), (61321, 56222), (75424, 77260), (4916, 81929), (68892, 81531), (28096, 28548), (62016, 107), (8593, 12030), (66743, 36772), (60174, 15106), (52844, 1923), (34768, 22065), (88988, 62910), (79214, 2998), (25675, 31376), (69959, 3614), (43885, 31708), (12206, 46548), (69924, 19343), (12984, 38980), (58250, 69438), (2580, 48684), (38112, 37124), (21842, 43150), (59384, 21921), (19908, 46678), (73396, 79529), (8274, 1557), (36975, 65519), (81069, 18712), (13692, 9148), (60617, 84762), (75749, 66154), (80375, 24553), (4257, 47056), (76880, 7687), (40714, 43448), (79112, 74791), (33119, 72730), (17670, 89183), (51614, 3921), (21247, 39857), (86756, 67673), (32792, 70035), (5917, 7197), (1762, 23130), (6455, 63664), (32806, 3729), (60469, 20511), (12522, 15149), (98106, 79338), (84754, 11162), (52058, 17973), (28789, 1521), (32766, 36325), (78914, 40453), (70297, 71854), (9313, 45190), (54559, 66227), (22342, 43860), (44152, 84294), (36913, 93173), (88523, 36338), (82234, 71140), (8328, 22947), (73250, 88125), (74356, 16820), (94472, 37349), (23126, 87806), (40315, 88729), (19935, 19145), (93312, 65719), (8477, 33108), (86660, 69525), (75557, 66964), (60437, 57494), (94419, 42524), (95372, 72274), (49866, 85685), (96808, 39404), (62961, 72507), (38634, 70815), (91379, 42430), (66359, 98699), (24382, 4186), (4003, 77760), (87840, 75265), (57641, 68871), (9773, 15942), (5664, 51289), (47923, 31308), (58632, 82468), (14097, 71829), (1838, 97710), (70433, 11364), (82363, 97879), (25257, 20615), (18249, 6758), (98581, 13639), (3290, 72449), (74546, 79380), (97254, 44448), (80316, 31760), (40516, 94809), (14444, 88981), (9693, 10259), (83795, 95485), (70201, 81014), (66644, 16761), (35529, 82718), (75774, 73476), (80139, 3957), (34803, 80689), (92085, 46499), (97871, 8004), (67369, 11354), (43578, 81596), (94695, 44963), (93741, 41629), (16005, 96652), (37918, 69012), (89832, 56041), (51798, 32386), (89749, 27647), (76279, 7990), (31746, 1346), (40841, 20480), (99942, 24473), (78495, 99194), (13588, 57088), (22183, 42297), (82707, 34435), (45026, 12747), (8000, 93211), (40453, 13025), (44100, 39880), (83900, 56474), (87691, 42802), (82000, 63867), (76627, 84731), (112, 92774), (34749, 97737), (59262, 57169), (95571, 44144), (74310, 68970), (63232, 92744), (53698, 21610), (39969, 75475), (39942, 28713), (81230, 50140), (97953, 96528), (86144, 48041), (96677, 49677), (22051, 48183), (33436, 54784), (5553, 11920), (67057, 17115), (57736, 72309), (8086, 85329), (72623, 94949), (13096, 48992), (63153, 56337), (30462, 1036), (75000, 24048), (62635, 50128), (91480, 83131), (25929, 79809), (96237, 76974), (59645, 20603), (31850, 29754), (91070, 36568), (30191, 33785), (86776, 67259), (49073, 39179), (16121, 73834), (84217, 52951), (95866, 47244), (63326, 73460), (134, 91953), (48189, 86069), (42971, 3700), (28643, 10479), (80151, 7446), (78798, 2655), (39135, 69364), (80244, 24904), (22818, 74964), (26753, 82419), (16858, 5212), (79891, 11215), (785, 46103), (12559, 24617), (73601, 71490), (70342, 7099), (73330, 6665), (11903, 28194), (16375, 37746), (86132, 51788), (90345, 68366), (5464, 78338), (23569, 83141), (17904, 94046), (35868, 60017), (22591, 93373), (70584, 72116), (49331, 34312), (16180, 91286), (58494, 65441), (9336, 52671), (32523, 26734), (40205, 83549), (28810, 96876), (44703, 38944), (46981, 37157), (8582, 7529), (59718, 71700), (62545, 73716), (6531, 23200), (30528, 59720), (57152, 84660), (16232, 67946), (60446, 45983), (68737, 54959), (57795, 73107), (26930, 35938), (9844, 44760), (3716, 79020), (99126, 8264), (66120, 16151), (50616, 25765), (93340, 95875), (34103, 88003), (14879, 99758), (49188, 6087), (89858, 42861), (36730, 72076), (25069, 26403), (98183, 48108), (3229, 5367), (59306, 80078), (61144, 58598), (72600, 98765), (57701, 23177), (10176, 11553), (82964, 13697), (7788, 28538), (39943, 97491), (56261, 17781), (2458, 1892), (6679, 45554), (42171, 66222), (24420, 44115), (35852, 41965), (50196, 49555), (34718, 60734), (6932, 61638), (69472, 56723), (489, 97620), (41335, 90578), (1333, 92787), (97883, 64754), (14208, 22097), (75776, 5938), (67446, 61518), (58743, 45162), (34749, 81243), (71451, 91991), (27804, 41836), (45274, 8039), (17593, 24498), (8801, 38559), (87460, 7109), (50075, 18284), (84043, 82146), (62932, 25018), (89647, 56768), (59920, 80801), (56357, 35142), (97376, 58181), (70715, 91103), (90829, 78985), (29776, 13275), (30546, 42320), (99266, 35340), (21234, 61062), (39239, 10745), (45990, 715), (47047, 6619), (4270, 94575), (90009, 72203), (25629, 2691), (67926, 89112), (46990, 61101), (22355, 69536), (1977, 56723), (54681, 34041), (83819, 7024), (81235, 7093), (16659, 87135), (49384, 32135), (42204, 17362), (90585, 70374), (51255, 1), (31600, 70085), (90189, 95778), (57349, 87789), (83384, 93771), (20718, 15529), (10644, 53591), (84103, 62101), (91340, 48382), (82854, 84420), (12561, 53517), (64835, 45362), (54154, 75841), (46498, 31175), (75035, 49552), (9306, 53403), (68851, 49139), (13463, 42107), (2322, 36695), (55953, 12098), (60656, 80482), (78987, 19632), (31228, 18523), (98972, 80489), (32367, 98405), (25139, 5164), (5692, 60610), (36535, 70097), (80542, 74320), (87984, 46750), (98201, 41341), (35217, 46813), (81795, 69057), (83927, 41032), (60149, 26087), (39954, 48361), (64485, 61448), (87185, 14580), (74559, 93251), (88544, 83366), (74015, 15864), (78623, 69719), (16941, 80710), (16315, 58313), (47277, 59107), (16067, 66290), (63906, 59891), (20754, 67817), (44428, 10652), (95960, 99045), (52163, 26221), (65566, 22057), (26836, 38898), (57107, 79274), (39020, 74857), (53540, 84159), (76646, 44324), (27967, 40171), (28710, 56332), (84036, 28711), (68742, 57241), (40535, 34737), (62681, 85386), (30472, 58405), (85086, 33013), (67059, 47481), (30441, 55098), (97892, 71991), (90296, 42905), (22441, 18863), (19606, 77242), (11206, 58380), (23901, 49962), (84094, 33761), (64400, 28093), (64228, 94543), (71874, 20871), (25385, 73117), (63398, 20999), (77547, 51893), (80783, 65858), (39807, 80754), (10336, 90318), (7826, 55346), (30206, 10711), (94411, 67364), (33509, 14329), (65350, 17006), (65999, 55699), (82753, 61081), (38851, 11896), (15155, 48635), (19985, 75204), (37144, 5344), (26173, 39587), (61111, 30966), (16180, 22987), (60707, 43599), (30136, 74118), (7880, 43857), (97445, 30233), (62700, 24828), (90914, 89452), (64131, 56925), (25259, 39132), (47104, 43950), (93891, 21913), (84573, 91029), (8604, 79858), (33141, 25534), (12468, 90413), (97063, 76359), (80826, 26061), (64013, 99099), (82158, 38882), (25799, 7564), (25477, 69847), (73374, 58520), (48230, 9453), (91424, 72273), (64893, 11750), (46753, 48434), (15974, 94633), (14872, 27027), (14527, 21313), (25660, 64644), (54196, 15138), (6313, 10911), (36168, 47170), (45346, 76), (10305, 60286), (65283, 39977), (21804, 37972), (65389, 86954), (90674, 64458), (15838, 22392), (43540, 42503), (49584, 67828), (56711, 87887), (40075, 73696), (23832, 91552), (39002, 65562), (20451, 64664), (70783, 92171), (29319, 57694), (56217, 44247), (52856, 57873), (80560, 90902), (31068, 11280), (46996, 34739), (57527, 4554), (8410, 25816), (12269, 38319), (88054, 49939), (337, 13231), (56432, 68236), (74841, 21476), (96006, 15712), (87145, 91660), (58090, 55111), (10310, 79789), (5734, 79710), (98992, 69026), (77033, 5734), (43338, 42635), (23898, 28669), (62708, 81652), (41279, 51722), (93444, 26355), (62046, 52199), (71492, 58736), (7379, 62581), (8592, 71885), (75026, 40387), (46696, 3939), (9787, 88907), (86356, 363), (97479, 20358), (77363, 65553), (44036, 22178), (98279, 64612), (3615, 411), (77003, 93018), (41605, 88489), (55992, 83614), (19493, 21633), (34639, 97064), (94602, 89289), (45853, 26299), (84170, 73386), (9221, 51439), (41513, 68166), (37170, 17690), (82511, 59246), (96674, 27574), (99301, 45675), (42716, 41520), (56623, 49130), (84100, 76804), (73855, 97007), (73303, 26912), (37151, 23837), (49190, 97104), (23487, 45628), (87763, 46550), (65111, 92605), (80481, 8151), (83949, 18930), (81749, 27244), (37449, 3023), (28303, 51545), (96441, 93242), (22082, 43254), (35135, 68407), (37712, 48709), (5111, 26774), (15532, 74246), (93605, 83583), (21491, 66472), (38922, 53076), (55455, 54432), (955, 44063), (311, 91630), (53554, 4522), (29927, 65668), (7525, 16035), (44093, 76745), (21481, 78198), (76875, 5306), (56126, 76437), (96534, 16880), (85600, 68336), (4479, 81002), (80414, 11593), (8186, 61147), (5624, 32879), (79312, 20995), (40407, 41512), (91261, 66022), (93228, 75364), (21136, 40111), (92148, 60681), (42549, 7944), (60157, 15040), (63562, 88365), (69056, 72713), (78263, 89223), (3776, 33039), (30042, 59984), (64567, 20977), (24720, 39157), (63582, 75653), (45363, 20249), (58093, 53833), (27918, 93306), (25791, 92686), (15904, 862), (72093, 19257), (64125, 88986), (41717, 27989), (43165, 98675), (76840, 48170), (64508, 3535), (91964, 33435), (96686, 88673), (66648, 64594), (17927, 30539), (73615, 22800), (18580, 48077), (59803, 48202), (76805, 89886), (2744, 52965), (55596, 22519), (35358, 11629), (83029, 80047), (36120, 91930), (26066, 23035), (48857, 14268), (63020, 26197), (60623, 23252), (34911, 72754), (34808, 21593), (64067, 58963), (34509, 8739), (52686, 96405), (98282, 10463), (6495, 64680), (59016, 86968), (33928, 51222), (39609, 84992), (67603, 89875), (14723, 16144), (30751, 46856), (76874, 75024), (89584, 58806), (51278, 4113), (27187, 93483), (80039, 52159), (6132, 25127), (42358, 77498), (33838, 79064), (74147, 76851), (39752, 27366), (44888, 9809), (10887, 4135), (22303, 36417), (58690, 34613), (53998, 74014), (71567, 32438), (65110, 93406), (77365, 41299), (18044, 70636), (77346, 21236), (78408, 245), (57704, 34662), (75258, 64730), (96992, 15533), (56010, 60769), (69163, 4826), (88709, 20725), (33197, 69743), (97169, 83194), (75277, 53343), (14531, 64740), (19997, 4752), (74016, 55946), (55290, 63626), (32533, 32920), (32946, 74610), (12386, 33853), (34825, 35374), (28772, 32716), (17280, 42683), (54184, 34332), (29964, 16203), (65767, 61448), (29133, 35728), (6861, 14160), (65483, 40224), (78335, 76002), (3061, 40615), (11780, 87517), (46135, 73448), (10920, 72592), (15696, 28810), (44154, 64134), (59365, 27248), (76601, 39862), (68264, 30019), (48572, 54575), (59499, 85796), (35064, 23789), (57028, 83545), (33911, 8463), (21827, 67966), (15983, 69649), (13919, 20584), (82742, 67956), (75457, 45767), (55394, 62309), (6099, 67510), (58078, 9594), (24511, 83149), (24781, 79624), (39745, 777), (92023, 40085), (22889, 37179), (17919, 28607), (79865, 72682), (99829, 38190), (21273, 21278), (88299, 23433), (88887, 48163), (62993, 61567), (82107, 84224), (65049, 61245), (75113, 93564), (81562, 7874), (32314, 32313), (3979, 46996), (40558, 93278), (58758, 68163), (40502, 58941), (76961, 65762), (48032, 36117), (64712, 9137), (12092, 56665), (12315, 66581), (20954, 29083), (57317, 48290), (23534, 86828), (4869, 35950), (26993, 24840), (93007, 45049), (18009, 20350), (43053, 71248), (47320, 66119), (50898, 96627), (669, 40018), (89236, 44039), (47375, 63306), (61906, 6658), (2672, 84546), (59778, 72319), (14497, 71952), (42420, 87023), (96465, 46140), (32857, 22772), (4985, 35125), (61918, 28016), (90275, 24406), (49799, 10811), (74137, 63345), (26135, 86306), (92971, 65541), (40134, 95892), (38554, 46307), (48113, 16343), (63990, 66283), (17793, 49570), (21736, 79819), (13831, 27523), (8939, 93929), (96577, 4909), (38583, 32781), (13701, 24436), (43444, 56054), (17166, 32346), (57202, 26264), (82858, 75049), (46317, 95666), (54911, 68161), (3894, 38521), (26456, 30270), (65214, 35331), (41143, 13109), (85441, 48899), (93226, 25027), (77045, 81171), (30345, 79232), (71167, 40854), (58761, 56824), (89047, 85314), (31686, 81947), (74946, 60661), (49903, 13625), (76341, 69067), (46963, 88891), (97223, 5921), (52143, 9828), (17413, 42731), (30236, 93426), (14540, 17652), (52251, 97233), (41581, 30097), (28771, 46426), (36260, 45179), (4068, 16410), (3146, 95055), (5993, 88855), (46103, 30022), (26667, 18756), (54576, 13438), (12800, 11258), (80761, 44979), (59811, 76627), (77917, 87270), (46286, 28657), (30609, 86852), (15200, 28936), (86331, 34195), (98461, 55054), (91760, 62792), (91551, 70192), (96030, 78205), (8254, 27057), (600, 37830), (58635, 65506), (81661, 73708), (11225, 24255), (15830, 9029), (84384, 46190), (31344, 25765), (25670, 30716), (88507, 19484), (28207, 45941), (91874, 15786), (10094, 10934), (38013, 2179), (14558, 36415), (65079, 48850), (65486, 85046), (54958, 60275), (99800, 96623), (68895, 99829), (3708, 75830), (96368, 22631), (99411, 50094), (56888, 3883), (87288, 45604), (64512, 84543), (45565, 14170), (77114, 15132), (31800, 70333), (57775, 40548), (31788, 67511), (51929, 13684), (53736, 81543), (84251, 86303), (63823, 83258), (77539, 61381), (43570, 39418), (79859, 34773), (8595, 64524), (97242, 9283), (15530, 84591), (75535, 65546), (16516, 50162), (58815, 1815), (34897, 82920), (66215, 81262), (81487, 4902), (64039, 25703), (78006, 90468), (3081, 26910), (58159, 4777), (73715, 36375), (69189, 60971), (18169, 39587), (57960, 57668), (6582, 63707), (11155, 47930), (70829, 92266), (6294, 92305), (2188, 6419), (17141, 54972), (60240, 35276), (10788, 29414), (17464, 76377), (3994, 17227), (12039, 24992), (1340, 77467), (1212, 41758), (52186, 80763), (970, 78819), (92897, 68714), (6349, 77016), (22069, 77732), (78209, 72708), (71986, 56770), (8580, 87225), (97505, 63546), (67459, 39771), (50707, 57066), (68226, 54176), (65425, 27407), (57723, 19288), (56974, 90449), (55878, 1264), (46939, 79863), (34868, 4652), (39872, 78482), (92657, 20961), (99690, 28825), (33761, 52922), (73738, 64995), (92092, 3237), (2463, 45045), (43984, 69864), (60146, 5333), (58127, 79082), (84395, 73949), (50818, 68457), (48585, 47420), (60878, 67337), (16573, 30621), (46524, 14168), (87995, 44854), (73143, 77177), (33967, 37276), (95038, 17670), (69022, 16038), (58485, 90526), (1705, 1443), (97969, 40011), (14719, 42770), (8695, 27192), (47546, 51349), (75263, 24419), (25420, 66286), (39198, 41401), (77896, 85583), (28265, 76766), (88836, 48759), (47768, 39582), (65103, 3167), (92171, 85360), (1549, 79296), (71725, 16645), (87349, 29290), (66201, 61712), (43525, 70338), (99025, 63090), (3687, 79963), (63600, 92088), (2480, 1359), (31384, 63603), (29650, 24391), (8552, 82260), (16729, 29139), (26503, 4767), (88945, 19824), (66695, 95696), (84016, 35417), (71521, 22206), (88433, 55606), (66380, 81316), (30573, 36000), (85223, 20494), (99672, 82813), (65500, 78258), (55817, 98414), (43248, 53800), (62787, 21018), (48981, 36258), (41216, 98585), (18576, 18004), (27272, 72860), (76774, 87664), (26737, 11514), (24472, 42538), (5860, 81355), (29066, 10012), (75308, 28561), (23609, 10007), (10007, 19146), (15568, 1487), (80743, 85294), (11207, 90623), (61258, 63879), (34363, 59005), (74884, 2528), (26604, 52738), (33304, 1202), (20381, 18984), (81968, 92425), (4407, 84677), (2112, 79756), (46970, 4367), (36854, 23482), (88346, 75107), (10643, 31806), (21351, 5590), (69317, 53292), (76711, 10085), (70333, 90592), (88818, 822), (23927, 48141), (84710, 33870), (96932, 22686), (5783, 87468), (7785, 11585), (49497, 33764), (13506, 55969), (37840, 78455), (21532, 22292), (97306, 42065), (6579, 40749), (2593, 4995), (81985, 23611), (63888, 98317), (44975, 83777), (57688, 42688), (641, 45787), (7316, 1967), (43837, 18274), (89994, 32770), (4285, 50388), (84699, 41841), (19564, 20683), (76027, 62278), (26140, 11288), (39656, 79954), (16718, 17335), (11583, 21283), (55441, 32178), (6810, 87225), (27191, 54323), (53406, 31512), (48003, 80077), (78497, 29570), (39140, 66619), (12651, 44576), (1761, 88410), (47139, 20766), (39183, 69367), (80479, 23285), (1568, 78535), (18476, 35058), (93551, 81063), (12059, 60021), (23356, 26572), (79975, 35434), (82230, 67436), (20243, 92343), (47809, 10634), (69537, 60167), (3873, 77404), (1136, 27956), (17470, 24156), (35849, 19150), (74760, 37961), (36660, 44448), (36009, 96619), (87110, 84921), (16080, 60637), (36046, 17351), (96403, 99978), (11060, 68629), (36081, 23464), (4684, 11817), (50126, 82936), (55262, 54135), (53717, 66293), (58028, 28065), (92791, 99766), (46266, 77711), (61912, 65782), (38677, 41158), (4001, 46340), (70987, 12784), (14819, 42857), (78985, 99956), (79737, 42497), (55305, 7329), (64103, 24170), (49093, 22115), (2465, 97282), (29009, 15663), (80976, 86477), (16439, 56685), (53482, 15293), (5038, 5991), (67060, 84201), (54350, 38095), (67539, 68292), (26464, 64908), (92909, 12867), (83517, 26474), (76081, 85247), (23250, 66616), (20783, 34330), (43074, 10165), (93968, 70375), (83802, 70820), (19871, 63094), (35699, 36506), (23905, 2401), (27847, 31968), (76714, 44112), (62599, 32720), (10362, 81985), (35708, 2090), (13071, 39035), (71851, 59493), (62833, 48082), (77164, 22804), (6469, 43229), (92051, 3719), (51910, 77689), (91470, 63253), (57914, 57836), (98819, 97813), (35975, 488), (51431, 34061), (45414, 85971), (56563, 93517), (40789, 53103), (9242, 20814), (784, 22584), (8740, 56894), (28504, 378), (8287, 96930), (74232, 97496), (61565, 7904), (9779, 45122), (33767, 48471), (16766, 10722), (47764, 478), (14374, 30099), (89134, 19977), (60860, 93201), (71123, 29840), (57159, 34410), (82411, 99537), (11307, 3733), (70264, 43028), (30418, 19372), (46543, 31506), (33043, 98980), (21137, 63374), (85640, 36957), (6790, 60751), (78771, 43700), (33808, 38263), (27232, 35152), (39925, 5062), (3120, 65621), (39319, 6795), (77468, 94964), (10481, 43009), (24237, 2103), (16837, 55667), (43846, 2874), (78786, 66811), (92185, 62395), (26318, 87942), (6208, 80815), (66952, 71885), (51435, 25450), (21443, 69801), (92554, 81977), (58912, 82288), (59681, 46177), (60397, 65866), (72065, 13318), (2848, 73852), (7577, 83238), (209, 40659), (72103, 15266), (23365, 67286), (14600, 29269), (85541, 63289), (25427, 54812), (22967, 54965), (81525, 27126), (20473, 55455), (84067, 25794), (46798, 79332), (93825, 74677), (447, 5904), (65661, 92916), (54428, 76482), (1025, 34415), (63761, 30038), (93025, 15090), (98807, 93426), (57562, 59615), (84884, 30620), (75066, 71824), (51199, 37934), (95530, 15260), (513, 98278), (62995, 28267), (47535, 69776), (39266, 4696), (14742, 13225), (44268, 16548), (45976, 41680), (99638, 30285), (85609, 5578), (28156, 12884), (76188, 86573), (44639, 15480), (86789, 72636), (18702, 13337), (96638, 59398), (90988, 26909), (85069, 95193), (67262, 38337), (51694, 19659), (93341, 80988), (48733, 88460), (55630, 22866), (96203, 10316), (30644, 68318), (79292, 63136), (60185, 73681), (60474, 19078), (48721, 82811), (19713, 99527), (17537, 55235), (296, 8353), (58691, 72158), (66734, 92490), (87642, 7174), (78285, 35337), (19503, 14273), (10713, 64116), (85966, 98738), (56561, 99347), (14869, 89963), (95126, 30748), (89272, 79060), (69888, 7193), (32583, 74564), (95542, 52128), (78360, 42675), (86062, 68404), (38732, 21411), (92935, 45415), (99027, 83925), (73232, 37405), From 7f09bb82642ad72371ec461d7ba094df94e4dcca Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 10 Jun 2020 12:52:41 +0200 Subject: [PATCH 28/29] Replase back/forward quotes and apostrophes by straight --- .../table-engines/special/distributed.md | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index bc2a4dc483d..1c4c6299b51 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -10,7 +10,7 @@ Reading is automatically parallelized. During a read, the table indexes on remot The Distributed engine accepts parameters: -- the cluster name in the server’s config file +- the cluster name in the server's config file - the name of a remote database @@ -31,13 +31,13 @@ Example: Distributed(logs, default, hits[, sharding_key[, policy_name]]) ``` -Data will be read from all servers in the ‘logs’ cluster, from the default.hits table located on every server in the cluster. +Data will be read from all servers in the `logs` cluster, from the default.hits table located on every server in the cluster. Data is not only read but is partially processed on the remote servers (to the extent that this is possible). For example, for a query with GROUP BY, data will be aggregated on remote servers, and the intermediate states of aggregate functions will be sent to the requestor server. Then data will be further aggregated. Instead of the database name, you can use a constant expression that returns a string. For example: currentDatabase(). -logs – The cluster name in the server’s config file. +logs – The cluster name in the server's config file. Clusters are set like this: @@ -75,15 +75,15 @@ Clusters are set like this: ``` -Here a cluster is defined with the name ‘logs’ that consists of two shards, each of which contains two replicas. +Here a cluster is defined with the name `logs` that consists of two shards, each of which contains two replicas. Shards refer to the servers that contain different parts of the data (in order to read all the data, you must access all the shards). Replicas are duplicating servers (in order to read all the data, you can access the data on any one of the replicas). Cluster names must not contain dots. The parameters `host`, `port`, and optionally `user`, `password`, `secure`, `compression` are specified for each server: -- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn’t start. If you change the DNS record, restart the server. -- `port` – The TCP port for messenger activity (‘tcp\_port’ in the config, usually set to 9000). Do not confuse it with http\_port. +- `host` – The address of the remote server. You can use either the domain or the IPv4 or IPv6 address. If you specify the domain, the server makes a DNS request when it starts, and the result is stored as long as the server is running. If the DNS request fails, the server doesn't start. If you change the DNS record, restart the server. +- `port` – The TCP port for messenger activity (`tcp_port` in the config, usually set to 9000). Do not confuse it with http\_port. - `user` – Name of the user for connecting to a remote server. Default value: default. This user must have access to connect to the specified server. Access is configured in the users.xml file. For more information, see the section [Access rights](../../../operations/access-rights.md). - `password` – The password for connecting to a remote server (not masked). Default value: empty string. - `secure` - Use ssl for connection, usually you also should define `port` = 9440. Server should listen on `9440` and have correct certificates. @@ -97,44 +97,44 @@ You can specify just one of the shards (in this case, query processing should be You can specify as many clusters as you wish in the configuration. -To view your clusters, use the ‘system.clusters’ table. +To view your clusters, use the `system.clusters` table. -The Distributed engine allows working with a cluster like a local server. However, the cluster is inextensible: you must write its configuration in the server config file (even better, for all the cluster’s servers). +The Distributed engine allows working with a cluster like a local server. However, the cluster is inextensible: you must write its configuration in the server config file (even better, for all the cluster's servers). -The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don’t need to create a Distributed table – use the ‘remote’ table function instead. See the section [Table functions](../../../sql-reference/table-functions/index.md). +The Distributed engine requires writing clusters to the config file. Clusters from the config file are updated on the fly, without restarting the server. If you need to send a query to an unknown set of shards and replicas each time, you don't need to create a Distributed table – use the `remote` table function instead. See the section [Table functions](../../../sql-reference/table-functions/index.md). There are two methods for writing data to a cluster: -First, you can define which servers to write which data to and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table “looks at”. This is the most flexible solution as you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. This is also the most optimal solution since data can be written to different shards completely independently. +First, you can define which servers to write which data to and perform the write directly on each shard. In other words, perform INSERT in the tables that the distributed table "looks at". This is the most flexible solution as you can use any sharding scheme, which could be non-trivial due to the requirements of the subject area. This is also the most optimal solution since data can be written to different shards completely independently. -Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across the servers itself. In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it doesn’t mean anything in this case. +Second, you can perform INSERT in a Distributed table. In this case, the table will distribute the inserted data across the servers itself. In order to write to a Distributed table, it must have a sharding key set (the last parameter). In addition, if there is only one shard, the write operation works without specifying the sharding key, since it doesn't mean anything in this case. Each shard can have a weight defined in the config file. By default, the weight is equal to one. Data is distributed across shards in the amount proportional to the shard weight. For example, if there are two shards and the first has a weight of 9 while the second has a weight of 10, the first will be sent 9 / 19 parts of the rows, and the second will be sent 10 / 19. -Each shard can have the ‘internal\_replication’ parameter defined in the config file. +Each shard can have the `internal_replication` parameter defined in the config file. -If this parameter is set to ‘true’, the write operation selects the first healthy replica and writes data to it. Use this alternative if the Distributed table “looks at” replicated tables. In other words, if the table where data will be written is going to replicate them itself. +If this parameter is set to `true`, the write operation selects the first healthy replica and writes data to it. Use this alternative if the Distributed table "looks at" replicated tables. In other words, if the table where data will be written is going to replicate them itself. -If it is set to ‘false’ (the default), data is written to all replicas. In essence, this means that the Distributed table replicates data itself. This is worse than using replicated tables, because the consistency of replicas is not checked, and over time they will contain slightly different data. +If it is set to `false` (the default), data is written to all replicas. In essence, this means that the Distributed table replicates data itself. This is worse than using replicated tables, because the consistency of replicas is not checked, and over time they will contain slightly different data. -To select the shard that a row of data is sent to, the sharding expression is analyzed, and its remainder is taken from dividing it by the total weight of the shards. The row is sent to the shard that corresponds to the half-interval of the remainders from ‘prev\_weight’ to ‘prev\_weights + weight’, where ‘prev\_weights’ is the total weight of the shards with the smallest number, and ‘weight’ is the weight of this shard. For example, if there are two shards, and the first has a weight of 9 while the second has a weight of 10, the row will be sent to the first shard for the remainders from the range \[0, 9), and to the second for the remainders from the range \[9, 19). +To select the shard that a row of data is sent to, the sharding expression is analyzed, and its remainder is taken from dividing it by the total weight of the shards. The row is sent to the shard that corresponds to the half-interval of the remainders from `prev_weight` to `prev_weights + weight`, where `prev_weights` is the total weight of the shards with the smallest number, and `weight` is the weight of this shard. For example, if there are two shards, and the first has a weight of 9 while the second has a weight of 10, the row will be sent to the first shard for the remainders from the range \[0, 9), and to the second for the remainders from the range \[9, 19). -The sharding expression can be any expression from constants and table columns that returns an integer. For example, you can use the expression ‘rand()’ for random distribution of data, or ‘UserID’ for distribution by the remainder from dividing the user’s ID (then the data of a single user will reside on a single shard, which simplifies running IN and JOIN by users). If one of the columns is not distributed evenly enough, you can wrap it in a hash function: intHash64(UserID). +The sharding expression can be any expression from constants and table columns that returns an integer. For example, you can use the expression `rand()` for random distribution of data, or `UserID` for distribution by the remainder from dividing the user's ID (then the data of a single user will reside on a single shard, which simplifies running IN and JOIN by users). If one of the columns is not distributed evenly enough, you can wrap it in a hash function: intHash64(UserID). -A simple reminder from the division is a limited solution for sharding and isn’t always appropriate. It works for medium and large volumes of data (dozens of servers), but not for very large volumes of data (hundreds of servers or more). In the latter case, use the sharding scheme required by the subject area, rather than using entries in Distributed tables. +A simple reminder from the division is a limited solution for sharding and isn't always appropriate. It works for medium and large volumes of data (dozens of servers), but not for very large volumes of data (hundreds of servers or more). In the latter case, use the sharding scheme required by the subject area, rather than using entries in Distributed tables. -SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don’t have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. +SELECT queries are sent to all the shards and work regardless of how data is distributed across the shards (they can be distributed completely randomly). When you add a new shard, you don't have to transfer the old data to it. You can write new data with a heavier weight – the data will be distributed slightly unevenly, but queries will work correctly and efficiently. You should be concerned about the sharding scheme in the following cases: - Queries are used that require joining data (IN or JOIN) by a specific key. If data is sharded by this key, you can use local IN or JOIN instead of GLOBAL IN or GLOBAL JOIN, which is much more efficient. -- A large number of servers is used (hundreds or more) with a large number of small queries (queries of individual clients - websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, as we’ve done in Yandex.Metrica, you can set up bi-level sharding: divide the entire cluster into “layers”, where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. Distributed tables are created for each layer, and a single shared distributed table is created for global queries. +- A large number of servers is used (hundreds or more) with a large number of small queries (queries of individual clients - websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, as we've done in Yandex.Metrica, you can set up bi-level sharding: divide the entire cluster into "layers", where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. Distributed tables are created for each layer, and a single shared distributed table is created for global queries. Data is written asynchronously. When inserted in the table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. The period for sending data is managed by the [distributed\_directory\_monitor\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) and [distributed\_directory\_monitor\_max\_sleep\_time\_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) settings. The `Distributed` engine sends each file with inserted data separately, but you can enable batch sending of files with the [distributed\_directory\_monitor\_batch\_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) setting. This setting improves cluster performance by better utilizing local server and network resources. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: `/var/lib/clickhouse/data/database/table/`. The number of threads performing background tasks can be set by [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) setting. -If the server ceased to exist or had a rough restart (for example, after a device failure) after an INSERT to a Distributed table, the inserted data might be lost. If a damaged data part is detected in the table directory, it is transferred to the ‘broken’ subdirectory and no longer used. +If the server ceased to exist or had a rough restart (for example, after a device failure) after an INSERT to a Distributed table, the inserted data might be lost. If a damaged data part is detected in the table directory, it is transferred to the `broken` subdirectory and no longer used. -When the max\_parallel\_replicas option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max\_parallel\_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). +When the `max_parallel_replicas` option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max\_parallel\_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). ## Virtual Columns {#virtual-columns} From 1effc16ae8e6269991177eb33490cf02cce172aa Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 10 Jun 2020 14:05:54 +0300 Subject: [PATCH 29/29] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 955f9d1a5d1..c7fbbb31ad1 100644 --- a/README.md +++ b/README.md @@ -15,5 +15,6 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events +* [ClickHouse Online Meetup (in Russian)](https://events.yandex.ru/events/click-house-onlajn-vs-18-06-2020) on June 18, 2020. * [ClickHouse Workshop in Novosibirsk](https://2020.codefest.ru/lecture/1628) on TBD date. * [Yandex C++ Open-Source Sprints in Moscow](https://events.yandex.ru/events/otkrytyj-kod-v-yandek-28-03-2020) on TBD date.