diff --git a/.clang-format b/.clang-format index 8416ba8ce72..c8b9672dc7d 100644 --- a/.clang-format +++ b/.clang-format @@ -52,7 +52,7 @@ IncludeCategories: ReflowComments: false AlignEscapedNewlinesLeft: false AlignEscapedNewlines: DontAlign -AlignTrailingComments: true +AlignTrailingComments: false # Not changed: AccessModifierOffset: -4 diff --git a/README.md b/README.md index 1014e3f059f..784ca00f5cc 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,3 @@ ClickHouse is an open-source column-oriented database management system that all * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any. * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. -## Upcoming Events - -* [ClickHouse Meetup in Athens](https://www.meetup.com/Athens-Big-Data/events/268379195/) on March 5. diff --git a/base/common/iostream_debug_helpers.h b/base/common/iostream_debug_helpers.h index 51a4b969d8a..4362d832970 100644 --- a/base/common/iostream_debug_helpers.h +++ b/base/common/iostream_debug_helpers.h @@ -80,7 +80,6 @@ dumpImpl(Out & out, T && x) } - /// Tuple, pair template Out & dumpTupleImpl(Out & out, T && x) diff --git a/base/ext/singleton.h b/base/ext/singleton.h deleted file mode 100644 index 8392ae7fc01..00000000000 --- a/base/ext/singleton.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include - -namespace ext -{ - -/** Thread-unsafe singleton. It works simply like a global variable. - * Supports deinitialization. - * - * In most of the cases, you don't need this class. - * Use "Meyers Singleton" instead: static T & instance() { static T x; return x; } - */ - -template -class Singleton -{ -public: - Singleton() - { - if (!instance) - instance = std::make_unique(); - } - - T * operator->() - { - return instance.get(); - } - - static bool isInitialized() - { - return !!instance; - } - - static void reset() - { - instance.reset(); - } - -private: - inline static std::unique_ptr instance{}; -}; - -} diff --git a/base/glibc-compatibility/musl/clock_gettime.c b/base/glibc-compatibility/musl/clock_gettime.c index 574f9b83d15..b0807d5fd15 100644 --- a/base/glibc-compatibility/musl/clock_gettime.c +++ b/base/glibc-compatibility/musl/clock_gettime.c @@ -2,7 +2,6 @@ #include #include #include "atomic.h" -#include "musl_features.h" #include "syscall.h" #ifdef VDSO_CGT_SYM @@ -54,7 +53,7 @@ static void *volatile vdso_func = (void *)cgt_init; #endif -int __clock_gettime(clockid_t clk, struct timespec *ts) +int clock_gettime(clockid_t clk, struct timespec *ts) { int r; @@ -104,5 +103,3 @@ int __clock_gettime(clockid_t clk, struct timespec *ts) return __syscall_ret(r); #endif } - -weak_alias(__clock_gettime, clock_gettime); diff --git a/base/glibc-compatibility/musl/clock_nanosleep.c b/base/glibc-compatibility/musl/clock_nanosleep.c index bf71a5e84ac..7c7a9f78288 100644 --- a/base/glibc-compatibility/musl/clock_nanosleep.c +++ b/base/glibc-compatibility/musl/clock_nanosleep.c @@ -1,10 +1,9 @@ #include #include #include -#include "musl_features.h" #include "syscall.h" -int __clock_nanosleep(clockid_t clk, int flags, const struct timespec * req, struct timespec * rem) +int clock_nanosleep(clockid_t clk, int flags, const struct timespec * req, struct timespec * rem) { if (clk == CLOCK_THREAD_CPUTIME_ID) return EINVAL; @@ -23,5 +22,3 @@ int __clock_nanosleep(clockid_t clk, int flags, const struct timespec * req, str pthread_setcanceltype(old_cancel_type, NULL); return status; } - -weak_alias(__clock_nanosleep, clock_nanosleep); diff --git a/base/glibc-compatibility/musl/musl_features.h b/base/glibc-compatibility/musl/musl_features.h index b656efcf4d6..11be55d68fa 100644 --- a/base/glibc-compatibility/musl/musl_features.h +++ b/base/glibc-compatibility/musl/musl_features.h @@ -2,7 +2,4 @@ #define weak __attribute__((__weak__)) #define hidden __attribute__((__visibility__("hidden"))) -#define weak_alias(old, new) \ - extern __typeof(old) new __attribute__((__weak__, __alias__(#old))) - #define predict_false(x) __builtin_expect(x, 0) diff --git a/base/glibc-compatibility/musl/x86_64/syscall.s b/base/glibc-compatibility/musl/x86_64/syscall.s index c4bee804ffa..8edce455728 100644 --- a/base/glibc-compatibility/musl/x86_64/syscall.s +++ b/base/glibc-compatibility/musl/x86_64/syscall.s @@ -2,6 +2,7 @@ .hidden __syscall .type __syscall,@function __syscall: +.cfi_startproc movq %rdi,%rax movq %rsi,%rdi movq %rdx,%rsi @@ -11,3 +12,4 @@ __syscall: movq 8(%rsp),%r9 syscall ret +.cfi_endproc diff --git a/base/memcpy/impl/FastMemcpy.h b/base/memcpy/impl/FastMemcpy.h index 09dd4256f18..a70a6bfc388 100644 --- a/base/memcpy/impl/FastMemcpy.h +++ b/base/memcpy/impl/FastMemcpy.h @@ -39,7 +39,6 @@ typedef __attribute__((__aligned__(1))) uint32_t uint32_unaligned_t; typedef __attribute__((__aligned__(1))) uint64_t uint64_unaligned_t; - //--------------------------------------------------------------------- // fast copy for different sizes //--------------------------------------------------------------------- @@ -694,4 +693,3 @@ static INLINE void* memcpy_fast(void *destination, const void *source, size_t si #endif - diff --git a/base/mysqlxx/CMakeLists.txt b/base/mysqlxx/CMakeLists.txt index 2d2ad75628d..b85d3371336 100644 --- a/base/mysqlxx/CMakeLists.txt +++ b/base/mysqlxx/CMakeLists.txt @@ -8,6 +8,7 @@ add_library (mysqlxx src/Row.cpp src/Value.cpp src/Pool.cpp + src/PoolFactory.cpp src/PoolWithFailover.cpp include/mysqlxx/Connection.h @@ -15,6 +16,7 @@ add_library (mysqlxx include/mysqlxx/mysqlxx.h include/mysqlxx/Null.h include/mysqlxx/Pool.h + include/mysqlxx/PoolFactory.h include/mysqlxx/PoolWithFailover.h include/mysqlxx/Query.h include/mysqlxx/ResultBase.h diff --git a/base/mysqlxx/include/mysqlxx/Pool.h b/base/mysqlxx/include/mysqlxx/Pool.h index 5261ffab017..b2b04f3bdfc 100644 --- a/base/mysqlxx/include/mysqlxx/Pool.h +++ b/base/mysqlxx/include/mysqlxx/Pool.h @@ -198,6 +198,8 @@ public: return description; } + void removeConnection(Connection * data); + protected: /// Number of MySQL connections which are created at launch. unsigned default_connections; diff --git a/base/mysqlxx/include/mysqlxx/PoolFactory.h b/base/mysqlxx/include/mysqlxx/PoolFactory.h new file mode 100644 index 00000000000..a21c2baca2f --- /dev/null +++ b/base/mysqlxx/include/mysqlxx/PoolFactory.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include + +#include + + +#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS 1 +#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS 16 +#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES 3 + + +namespace mysqlxx +{ + +/* + * PoolFactory.h + * This class is a helper singleton to mutualize connections to MySQL. + */ +class PoolFactory final : private boost::noncopyable +{ +public: + static PoolFactory & instance(); + + PoolFactory(const PoolFactory &) = delete; + + /** Allocates a PoolWithFailover to connect to MySQL. */ + PoolWithFailover Get(const std::string & config_name, + unsigned default_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS, + unsigned max_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS, + size_t max_tries = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES); + + /** Allocates a PoolWithFailover to connect to MySQL. */ + PoolWithFailover Get(const Poco::Util::AbstractConfiguration & config, + const std::string & config_name, + unsigned default_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS, + unsigned max_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS, + size_t max_tries = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES); + + void reset(); + + + ~PoolFactory() = default; + PoolFactory& operator=(const PoolFactory &) = delete; + +private: + PoolFactory(); + + struct Impl; + std::unique_ptr impl; +}; + +} diff --git a/base/mysqlxx/include/mysqlxx/PoolWithFailover.h b/base/mysqlxx/include/mysqlxx/PoolWithFailover.h index 21b27ebd4fe..fe151240fa5 100644 --- a/base/mysqlxx/include/mysqlxx/PoolWithFailover.h +++ b/base/mysqlxx/include/mysqlxx/PoolWithFailover.h @@ -77,6 +77,10 @@ namespace mysqlxx size_t max_tries; /// Mutex for set of replicas. std::mutex mutex; + std::string config_name; + + /// Can the Pool be shared + bool shareable; public: using Entry = Pool::Entry; @@ -100,8 +104,6 @@ namespace mysqlxx PoolWithFailover(const PoolWithFailover & other); - PoolWithFailover & operator=(const PoolWithFailover &) = delete; - /** Allocates a connection to use. */ Entry Get(); }; diff --git a/base/mysqlxx/include/mysqlxx/Value.h b/base/mysqlxx/include/mysqlxx/Value.h index 4b0c6c7cbfa..2b3465d52d1 100644 --- a/base/mysqlxx/include/mysqlxx/Value.h +++ b/base/mysqlxx/include/mysqlxx/Value.h @@ -23,26 +23,26 @@ namespace mysqlxx class ResultBase; -/** Представляет одно значение, считанное из MySQL. - * Объект сам не хранит данные, а является всего лишь обёрткой над парой (const char *, size_t). - * Если уничтожить UseQueryResult/StoreQueryResult или Connection, - * или считать следующий Row при использовании UseQueryResult, то объект станет некорректным. - * Позволяет преобразовать значение (распарсить) в различные типы данных: - * - с помощью функций вида getUInt(), getString(), ... (рекомендуется); - * - с помощью шаблонной функции get(), которая специализирована для многих типов (для шаблонного кода); - * - шаблонная функция get работает также для всех типов, у которых есть конструктор из Value - * (это сделано для возможности расширения); - * - с помощью operator Type() - но этот метод реализован лишь для совместимости и не рекомендуется - * к использованию, так как неудобен (часто возникают неоднозначности). +/** Represents a single value read from MySQL. + * It doesn't owns the value. It's just a wrapper of a pair (const char *, size_t). + * If the UseQueryResult/StoreQueryResult or Connection is destroyed, + * or you have read the next Row while using UseQueryResult, then the object is invalidated. + * Allows to transform (parse) the value to various data types: + * - with getUInt(), getString(), ... (recommended); + * - with template function get() that is specialized for multiple data types; + * - the template function get also works for all types that can be constructed from Value + * (it is an extension point); + * - with operator Type() - this is done for compatibility and not recommended because ambiguities possible. * - * При ошибке парсинга, выкидывается исключение. - * При попытке достать значение, которое равно nullptr, выкидывается исключение - * - используйте метод isNull() для проверки. + * On parsing error, exception is thrown. + * When trying to extract a value that is nullptr, exception is thrown + * - use isNull() method to check. * - * Во всех распространённых системах, time_t - это всего лишь typedef от Int64 или Int32. - * Для того, чтобы можно было писать row[0].get(), ожидая, что значение вида '2011-01-01 00:00:00' - * корректно распарсится согласно текущей тайм-зоне, сделано так, что метод getUInt и соответствующие методы get<>() - * также умеют парсить дату и дату-время. + * As time_t is just an alias for integer data type + * to allow to write row[0].get(), and expect that the values like '2011-01-01 00:00:00' + * will be successfully parsed according to the current time zone, + * the getUInt method and the corresponding get<>() methods + * are capable of parsing Date and DateTime. */ class Value { @@ -166,7 +166,7 @@ private: else throwException("Cannot parse DateTime"); - return 0; /// чтобы не было warning-а. + return 0; /// avoid warning. } @@ -184,7 +184,7 @@ private: else throwException("Cannot parse Date"); - return 0; /// чтобы не было warning-а. + return 0; /// avoid warning. } @@ -231,7 +231,7 @@ private: double readFloatText(const char * buf, size_t length) const; /// Выкинуть исключение с подробной информацией - void throwException(const char * text) const; + [[noreturn]] void throwException(const char * text) const; }; diff --git a/base/mysqlxx/src/Pool.cpp b/base/mysqlxx/src/Pool.cpp index 410ac062039..b8216c254e6 100644 --- a/base/mysqlxx/src/Pool.cpp +++ b/base/mysqlxx/src/Pool.cpp @@ -22,15 +22,20 @@ void Pool::Entry::incrementRefCount() if (!data) return; ++data->ref_count; - mysql_thread_init(); + if (data->ref_count == 1) + mysql_thread_init(); } void Pool::Entry::decrementRefCount() { if (!data) return; - --data->ref_count; - mysql_thread_end(); + if (data->ref_count > 0) + { + --data->ref_count; + if (data->ref_count == 0) + mysql_thread_end(); + } } @@ -169,14 +174,24 @@ Pool::Entry Pool::tryGet() return Entry(); } +void Pool::removeConnection(Connection* connection) +{ + std::lock_guard lock(mutex); + if (connection) + { + if (connection->ref_count > 0) + { + connection->conn.disconnect(); + connection->ref_count = 0; + } + connections.remove(connection); + } +} + void Pool::Entry::disconnect() { - if (data) - { - decrementRefCount(); - data->conn.disconnect(); - } + pool->removeConnection(data); } diff --git a/base/mysqlxx/src/PoolFactory.cpp b/base/mysqlxx/src/PoolFactory.cpp new file mode 100644 index 00000000000..8f07e22671f --- /dev/null +++ b/base/mysqlxx/src/PoolFactory.cpp @@ -0,0 +1,122 @@ +#include +#include +#include + +namespace mysqlxx +{ + +struct PoolFactory::Impl +{ + // Cache of already affected pools identified by their config name + std::map> pools; + + // Cache of Pool ID (host + port + user +...) cibling already established shareable pool + std::map pools_by_ids; + + /// Protect pools and pools_by_ids caches + std::mutex mutex; +}; + +PoolWithFailover PoolFactory::Get(const std::string & config_name, unsigned default_connections, + unsigned max_connections, size_t max_tries) +{ + return Get(Poco::Util::Application::instance().config(), config_name, default_connections, max_connections, max_tries); +} + +/// Duplicate of code from StringUtils.h. Copied here for less dependencies. +static bool startsWith(const std::string & s, const char * prefix) +{ + return s.size() >= strlen(prefix) && 0 == memcmp(s.data(), prefix, strlen(prefix)); +} + +static std::string getPoolEntryName(const Poco::Util::AbstractConfiguration & config, + const std::string & config_name) +{ + bool shared = config.getBool(config_name + ".share_connection", false); + + // Not shared no need to generate a name the pool won't be stored + if (!shared) + return ""; + + std::string entry_name = ""; + std::string host = config.getString(config_name + ".host", ""); + std::string port = config.getString(config_name + ".port", ""); + std::string user = config.getString(config_name + ".user", ""); + std::string db = config.getString(config_name + ".db", ""); + std::string table = config.getString(config_name + ".table", ""); + + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(config_name, keys); + + if (config.has(config_name + ".replica")) + { + Poco::Util::AbstractConfiguration::Keys replica_keys; + config.keys(config_name, replica_keys); + for (const auto & replica_config_key : replica_keys) + { + /// There could be another elements in the same level in configuration file, like "user", "port"... + if (startsWith(replica_config_key, "replica")) + { + std::string replica_name = config_name + "." + replica_config_key; + std::string tmp_host = config.getString(replica_name + ".host", host); + std::string tmp_port = config.getString(replica_name + ".port", port); + std::string tmp_user = config.getString(replica_name + ".user", user); + entry_name += (entry_name.empty() ? "" : "|") + tmp_user + "@" + tmp_host + ":" + tmp_port + "/" + db; + } + } + } + else + { + entry_name = user + "@" + host + ":" + port + "/" + db; + } + return entry_name; +} + +PoolWithFailover PoolFactory::Get(const Poco::Util::AbstractConfiguration & config, + const std::string & config_name, unsigned default_connections, unsigned max_connections, size_t max_tries) +{ + + std::lock_guard lock(impl->mutex); + if (auto entry = impl->pools.find(config_name); entry != impl->pools.end()) + { + return *(entry->second.get()); + } + else + { + std::string entry_name = getPoolEntryName(config, config_name); + if (auto id = impl->pools_by_ids.find(entry_name); id != impl->pools_by_ids.end()) + { + entry = impl->pools.find(id->second); + std::shared_ptr pool = entry->second; + impl->pools.insert_or_assign(config_name, pool); + return *pool; + } + + auto pool = std::make_shared(config, config_name, default_connections, max_connections, max_tries); + // Check the pool will be shared + if (!entry_name.empty()) + { + // Store shared pool + impl->pools.insert_or_assign(config_name, pool); + impl->pools_by_ids.insert_or_assign(entry_name, config_name); + } + return *(pool.get()); + } +} + +void PoolFactory::reset() +{ + std::lock_guard lock(impl->mutex); + impl->pools.clear(); + impl->pools_by_ids.clear(); +} + +PoolFactory::PoolFactory() : impl(std::make_unique()) {} + +PoolFactory & PoolFactory::instance() +{ + static PoolFactory ret; + return ret; +} + +} diff --git a/base/mysqlxx/src/PoolWithFailover.cpp b/base/mysqlxx/src/PoolWithFailover.cpp index dd89f1596d3..8306922b0e5 100644 --- a/base/mysqlxx/src/PoolWithFailover.cpp +++ b/base/mysqlxx/src/PoolWithFailover.cpp @@ -15,6 +15,7 @@ PoolWithFailover::PoolWithFailover(const Poco::Util::AbstractConfiguration & cfg const unsigned max_connections, const size_t max_tries) : max_tries(max_tries) { + shareable = cfg.getBool(config_name + ".share_connection", false); if (cfg.has(config_name + ".replica")) { Poco::Util::AbstractConfiguration::Keys replica_keys; @@ -48,15 +49,22 @@ PoolWithFailover::PoolWithFailover(const std::string & config_name, const unsign {} PoolWithFailover::PoolWithFailover(const PoolWithFailover & other) - : max_tries{other.max_tries} + : max_tries{other.max_tries}, config_name{other.config_name}, shareable{other.shareable} { - for (const auto & priority_replicas : other.replicas_by_priority) + if (shareable) { - Replicas replicas; - replicas.reserve(priority_replicas.second.size()); - for (const auto & pool : priority_replicas.second) - replicas.emplace_back(std::make_shared(*pool)); - replicas_by_priority.emplace(priority_replicas.first, std::move(replicas)); + replicas_by_priority = other.replicas_by_priority; + } + else + { + for (const auto & priority_replicas : other.replicas_by_priority) + { + Replicas replicas; + replicas.reserve(priority_replicas.second.size()); + for (const auto & pool : priority_replicas.second) + replicas.emplace_back(std::make_shared(*pool)); + replicas_by_priority.emplace(priority_replicas.first, std::move(replicas)); + } } } @@ -81,7 +89,7 @@ PoolWithFailover::Entry PoolWithFailover::Get() try { - Entry entry = pool->tryGet(); + Entry entry = shareable ? pool->Get() : pool->tryGet(); if (!entry.isNull()) { diff --git a/contrib/base64 b/contrib/base64 index 5257626d2be..95ba56a9b04 160000 --- a/contrib/base64 +++ b/contrib/base64 @@ -1 +1 @@ -Subproject commit 5257626d2be17a3eb23f79be17fe55ebba394ad2 +Subproject commit 95ba56a9b041f9933f5cd2bbb2ee4e083468c20a diff --git a/contrib/poco b/contrib/poco index 1f3e4638f25..860574c9398 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 1f3e4638f250ad4d028a2499af20d4185463e07d +Subproject commit 860574c93980d887a89df141edd9ca2fb0024fa3 diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index be8ee14b78d..963b3251084 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -105,6 +105,7 @@ namespace ErrorCodes extern const int UNEXPECTED_PACKET_FROM_SERVER; extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED; extern const int INVALID_USAGE_OF_INPUT; + extern const int DEADLOCK_AVOIDED; } @@ -906,9 +907,34 @@ private: query = serializeAST(*parsed_query); } - connection->sendQuery(connection_parameters.timeouts, query, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true); - sendExternalTables(); - receiveResult(); + static constexpr size_t max_retries = 10; + for (size_t retry = 0; retry < max_retries; ++retry) + { + try + { + connection->sendQuery( + connection_parameters.timeouts, + query, + query_id, + QueryProcessingStage::Complete, + &context.getSettingsRef(), + nullptr, + true); + + sendExternalTables(); + receiveResult(); + + break; + } + catch (const Exception & e) + { + /// Retry when the server said "Client should retry" and no rows has been received yet. + if (processed_rows == 0 && e.code() == ErrorCodes::DEADLOCK_AVOIDED && retry + 1 < max_retries) + continue; + + throw; + } + } } diff --git a/dbms/programs/client/Suggest.cpp b/dbms/programs/client/Suggest.cpp index cc31eed53e3..9058bafd138 100644 --- a/dbms/programs/client/Suggest.cpp +++ b/dbms/programs/client/Suggest.cpp @@ -9,29 +9,43 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int UNKNOWN_PACKET_FROM_SERVER; + extern const int DEADLOCK_AVOIDED; } void Suggest::load(const ConnectionParameters & connection_parameters, size_t suggestion_limit) { loading_thread = std::thread([connection_parameters, suggestion_limit, this] { - try + for (size_t retry = 0; retry < 10; ++retry) { - Connection connection( - connection_parameters.host, - connection_parameters.port, - connection_parameters.default_database, - connection_parameters.user, - connection_parameters.password, - "client", - connection_parameters.compression, - connection_parameters.security); + try + { + Connection connection( + connection_parameters.host, + connection_parameters.port, + connection_parameters.default_database, + connection_parameters.user, + connection_parameters.password, + "client", + connection_parameters.compression, + connection_parameters.security); - loadImpl(connection, connection_parameters.timeouts, suggestion_limit); - } - catch (...) - { - std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; + loadImpl(connection, connection_parameters.timeouts, suggestion_limit); + } + catch (const Exception & e) + { + /// Retry when the server said "Client should retry". + if (e.code() == ErrorCodes::DEADLOCK_AVOIDED) + continue; + + std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; + } + catch (...) + { + std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; + } + + break; } /// Note that keyword suggestions are available even if we cannot load data from server. diff --git a/dbms/programs/copier/Internals.h b/dbms/programs/copier/Internals.h index 14dc888a0b4..4e4142a035b 100644 --- a/dbms/programs/copier/Internals.h +++ b/dbms/programs/copier/Internals.h @@ -133,7 +133,6 @@ struct TaskStateWithOwner }; - struct ShardPriority { UInt8 is_remote = 1; diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index 5cfceaeb592..a769ea8d175 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -298,7 +298,7 @@ void LocalServer::processQueries() try { - executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {}, {}); + executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {}); } catch (...) { diff --git a/dbms/programs/performance-test/StopConditionsSet.cpp b/dbms/programs/performance-test/StopConditionsSet.cpp index 58d3383e81c..ab334b71244 100644 --- a/dbms/programs/performance-test/StopConditionsSet.cpp +++ b/dbms/programs/performance-test/StopConditionsSet.cpp @@ -60,5 +60,4 @@ void StopConditionsSet::report(UInt64 value, StopConditionsSet::StopCondition & } - } diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 387d441bc9b..eb83fc6f5d7 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -592,12 +592,14 @@ void HTTPHandler::processQuery( customizeContext(context); executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context, - [&response] (const String & content_type, const String & format) + [&response] (const String & current_query_id, const String & content_type, const String & format, const String & timezone) { response.setContentType(content_type); + response.add("X-ClickHouse-Query-Id", current_query_id); response.add("X-ClickHouse-Format", format); - }, - [&response] (const String & current_query_id) { response.add("X-ClickHouse-Query-Id", current_query_id); }); + response.add("X-ClickHouse-Timezone", timezone); + } + ); if (used_output.hasDelayed()) { diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index 3d4461a2317..bdd83eac64d 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -282,14 +282,9 @@ void MySQLHandler::comQuery(ReadBuffer & payload) } else { - bool with_output = false; - std::function set_content_type_and_format = [&with_output](const String &, const String &) -> void - { - with_output = true; - }; - String replacement_query = "select ''"; bool should_replace = false; + bool with_output = false; // Translate query from MySQL to ClickHouse. // This is a temporary workaround until ClickHouse supports the syntax "@@var_name". @@ -307,7 +302,13 @@ void MySQLHandler::comQuery(ReadBuffer & payload) ReadBufferFromString replacement(replacement_query); Context query_context = connection_context; - executeQuery(should_replace ? replacement : payload, *out, true, query_context, set_content_type_and_format, {}); + + executeQuery(should_replace ? replacement : payload, *out, true, query_context, + [&with_output](const String &, const String &, const String &, const String &) + { + with_output = true; + } + ); if (!with_output) packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index b1664286780..6e6d997d7ed 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -60,6 +60,7 @@ #include "TCPHandlerFactory.h" #include "Common/config_version.h" #include +#include #include "MySQLHandlerFactory.h" #if defined(OS_LINUX) @@ -219,6 +220,9 @@ int Server::main(const std::vector & /*args*/) CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); + if (ThreadFuzzer::instance().isEffective()) + LOG_WARNING(log, "ThreadFuzzer is enabled. Application will run slowly and unstable."); + /** Context contains all that query execution is dependent: * settings, available functions, data types, aggregate functions, databases... */ @@ -466,6 +470,8 @@ int Server::main(const std::vector & /*args*/) if (config->has("max_partition_size_to_drop")) global_context->setMaxPartitionSizeToDrop(config->getUInt64("max_partition_size_to_drop")); + + global_context->updateStorageConfiguration(*config); }, /* already_loaded = */ true); diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 243cd3bf086..dc6f0094b6b 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -503,7 +503,7 @@ void TCPHandler::processOrdinaryQuery() if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) { - /// Some time passed and there is a progress. + /// Some time passed. after_send_progress.restart(); sendProgress(); } @@ -539,6 +539,8 @@ void TCPHandler::processOrdinaryQuery() } state.io.onFinish(); + + sendProgress(); } @@ -546,8 +548,8 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads) { auto & pipeline = state.io.pipeline; - if (pipeline.getMaxThreads()) - num_threads = std::min(num_threads, pipeline.getMaxThreads()); + /// Reduce the number of threads to recommended value. + num_threads = std::min(num_threads, pipeline.getNumThreads()); /// Send header-block, to allow client to prepare output format for data to send. { @@ -658,6 +660,8 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads) } state.io.onFinish(); + + sendProgress(); } diff --git a/dbms/programs/server/config.d/listen.xml.disabled b/dbms/programs/server/config.d/listen.xml.disabled new file mode 100644 index 00000000000..de8c737ff75 --- /dev/null +++ b/dbms/programs/server/config.d/listen.xml.disabled @@ -0,0 +1,3 @@ + + :: + diff --git a/dbms/programs/server/config.d/tls.xml.disabled b/dbms/programs/server/config.d/tls.xml.disabled new file mode 100644 index 00000000000..c81f005f2ed --- /dev/null +++ b/dbms/programs/server/config.d/tls.xml.disabled @@ -0,0 +1,9 @@ + + 8443 + 9440 + + + + + + diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index ae15a583fcd..4239cb02523 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -3,25 +3,6 @@ NOTE: User and query level settings are set up in "users.xml" file. --> - - - - - - - trace @@ -250,6 +231,24 @@ + + + + + + + + + /external/ + + + + + + +
+ jbod1 +
+ + external + + + + + + + + jbod1 + + + external + + + + + + + +
+ jbod1 + jbod2 + 10485760 + +
+ + external + + + + + + + +
+ jbod1 +
+ + external + + + 0.7 + + + + + + + default + 2097152 + + + + external + 20971520 + + + + + + + + + + default + 0 + + + external + + + jbod1 + 1024 + + + jbod2 + 1024000000 + + + + + + + + + diff --git a/dbms/tests/integration/test_reloading_storage_configuration/configs/logs_config.xml b/dbms/tests/integration/test_reloading_storage_configuration/configs/logs_config.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/dbms/tests/integration/test_reloading_storage_configuration/configs/logs_config.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/dbms/tests/integration/test_reloading_storage_configuration/test.py b/dbms/tests/integration/test_reloading_storage_configuration/test.py new file mode 100644 index 00000000000..c9effcdd67a --- /dev/null +++ b/dbms/tests/integration/test_reloading_storage_configuration/test.py @@ -0,0 +1,472 @@ +import collections +import os +import re +import shutil +import time +import xml.etree.ElementTree as ET + +import pytest + +import helpers.client +import helpers.cluster + + +cluster = helpers.cluster.ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', + config_dir='configs', + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + stay_alive=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/jbod3:size=40M', '/jbod4:size=40M', '/external:size=200M'], + macros={"shard": 0, "replica": 1} ) + +node2 = cluster.add_instance('node2', + config_dir='configs', + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + stay_alive=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/jbod3:size=40M', '/jbod4:size=40M', '/external:size=200M'], + macros={"shard": 0, "replica": 2} ) + + +def get_log(node): + return node.exec_in_container(["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"]) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def start_over(): + shutil.copy(os.path.join(os.path.dirname(__file__), "configs/config.d/storage_configuration.xml"), os.path.join(node1.config_d_dir, "storage_configuration.xml")) + + for node in (node1, node2): + separate_configuration_path = os.path.join(node.config_d_dir, "separate_configuration.xml") + try: + os.remove(separate_configuration_path) + except: + """""" + + +def add_disk(node, name, path, separate_file=False): + separate_configuration_path = os.path.join(node.config_d_dir, "separate_configuration.xml") + + try: + if separate_file: + tree = ET.parse(separate_configuration_path) + else: + tree = ET.parse(os.path.join(node.config_d_dir, "storage_configuration.xml")) + except: + tree = ET.ElementTree(ET.fromstring('')) + root = tree.getroot() + new_disk = ET.Element(name) + new_path = ET.Element("path") + new_path.text = path + new_disk.append(new_path) + root.find("storage_configuration").find("disks").append(new_disk) + if separate_file: + tree.write(separate_configuration_path) + else: + tree.write(os.path.join(node.config_d_dir, "storage_configuration.xml")) + + +def add_policy(node, name, volumes): + tree = ET.parse(os.path.join(node.config_d_dir, "storage_configuration.xml")) + root = tree.getroot() + new_policy = ET.Element(name) + new_volumes = ET.Element("volumes") + for volume, disks in volumes.items(): + new_volume = ET.Element(volume) + for disk in disks: + new_disk = ET.Element("disk") + new_disk.text = disk + new_volume.append(new_disk) + new_volumes.append(new_volume) + new_policy.append(new_volumes) + root.find("storage_configuration").find("policies").append(new_policy) + tree.write(os.path.join(node.config_d_dir, "storage_configuration.xml")) + + +def test_add_disk(started_cluster): + try: + name = "test_add_disk" + engine = "MergeTree()" + + start_over() + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + assert "jbod3" not in set(node1.query("SELECT name FROM system.disks").splitlines()) + + add_disk(node1, "jbod3", "/jbod3/") + node1.query("SYSTEM RELOAD CONFIG") + + assert "jbod3" in set(node1.query("SELECT name FROM system.disks").splitlines()) + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_add_disk_to_separate_config(started_cluster): + try: + name = "test_add_disk" + engine = "MergeTree()" + + start_over() + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + assert "jbod3" not in set(node1.query("SELECT name FROM system.disks").splitlines()) + + add_disk(node1, "jbod3", "/jbod3/", separate_file=True) + node1.query("SYSTEM RELOAD CONFIG") + + assert "jbod3" in set(node1.query("SELECT name FROM system.disks").splitlines()) + start_over() + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_add_policy(started_cluster): + try: + name = "test_add_policy" + engine = "MergeTree()" + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + add_policy(node1, "cool_policy", {"volume1": ["jbod3", "jbod4"]}) + node1.query("SYSTEM RELOAD CONFIG") + + disks = set(node1.query("SELECT name FROM system.disks").splitlines()) + assert "cool_policy" in set(node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) + assert {"volume1"} == set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + assert {"['jbod3','jbod4']"} == set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_new_policy_works(started_cluster): + try: + name = "test_new_policy_works" + engine = "MergeTree()" + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) + node1.query("SYSTEM RELOAD CONFIG") + + # Incompatible storage policy. + with pytest.raises(helpers.client.QueryRuntimeException): + node1.query(""" + ALTER TABLE {name} MODIFY SETTING storage_policy='cool_policy' + """.format(name=name)) + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "cool_policy", collections.OrderedDict([("volume1", ["jbod3"]), ("main", ["jbod1", "jbod2"]), ("external", ["external"])])) + node1.query("SYSTEM RELOAD CONFIG") + + node1.query(""" + ALTER TABLE {name} MODIFY SETTING storage_policy='cool_policy' + """.format(name=name)) + + node1.query(""" + INSERT INTO TABLE {name} VALUES (1) + """.format(name=name)) + assert {"jbod3"} == set(node1.query("SELECT disk_name FROM system.parts WHERE active = 1 AND table = '{name}'".format(name=name)).splitlines()) + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_add_volume_to_policy(started_cluster): + try: + name = "test_add_volume_to_policy" + engine = "MergeTree()" + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "cool_policy", collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])])) + node1.query("SYSTEM RELOAD CONFIG") + + volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + assert {"volume1", "volume2"} == volumes + assert {"['jbod3']", "['jbod4']"} == disks_sets + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_add_disk_to_policy(started_cluster): + try: + name = "test_add_disk_to_policy" + engine = "MergeTree()" + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "cool_policy", {"volume1": ["jbod3","jbod4"]}) + node1.query("SYSTEM RELOAD CONFIG") + + volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + assert {"volume1"} == volumes + assert {"['jbod3','jbod4']"} == disks_sets + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_remove_disk(started_cluster): + try: + name = "test_remove_disk" + engine = "MergeTree()" + + start_over() + add_disk(node1, "remove_disk_jbod3", "/jbod3/") + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + assert "remove_disk_jbod3" in set(node1.query("SELECT name FROM system.disks").splitlines()) + + start_over() + node1.query("SYSTEM RELOAD CONFIG") + + assert "remove_disk_jbod3" in set(node1.query("SELECT name FROM system.disks").splitlines()) + assert re.search("Warning.*remove_disk_jbod3", get_log(node1)) + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_remove_policy(started_cluster): + try: + name = "test_remove_policy" + engine = "MergeTree()" + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "remove_policy_cool_policy", {"volume1": ["jbod3", "jbod4"]}) + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + assert "remove_policy_cool_policy" in set(node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + node1.query("SYSTEM RELOAD CONFIG") + + assert "remove_policy_cool_policy" in set(node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) + assert re.search("Error.*remove_policy_cool_policy", get_log(node1)) + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_remove_volume_from_policy(started_cluster): + try: + name = "test_remove_volume_from_policy" + engine = "MergeTree()" + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "test_remove_volume_from_policy_cool_policy", collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])])) + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + assert {"volume1", "volume2"} == volumes + assert {"['jbod3']", "['jbod4']"} == disks_sets + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) + node1.query("SYSTEM RELOAD CONFIG") + + volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + assert {"volume1", "volume2"} == volumes + assert {"['jbod3']", "['jbod4']"} == disks_sets + assert re.search("Error.*test_remove_volume_from_policy_cool_policy", get_log(node1)) + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" + + +def test_remove_disk_from_policy(started_cluster): + try: + name = "test_remove_disk_from_policy" + engine = "MergeTree()" + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "test_remove_disk_from_policy_cool_policy", {"volume1": ["jbod3","jbod4"]}) + node1.restart_clickhouse(kill=True) + time.sleep(2) + + node1.query(""" + CREATE TABLE {name} ( + d UInt64 + ) ENGINE = {engine} + ORDER BY d + SETTINGS storage_policy='jbods_with_external' + """.format(name=name, engine=engine)) + + volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + assert {"volume1"} == volumes + assert {"['jbod3','jbod4']"} == disks_sets + + start_over() + add_disk(node1, "jbod3", "/jbod3/") + add_disk(node1, "jbod4", "/jbod4/") + add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) + node1.query("SYSTEM RELOAD CONFIG") + + volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + assert {"volume1"} == volumes + assert {"['jbod3','jbod4']"} == disks_sets + assert re.search("Error.*test_remove_disk_from_policy_cool_policy", get_log(node1)) + + finally: + try: + node1.query("DROP TABLE IF EXISTS {}".format(name)) + except: + """""" diff --git a/dbms/tests/integration/test_storage_kafka/test.py b/dbms/tests/integration/test_storage_kafka/test.py index 8c4f2fbc9ef..3242b59bca3 100644 --- a/dbms/tests/integration/test_storage_kafka/test.py +++ b/dbms/tests/integration/test_storage_kafka/test.py @@ -7,6 +7,7 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV from helpers.client import QueryRuntimeException +from helpers.network import PartitionManager import json import subprocess @@ -34,6 +35,7 @@ instance = cluster.add_instance('instance', config_dir='configs', main_configs=['configs/kafka.xml', 'configs/log_conf.xml' ], with_kafka=True, + with_zookeeper=True, clickhouse_path_dir='clickhouse_path') kafka_id = '' @@ -1106,6 +1108,90 @@ def test_kafka_rebalance(kafka_cluster): assert result == 1, 'Messages from kafka get duplicated!' +@pytest.mark.timeout(1200) +def test_kafka_no_holes_when_write_suffix_failed(kafka_cluster): + messages = [json.dumps({'key': j+1, 'value': 'x' * 300}) for j in range(22)] + kafka_produce('no_holes_when_write_suffix_failed', messages) + + instance.query(''' + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + + CREATE TABLE test.kafka (key UInt64, value String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'no_holes_when_write_suffix_failed', + kafka_group_name = 'no_holes_when_write_suffix_failed', + kafka_format = 'JSONEachRow', + kafka_max_block_size = 20; + + CREATE TABLE test.view (key UInt64, value String) + ENGINE = ReplicatedMergeTree('/clickhouse/kafkatest/tables/no_holes_when_write_suffix_failed', 'node1') + ORDER BY key; + + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.kafka + WHERE NOT sleepEachRow(1); + ''') + # the tricky part here is that disconnect should happen after write prefix, but before write suffix + # so i use sleepEachRow + with PartitionManager() as pm: + time.sleep(12) + pm.drop_instance_zk_connections(instance) + time.sleep(20) + pm.heal_all + + # connection restored and it will take a while until next block will be flushed + # it takes years on CI :\ + time.sleep(90) + + # as it's a bit tricky to hit the proper moment - let's check in logs if we did it correctly + assert instance.contains_in_log("ZooKeeper session has been expired.: while write prefix to view") + + result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.view') + print(result) + + # kafka_cluster.open_bash_shell('instance') + + instance.query(''' + DROP TABLE test.consumer; + DROP TABLE test.view; + ''') + + assert TSV(result) == TSV('22\t22\t22') + + +@pytest.mark.timeout(120) +def test_exception_from_destructor(kafka_cluster): + instance.query(''' + CREATE TABLE test.kafka (key UInt64, value String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'xyz', + kafka_group_name = '', + kafka_format = 'JSONEachRow'; + ''') + instance.query_and_get_error(''' + SELECT * FROM test.kafka; + ''') + instance.query(''' + DROP TABLE test.kafka; + ''') + + instance.query(''' + CREATE TABLE test.kafka (key UInt64, value String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'xyz', + kafka_group_name = '', + kafka_format = 'JSONEachRow'; + ''') + instance.query(''' + DROP TABLE test.kafka; + ''') + + kafka_cluster.open_bash_shell('instance') + assert TSV(instance.query('SELECT 1')) == TSV('1') if __name__ == '__main__': diff --git a/dbms/tests/integration/test_ttl_replicated/test.py b/dbms/tests/integration/test_ttl_replicated/test.py index 8ab470e6f60..03dd1d2aa89 100644 --- a/dbms/tests/integration/test_ttl_replicated/test.py +++ b/dbms/tests/integration/test_ttl_replicated/test.py @@ -60,21 +60,25 @@ def test_ttl_many_columns(started_cluster): ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0; '''.format(replica=node.name)) - node1.query("SYSTEM STOP MERGES test_ttl_2") - node2.query("SYSTEM STOP MERGES test_ttl_2") + node1.query("SYSTEM STOP TTL MERGES test_ttl_2") + node2.query("SYSTEM STOP TTL MERGES test_ttl_2") node1.query("INSERT INTO test_ttl_2 VALUES (toDateTime('2000-10-10 00:00:00'), 1, 2, 3, 4, 5)") node1.query("INSERT INTO test_ttl_2 VALUES (toDateTime('2100-10-10 10:00:00'), 6, 7, 8, 9, 10)") + node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) + # Check that part will appear in result of merge node1.query("SYSTEM STOP FETCHES test_ttl_2") node2.query("SYSTEM STOP FETCHES test_ttl_2") - node1.query("SYSTEM START MERGES test_ttl_2") - node2.query("SYSTEM START MERGES test_ttl_2") + node1.query("SYSTEM START TTL MERGES test_ttl_2") + node2.query("SYSTEM START TTL MERGES test_ttl_2") time.sleep(1) # sleep to allow use ttl merge selector for second time node1.query("OPTIMIZE TABLE test_ttl_2 FINAL", timeout=5) + + node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) expected = "1\t0\t0\t0\t0\n6\t7\t8\t9\t10\n" assert TSV(node1.query("SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id")) == TSV(expected) diff --git a/dbms/tests/performance/arithmetic.xml b/dbms/tests/performance/arithmetic.xml index 066173d8635..348273bebbf 100644 --- a/dbms/tests/performance/arithmetic.xml +++ b/dbms/tests/performance/arithmetic.xml @@ -35,6 +35,8 @@ multiply divide intDivOrZero + modulo + moduloOrZero diff --git a/dbms/tests/performance/empty_string_deserialization.xml b/dbms/tests/performance/empty_string_deserialization.xml index 99e7470c5e6..c56f67ab274 100644 --- a/dbms/tests/performance/empty_string_deserialization.xml +++ b/dbms/tests/performance/empty_string_deserialization.xml @@ -12,7 +12,7 @@ --> CREATE TABLE empty_strings (s String) ENGINE = Log; - INSERT INTO empty_strings SELECT '' FROM numbers(1000000000); + INSERT INTO empty_strings SELECT '' FROM numbers_mt(100000000); SELECT count() FROM empty_strings diff --git a/dbms/tests/performance/generate_table_function.xml b/dbms/tests/performance/generate_table_function.xml new file mode 100644 index 00000000000..4674b81af99 --- /dev/null +++ b/dbms/tests/performance/generate_table_function.xml @@ -0,0 +1,30 @@ + + loop + + + + 4000 + 10000 + + + + SELECT COUNT(*) FROM (SELECT * FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('f32 Float32, f64 Float64', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Tuple(Int32, Int64)', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(Int8)', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(Nullable(Int32))', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Tuple(Int32, Array(Int64))', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Nullable(String)', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(String)', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i UUID', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(Nullable(UUID))', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i FixedString(4)', 10, 10, 1) LIMIT 100000); + SELECT COUNT(*) FROM (SELECT * FROM generate('i String', 10, 10, 1) LIMIT 100000); + diff --git a/dbms/tests/performance/logical_functions_large.xml b/dbms/tests/performance/logical_functions_large.xml index bd77c8e0b3f..b90023c3f60 100644 --- a/dbms/tests/performance/logical_functions_large.xml +++ b/dbms/tests/performance/logical_functions_large.xml @@ -18,16 +18,16 @@ SELECT count() FROM - (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3, materialize(1) AS x4, materialize(1) AS x5, materialize(1) AS x6, materialize(1) AS x7, materialize(1) AS x8, materialize(1) AS x9, materialize(1) AS x10 FROM numbers(40000000)) + (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3, materialize(1) AS x4, materialize(1) AS x5, materialize(1) AS x6, materialize(1) AS x7, materialize(1) AS x8, materialize(1) AS x9, materialize(1) AS x10 FROM numbers(20000000)) WHERE NOT ignore(and(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10)) SELECT count() FROM - (SELECT materialize(0) AS x1, materialize(0) AS x2, materialize(0) AS x3, materialize(0) AS x4, materialize(0) AS x5, materialize(0) AS x6, materialize(0) AS x7, materialize(0) AS x8, materialize(0) AS x9, materialize(0) AS x10 FROM numbers(200000000)) + (SELECT materialize(0) AS x1, materialize(0) AS x2, materialize(0) AS x3, materialize(0) AS x4, materialize(0) AS x5, materialize(0) AS x6, materialize(0) AS x7, materialize(0) AS x8, materialize(0) AS x9, materialize(0) AS x10 FROM numbers(150000000)) WHERE NOT ignore(and(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10)) SELECT count() FROM - (SELECT materialize(0) AS x1, materialize(0) AS x2, materialize(0) AS x3, materialize(0) AS x4, materialize(0) AS x5, materialize(0) AS x6, materialize(0) AS x7, materialize(0) AS x8, materialize(0) AS x9, materialize(0) AS x10 FROM numbers(40000000)) + (SELECT materialize(0) AS x1, materialize(0) AS x2, materialize(0) AS x3, materialize(0) AS x4, materialize(0) AS x5, materialize(0) AS x6, materialize(0) AS x7, materialize(0) AS x8, materialize(0) AS x9, materialize(0) AS x10 FROM numbers(20000000)) WHERE NOT ignore(or(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10)) SELECT count() FROM - (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3, materialize(1) AS x4, materialize(1) AS x5, materialize(1) AS x6, materialize(1) AS x7, materialize(1) AS x8, materialize(1) AS x9, materialize(1) AS x10 FROM numbers(200000000)) + (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3, materialize(1) AS x4, materialize(1) AS x5, materialize(1) AS x6, materialize(1) AS x7, materialize(1) AS x8, materialize(1) AS x9, materialize(1) AS x10 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10)) SELECT count() FROM (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3, materialize(1) AS x4, materialize(1) AS x5, materialize(1) AS x6, materialize(1) AS x7, materialize(1) AS x8, materialize(1) AS x9, materialize(1) AS x10 FROM numbers(500000000)) @@ -43,9 +43,9 @@ (SELECT materialize(toUInt8(0)) AS x1, materialize(toUInt16(0)) AS x2, materialize(toUInt32(0)) AS x3, materialize(toUInt64(0)) AS x4, materialize(toInt8(0)) AS x5, materialize(toInt16(0)) AS x6, materialize(toInt32(0)) AS x7, materialize(toInt64(0)) AS x8, materialize(toFloat32(0)) AS x9, materialize(toFloat64(0)) AS x10 FROM numbers(20000000)) WHERE NOT ignore(or(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10)) SELECT count() FROM - (SELECT materialize(toUInt8(1)) AS x1, materialize(toUInt16(1)) AS x2, materialize(toUInt32(1)) AS x3, materialize(toUInt64(1)) AS x4, materialize(toInt8(1)) AS x5, materialize(toInt16(1)) AS x6, materialize(toInt32(1)) AS x7, materialize(toInt64(1)) AS x8, materialize(toFloat32(1)) AS x9, materialize(toFloat64(1)) AS x10 FROM numbers(40000000)) + (SELECT materialize(toUInt8(1)) AS x1, materialize(toUInt16(1)) AS x2, materialize(toUInt32(1)) AS x3, materialize(toUInt64(1)) AS x4, materialize(toInt8(1)) AS x5, materialize(toInt16(1)) AS x6, materialize(toInt32(1)) AS x7, materialize(toInt64(1)) AS x8, materialize(toFloat32(1)) AS x9, materialize(toFloat64(1)) AS x10 FROM numbers(20000000)) WHERE NOT ignore(or(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10)) SELECT count() FROM - (SELECT materialize(toUInt8(1)) AS x1, materialize(toUInt16(1)) AS x2, materialize(toUInt32(1)) AS x3, materialize(toUInt64(1)) AS x4, materialize(toInt8(1)) AS x5, materialize(toInt16(1)) AS x6, materialize(toInt32(1)) AS x7, materialize(toInt64(1)) AS x8, materialize(toFloat32(1)) AS x9, materialize(toFloat64(1)) AS x10 FROM numbers(50000000)) + (SELECT materialize(toUInt8(1)) AS x1, materialize(toUInt16(1)) AS x2, materialize(toUInt32(1)) AS x3, materialize(toUInt64(1)) AS x4, materialize(toInt8(1)) AS x5, materialize(toInt16(1)) AS x6, materialize(toInt32(1)) AS x7, materialize(toInt64(1)) AS x8, materialize(toFloat32(1)) AS x9, materialize(toFloat64(1)) AS x10 FROM numbers(20000000)) WHERE NOT ignore(xor(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10)) diff --git a/dbms/tests/performance/logical_functions_medium.xml b/dbms/tests/performance/logical_functions_medium.xml index daf11737aa4..0f6de7ea23e 100644 --- a/dbms/tests/performance/logical_functions_medium.xml +++ b/dbms/tests/performance/logical_functions_medium.xml @@ -17,26 +17,26 @@ 1 - SELECT count() FROM (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3 FROM numbers(200000000)) WHERE NOT ignore(and(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(0) AS x1, materialize(0) AS x2, materialize(0) AS x3 FROM numbers(400000000)) WHERE NOT ignore(and(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(0) AS x1, materialize(0) AS x2, materialize(0) AS x3 FROM numbers(200000000)) WHERE NOT ignore(or(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3 FROM numbers(200000000)) WHERE NOT ignore(or(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3 FROM numbers(100000000)) WHERE NOT ignore(and(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(0) AS x1, materialize(0) AS x2, materialize(0) AS x3 FROM numbers(200000000)) WHERE NOT ignore(and(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(0) AS x1, materialize(0) AS x2, materialize(0) AS x3 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(1) AS x1, materialize(1) AS x2, materialize(1) AS x3 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(1) AS x1, materialize(toUInt32(1)) AS x2, materialize(toUInt64(1)) AS x3 FROM numbers(100000000)) WHERE NOT ignore(and(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(0) AS x1, materialize(toUInt32(0)) AS x2, materialize(toUInt64(0)) AS x3 FROM numbers(200000000)) WHERE NOT ignore(and(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(0) AS x1, materialize(toUInt32(0)) AS x2, materialize(toUInt64(0)) AS x3 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(1) AS x1, materialize(toUInt32(1)) AS x2, materialize(toUInt64(1)) AS x3 FROM numbers(200000000)) WHERE NOT ignore(or(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(1) AS x1, materialize(toUInt32(1)) AS x2, materialize(toUInt64(1)) AS x3 FROM numbers(50000000)) WHERE NOT ignore(and(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(0) AS x1, materialize(toUInt32(0)) AS x2, materialize(toUInt64(0)) AS x3 FROM numbers(100000000)) WHERE NOT ignore(and(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(0) AS x1, materialize(toUInt32(0)) AS x2, materialize(toUInt64(0)) AS x3 FROM numbers(50000000)) WHERE NOT ignore(or(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(1) AS x1, materialize(toUInt32(1)) AS x2, materialize(toUInt64(1)) AS x3 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(toNullable(1)) AS x1, materialize(toNullable(1)) AS x2, materialize(toNullable(1)) AS x3 FROM numbers(30000000)) WHERE NOT ignore(and(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(toNullable(0)) AS x1, materialize(toNullable(0)) AS x2, materialize(toNullable(0)) AS x3 FROM numbers(100000000)) WHERE NOT ignore(and(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(toNullable(0)) AS x1, materialize(toNullable(0)) AS x2, materialize(toNullable(0)) AS x3 FROM numbers(50000000)) WHERE NOT ignore(or(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(toNullable(1)) AS x1, materialize(toNullable(1)) AS x2, materialize(toNullable(1)) AS x3 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(toNullable(1)) AS x1, materialize(toNullable(1)) AS x2, materialize(toNullable(1)) AS x3 FROM numbers(20000000)) WHERE NOT ignore(and(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(toNullable(0)) AS x1, materialize(toNullable(0)) AS x2, materialize(toNullable(0)) AS x3 FROM numbers(50000000)) WHERE NOT ignore(and(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(toNullable(0)) AS x1, materialize(toNullable(0)) AS x2, materialize(toNullable(0)) AS x3 FROM numbers(20000000)) WHERE NOT ignore(or(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(toNullable(1)) AS x1, materialize(toNullable(1)) AS x2, materialize(toNullable(1)) AS x3 FROM numbers(50000000)) WHERE NOT ignore(or(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(toNullable(toUInt16(1))) AS x1, materialize(toNullable(toUInt32(1))) AS x2, materialize(toNullable(toUInt64(1))) AS x3 FROM numbers(30000000)) WHERE NOT ignore(and(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(toNullable(toUInt16(0))) AS x1, materialize(toNullable(toUInt32(0))) AS x2, materialize(toNullable(toUInt64(0))) AS x3 FROM numbers(100000000)) WHERE NOT ignore(and(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(toNullable(toUInt16(0))) AS x1, materialize(toNullable(toUInt32(0))) AS x2, materialize(toNullable(toUInt64(0))) AS x3 FROM numbers(50000000)) WHERE NOT ignore(or(x1,x2,x3)) - SELECT count() FROM (SELECT materialize(toNullable(toUInt16(1))) AS x1, materialize(toNullable(toUInt32(1))) AS x2, materialize(toNullable(toUInt64(1))) AS x3 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(toNullable(toUInt16(1))) AS x1, materialize(toNullable(toUInt32(1))) AS x2, materialize(toNullable(toUInt64(1))) AS x3 FROM numbers(20000000)) WHERE NOT ignore(and(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(toNullable(toUInt16(0))) AS x1, materialize(toNullable(toUInt32(0))) AS x2, materialize(toNullable(toUInt64(0))) AS x3 FROM numbers(50000000)) WHERE NOT ignore(and(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(toNullable(toUInt16(0))) AS x1, materialize(toNullable(toUInt32(0))) AS x2, materialize(toNullable(toUInt64(0))) AS x3 FROM numbers(20000000)) WHERE NOT ignore(or(x1,x2,x3)) + SELECT count() FROM (SELECT materialize(toNullable(toUInt16(1))) AS x1, materialize(toNullable(toUInt32(1))) AS x2, materialize(toNullable(toUInt64(1))) AS x3 FROM numbers(50000000)) WHERE NOT ignore(or(x1,x2,x3)) CREATE DATABASE IF NOT EXISTS test_logical_functions @@ -46,9 +46,9 @@ CREATE TABLE test_logical_functions._4_1_Nullable_UInt8 (x1 Nullable(UInt8), x2 Nullable(UInt8), x3 Nullable(UInt8), x4 Nullable(UInt8)) ENGINE = Memory CREATE TABLE test_logical_functions._4_1_Nullable_Mixed (x1 Nullable(UInt16), x2 Nullable(UInt32), x3 Nullable(UInt64), x4 Nullable(Float64)) ENGINE = Memory INSERT INTO test_logical_functions._4_1_UInt8 SELECT materialize(1), materialize(1), materialize(1), materialize(1) FROM numbers(500000000) - INSERT INTO test_logical_functions._4_1_Mixed SELECT materialize(1), materialize(1), materialize(1), materialize(1) FROM numbers(200000000) - INSERT INTO test_logical_functions._4_1_Nullable_UInt8 SELECT materialize(1), materialize(1), materialize(1), materialize(1) FROM numbers(200000000) - INSERT INTO test_logical_functions._4_1_Nullable_Mixed SELECT materialize(1), materialize(1), materialize(1), materialize(1.0) FROM numbers(100000000) + INSERT INTO test_logical_functions._4_1_Mixed SELECT materialize(1), materialize(1), materialize(1), materialize(1) FROM numbers(150000000) + INSERT INTO test_logical_functions._4_1_Nullable_UInt8 SELECT materialize(1), materialize(1), materialize(1), materialize(1) FROM numbers(150000000) + INSERT INTO test_logical_functions._4_1_Nullable_Mixed SELECT materialize(1), materialize(1), materialize(1), materialize(1.0) FROM numbers(50000000) SELECT count() FROM test_logical_functions._4_1_UInt8 WHERE NOT ignore(xor(x1,x2,x3,x4)) SELECT count() FROM test_logical_functions._4_1_Mixed WHERE NOT ignore(xor(x1,x2,x3,x4)) diff --git a/dbms/tests/performance/logical_functions_small.xml b/dbms/tests/performance/logical_functions_small.xml index fce5cf0f9d1..a2c90346ed6 100644 --- a/dbms/tests/performance/logical_functions_small.xml +++ b/dbms/tests/performance/logical_functions_small.xml @@ -17,6 +17,7 @@ 1 + @@ -25,8 +26,8 @@ CREATE TABLE test_logical_functions._2_0_UInt8 (x1 UInt8, x2 UInt8) ENGINE = Memory CREATE TABLE test_logical_functions._2_1_UInt8 (x1 UInt8, x2 UInt8) ENGINE = Memory - INSERT INTO test_logical_functions._2_0_UInt8 SELECT materialize(0) AS x1, materialize(0) AS x2 FROM numbers(2000000000) - INSERT INTO test_logical_functions._2_1_UInt8 SELECT materialize(1) AS x1, materialize(1) AS x2 FROM numbers(2000000000) + INSERT INTO test_logical_functions._2_0_UInt8 SELECT materialize(0) AS x1, materialize(0) AS x2 FROM numbers(1000000000) + INSERT INTO test_logical_functions._2_1_UInt8 SELECT materialize(1) AS x1, materialize(1) AS x2 FROM numbers(1000000000) SELECT count() FROM test_logical_functions._2_1_UInt8 WHERE NOT ignore(and(x1,x2)) SELECT count() FROM test_logical_functions._2_0_UInt8 WHERE NOT ignore(and(x1,x2)) @@ -34,20 +35,20 @@ SELECT count() FROM test_logical_functions._2_1_UInt8 WHERE NOT ignore(or(x1,x2)) SELECT count() FROM test_logical_functions._2_1_UInt8 WHERE NOT ignore(xor(x1,x2)) - SELECT count() FROM (SELECT toNullable(materialize(1)) AS x1, toNullable(materialize(1)) AS x2 FROM numbers(50000000)) WHERE NOT ignore(and(x1,x2)) - SELECT count() FROM (SELECT toNullable(materialize(0)) AS x1, toNullable(materialize(0)) AS x2 FROM numbers(50000000)) WHERE NOT ignore(and(x1,x2)) - SELECT count() FROM (SELECT toNullable(materialize(0)) AS x1, toNullable(materialize(0)) AS x2 FROM numbers(50000000)) WHERE NOT ignore(or(x1,x2)) - SELECT count() FROM (SELECT toNullable(materialize(1)) AS x1, toNullable(materialize(1)) AS x2 FROM numbers(50000000)) WHERE NOT ignore(or(x1,x2)) - SELECT count() FROM (SELECT toNullable(materialize(1)) AS x1, toNullable(materialize(1)) AS x2 FROM numbers(200000000)) WHERE NOT ignore(xor(x1,x2)) + SELECT count() FROM (SELECT toNullable(materialize(1)) AS x1, toNullable(materialize(1)) AS x2 FROM numbers(30000000)) WHERE NOT ignore(and(x1,x2)) + SELECT count() FROM (SELECT toNullable(materialize(0)) AS x1, toNullable(materialize(0)) AS x2 FROM numbers(40000000)) WHERE NOT ignore(and(x1,x2)) + SELECT count() FROM (SELECT toNullable(materialize(0)) AS x1, toNullable(materialize(0)) AS x2 FROM numbers(40000000)) WHERE NOT ignore(or(x1,x2)) + SELECT count() FROM (SELECT toNullable(materialize(1)) AS x1, toNullable(materialize(1)) AS x2 FROM numbers(30000000)) WHERE NOT ignore(or(x1,x2)) + SELECT count() FROM (SELECT toNullable(materialize(1)) AS x1, toNullable(materialize(1)) AS x2 FROM numbers(100000000)) WHERE NOT ignore(xor(x1,x2)) SELECT count() FROM (SELECT materialize(toUInt16(1)) AS x1, materialize(toUInt32(1)) AS x2 FROM numbers(200000000)) WHERE NOT ignore(and(x1,x2)) - SELECT count() FROM (SELECT materialize(toUInt32(1)) AS x1, materialize(toUInt64(1)) AS x2 FROM numbers(200000000)) WHERE NOT ignore(and(x1,x2)) - SELECT count() FROM (SELECT materialize(toUInt64(1)) AS x1, materialize(0.1) AS x2 FROM numbers(200000000)) WHERE NOT ignore(and(x1,x2)) + SELECT count() FROM (SELECT materialize(toUInt32(1)) AS x1, materialize(toUInt64(1)) AS x2 FROM numbers(100000000)) WHERE NOT ignore(and(x1,x2)) + SELECT count() FROM (SELECT materialize(toUInt64(1)) AS x1, materialize(0.1) AS x2 FROM numbers(100000000)) WHERE NOT ignore(and(x1,x2)) SELECT count() FROM (SELECT materialize(toUInt16(1)) AS x1, materialize(toUInt32(1)) AS x2 FROM numbers(200000000)) WHERE NOT ignore(or(x1,x2)) - SELECT count() FROM (SELECT materialize(toUInt32(1)) AS x1, materialize(toUInt64(1)) AS x2 FROM numbers(200000000)) WHERE NOT ignore(or(x1,x2)) - SELECT count() FROM (SELECT materialize(toUInt64(1)) AS x1, materialize(0.1) AS x2 FROM numbers(200000000)) WHERE NOT ignore(or(x1,x2)) - SELECT count() FROM (SELECT materialize(toUInt64(1)) AS x1, materialize(toUInt64(1)) AS x2 FROM numbers(200000000)) WHERE NOT ignore(xor(x1,x2)) + SELECT count() FROM (SELECT materialize(toUInt32(1)) AS x1, materialize(toUInt64(1)) AS x2 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2)) + SELECT count() FROM (SELECT materialize(toUInt64(1)) AS x1, materialize(0.1) AS x2 FROM numbers(100000000)) WHERE NOT ignore(or(x1,x2)) + SELECT count() FROM (SELECT materialize(toUInt64(1)) AS x1, materialize(toUInt64(1)) AS x2 FROM numbers(100000000)) WHERE NOT ignore(xor(x1,x2)) DROP TABLE test_logical_functions._2_0_UInt8 DROP TABLE test_logical_functions._2_1_UInt8 diff --git a/dbms/tests/performance/math.xml b/dbms/tests/performance/math.xml index f4d31713a08..280f0821964 100644 --- a/dbms/tests/performance/math.xml +++ b/dbms/tests/performance/math.xml @@ -14,6 +14,19 @@ + + func_slow + + atan + cbrt + cos + lgamma + tgamma + log10 + sin + tan + + func @@ -22,26 +35,25 @@ exp2 log2 exp10 - log10 sqrt - cbrt erf erfc - lgamma - tgamma - sin - cos - tan asin acos - atan sigmoid tanh + + arg + + toFloat64(number) + toFloat32(number) + number + + - SELECT count() FROM numbers(100000000) WHERE NOT ignore({func}(toFloat64(number))) - SELECT count() FROM numbers(100000000) WHERE NOT ignore({func}(toFloat32(number))) - SELECT count() FROM numbers(100000000) WHERE NOT ignore({func}(number)) + SELECT count() FROM numbers(100000000) WHERE NOT ignore({func}({arg})) + SELECT count() FROM numbers(10000000) WHERE NOT ignore({func_slow}({arg})) diff --git a/dbms/tests/performance/merge_tree_many_partitions_2.xml b/dbms/tests/performance/merge_tree_many_partitions_2.xml index 6a48fcc50ac..35d158abf83 100644 --- a/dbms/tests/performance/merge_tree_many_partitions_2.xml +++ b/dbms/tests/performance/merge_tree_many_partitions_2.xml @@ -2,7 +2,7 @@ loop CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x - INSERT INTO bad_partitions (x) SELECT * FROM numbers(5000) + INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000) diff --git a/dbms/tests/performance/merge_tree_simple_select.xml b/dbms/tests/performance/merge_tree_simple_select.xml index 5e482e0d44b..5600e12a5db 100644 --- a/dbms/tests/performance/merge_tree_simple_select.xml +++ b/dbms/tests/performance/merge_tree_simple_select.xml @@ -9,7 +9,7 @@ CREATE TABLE simple_mergetree (EventDate Date, x UInt64) ENGINE = MergeTree ORDER BY x - INSERT INTO simple_mergetree SELECT number, today() + intDiv(number, 10000000) FROM numbers(1000000000) + INSERT INTO simple_mergetree SELECT number, today() + intDiv(number, 10000000) FROM numbers_mt(100000000) OPTIMIZE TABLE simple_mergetree FINAL SELECT count() FROM simple_mergetree diff --git a/dbms/tests/performance/ngram_distance.xml b/dbms/tests/performance/ngram_distance.xml index 5b292bd0e25..78da4d55d0e 100644 --- a/dbms/tests/performance/ngram_distance.xml +++ b/dbms/tests/performance/ngram_distance.xml @@ -25,22 +25,33 @@ - SELECT DISTINCT URL,Title, ngramDistance(Title, URL) AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT SearchPhrase,Title, ngramDistance(Title, SearchPhrase) AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT Title, ngramDistance(Title, 'what is love') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT Title, ngramDistance(Title, 'baby dont hurt me') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT Title, ngramDistance(Title, 'no more') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT Title, ngramDistanceCaseInsensitive(Title, 'wHAt Is lovE') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT Title, ngramDistanceCaseInsensitive(Title, 'BABY DonT hUrT me') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT Title, ngramDistanceCaseInsensitive(Title, 'nO MOrE') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT URL,Title, ngramDistanceUTF8(Title, URL) AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT SearchPhrase,Title, ngramDistanceUTF8(Title, SearchPhrase) AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT Title, ngramDistanceUTF8(Title, 'метрика') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT URL, ngramDistanceUTF8(URL, 'как дела') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT URL, ngramDistanceUTF8(URL, 'чем занимаешься') AS distance FROM hits_10m_single FORMAT Null + + + small_table + + + hits_10m_single + + + - SELECT DISTINCT Title, ngramDistanceCaseInsensitiveUTF8(Title, 'Метрика') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT URL, ngramDistanceCaseInsensitiveUTF8(URL, 'как дЕлА') AS distance FROM hits_10m_single FORMAT Null - SELECT DISTINCT URL, ngramDistanceCaseInsensitiveUTF8(URL, 'Чем зАнимаешЬся') AS distance FROM hits_10m_single FORMAT Null + + SELECT ngramDistance(Title, URL) AS distance FROM {small_table} FORMAT Null + SELECT ngramDistance(Title, SearchPhrase) AS distance FROM {small_table} FORMAT Null + SELECT ngramDistance(Title, 'what is love') AS distance FROM hits_10m_single FORMAT Null + SELECT ngramDistance(Title, 'baby dont hurt me') AS distance FROM hits_10m_single FORMAT Null + SELECT ngramDistance(Title, 'no more') AS distance FROM hits_10m_single FORMAT Null + SELECT ngramDistanceCaseInsensitive(Title, 'wHAt Is lovE') AS distance FROM hits_10m_single FORMAT Null + SELECT ngramDistanceCaseInsensitive(Title, 'BABY DonT hUrT me') AS distance FROM hits_10m_single FORMAT Null + SELECT ngramDistanceCaseInsensitive(Title, 'nO MOrE') AS distance FROM hits_10m_single FORMAT Null + SELECT ngramDistanceUTF8(Title, URL) AS distance FROM {small_table} FORMAT Null + SELECT ngramDistanceUTF8(Title, SearchPhrase) AS distance FROM {small_table} FORMAT Null + SELECT ngramDistanceUTF8(Title, 'метрика') AS distance FROM hits_10m_single FORMAT Null + SELECT ngramDistanceUTF8(URL, 'как дела') AS distance FROM {small_table} FORMAT Null + SELECT ngramDistanceUTF8(URL, 'чем занимаешься') AS distance FROM {small_table} FORMAT Null + + SELECT ngramDistanceCaseInsensitiveUTF8(Title, 'Метрика') AS distance FROM hits_10m_single FORMAT Null + SELECT ngramDistanceCaseInsensitiveUTF8(URL, 'как дЕлА') AS distance FROM {small_table} FORMAT Null + SELECT ngramDistanceCaseInsensitiveUTF8(URL, 'Чем зАнимаешЬся') AS distance FROM {small_table} FORMAT Null diff --git a/dbms/tests/performance/prewhere.xml b/dbms/tests/performance/prewhere.xml index e9a7c4c5a7f..e4c9cc749ff 100644 --- a/dbms/tests/performance/prewhere.xml +++ b/dbms/tests/performance/prewhere.xml @@ -21,5 +21,5 @@ 1 - SELECT Title, URL FROM hits_10m_single PREWHERE WatchID % 2 = 1 WHERE UserID = 10000 FORMAT Null + SELECT Title FROM hits_10m_single PREWHERE WatchID % 2 = 1 WHERE UserID = 10000 FORMAT Null diff --git a/dbms/tests/performance/set_index.xml b/dbms/tests/performance/set_index.xml new file mode 100644 index 00000000000..4e24b7ccd79 --- /dev/null +++ b/dbms/tests/performance/set_index.xml @@ -0,0 +1,21 @@ + + loop + + CREATE TABLE test_in (`a` UInt32) ENGINE = MergeTree() ORDER BY a + INSERT INTO test_in SELECT number FROM numbers(500000000) + + + + 8000 + + + 7000 + 20000 + + + + + SELECT count() FROM test_in WHERE a IN (SELECT rand(1) FROM numbers(100000)) SETTINGS max_rows_to_read = 1, read_overflow_mode = 'break' + + DROP TABLE IF EXISTS test_in + diff --git a/dbms/tests/performance/trim_whitespace.xml b/dbms/tests/performance/trim_whitespace.xml index e886359a368..2038d8f5647 100644 --- a/dbms/tests/performance/trim_whitespace.xml +++ b/dbms/tests/performance/trim_whitespace.xml @@ -1,8 +1,15 @@ loop - CREATE TABLE IF NOT EXISTS whitespaces(value String) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY tuple() - INSERT INTO whitespaces SELECT value FROM (SELECT arrayStringConcat(groupArray(' ')) AS spaces, concat(spaces, toString(any(number)), spaces) AS value FROM numbers(100000000) GROUP BY pow(number, intHash32(number) % 4) % 12345678) + + create table if not exists whitespaces + engine = MergeTree() partition by tuple() order by tuple() + as + with 32 - log2(intHash32(number)) + 1 as num_spaces, + repeat(' ', toUInt32(num_spaces)) as spaces + select spaces || toString(number) || spaces value + from numbers_mt(100000000); + @@ -15,10 +22,14 @@ func - value trimLeft(value) trimRight(value) trimBoth(value) + + + + func_slow + replaceRegexpOne(value, '^ *', '') replaceRegexpOne(value, ' *$', '') replaceRegexpAll(value, '^ *| *$', '') @@ -26,7 +37,8 @@ - SELECT count() FROM whitespaces WHERE NOT ignore({func}) + SELECT {func} FROM whitespaces FORMAT Null + SELECT {func_slow} FROM whitespaces LIMIT 10000000 FORMAT Null DROP TABLE IF EXISTS whitespaces diff --git a/dbms/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.sql b/dbms/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.sql index 3ba1af5d547..4679c6f5a4e 100644 --- a/dbms/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.sql +++ b/dbms/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper.sql @@ -1,5 +1,8 @@ DROP TABLE IF EXISTS test.replicated_alter1; DROP TABLE IF EXISTS test.replicated_alter2; + +SET replication_alter_partitions_sync = 2; + CREATE TABLE test.replicated_alter1 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test/alter', 'r1', d, k, 8192); CREATE TABLE test.replicated_alter2 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test/alter', 'r2', d, k, 8192); diff --git a/dbms/tests/queries/0_stateless/00079_defaulted_columns.reference b/dbms/tests/queries/0_stateless/00079_defaulted_columns.reference index 03fd13f9044..53c257874c9 100644 --- a/dbms/tests/queries/0_stateless/00079_defaulted_columns.reference +++ b/dbms/tests/queries/0_stateless/00079_defaulted_columns.reference @@ -5,7 +5,6 @@ col3 UInt64 MATERIALIZED col1 + 2 col4 UInt64 ALIAS col1 + 3 10 11 12 13 -99 payload String date Date MATERIALIZED today() key UInt64 MATERIALIZED 0 * rand() diff --git a/dbms/tests/queries/0_stateless/00079_defaulted_columns.sql b/dbms/tests/queries/0_stateless/00079_defaulted_columns.sql index 77178478a4d..617dc2435c4 100644 --- a/dbms/tests/queries/0_stateless/00079_defaulted_columns.sql +++ b/dbms/tests/queries/0_stateless/00079_defaulted_columns.sql @@ -11,10 +11,7 @@ select * from defaulted; select col3, col4 from defaulted; drop table defaulted; -create table defaulted (col1 Int8, col2 UInt64 default (SELECT dummy+99 from system.one)) engine=Memory; -insert into defaulted (col1) values (0); -select col2 from defaulted; -drop table defaulted; +create table defaulted (col1 Int8, col2 UInt64 default (SELECT dummy+99 from system.one)) engine=Memory; --{serverError 116} create table defaulted (payload String, date materialized today(), key materialized 0 * rand()) engine=MergeTree(date, key, 8192); desc table defaulted; diff --git a/dbms/tests/queries/0_stateless/00090_union_race_conditions_1.sh b/dbms/tests/queries/0_stateless/00090_union_race_conditions_1.sh index b996f0d58f2..ab8b89daa6d 100755 --- a/dbms/tests/queries/0_stateless/00090_union_race_conditions_1.sh +++ b/dbms/tests/queries/0_stateless/00090_union_race_conditions_1.sh @@ -13,6 +13,6 @@ echo " INSERT INTO two_blocks VALUES ('2000-01-02'); " | $CLICKHOUSE_CLIENT -n -for i in {1..10}; do seq 1 100 | sed 's/.*/SELECT count() FROM (SELECT * FROM two_blocks);/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 | grep -vE '^2$' && echo 'Fail!' && break; echo -n '.'; done; echo +for i in {1..10}; do seq 1 100 | sed 's/.*/SELECT count() FROM (SELECT * FROM two_blocks);/' | $CLICKHOUSE_CLIENT -n | grep -vE '^2$' && echo 'Fail!' && break; echo -n '.'; done; echo echo "DROP TABLE two_blocks;" | $CLICKHOUSE_CLIENT -n diff --git a/dbms/tests/queries/0_stateless/00091_union_race_conditions_2.sh b/dbms/tests/queries/0_stateless/00091_union_race_conditions_2.sh index af7e1c25dd5..8018debb6b9 100755 --- a/dbms/tests/queries/0_stateless/00091_union_race_conditions_2.sh +++ b/dbms/tests/queries/0_stateless/00091_union_race_conditions_2.sh @@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -o errexit set -o pipefail -for i in {1..10}; do seq 1 100 | sed 's/.*/SELECT count() FROM (SELECT * FROM (SELECT * FROM system.numbers_mt LIMIT 111) LIMIT 55);/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 --max_block_size=1 | grep -vE '^55$' && echo 'Fail!' && break; echo -n '.'; done; echo +for i in {1..10}; do seq 1 100 | sed 's/.*/SELECT count() FROM (SELECT * FROM (SELECT * FROM system.numbers_mt LIMIT 111) LIMIT 55);/' | $CLICKHOUSE_CLIENT -n --max_block_size=1 | grep -vE '^55$' && echo 'Fail!' && break; echo -n '.'; done; echo diff --git a/dbms/tests/queries/0_stateless/00092_union_race_conditions_3.sh b/dbms/tests/queries/0_stateless/00092_union_race_conditions_3.sh index bd28a69be58..7958d11e35b 100755 --- a/dbms/tests/queries/0_stateless/00092_union_race_conditions_3.sh +++ b/dbms/tests/queries/0_stateless/00092_union_race_conditions_3.sh @@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -o errexit set -o pipefail -for i in {1..10}; do seq 1 100 | sed 's/.*/SELECT * FROM (SELECT * FROM system.numbers_mt LIMIT 111) LIMIT 55;/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 --max_block_size=1 | wc -l | grep -vE '^5500$' && echo 'Fail!' && break; echo -n '.'; done; echo +for i in {1..10}; do seq 1 100 | sed 's/.*/SELECT * FROM (SELECT * FROM system.numbers_mt LIMIT 111) LIMIT 55;/' | $CLICKHOUSE_CLIENT -n --max_block_size=1 | wc -l | grep -vE '^5500$' && echo 'Fail!' && break; echo -n '.'; done; echo diff --git a/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh b/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh index 2d255a0c2f6..0554124c36f 100755 --- a/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh +++ b/dbms/tests/queries/0_stateless/00093_union_race_conditions_4.sh @@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -o errexit set -o pipefail -for i in {1..10}; do seq 1 10 | sed 's/.*/SELECT 1 % ((number + 500) % 1000) FROM numbers_mt(1000);/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 --max_block_size=1 >/dev/null 2>&1 && echo 'Fail!' && break; echo -n '.'; done; echo +for i in {1..10}; do seq 1 10 | sed 's/.*/SELECT 1 % ((number + 500) % 1000) FROM numbers_mt(1000);/' | $CLICKHOUSE_CLIENT -n --max_block_size=1 >/dev/null 2>&1 && echo 'Fail!' && break; echo -n '.'; done; echo diff --git a/dbms/tests/queries/0_stateless/00094_union_race_conditions_5.sh b/dbms/tests/queries/0_stateless/00094_union_race_conditions_5.sh index a0fe4badec6..ed773ae05f8 100755 --- a/dbms/tests/queries/0_stateless/00094_union_race_conditions_5.sh +++ b/dbms/tests/queries/0_stateless/00094_union_race_conditions_5.sh @@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -o errexit set -o pipefail -for i in {1..10}; do seq 1 100 | sed 's/.*/SELECT * FROM system.numbers_mt LIMIT 111;/' | $CLICKHOUSE_CLIENT -n --receive_timeout=1 --max_block_size=$(($RANDOM % 123 + 1)) | wc -l | grep -vE '^11100$' && echo 'Fail!' && break; echo -n '.'; done; echo +for i in {1..10}; do seq 1 100 | sed 's/.*/SELECT * FROM system.numbers_mt LIMIT 111;/' | $CLICKHOUSE_CLIENT -n --max_block_size=$(($RANDOM % 123 + 1)) | wc -l | grep -vE '^11100$' && echo 'Fail!' && break; echo -n '.'; done; echo diff --git a/dbms/tests/queries/0_stateless/00265_content_type_and_format.sh b/dbms/tests/queries/0_stateless/00265_content_type_and_format.sh deleted file mode 100755 index 2a36a17c6a1..00000000000 --- a/dbms/tests/queries/0_stateless/00265_content_type_and_format.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&default_format=JSONCompact" --data-binary @- <<< "SELECT 1" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT JSON" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT TabSeparated" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT Vertical" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT Native" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; -${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT RowBinary" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' | sed 's/\r$//' | sort; diff --git a/dbms/tests/queries/0_stateless/00265_content_type_and_format.reference b/dbms/tests/queries/0_stateless/00265_http_content_type_format_timezone.reference similarity index 64% rename from dbms/tests/queries/0_stateless/00265_content_type_and_format.reference rename to dbms/tests/queries/0_stateless/00265_http_content_type_format_timezone.reference index dbe9ebc0f58..9de2c3d4ba4 100644 --- a/dbms/tests/queries/0_stateless/00265_content_type_and_format.reference +++ b/dbms/tests/queries/0_stateless/00265_http_content_type_format_timezone.reference @@ -1,14 +1,21 @@ < Content-Type: application/json; charset=UTF-8 < X-ClickHouse-Format: JSONCompact +< X-ClickHouse-Timezone: CLICKHOUSE_TIMEZONE < Content-Type: application/json; charset=UTF-8 < X-ClickHouse-Format: JSON +< X-ClickHouse-Timezone: CLICKHOUSE_TIMEZONE < Content-Type: text/tab-separated-values; charset=UTF-8 < X-ClickHouse-Format: TabSeparated +< X-ClickHouse-Timezone: CLICKHOUSE_TIMEZONE < Content-Type: text/tab-separated-values; charset=UTF-8 < X-ClickHouse-Format: TabSeparated +< X-ClickHouse-Timezone: CLICKHOUSE_TIMEZONE < Content-Type: text/plain; charset=UTF-8 < X-ClickHouse-Format: Vertical +< X-ClickHouse-Timezone: CLICKHOUSE_TIMEZONE < Content-Type: application/octet-stream < X-ClickHouse-Format: Native +< X-ClickHouse-Timezone: CLICKHOUSE_TIMEZONE < Content-Type: application/octet-stream < X-ClickHouse-Format: RowBinary +< X-ClickHouse-Timezone: CLICKHOUSE_TIMEZONE diff --git a/dbms/tests/queries/0_stateless/00265_http_content_type_format_timezone.sh b/dbms/tests/queries/0_stateless/00265_http_content_type_format_timezone.sh new file mode 100755 index 00000000000..2ffadacd81a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00265_http_content_type_format_timezone.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +CLICKHOUSE_TIMEZONE_ESCAPED=$($CLICKHOUSE_CLIENT --query="SELECT timezone()" | sed 's/[]\/$*.^+:()[]/\\&/g') + +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&default_format=JSONCompact" --data-binary @- <<< "SELECT 1" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' -e '< X-ClickHouse-Timezone' | sed "s|$CLICKHOUSE_TIMEZONE_ESCAPED|CLICKHOUSE_TIMEZONE|" | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT JSON" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' -e '< X-ClickHouse-Timezone' | sed "s|$CLICKHOUSE_TIMEZONE_ESCAPED|CLICKHOUSE_TIMEZONE|" | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' -e '< X-ClickHouse-Timezone' | sed "s|$CLICKHOUSE_TIMEZONE_ESCAPED|CLICKHOUSE_TIMEZONE|" | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT TabSeparated" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' -e '< X-ClickHouse-Timezone' | sed "s|$CLICKHOUSE_TIMEZONE_ESCAPED|CLICKHOUSE_TIMEZONE|" | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT Vertical" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' -e '< X-ClickHouse-Timezone' | sed "s|$CLICKHOUSE_TIMEZONE_ESCAPED|CLICKHOUSE_TIMEZONE|" | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT Native" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' -e '< X-ClickHouse-Timezone' | sed "s|$CLICKHOUSE_TIMEZONE_ESCAPED|CLICKHOUSE_TIMEZONE|" | sed 's/\r$//' | sort; +${CLICKHOUSE_CURL} -vsS ${CLICKHOUSE_URL} --data-binary @- <<< "SELECT 1 FORMAT RowBinary" 2>&1 | grep -e '< Content-Type' -e '< X-ClickHouse-Format' -e '< X-ClickHouse-Timezone' | sed "s|$CLICKHOUSE_TIMEZONE_ESCAPED|CLICKHOUSE_TIMEZONE|" | sed 's/\r$//' | sort; diff --git a/dbms/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app.sh b/dbms/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app.sh index 0d4abee9a06..a6eaa603b7f 100755 --- a/dbms/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app.sh +++ b/dbms/tests/queries/0_stateless/00385_storage_file_and_clickhouse-local_app.sh @@ -48,14 +48,8 @@ echo echo ${CLICKHOUSE_LOCAL} -q "CREATE TABLE sophisticated_default ( - a UInt8 DEFAULT - ( - SELECT number FROM system.numbers LIMIT 3,1 - ), - b UInt8 ALIAS - ( - SELECT dummy+9 FROM system.one - ), + a UInt8 DEFAULT 3, + b UInt8 ALIAS a + 5, c UInt8 ) ENGINE = Memory; SELECT count() FROM system.tables WHERE name='sophisticated_default';" diff --git a/dbms/tests/queries/0_stateless/00501_http_head.sh b/dbms/tests/queries/0_stateless/00501_http_head.sh index 7251fc2cf21..8103889626b 100755 --- a/dbms/tests/queries/0_stateless/00501_http_head.sh +++ b/dbms/tests/queries/0_stateless/00501_http_head.sh @@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh ( ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}&query=SELECT%201"; - ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}&query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Server-Display-Name:" | grep -v "X-ClickHouse-Query-Id:" | grep -v "X-ClickHouse-Format:" + ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}&query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Server-Display-Name:" | grep -v "X-ClickHouse-Query-Id:" | grep -v "X-ClickHouse-Format:" | grep -v "X-ClickHouse-Timezone:" if [[ `${CLICKHOUSE_CURL} -sS -X POST -I "${CLICKHOUSE_URL}&query=SELECT+1" | grep -c '411 Length Required'` -ne 1 ]]; then echo FAIL diff --git a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.reference b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.reference index 9f61a33663e..8c3fc467d57 100644 --- a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.reference +++ b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.reference @@ -2,4 +2,4 @@ 0 1 1 1 0 -1 1 1 +1 1 diff --git a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index 1bf995a1a7e..15961eac73f 100755 --- a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -63,7 +63,6 @@ $CLICKHOUSE_CLIENT $settings -q " SELECT -- max(thread_realtime), $query_elapsed, max(thread_time_user_system_io), 0.9 * $query_elapsed <= max(thread_realtime) AND max(thread_realtime) <= 1.1 * $query_elapsed, - 0.7 * $query_elapsed <= max(thread_time_user_system_io) AND max(thread_time_user_system_io) <= 1.3 * $query_elapsed, uniqExact(thread_id) = $threads FROM ( diff --git a/dbms/tests/queries/0_stateless/00732_quorum_insert_simple_test_1_parts_zookeeper.reference b/dbms/tests/queries/0_stateless/00732_quorum_insert_simple_test_1_parts_zookeeper.reference index 128dde7896d..fe6c0ad10f2 100644 --- a/dbms/tests/queries/0_stateless/00732_quorum_insert_simple_test_1_parts_zookeeper.reference +++ b/dbms/tests/queries/0_stateless/00732_quorum_insert_simple_test_1_parts_zookeeper.reference @@ -2,4 +2,10 @@ 2 1 2 +1 2 +1 +2 +1 +10 +10 diff --git a/dbms/tests/queries/0_stateless/00732_quorum_insert_simple_test_1_parts_zookeeper.sql b/dbms/tests/queries/0_stateless/00732_quorum_insert_simple_test_1_parts_zookeeper.sql index 19ca74a4c90..6c28a8e0553 100644 --- a/dbms/tests/queries/0_stateless/00732_quorum_insert_simple_test_1_parts_zookeeper.sql +++ b/dbms/tests/queries/0_stateless/00732_quorum_insert_simple_test_1_parts_zookeeper.sql @@ -17,7 +17,18 @@ SELECT x FROM quorum2 ORDER BY x; OPTIMIZE TABLE quorum1 PARTITION '2018-11-15' FINAL; +-- everything works fine after merge +SELECT x FROM quorum1 ORDER BY x; +SELECT x FROM quorum2 ORDER BY x; + SELECT count(*) FROM system.parts WHERE active AND database = currentDatabase() AND table='quorum1'; +INSERT INTO quorum1 VALUES (3, '2018-11-15'); +INSERT INTO quorum1 VALUES (4, '2018-11-15'); + +-- and after we add new parts +SELECT sum(x) FROM quorum1; +SELECT sum(x) FROM quorum2; + DROP TABLE IF EXISTS quorum1; DROP TABLE IF EXISTS quorum2; diff --git a/dbms/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql b/dbms/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql index 95092d55048..f7ea0c1c0d3 100644 --- a/dbms/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql +++ b/dbms/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql @@ -1,9 +1,9 @@ SET send_logs_level = 'none'; -DROP TABLE IF EXISTS test.old_style; -CREATE TABLE test.old_style(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/old_style', 'r1', d, x, 8192); -ALTER TABLE test.old_style ADD COLUMN y UInt32, MODIFY ORDER BY (x, y); -- { serverError 36 } -DROP TABLE test.old_style; +DROP TABLE IF EXISTS old_style; +CREATE TABLE old_style(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/old_style', 'r1', d, x, 8192); +ALTER TABLE old_style ADD COLUMN y UInt32, MODIFY ORDER BY (x, y); -- { serverError 36 } +DROP TABLE old_style; DROP TABLE IF EXISTS test.summing_r1; DROP TABLE IF EXISTS test.summing_r2; @@ -40,8 +40,11 @@ SELECT '*** Check SHOW CREATE TABLE ***'; SHOW CREATE TABLE test.summing_r2; DETACH TABLE test.summing_r2; -ALTER TABLE test.summing_r1 ADD COLUMN t UInt32 AFTER z, MODIFY ORDER BY (x, y, t * t); -- { serverError 341 } +ALTER TABLE test.summing_r1 ADD COLUMN t UInt32 AFTER z, MODIFY ORDER BY (x, y, t * t) SETTINGS replication_alter_partitions_sync = 2; -- { serverError 341 } ATTACH TABLE test.summing_r2; + +SELECT sleep(1) Format Null; + SELECT '*** Check SHOW CREATE TABLE after offline ALTER ***'; SHOW CREATE TABLE test.summing_r2; diff --git a/dbms/tests/queries/0_stateless/00808_not_optimize_predicate.sql b/dbms/tests/queries/0_stateless/00808_not_optimize_predicate.sql index 542bea690e8..b4204f0c3bb 100644 --- a/dbms/tests/queries/0_stateless/00808_not_optimize_predicate.sql +++ b/dbms/tests/queries/0_stateless/00808_not_optimize_predicate.sql @@ -10,7 +10,7 @@ SET enable_optimize_predicate_expression = 1; SELECT '-------ENABLE OPTIMIZE PREDICATE-------'; SELECT * FROM (SELECT * FROM test_00808 FINAL) WHERE id = 1; -SELECT * FROM (SELECT * FROM test_00808 LIMIT 1) WHERE id = 1; +SELECT * FROM (SELECT * FROM test_00808 ORDER BY id LIMIT 1) WHERE id = 1; SELECT * FROM (SELECT id FROM test_00808 GROUP BY id LIMIT 1 BY id) WHERE id = 1; SET force_primary_key = 1; diff --git a/dbms/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.sql b/dbms/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.sql index a08d6a20a87..2c13d2de166 100644 --- a/dbms/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.sql +++ b/dbms/tests/queries/0_stateless/00836_indices_alter_replicated_zookeeper.sql @@ -3,6 +3,7 @@ DROP TABLE IF EXISTS test.minmax_idx_r; DROP TABLE IF EXISTS test.minmax_idx2; DROP TABLE IF EXISTS test.minmax_idx2_r; +SET replication_alter_partitions_sync = 2; CREATE TABLE test.minmax_idx ( @@ -109,4 +110,4 @@ SELECT * FROM test.minmax_idx2_r WHERE u64 * i32 >= 2 ORDER BY (u64, i32); DROP TABLE test.minmax_idx; DROP TABLE test.minmax_idx_r; DROP TABLE test.minmax_idx2; -DROP TABLE test.minmax_idx2_r; \ No newline at end of file +DROP TABLE test.minmax_idx2_r; diff --git a/dbms/tests/queries/0_stateless/00910_zookeeper_test_alter_compression_codecs.sql b/dbms/tests/queries/0_stateless/00910_zookeeper_test_alter_compression_codecs.sql index 9a27429d180..d9ca0595fa5 100644 --- a/dbms/tests/queries/0_stateless/00910_zookeeper_test_alter_compression_codecs.sql +++ b/dbms/tests/queries/0_stateless/00910_zookeeper_test_alter_compression_codecs.sql @@ -1,65 +1,68 @@ SET send_logs_level = 'none'; +SET replication_alter_partitions_sync = 2; -DROP TABLE IF EXISTS test.alter_compression_codec1; -DROP TABLE IF EXISTS test.alter_compression_codec2; +DROP TABLE IF EXISTS alter_compression_codec1; +DROP TABLE IF EXISTS alter_compression_codec2; -CREATE TABLE test.alter_compression_codec1 ( +CREATE TABLE alter_compression_codec1 ( somedate Date CODEC(LZ4), id UInt64 CODEC(NONE) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/alter_compression_codecs', '1') PARTITION BY somedate ORDER BY id; -CREATE TABLE test.alter_compression_codec2 ( +CREATE TABLE alter_compression_codec2 ( somedate Date CODEC(LZ4), id UInt64 CODEC(NONE) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/alter_compression_codecs', '2') PARTITION BY somedate ORDER BY id; -INSERT INTO test.alter_compression_codec1 VALUES('2018-01-01', 1); -INSERT INTO test.alter_compression_codec1 VALUES('2018-01-01', 2); -SYSTEM SYNC REPLICA test.alter_compression_codec2; +INSERT INTO alter_compression_codec1 VALUES('2018-01-01', 1); +INSERT INTO alter_compression_codec1 VALUES('2018-01-01', 2); +SYSTEM SYNC REPLICA alter_compression_codec2; -SELECT * FROM test.alter_compression_codec1 ORDER BY id; -SELECT * FROM test.alter_compression_codec2 ORDER BY id; +SELECT * FROM alter_compression_codec1 ORDER BY id; +SELECT * FROM alter_compression_codec2 ORDER BY id; -ALTER TABLE test.alter_compression_codec1 ADD COLUMN alter_column String DEFAULT 'default_value' CODEC(ZSTD); -SYSTEM SYNC REPLICA test.alter_compression_codec2; +ALTER TABLE alter_compression_codec1 ADD COLUMN alter_column String DEFAULT 'default_value' CODEC(ZSTD); +SYSTEM SYNC REPLICA alter_compression_codec1; +SYSTEM SYNC REPLICA alter_compression_codec2; -SELECT compression_codec FROM system.columns WHERE database = 'test' AND table = 'alter_compression_codec1' AND name = 'alter_column'; -SELECT compression_codec FROM system.columns WHERE database = 'test' AND table = 'alter_compression_codec2' AND name = 'alter_column'; +SELECT compression_codec FROM system.columns WHERE table = 'alter_compression_codec1' AND name = 'alter_column'; +SELECT compression_codec FROM system.columns WHERE table = 'alter_compression_codec2' AND name = 'alter_column'; -INSERT INTO test.alter_compression_codec1 VALUES('2018-01-01', 3, '3'); -INSERT INTO test.alter_compression_codec1 VALUES('2018-01-01', 4, '4'); -SYSTEM SYNC REPLICA test.alter_compression_codec2; +INSERT INTO alter_compression_codec1 VALUES('2018-01-01', 3, '3'); +INSERT INTO alter_compression_codec1 VALUES('2018-01-01', 4, '4'); +SYSTEM SYNC REPLICA alter_compression_codec1; +SYSTEM SYNC REPLICA alter_compression_codec2; -SELECT * FROM test.alter_compression_codec1 ORDER BY id; -SELECT * FROM test.alter_compression_codec2 ORDER BY id; +SELECT * FROM alter_compression_codec1 ORDER BY id; +SELECT * FROM alter_compression_codec2 ORDER BY id; -ALTER TABLE test.alter_compression_codec1 MODIFY COLUMN alter_column CODEC(NONE); -SELECT compression_codec FROM system.columns WHERE database = 'test' AND table = 'alter_compression_codec1' AND name = 'alter_column'; -SELECT compression_codec FROM system.columns WHERE database = 'test' AND table = 'alter_compression_codec2' AND name = 'alter_column'; +ALTER TABLE alter_compression_codec1 MODIFY COLUMN alter_column CODEC(NONE); +SELECT compression_codec FROM system.columns WHERE table = 'alter_compression_codec1' AND name = 'alter_column'; +SELECT compression_codec FROM system.columns WHERE table = 'alter_compression_codec2' AND name = 'alter_column'; -INSERT INTO test.alter_compression_codec2 VALUES('2018-01-01', 5, '5'); -INSERT INTO test.alter_compression_codec2 VALUES('2018-01-01', 6, '6'); -SYSTEM SYNC REPLICA test.alter_compression_codec1; -SELECT * FROM test.alter_compression_codec1 ORDER BY id; -SELECT * FROM test.alter_compression_codec2 ORDER BY id; +INSERT INTO alter_compression_codec2 VALUES('2018-01-01', 5, '5'); +INSERT INTO alter_compression_codec2 VALUES('2018-01-01', 6, '6'); +SYSTEM SYNC REPLICA alter_compression_codec1; +SELECT * FROM alter_compression_codec1 ORDER BY id; +SELECT * FROM alter_compression_codec2 ORDER BY id; -ALTER TABLE test.alter_compression_codec1 MODIFY COLUMN alter_column CODEC(ZSTD, LZ4HC, LZ4, LZ4, NONE); -SYSTEM SYNC REPLICA test.alter_compression_codec1; -SYSTEM SYNC REPLICA test.alter_compression_codec2; -SELECT compression_codec FROM system.columns WHERE database = 'test' AND table = 'alter_compression_codec1' AND name = 'alter_column'; -SELECT compression_codec FROM system.columns WHERE database = 'test' AND table = 'alter_compression_codec2' AND name = 'alter_column'; +ALTER TABLE alter_compression_codec1 MODIFY COLUMN alter_column CODEC(ZSTD, LZ4HC, LZ4, LZ4, NONE); +SYSTEM SYNC REPLICA alter_compression_codec1; +SYSTEM SYNC REPLICA alter_compression_codec2; +SELECT compression_codec FROM system.columns WHERE table = 'alter_compression_codec1' AND name = 'alter_column'; +SELECT compression_codec FROM system.columns WHERE table = 'alter_compression_codec2' AND name = 'alter_column'; -INSERT INTO test.alter_compression_codec1 VALUES('2018-01-01', 7, '7'); -INSERT INTO test.alter_compression_codec2 VALUES('2018-01-01', 8, '8'); -SYSTEM SYNC REPLICA test.alter_compression_codec2; -SYSTEM SYNC REPLICA test.alter_compression_codec1; -SELECT * FROM test.alter_compression_codec1 ORDER BY id; -SELECT * FROM test.alter_compression_codec2 ORDER BY id; +INSERT INTO alter_compression_codec1 VALUES('2018-01-01', 7, '7'); +INSERT INTO alter_compression_codec2 VALUES('2018-01-01', 8, '8'); +SYSTEM SYNC REPLICA alter_compression_codec2; +SYSTEM SYNC REPLICA alter_compression_codec1; +SELECT * FROM alter_compression_codec1 ORDER BY id; +SELECT * FROM alter_compression_codec2 ORDER BY id; -ALTER TABLE test.alter_compression_codec1 MODIFY COLUMN alter_column FixedString(100); -SYSTEM SYNC REPLICA test.alter_compression_codec2; -SELECT compression_codec FROM system.columns WHERE database = 'test' AND table = 'alter_compression_codec1' AND name = 'alter_column'; -SELECT compression_codec FROM system.columns WHERE database = 'test' AND table = 'alter_compression_codec2' AND name = 'alter_column'; +ALTER TABLE alter_compression_codec1 MODIFY COLUMN alter_column FixedString(100); +SYSTEM SYNC REPLICA alter_compression_codec2; +SELECT compression_codec FROM system.columns WHERE table = 'alter_compression_codec1' AND name = 'alter_column'; +SELECT compression_codec FROM system.columns WHERE table = 'alter_compression_codec2' AND name = 'alter_column'; -DROP TABLE IF EXISTS test.alter_compression_codec1; -DROP TABLE IF EXISTS test.alter_compression_codec2; +DROP TABLE IF EXISTS alter_compression_codec1; +DROP TABLE IF EXISTS alter_compression_codec2; diff --git a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql index f2e453c99d2..08c9bbd8004 100644 --- a/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql +++ b/dbms/tests/queries/0_stateless/00980_zookeeper_merge_tree_alter_settings.sql @@ -1,6 +1,8 @@ DROP TABLE IF EXISTS replicated_table_for_alter1; DROP TABLE IF EXISTS replicated_table_for_alter2; +SET replication_alter_partitions_sync = 2; + CREATE TABLE replicated_table_for_alter1 ( id UInt64, Data String diff --git a/dbms/tests/queries/0_stateless/00996_limit_with_ties.reference b/dbms/tests/queries/0_stateless/00996_limit_with_ties.reference index aa5d102bc9b..ecbbaa76f7a 100644 --- a/dbms/tests/queries/0_stateless/00996_limit_with_ties.reference +++ b/dbms/tests/queries/0_stateless/00996_limit_with_ties.reference @@ -50,3 +50,6 @@ 2 2 * +100 +100 +100 diff --git a/dbms/tests/queries/0_stateless/00996_limit_with_ties.sql b/dbms/tests/queries/0_stateless/00996_limit_with_ties.sql index 3e4813bc6b5..8d7aedc2cb0 100644 --- a/dbms/tests/queries/0_stateless/00996_limit_with_ties.sql +++ b/dbms/tests/queries/0_stateless/00996_limit_with_ties.sql @@ -1,8 +1,6 @@ DROP TABLE IF EXISTS ties; CREATE TABLE ties (a Int) ENGINE = Memory; --- SET experimental_use_processors=1; - INSERT INTO ties VALUES (1), (1), (2), (2), (2), (2) (3), (3); SELECT a FROM ties order by a limit 1 with ties; @@ -32,4 +30,9 @@ SELECT '*'; SELECT a FROM ties order by a limit 3, 2 with ties; SELECT '*'; +select count() from (select number > 100 from numbers(2000) order by number > 100 limit 1, 7 with ties); --TODO replace "number > 100" with "number > 100 as n" +select count() from (select number, number < 100 from numbers(2000) order by number < 100 desc limit 10 with ties); +SET max_block_size = 5; +select count() from (select number < 100, number from numbers(2000) order by number < 100 desc limit 10 with ties); + DROP TABLE ties; diff --git a/dbms/tests/queries/0_stateless/01019_parallel_parsing_cancel.sh b/dbms/tests/queries/0_stateless/01019_parallel_parsing_cancel.sh index 94d4be1d798..19d2b487955 100755 --- a/dbms/tests/queries/0_stateless/01019_parallel_parsing_cancel.sh +++ b/dbms/tests/queries/0_stateless/01019_parallel_parsing_cancel.sh @@ -40,8 +40,8 @@ function thread2() export -f thread1; export -f thread2; -bash -c thread1 & -bash -c thread2 & +bash -c thread1 > /dev/null 2>&1 & +bash -c thread2 > /dev/null 2>&1 & wait echo OK diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference new file mode 100644 index 00000000000..7049ddce9a4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference @@ -0,0 +1,10 @@ +14850 +14850 +59700 +59700 +59700 +0 +6000 +6000 +2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 +2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql new file mode 100644 index 00000000000..3d4f1e6e044 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql @@ -0,0 +1,71 @@ +DROP TABLE IF EXISTS test_alter_on_mutation; + +CREATE TABLE test_alter_on_mutation +( + date Date, + key UInt64, + value String +) +ENGINE ReplicatedMergeTree('/clickhouse/tables/test_alter_on_mutation', '1') +ORDER BY key PARTITION BY date; + +INSERT INTO test_alter_on_mutation select toDate('2020-01-05'), number, toString(number) from system.numbers limit 100; +INSERT INTO test_alter_on_mutation select toDate('2020-01-06'), number, toString(number) from system.numbers limit 100; +INSERT INTO test_alter_on_mutation select toDate('2020-01-07'), number, toString(number) from system.numbers limit 100; + +SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; + +ALTER TABLE test_alter_on_mutation MODIFY COLUMN value UInt64; + +SELECT sum(value) from test_alter_on_mutation; + +INSERT INTO test_alter_on_mutation select toDate('2020-01-05'), number, toString(number) from system.numbers limit 100, 100; +INSERT INTO test_alter_on_mutation select toDate('2020-01-06'), number, toString(number) from system.numbers limit 100, 100; +INSERT INTO test_alter_on_mutation select toDate('2020-01-07'), number, toString(number) from system.numbers limit 100, 100; + +OPTIMIZE TABLE test_alter_on_mutation FINAL; + +SELECT sum(value) from test_alter_on_mutation; + +ALTER TABLE test_alter_on_mutation MODIFY COLUMN value String; + +SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; + +OPTIMIZE TABLE test_alter_on_mutation FINAL; + +SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; + +ALTER TABLE test_alter_on_mutation ADD COLUMN value1 Float64; + +SELECT sum(value1) from test_alter_on_mutation; + +ALTER TABLE test_alter_on_mutation DROP COLUMN value; + +SELECT sum(value) from test_alter_on_mutation; -- {serverError 47} + +ALTER TABLE test_alter_on_mutation ADD COLUMN value String DEFAULT '10'; + +SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; + +-- TODO(alesap) +OPTIMIZE table test_alter_on_mutation FINAL; + +ALTER TABLE test_alter_on_mutation MODIFY COLUMN value UInt64 DEFAULT 10; + +SELECT sum(value) from test_alter_on_mutation; + +DROP TABLE IF EXISTS test_alter_on_mutation; + +DROP TABLE IF EXISTS nested_alter; + +CREATE TABLE nested_alter (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), `s` String DEFAULT '0') ENGINE = ReplicatedMergeTree('/clickhouse/tables/nested_alter', 'r2', d, k, 8192); + +INSERT INTO nested_alter VALUES ('2015-01-01', 6,38,'2014-07-15 13:26:50',[10,20,30],['asd','qwe','qwe'],['2000-01-01','2000-01-01','2000-01-03'],'100500'); + +SELECT * FROM nested_alter; + +ALTER TABLE nested_alter DROP COLUMN `n.d`; + +SELECT * FROM nested_alter; + +DROP TABLE nested_alter; diff --git a/dbms/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference b/dbms/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference new file mode 100644 index 00000000000..198f79cf9a4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference @@ -0,0 +1,8 @@ +Wrong column name. +CREATE TABLE default.table_for_bad_alters (`key` UInt64, `value1` UInt8, `value2` String) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters (`key` UInt64, `value1` UInt8, `value2` UInt32) ENGINE = ReplicatedMergeTree(\'/clickhouse/tables/table_for_bad_alters\', \'1\') ORDER BY key SETTINGS index_granularity = 8192 +syntax error at begin of string. +7 +Hello +World +Wrong index name. diff --git a/dbms/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh new file mode 100755 index 00000000000..b00215271fc --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_bad_alters"; + +$CLICKHOUSE_CLIENT -n --query "CREATE TABLE table_for_bad_alters ( + key UInt64, + value1 UInt8, + value2 String +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/table_for_bad_alters', '1') +ORDER BY key;" + +$CLICKHOUSE_CLIENT --query "INSERT INTO table_for_bad_alters VALUES(1, 1, 'Hello');" +$CLICKHOUSE_CLIENT --query "ALTER TABLE table_for_bad_alters MODIFY COLUMN value1 UInt32, DROP COLUMN non_existing_column" 2>&1 | grep -o "Wrong column name." | uniq +$CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE table_for_bad_alters;" # nothing changed + +$CLICKHOUSE_CLIENT --query "ALTER TABLE table_for_bad_alters MODIFY COLUMN value2 UInt32 SETTINGS replication_alter_partitions_sync=0;" + +sleep 2 + +while [[ $($CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='0000000000'" 2>&1) ]]; do + sleep 1 +done + +$CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE table_for_bad_alters;" # Type changed, but we can revert back + +$CLICKHOUSE_CLIENT --query "INSERT INTO table_for_bad_alters VALUES(2, 2, 7)" + +$CLICKHOUSE_CLIENT --query "SELECT distinct(value2) FROM table_for_bad_alters" 2>&1 | grep -o 'syntax error at begin of string.' | uniq + +$CLICKHOUSE_CLIENT --query "ALTER TABLE table_for_bad_alters MODIFY COLUMN value2 String SETTINGS replication_alter_partitions_sync=2" + +$CLICKHOUSE_CLIENT --query "INSERT INTO table_for_bad_alters VALUES(3, 3, 'World')" + +$CLICKHOUSE_CLIENT --query "SELECT value2 FROM table_for_bad_alters ORDER BY value2" + +$CLICKHOUSE_CLIENT --query "ALTER TABLE table_for_bad_alters DROP INDEX idx2" 2>&1 | grep -o 'Wrong index name.' | uniq + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_bad_alters" diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.reference b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.reference new file mode 100644 index 00000000000..af33a5bfc3f --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.reference @@ -0,0 +1,9 @@ +Starting alters +Finishing alters +Equal number of columns +0 +0 +0 +0 +0 +0 diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh new file mode 100755 index 00000000000..7a71df32c8e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +REPLICAS=3 + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_add_drop_$i" +done + + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/concurrent_alter_add_drop_column', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" +done + +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_1 SELECT number, number + 10 from numbers(100000)" + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_add_drop_$i" +done + + +function alter_thread() +{ + TYPES=(Float64 String UInt8 UInt32) + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + ADD=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA ADD COLUMN value$ADD UInt32 DEFAULT 42 SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + DROP=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA DROP COLUMN value$DROP SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM + done +} + + +function optimize_thread() +{ + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE concurrent_alter_add_drop_$REPLICA FINAL SETTINGS replication_alter_partitions_sync=0"; + sleep 0.$RANDOM + done +} + +function insert_thread() +{ + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_$REPLICA VALUES($RANDOM, 7)" + sleep 0.$RANDOM + done +} + + +echo "Starting alters" +export -f alter_thread; +export -f optimize_thread; +export -f insert_thread; + + +TIMEOUT=30 + +# Sometimes we detach and attach tables +timeout $TIMEOUT bash -c alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c alter_thread 2> /dev/null & + +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & + +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & + + +wait + +echo "Finishing alters" + +columns1=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_1'" 2> /dev/null) +columns2=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_2'" 2> /dev/null) +columns3=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_3'" 2> /dev/null) + +while [ "$columns1" != "$columns2" ] || [ "$columns2" != "$columns3" ]; do + columns1=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_1'" 2> /dev/null) + columns2=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_2'" 2> /dev/null) + columns3=$($CLICKHOUSE_CLIENT --query "select count() from system.columns where table='concurrent_alter_add_drop_3'" 2> /dev/null) + + sleep 1 +done + +echo "Equal number of columns" + +# This alter will finish all previous, but replica 1 maybe still not up-to-date +while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_1 MODIFY COLUMN value0 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do + sleep 1 +done + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_add_drop_$i" + $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE is_done = 0 and table = 'concurrent_alter_add_drop_$i'" + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE is_done = 0 and table = 'concurrent_alter_add_drop_$i'" + $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.replication_queue WHERE table = 'concurrent_alter_add_drop_$i'" + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_add_drop_$i'" + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_add_drop_$i" +done diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.reference b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.reference new file mode 100644 index 00000000000..ed688ba26d2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.reference @@ -0,0 +1,11 @@ +1725 +1725 +1725 +Starting alters +Finishing alters +1 +0 +1 +0 +1 +0 diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh new file mode 100755 index 00000000000..319c65cc5a7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +REPLICAS=3 + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_detach_$i" +done + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/concurrent_alter_detach', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" +done + +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10)" +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10, 40)" + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_detach_$i" +done + +INITIAL_SUM=`$CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_detach_1"` + +# This alters mostly requires not only metadata change +# but also conversion of data. Also they are all compatible +# between each other, so can be executed concurrently. +function correct_alter_thread() +{ + TYPES=(Float64 String UInt8 UInt32) + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM + done +} + +# This thread add some data to table. After we finish we can check, that +# all our data have same types. +# insert queries will fail sometime because of wrong types. +function insert_thread() +{ + + VALUES=(7.0 7 '7') + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" + sleep 0.$RANDOM + done +} + +function detach_attach_thread() +{ + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_alter_detach_$REPLICA" + sleep 0.$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$REPLICA" + done +} + +echo "Starting alters" +export -f correct_alter_thread; +export -f insert_thread; +export -f detach_attach_thread; + +TIMEOUT=15 + +# Sometimes we detach and attach tables +timeout $TIMEOUT bash -c detach_attach_thread 2> /dev/null & + +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & + +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & + +wait + +echo "Finishing alters" + +sleep 1 + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$i" 2> /dev/null +done + + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_detach_$i" +done + + +# This alter will finish all previous, but replica 1 maybe still not up-to-date +while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do + sleep 1 +done + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_detach_$i" + $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_detach_$i" + $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE is_done=0 and table = 'concurrent_alter_detach_$i'" # all mutations have to be done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE is_done=0 and table = 'concurrent_alter_detach_$i'" # all mutations have to be done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_detach_$i'" # all mutations have to be done + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_detach_$i" +done diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.referece b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.referece new file mode 100644 index 00000000000..be716831212 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.referece @@ -0,0 +1,12 @@ +1725 +1725 +1725 +1725 +1725 +Starting alters +Finishing alters +1 +1 +1 +1 +1 diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference new file mode 100644 index 00000000000..ff9c6824f00 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference @@ -0,0 +1,17 @@ +1725 +1725 +1725 +1725 +1725 +Starting alters +Finishing alters +1 +0 +1 +0 +1 +0 +1 +0 +1 +0 diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh new file mode 100755 index 00000000000..31643d21815 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +REPLICAS=5 + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" +done + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" +done + +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10)" +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10, 40)" + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_mt_$i" +done + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_mt_$i" +done + +INITIAL_SUM=`$CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_mt_1"` + +# This alters mostly requires not only metadata change +# but also conversion of data. Also they are all compatible +# between each other, so can be executed concurrently. +function correct_alter_thread() +{ + TYPES=(Float64 String UInt8 UInt32) + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM + done +} + +# This thread add some data to table. After we finish we can check, that +# all our data have same types. +# insert queries will fail sometime because of wrong types. +function insert_thread() +{ + + VALUES=(7.0 7 '7') + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" + sleep 0.$RANDOM + done +} + +# Some select load, to be sure, that our selects work in concurrent execution with alters +function select_thread() +{ + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) FROM concurrent_alter_mt_$REPLICA" 1>/dev/null + sleep 0.$RANDOM + done +} + + +echo "Starting alters" +export -f correct_alter_thread; +export -f insert_thread; +export -f select_thread; + + +TIMEOUT=30 + + +# Selects should run successfully +timeout $TIMEOUT bash -c select_thread & +timeout $TIMEOUT bash -c select_thread & +timeout $TIMEOUT bash -c select_thread & + + +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & + + +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & + +wait + +echo "Finishing alters" + +# This alter will finish all previous, but replica 1 maybe still not up-to-date +while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do + sleep 1 +done + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_mt_$i" + $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_mt_$i" + $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE is_done=0 and table = 'concurrent_alter_mt_$i'" # all mutations have to be done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE is_done=0 and table = 'concurrent_alter_mt_$i'" + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_mt_$i'" + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" +done diff --git a/dbms/tests/queries/0_stateless/01081_keywords_formatting.reference b/dbms/tests/queries/0_stateless/01081_keywords_formatting.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/dbms/tests/queries/0_stateless/01081_keywords_formatting.reference @@ -0,0 +1 @@ +2 diff --git a/dbms/tests/queries/0_stateless/01081_keywords_formatting.sql b/dbms/tests/queries/0_stateless/01081_keywords_formatting.sql new file mode 100644 index 00000000000..6044f383c10 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01081_keywords_formatting.sql @@ -0,0 +1 @@ +SELECT (1 AS `interval`) + `interval`; diff --git a/dbms/tests/queries/0_stateless/01084_defaults_on_aliases.reference b/dbms/tests/queries/0_stateless/01084_defaults_on_aliases.reference new file mode 100644 index 00000000000..9b39b07db94 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01084_defaults_on_aliases.reference @@ -0,0 +1,6 @@ +1 1 +1 1 1 +2 2 4 +2 2 2 4 +3 3 9 +3 3 3 9 27 diff --git a/dbms/tests/queries/0_stateless/01084_defaults_on_aliases.sql b/dbms/tests/queries/0_stateless/01084_defaults_on_aliases.sql new file mode 100644 index 00000000000..2e4be37cc73 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01084_defaults_on_aliases.sql @@ -0,0 +1,30 @@ +DROP TABLE IF EXISTS table_with_defaults_on_aliases; + +CREATE TABLE table_with_defaults_on_aliases (col1 UInt32, col2 ALIAS col1, col3 DEFAULT col2) Engine = MergeTree() ORDER BY tuple(); + +INSERT INTO table_with_defaults_on_aliases (col1) VALUES (1); + +SELECT * FROM table_with_defaults_on_aliases WHERE col1 = 1; + +SELECT col1, col2, col3 FROM table_with_defaults_on_aliases WHERE col1 = 1; + +ALTER TABLE table_with_defaults_on_aliases ADD COLUMN col4 UInt64 DEFAULT col2 * col3; + +INSERT INTO table_with_defaults_on_aliases (col1) VALUES (2); + +SELECT * FROM table_with_defaults_on_aliases WHERE col1 = 2; + +SELECT col1, col2, col3, col4 FROM table_with_defaults_on_aliases WHERE col1 = 2; + +ALTER TABLE table_with_defaults_on_aliases ADD COLUMN col5 UInt64 ALIAS col2 * col4; + +INSERT INTO table_with_defaults_on_aliases (col1) VALUES (3); + +SELECT * FROM table_with_defaults_on_aliases WHERE col1 = 3; + +SELECT col1, col2, col3, col4, col5 FROM table_with_defaults_on_aliases WHERE col1 = 3; + + +ALTER TABLE table_with_defaults_on_aliases ADD COLUMN col6 UInt64 MATERIALIZED col2 * col4; + +DROP TABLE IF EXISTS table_with_defaults_on_aliases; diff --git a/dbms/tests/queries/0_stateless/01086_modulo_or_zero.reference b/dbms/tests/queries/0_stateless/01086_modulo_or_zero.reference new file mode 100644 index 00000000000..627e1097cda --- /dev/null +++ b/dbms/tests/queries/0_stateless/01086_modulo_or_zero.reference @@ -0,0 +1,5 @@ +1 +1 +1 +1 +1 diff --git a/dbms/tests/queries/0_stateless/01086_modulo_or_zero.sql b/dbms/tests/queries/0_stateless/01086_modulo_or_zero.sql new file mode 100644 index 00000000000..97068c2ff86 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01086_modulo_or_zero.sql @@ -0,0 +1,5 @@ +select moduloOrZero(0, 0) = 0; +select moduloOrZero(-128, -1) = 0; +select moduloOrZero(-127, -1) = 0; +select moduloOrZero(1, 1) = 0; +select moduloOrZero(5, 3) = 2; diff --git a/dbms/tests/queries/0_stateless/01087_storage_generate.reference b/dbms/tests/queries/0_stateless/01087_storage_generate.reference new file mode 100644 index 00000000000..33e5fdd18ee --- /dev/null +++ b/dbms/tests/queries/0_stateless/01087_storage_generate.reference @@ -0,0 +1,103 @@ +100 +- +[] -183162.1041 ('2074-01-15 19:36:42.413','0345f8ad-8936-8cc9-9ff2-394f225fc318') +[3] -39049.0845 ('2045-07-04 15:01:09.380','f79d47d8-4030-9916-54b9-495a5ccc7202') +[-95,-104] -14324.1685 ('2023-10-02 06:05:53.887','ea94157b-737b-a272-acd5-c7ab9c6f07c3') +[] -182420.0194 ('2002-01-08 01:42:50.396','ac579c0f-8523-144f-aa4c-c9587cc27144') +[] 127157.2100 ('2006-11-23 14:25:39.542','d07c5204-ef95-6804-83df-01dedaf32522') +[28] -198600.4267 ('2060-09-25 02:57:41.504','b13ff007-c245-d737-85b2-1fa003e57127') +[] -87232.0739 ('2027-05-12 20:26:59.405','a2f2cbf4-b11b-6976-7b91-14b6964acbe2') +[] -110349.8987 ('2042-11-01 10:51:30.039','445a77b5-0a27-3485-8dd8-c7cc35d2692f') +[32,73] 123253.5669 ('2016-06-21 00:23:02.917','c12095e6-b82c-d81c-4629-acd80e02b080') +[-109,85] 34320.8302 ('2080-12-31 16:49:32.509','ebbbe70a-0321-ff18-89de-2bc9a9e4c454') +[68,76] 160458.5593 ('2030-05-23 03:33:29.681','805b0a62-9ada-a47e-2d5e-63cb5923549c') +[69] -189322.2887 ('2084-06-19 03:31:58.508','429df3a4-ff18-28d5-9ad8-dcdd78e8b1ae') +[] 189158.4731 ('1985-12-21 16:36:51.092','d63c5cbb-9418-ce59-000c-056f88157bfa') +[-120] 114890.5905 ('2089-02-19 22:30:18.216','2bc4860a-7214-300a-851e-b61011c346ef') +[] 54493.1631 ('2062-08-18 04:57:01.348','c00d218a-913f-b657-1ff9-99927741f7ab') +[103] 88891.6006 ('2071-05-23 21:46:45.644','036d2746-f7aa-b5a4-b716-b8e8a5e041da') +[28] -41173.9863 ('2044-03-11 10:06:47.659','6bf54ef0-9bad-54d4-5ca3-02d79883b697') +[] -64809.0652 ('2010-11-02 23:46:46.150','ae5cafb4-fe3c-71a5-9a76-0314c44180de') +[125,74] 28139.7661 ('1989-07-19 22:10:13.477','ee9b8173-4426-8615-97eb-a904266847e5') +[-101] -29032.2527 ('2052-08-13 08:55:15.045','ab8ee358-ff53-de7e-f012-cd0eed754ff2') +[82] 170334.6151 ('2034-02-23 18:50:46.847','f1402791-1d23-e56d-25f0-5a51a3cb245a') +[27,56] 168648.5067 ('2004-05-31 19:47:19.448','5019365e-f74d-b31e-aacb-63d8189e3e3e') +[-74] 89577.4738 ('1997-01-15 09:05:57.691','d144325c-24a9-411e-cc10-78b0637f75a7') +[100] 178532.5772 ('2055-04-22 16:20:11.498','41439c27-fba1-1b66-13a7-cf79fded4d9a') +[32] 123101.7871 ('2103-02-18 15:53:42.748','25b804d3-a73e-ed14-e6e1-eafb0d9473cd') +[-115,-85] -208371.1662 ('2039-10-22 18:06:50.235','41fba85a-5080-48bb-e18d-a8af04a890aa') +[-12,101] -7791.5577 ('2073-07-27 11:35:45.239','c00be55f-59ac-762c-af0a-9e33cf30a1f4') +[-127] -18602.1918 ('2024-02-05 19:54:00.798','ea85cbed-66f2-197b-4e63-dfbdcd306cce') +[-78,125] -112158.3556 ('2016-08-12 06:46:17.173','15809e25-b003-010b-c63c-9e880568736a') +[] 151191.1081 ('1982-11-12 17:59:10.171','d6bbbe2c-fca0-53c8-22a6-de9e0715d3cc') +[46] 58689.9611 ('2085-04-13 09:13:07.230','fe4be382-eb78-4cf9-fa57-c6eccf955419') +[-4,42] -88292.1046 ('1980-03-06 08:29:12.503','2633522e-ff9c-b837-1b9b-6559875c13b0') +[-2] 64983.6649 ('2034-07-07 11:20:23.903','d19f5c4d-e444-2e5b-a55d-5280d1760b94') +[-31,-110] -25638.6649 ('2025-05-17 21:45:25.519','3654a15e-bfa3-6075-b5b8-07e25310de1f') +[25] -140469.2476 ('2083-12-13 23:55:25.450','940f7441-ae40-d810-f6c3-e2fff468050c') +[-99] 128186.7318 ('1995-09-15 04:26:33.803','4cfd264f-ff00-4190-929c-b675826607d3') +[] -52961.0340 ('2046-03-19 14:15:50.245','314de821-308b-c61c-e256-9f6afed5d4f3') +[] -155852.9334 ('2052-01-29 06:31:08.957','5be2ccd2-b5bb-921d-5b5e-4a0e22385de7') +[-74,81] 29366.0091 ('1978-03-05 19:24:49.193','ab9b6a39-89ac-9280-c76f-60d598ce65c6') +[9] 56134.8951 ('2104-04-03 10:27:33.053','339a7f6d-0e0b-e039-78c0-2d045457d821') +[-61] 68841.1188 ('2059-07-26 12:14:33.197','c817bbb3-d091-b73c-1b9a-53f8a03bffb6') +[89] 168034.0459 ('2028-05-29 08:02:02.393','bc217a73-e802-1772-80b5-d8c827295799') +[124] 11648.6762 ('2084-12-25 12:10:35.676','77390177-1dd6-a5c0-dd35-4f85e38bcb2c') +[-47,-125] -120893.6705 ('2012-10-18 22:52:57.524','472814b2-4033-c5a5-7d86-fb36079e88fb') +[35] 153250.6252 ('2006-11-06 00:05:25.456','de0d6ed9-eca6-e01e-eb1c-c46c8ad6e33e') +[-43,70] -141086.3184 ('2013-02-03 23:07:11.759','65d48b24-cdc0-f7db-cb16-d0ad03279bcc') +[120,-57] -93351.1404 ('2000-02-03 14:39:00.466','6991722b-90dc-e9dd-c5e7-f28bd1d4f0d8') +[34,43] 187780.4567 ('2014-02-10 05:22:19.250','3db77bc5-d877-b22e-6667-955bf36d2e08') +[73] -90148.5697 ('2014-10-05 18:34:31.419','5a0f919e-38c9-0a68-e805-977db04d0acb') +[] -179121.0029 ('2077-01-23 07:57:55.365','fcf79336-a6dc-44fd-8c78-7e74e07b60fa') +[-69,120] 119321.8003 ('1989-07-01 13:11:35.185','92f6a362-250c-cfcd-acd7-99399cbf88ad') +[] 208864.8324 ('1991-02-17 03:04:00.682','b0dc8e88-ea6f-c2da-c116-3e4873dc8d54') +[22,-14] -127735.4391 ('2036-08-10 08:33:03.806','5ab1ab2b-913d-ff8a-6f8f-86387e77ed5c') +[83,-70] -142476.9847 ('2074-11-22 19:27:13.085','51b9d30a-3b10-265c-4086-1ac35b634ec7') +[] -128052.2443 ('2088-01-02 10:58:36.999','745e8226-d906-7fb3-33f4-9a079037bdcd') +[12,-116] -88390.1399 ('2074-02-18 17:46:45.208','fb5f827e-1809-6cab-2855-d45df20ecd92') +[] -84110.2097 ('2039-03-24 17:08:15.660','88e18c93-6276-d176-dad1-7db72e340ca7') +[] 202866.8175 ('2104-01-25 13:42:41.758','10faa33e-d383-c6b3-399d-44c06ebb00f5') +[-21] 151775.1601 ('1995-10-20 15:44:53.296','7ccaf135-787d-2ac0-09c0-7545c798ee14') +[-19] -15498.5738 ('2097-08-02 18:34:16.406','cf97f268-02c0-24fc-bbf3-c7b272632c14') +[116] -72670.9713 ('2020-08-31 18:10:41.904','f9cdd931-e2ed-0584-d4b9-67a6df717a4c') +[] 124014.7040 ('1975-07-23 11:17:25.176','ccf33ba5-8fd8-c8b5-ccc4-a9cb892d4b55') +[-56] -204745.8115 ('2037-11-13 01:03:12.923','6dc83c7b-7782-57b4-a293-18ca8aba331d') +[] -28535.2534 ('2105-04-07 20:51:09.990','0d9f3a2f-d4f2-a330-7b6e-001ea3aacbde') +[-124,-128] -31519.7583 ('1993-02-14 23:06:10.338','a073dafb-6f1f-273e-acf9-88200f82af6d') +[46] -154950.9257 ('2032-06-04 23:16:16.051','e6aa3b80-9f53-6c10-0cc8-622622f964b4') +[] 206914.3454 ('2003-10-05 10:44:30.786','137ed3be-2d40-d1c1-7aff-b32f7e21c0da') +[-47] 91521.1349 ('2006-09-01 04:06:32.496','52e4ef43-9379-4864-8f63-8e205875a096') +[121] 161456.7813 ('2027-11-03 10:20:30.670','f1abbd17-f399-657c-1a47-1dd627578b53') +[99] -127959.4741 ('2084-08-18 06:04:41.942','2a3b92c3-75ed-bd20-5a77-b77cbe1ce479') +[-97] 82020.4570 ('2061-10-25 06:16:50.814','8625d479-6e81-318f-5077-a9deb13c50e0') +[71] -121599.1388 ('2010-04-02 11:05:18.877','0ec279cf-c9b2-dc65-40c0-2d0f390b1102') +[] 98975.6469 ('2049-03-06 08:56:25.010','845340d7-a1df-9ddf-b737-9eb90ca6344c') +[92,81] 135864.7854 ('2040-12-30 21:17:28.184','ea224755-198e-c9ae-c59b-0517a7459d7c') +[81] -154620.5037 ('1984-06-07 02:36:28.734','52d3b727-043f-1d43-6f48-51e8abdc2127') +[38] 33379.3375 ('2057-10-19 17:03:44.317','e709bfc2-0915-9e4e-4d01-c10b24795e30') +[] 7491.1071 ('1971-04-29 09:30:25.245','26bcd2ab-6d0b-fc20-27eb-084c4248af7d') +[-122] -135635.3813 ('2010-03-04 23:05:25.982','66ed96eb-fc6e-653e-0353-ac4477ea60a6') +[] -174748.4115 ('2020-10-28 07:39:33.461','e17fa9ba-2595-c0f9-2f85-d6bbdc2f6f6a') +[72,106] 25749.2190 ('2008-06-15 04:03:39.682','0e47b616-da80-091e-664d-2a35bc57a480') +[-84,97] 109277.9244 ('1998-10-27 10:40:00.442','9488bce4-46d7-8249-78aa-540b8be43937') +[-120,-107] -64113.5210 ('2091-12-03 06:46:11.903','325fcb1c-8552-b434-b349-732d62be19f1') +[] -66141.6000 ('2085-10-05 08:08:11.830','4c66022b-75b9-b0a8-3897-b9de8ea851f1') +[-34,-102] -142314.4437 ('2038-01-27 12:04:29.739','91e9eb11-5679-02ef-6ea6-2c9fdcb12ed9') +[103] 96187.7213 ('1978-10-07 13:57:43.616','7c02e8e3-9e98-5043-8029-34e32ad1af61') +[] -21344.8423 ('2085-01-13 00:10:52.538','52cb36f8-987a-f414-7e0f-93ddccc5c377') +[16] -95098.4107 ('2074-02-19 18:56:00.878','821e4b10-f70a-4bee-ef0c-ac12eab994f3') +[21,86] 27954.7748 ('2033-10-18 03:15:38.815','bfe4d932-c5ed-45c0-9f50-72a6394d49af') +[] 149788.2085 ('2073-09-10 20:42:48.693','5e7d825e-5c88-7c89-4235-0e7934739a12') +[33,116] -148302.8732 ('2044-08-10 22:05:18.943','a53d4b07-5529-7472-3cca-3770f52b3648') +[] -98384.4505 ('2070-01-28 05:17:35.804','4833b839-51a3-87b8-7709-30676f697aa4') +[] -75597.1523 ('2075-02-04 19:24:01.477','d64becff-5c08-b0a0-e7f1-b86eaf5f1913') +[] 179005.6113 ('2100-05-27 21:54:12.965','d87ce81c-c471-b6b3-93b7-05225cb577be') +[] -134366.9213 ('2054-11-16 18:19:00.801','c348fced-6700-f0f6-cda0-14aef7ea6948') +[10] 82182.0343 ('2017-03-04 09:41:21.249','e19f0022-49ab-2d41-872d-be35669a79bc') +[-28] 90333.8564 ('2032-11-19 01:23:37.107','e2586be2-e968-21d0-d1b1-b438c55a59a3') +[-73] 185647.6735 ('2001-01-23 16:20:26.442','24b04f39-f272-24ff-538d-41e636a1a37a') +[-79,7] -87628.8007 ('2005-03-25 04:17:49.969','38a10e9d-7086-f358-8e50-c72b278bec42') +[119,-55] -208591.8591 ('1976-11-14 15:17:57.569','d0935dc7-7f56-71db-67f2-1b4e52770ba9') +[-108,-124] 181408.0349 ('2056-10-27 05:07:32.393','29d655c1-c35a-1245-25e2-65b4f233cb9c') +- diff --git a/dbms/tests/queries/0_stateless/01087_storage_generate.sql b/dbms/tests/queries/0_stateless/01087_storage_generate.sql new file mode 100644 index 00000000000..46d49dc165f --- /dev/null +++ b/dbms/tests/queries/0_stateless/01087_storage_generate.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=Generate(); +SELECT COUNT(*) FROM (SELECT * FROM test_table LIMIT 100); + +DROP TABLE IF EXISTS test_table; + +SELECT '-'; + +DROP TABLE IF EXISTS test_table_2; +CREATE TABLE test_table_2(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=Generate(3, 5, 10); + +SELECT * FROM test_table_2 LIMIT 100; + +SELECT '-'; + +DROP TABLE IF EXISTS test_table_2; + diff --git a/dbms/tests/queries/0_stateless/01087_table_function_generate.reference b/dbms/tests/queries/0_stateless/01087_table_function_generate.reference new file mode 100644 index 00000000000..08081a34bc9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01087_table_function_generate.reference @@ -0,0 +1,238 @@ +UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 +5443401583997919274 956654340036924402 2956613447 2041372187 46025 26509 247 -34 +14051730854243326159 340055300607421421 579798001 915264595 58925 22498 36 -57 +12126660396637528292 -9182366379883086416 535113873 -1583603936 45790 6066 230 91 +5198178071978083704 -3549936112074464250 3354362520 -1732019372 41330 -27737 13 -47 +9045663333607591872 -5069075924065328373 741246230 -1830932765 29642 -11720 41 7 +18192666371709191624 -5005976579831091773 671021725 1851158245 38613 -27838 57 3 +4333039311970693040 -7294587049092886539 2106347821 2101852759 24058 9107 85 94 +1398111012802844853 1131449717368086026 1687614855 -1193084417 9803 -18141 198 115 +15838944643191192696 6226099517671026657 1300309956 468322781 17216 -2375 184 -102 +15170414162889419078 3337938833953948518 3603117877 -1297530274 25534 8264 36 16 +- +Enum8(\'hello\' = 1, \'world\' = 5) +world +world +world +hello +hello +world +world +world +hello +world +- +Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5))) +['world','world','hello','hello','world','world','world'] +['world'] +['world','hello','world'] +[] +[] +['world','hello','hello','hello','world'] +['hello'] +['world','hello','hello','world','hello'] +['hello','world','hello','hello','world','world'] +['world','hello','world','hello','hello','world','world'] +- +Nullable(Enum16(\'o\' = -200, \'h\' = 1, \'w\' = 5)) +w +h +h +o +w +w +o +w +h +o +- +Date DateTime DateTime(\'Europe/Moscow\') +2031-03-05 2034-09-09 02:49:47 2061-06-26 03:46:01 +1972-10-06 1999-01-02 11:09:55 2064-03-18 05:47:09 +2004-01-16 2055-12-02 15:29:20 2090-08-18 23:04:46 +2061-07-14 2051-03-20 20:58:44 1973-04-20 21:20:34 +2063-04-13 2048-01-31 01:02:11 2051-02-07 03:11:54 +2106-02-07 2028-08-29 13:37:25 2054-10-20 03:48:21 +2026-11-24 2036-08-09 02:59:19 2065-10-12 06:39:38 +2106-02-07 2068-04-17 13:07:59 2101-04-03 08:48:59 +1997-11-15 1984-11-03 12:39:41 1998-04-01 17:38:08 +2008-09-11 2064-12-25 16:23:42 2031-10-18 03:20:14 +- +DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') +1988-05-16 19:00:01.447 2064-03-18 05:47:09.972361 2104-06-20 09:26:44.845879 +2076-04-17 18:22:00.873 1973-04-20 21:20:34.769886 2052-08-01 07:14:05.921510 +1991-04-07 13:55:25.230 2054-10-20 03:48:21.341514 2013-02-07 18:37:45.437737 +2023-06-24 16:54:15.821 2101-04-03 08:48:59.544378 2039-07-05 08:51:02.770005 +2084-03-05 21:04:37.956 2031-10-18 03:20:14.437888 2076-03-16 14:08:20.993528 +1999-01-02 11:09:55.187 2054-01-01 16:49:22.580109 1997-01-09 20:11:35.889758 +2051-03-20 20:58:44.360 1975-02-11 06:38:15.042546 2015-10-21 23:47:13.191963 +2028-08-29 13:37:25.531 1975-02-14 07:25:38.319928 2103-09-16 20:57:23.033927 +2068-04-17 13:07:59.759 2024-03-06 21:42:43.711891 2045-04-22 19:38:11.140126 +2064-12-25 16:23:42.781 2025-08-18 15:44:56.149625 2093-09-26 16:30:56.744858 +- +Float32 Float64 +2.3424705e38 5.304765772621186e307 +4.5936326e37 1.3693852957827914e308 +4.2396088e37 1.1817811347484115e308 +2.6575997e38 5.065787759860024e307 +5.8727575e37 8.815282962741328e307 +5.3163816e37 1.7729324649694315e308 +1.6688205e38 4.2226828718895e307 +1.3370661e38 1.3625030842560206e307 +1.0302116e38 1.5435548915708008e308 +2.8546838e38 1.4784044970034722e308 +- +Decimal32(4) Decimal64(8) Decimal64(8) +-133835.3849 87676267830.44260947 10041303591043480341650.6377217747572943 +57979.8001 -68015271123.73929132 -11658496611537681782723.8256877955807880 +53511.3873 -78637963449.98695195 16686303649199763212696.4854950355256776 +-94060.4776 90273888640.14252543 7993046724924589483272.0796323974797493 +74124.6230 20114310313.64207198 -4810540869033768101015.4448286464595642 +67102.1725 -60472921957.85611731 1764715777766465744700.9237855716355053 +210634.7821 -20967919098.37725326 -16938476260073815366594.8118263905360890 +168761.4855 -74544559691.08355371 -9350794626143586522954.2962771754340925 +130030.9956 -54650148153.48939189 -13456138041801265081736.4812607484010998 +-69184.9419 38286965773.25360062 11485126437992390872631.7990315807376230 +- +Tuple(Int32, Int64) +(-1338353849,5443401583997919274) +(579798001,-4395013219466225457) +(535113873,-6320083677072023324) +(-940604776,5198178071978083704) +(741246230,9045663333607591872) +(671021725,-254077702000359992) +(2106347821,4333039311970693040) +(1687614855,1398111012802844853) +(1300309956,-2607799430518358920) +(-691849419,-3276329910820132538) +- +Array(Int8) +[27,83,32,84,-29,-27,87] +[-1] +[-35,94,-55] +[] +[] +[45,-34,114,-54,-43] +[-6] +[75,64,-66,-115,-30] +[-78,-89,56,66,-109,35] +[-71,72,-9,36,-26,13,41] +- +Array(Nullable(Int32)) +[2041372187,915264595,-1583603936,-1732019372,-1830932765,1851158245,2101852759] +[-1193084417] +[468322781,-1297530274,-1407994935] +[] +[] +[-1321933267,-488197410,104178034,-1735625782,-1618897195] +[-1272422918] +[-153016757,891437888,1950049214,6580109,-1644079134] +[790042546,161321895,1074319928,161583938,515711891,1709750563] +[-149817671,1755521096,815845879,-51580892,1361921510,-1688868851,-1185529559] +- +Tuple(Int32, Array(Int64)) +(-1338353849,[5443401583997919274,-4395013219466225457,-6320083677072023324,5198178071978083704,9045663333607591872,-254077702000359992,4333039311970693040]) +(579798001,[1398111012802844853,-2607799430518358920,-3276329910820132538,956654340036924402,340055300607421421]) +(535113873,[]) +(-940604776,[-9182366379883086416,-3549936112074464250,-5069075924065328373,-5005976579831091773]) +(741246230,[-7294587049092886539]) +(671021725,[1131449717368086026,6226099517671026657,3337938833953948518,-104956130729581604,515805789944032293]) +(2106347821,[2731028582309582302,-8197314279937271385,7439592879615992239,-8726726222408049230,-4046170041070917399,-8162695179087422573,7147712321550951494,-2473105312361834401,2871941532606538254]) +(1687614855,[7045950974355040215,8128475529675984757,3862453874713979777,8584893221699499395,-4344095019439049735,7221768832555831190,5712009283210486481,8657278465574644253,-4620821897447975309]) +(1300309956,[-3580736586972265629]) +(-691849419,[7980379733974797651,-548434416689229144]) +- +Nullable(String) +;\\Sm\'sH +T +@^1 + + +7-f)$ +9 +)&}y3 +w5>+\'@ ++g+N^g$ +- +Array(String) +['Y9n(%ub','\\\'f%7','','X0O@','D','4^,~q','a(gmt6#{X'] +['@#q4?Q%\'.'] +['h','#B','{'] +[] +[] +['~vYP/4f9.',':,7u.0',';e.<','^O,i','3'] +['!y1/Z\'5D'] +['&- KDN%>[','>-xM./ B','?+//','M,.71QR#_','~N'] +['z9P/%m','7q\'!k','Q%] #.*3','U:&XeP{*',',','s.3'] +['+k.=%','8\'nb=P','-uY ,h8(w','=\'W$','','m<+%l','<~+@ Vw'] +- +UUID +4b8ad8e6-77fe-a02a-c301-c6c33e91d8cf +a84a8d61-8560-a0e4-4823-a36a537a8578 +7d88a6e0-3d74-cfc0-fc79-55ad1aebc5c8 +3c220c4b-fbd0-efb0-1367-168f02acd8b5 +dbcf3c3c-127c-bc78-d288-234e0a942946 +0d46b80d-bebc-93f2-04b8-1e7e84b1abed +8091ae6d-8194-3db0-cebc-17ea18786406 +b9a7064d-de99-e30b-ba87-32d2cd3e2dc3 +9ac46bba-2ba7-4ff5-0fb3-b785f4f0de0a +56678c86-2703-2fe1-2e52-bdaf2fce8366 +- +Array(Nullable(UUID)) +['4b8ad8e6-77fe-a02a-c301-c6c33e91d8cf','a84a8d61-8560-a0e4-4823-a36a537a8578','7d88a6e0-3d74-cfc0-fc79-55ad1aebc5c8','3c220c4b-fbd0-efb0-1367-168f02acd8b5','dbcf3c3c-127c-bc78-d288-234e0a942946','0d46b80d-bebc-93f2-04b8-1e7e84b1abed','8091ae6d-8194-3db0-cebc-17ea18786406'] +['b9a7064d-de99-e30b-ba87-32d2cd3e2dc3'] +['9ac46bba-2ba7-4ff5-0fb3-b785f4f0de0a','56678c86-2703-2fe1-2e52-bdaf2fce8366','fe8b1ef4-86dd-23dc-0728-82a212b42c25'] +[] +[] +['25e69006-a800-55de-8e3d-4a17f81a19a7','673ebe4e-af09-61af-86e4-70bca5481db2','c7d91dc9-0123-e8e9-8eb8-47fb80b35b93','6331c67c-7aba-2446-ddad-c3d24c8a985f','27db2f9b-92b4-220e-61c8-3f14833fe7d7'] +['70ce256d-7fca-cf75-359a-3155bba86b81'] +['7723aae6-820e-b583-c3b6-ac9887cab3f9','6438e07d-9a7a-4f96-4f45-222df77ea2d1','7824d4e5-6e37-2a1d-bfdf-8af768445673','ce4eab0b-b346-0363-6ec0-0116104b4d53','f86391cd-c2a3-d2a8-ea85-27ed3508504c'] +['b87d39f3-e56c-7128-7a66-4e516e2ce1c0','af25bac3-d662-673a-3516-022e687643ed','657c9997-3b29-f51c-7193-6a3b9a0c18eb','19bb38b5-6f97-a81c-2d56-57f189119a1a','0c7e416d-c669-dc04-1130-ff950fbbf44b','9705bc44-8d08-c734-6f47-8edcc1608a81'] +['7c20103f-659a-f845-399c-abdc8dc88ba0','bf479e85-1a0f-66c3-66fa-f6029e4ee2a8','f14af1a0-823c-b414-eb8e-e6b05b019868','ce353e45-2a9e-492c-1c54-d50459160ecf','1e232279-77ad-db7e-82f6-b4b3e30cdc2e','991111af-30a3-1ff7-e15a-023dfa0a8a6e','a749ef39-dc02-d05c-e8b2-129a7cccfd24'] +- +FixedString(4) +G +- +5S + T +W^ +-r +K +@ +8B +#H +- +String +;\\Sm\'sH +T +@^1 + + +7-f)$ +9 +)&}y3 +w5>+\'@ ++g+N^g$ +- +[27] -119308.4417 ('1998-04-01 17:38:08.539','4b8ad8e6-77fe-a02a-c301-c6c33e91d8cf') +[83] 46832.2781 ('1970-03-18 06:48:29.214','a84a8d61-8560-a0e4-4823-a36a537a8578') +[32] -129753.0274 ('1995-01-14 03:15:46.162','7d88a6e0-3d74-cfc0-fc79-55ad1aebc5c8') +[] -140799.4935 ('2004-01-17 09:12:08.895','3c220c4b-fbd0-efb0-1367-168f02acd8b5') +[] -132193.3267 ('1986-05-06 01:18:11.938','dbcf3c3c-127c-bc78-d288-234e0a942946') +[84] -48819.7410 ('2101-05-10 09:27:05.563','0d46b80d-bebc-93f2-04b8-1e7e84b1abed') +[-29] 10417.8034 ('1995-11-08 18:51:19.096','8091ae6d-8194-3db0-cebc-17ea18786406') +[-27] -173562.5782 ('2013-02-27 03:31:50.404','b9a7064d-de99-e30b-ba87-32d2cd3e2dc3') +[] -161889.7195 ('2068-07-13 23:42:17.445','9ac46bba-2ba7-4ff5-0fb3-b785f4f0de0a') +[87] -127242.2918 ('2033-08-04 15:06:45.865','56678c86-2703-2fe1-2e52-bdaf2fce8366') +- +[] 3608695403 ZL 109414.2847 h 2.2986075276244747e306 ('1985-05-10','2009-10-28 20:06:11','1993-01-03 17:51:52.981','b13ff007-c245-d737-85b2-1fa003e57127') . +[85] 4204173796 ], -199466.5471 h 1.1231803213254798e308 ('2075-04-03','1983-02-12 23:57:05','2060-06-06 20:15:08.751','a2f2cbf4-b11b-6976-7b91-14b6964acbe2') * +[-94,100] 32713522 8D$ 102255.5602 h 1.738807291208415e308 ('2029-07-12','2056-08-07 23:18:32','2081-01-25 13:13:30.589','445a77b5-0a27-3485-8dd8-c7cc35d2692f') +[] 4117557956 0b>+ 65942.4942 w 5.949505844751135e307 ('2048-03-05','2074-01-22 02:32:44','2073-12-04 05:05:06.955','c12095e6-b82c-d81c-4629-acd80e02b080')  +[] 1511604199 Il= -96352.6064 o 1.6472659147355216e308 ('2024-06-01','2024-12-26 00:54:40','2038-04-14 05:21:44.387','ebbbe70a-0321-ff18-89de-2bc9a9e4c454') Q +[-18] 2278197196 ~ 193977.7666 o 1.213689191969361e308 ('2060-10-04','1992-10-24 16:31:53','1983-06-10 08:51:48.294','805b0a62-9ada-a47e-2d5e-63cb5923549c') \t +[] 3761265784 N"(6 -59230.0369 o 1.2102282609858645e308 ('2106-02-07','2060-07-09 20:14:59','2007-03-17 04:51:09.288','429df3a4-ff18-28d5-9ad8-dcdd78e8b1ae') Y| +[] 66606254 6x&+ 130635.2269 o 1.1958868988757417e308 ('2088-10-07','2070-03-01 21:30:45','1978-05-22 14:28:52.523','d63c5cbb-9418-ce59-000c-056f88157bfa') у +[-27,-12] 4089193163 )+.8 -111081.7896 o 1.464035857434812e308 ('2106-02-07','2007-04-27 23:04:36','1987-07-21 04:32:01.821','2bc4860a-7214-300a-851e-b61011c346ef') # +[14,-43] 3638976325 #" 116961.4294 o 9.260305126207595e307 ('2042-06-11','2087-12-28 00:21:16','2071-04-01 21:44:13.058','c00d218a-913f-b657-1ff9-99927741f7ab') Fx +- diff --git a/dbms/tests/queries/0_stateless/01087_table_function_generate.sql b/dbms/tests/queries/0_stateless/01087_table_function_generate.sql new file mode 100644 index 00000000000..0329fa81bf1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01087_table_function_generate.sql @@ -0,0 +1,190 @@ +SELECT + toTypeName(ui64), toTypeName(i64), + toTypeName(ui32), toTypeName(i32), + toTypeName(ui16), toTypeName(i16), + toTypeName(ui8), toTypeName(i8) +FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') +LIMIT 1; +SELECT + ui64, i64, + ui32, i32, + ui16, i16, + ui8, i8 +FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)') +LIMIT 1; +SELECT + i +FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))') +LIMIT 1; +SELECT + i +FROM generate('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i)s +FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))') +LIMIT 1; +SELECT + i +FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT +toTypeName(d), toTypeName(dt), toTypeName(dtm) +FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')') +LIMIT 1; +SELECT +d, dt, dtm +FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT +toTypeName(dt64), toTypeName(dts64), toTypeName(dtms64) +FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')') +LIMIT 1; +SELECT +dt64, dts64, dtms64 +FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(f32), toTypeName(f64) +FROM generate('f32 Float32, f64 Float64') +LIMIT 1; +SELECT + f32, f64 +FROM generate('f32 Float32, f64 Float64', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(d32), toTypeName(d64), toTypeName(d64) +FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)') +LIMIT 1; +SELECT + d32, d64, d128 +FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Tuple(Int32, Int64)') +LIMIT 1; +SELECT + i +FROM generate('i Tuple(Int32, Int64)', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Array(Int8)') +LIMIT 1; +SELECT + i +FROM generate('i Array(Int8)', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Array(Nullable(Int32))') +LIMIT 1; +SELECT + i +FROM generate('i Array(Nullable(Int32))', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Tuple(Int32, Array(Int64))') +LIMIT 1; +SELECT + i +FROM generate('i Tuple(Int32, Array(Int64))', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Nullable(String)', 1) +LIMIT 1; +SELECT + i +FROM generate('i Nullable(String)', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Array(String)') +LIMIT 1; +SELECT + i +FROM generate('i Array(String)', 10, 10, 1) +LIMIT 10; + +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i UUID') +LIMIT 1; +SELECT + i +FROM generate('i UUID', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Array(Nullable(UUID))') +LIMIT 1; +SELECT + i +FROM generate('i Array(Nullable(UUID))', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i FixedString(4)') +LIMIT 1; +SELECT + i +FROM generate('i FixedString(4)', 10, 10, 1) +LIMIT 10; +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i String') +LIMIT 1; +SELECT + i +FROM generate('i String', 10, 10, 1) +LIMIT 10; +SELECT '-'; +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=Memory; +INSERT INTO test_table SELECT * FROM generate('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 2, 10, 1) +LIMIT 10; + +SELECT * FROM test_table; + +DROP TABLE IF EXISTS test_table; + +SELECT '-'; + +DROP TABLE IF EXISTS test_table_2; +CREATE TABLE test_table_2(a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16('h' = 1, 'w' = 5 , 'o' = -200)), f Float64, g Tuple(Date, DateTime, DateTime64, UUID), h FixedString(2)) ENGINE=Memory; +INSERT INTO test_table_2 SELECT * FROM generate('a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)), f Float64, g Tuple(Date, DateTime, DateTime64, UUID), h FixedString(2)', 3, 5, 10) +LIMIT 10; + +SELECT * FROM test_table_2; +SELECT '-'; + +DROP TABLE IF EXISTS test_table_2; + diff --git a/dbms/tests/queries/0_stateless/01088_array_slice_of_aggregate_functions.reference b/dbms/tests/queries/0_stateless/01088_array_slice_of_aggregate_functions.reference new file mode 100644 index 00000000000..57fed5c75ed --- /dev/null +++ b/dbms/tests/queries/0_stateless/01088_array_slice_of_aggregate_functions.reference @@ -0,0 +1 @@ +['\0\0\0\0\0'] diff --git a/dbms/tests/queries/0_stateless/01088_array_slice_of_aggregate_functions.sql b/dbms/tests/queries/0_stateless/01088_array_slice_of_aggregate_functions.sql new file mode 100644 index 00000000000..ba525f30228 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01088_array_slice_of_aggregate_functions.sql @@ -0,0 +1 @@ +select arraySlice(groupArray(x),1,1) as y from (select uniqState(number) as x from numbers(10) group by number); diff --git a/dbms/tests/queries/0_stateless/01090_fixed_string_bit_ops.reference b/dbms/tests/queries/0_stateless/01090_fixed_string_bit_ops.reference new file mode 100644 index 00000000000..5c8bb6fac36 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01090_fixed_string_bit_ops.reference @@ -0,0 +1,4 @@ +aca +acagac +aca +acagac diff --git a/dbms/tests/queries/0_stateless/01090_fixed_string_bit_ops.sql b/dbms/tests/queries/0_stateless/01090_fixed_string_bit_ops.sql new file mode 100644 index 00000000000..72ef1c3746c --- /dev/null +++ b/dbms/tests/queries/0_stateless/01090_fixed_string_bit_ops.sql @@ -0,0 +1,5 @@ +SELECT DISTINCT bitXor(materialize(toFixedString('abc', 3)), toFixedString('\x00\x01\x02', 3)) FROM numbers(10); +SELECT DISTINCT bitXor(materialize(toFixedString('abcdef', 6)), toFixedString('\x00\x01\x02\x03\x04\x05', 6)) FROM numbers(10); + +SELECT DISTINCT bitXor(toFixedString('\x00\x01\x02', 3), materialize(toFixedString('abc', 3))) FROM numbers(10); +SELECT DISTINCT bitXor(toFixedString('\x00\x01\x02\x03\x04\x05', 6), materialize(toFixedString('abcdef', 6))) FROM numbers(10); diff --git a/dbms/tests/queries/0_stateless/01090_zookeeper_mutations_and_insert_quorum.reference b/dbms/tests/queries/0_stateless/01090_zookeeper_mutations_and_insert_quorum.reference new file mode 100644 index 00000000000..bf8f7658af4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01090_zookeeper_mutations_and_insert_quorum.reference @@ -0,0 +1,3 @@ +9 +9 +0 diff --git a/dbms/tests/queries/0_stateless/01090_zookeeper_mutations_and_insert_quorum.sql b/dbms/tests/queries/0_stateless/01090_zookeeper_mutations_and_insert_quorum.sql new file mode 100644 index 00000000000..eb117ddf530 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01090_zookeeper_mutations_and_insert_quorum.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS mutations_and_quorum1; +DROP TABLE IF EXISTS mutations_and_quorum2; + +CREATE TABLE mutations_and_quorum1 (`server_date` Date, `something` String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/default/1/mutations_and_quorum', '1') PARTITION BY toYYYYMM(server_date) ORDER BY (server_date, something); +CREATE TABLE mutations_and_quorum2 (`server_date` Date, `something` String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/default/1/mutations_and_quorum', '2') PARTITION BY toYYYYMM(server_date) ORDER BY (server_date, something); + +SET insert_quorum=2; + +INSERT INTO mutations_and_quorum1 VALUES ('2019-01-01', 'test1'), ('2019-02-01', 'test2'), ('2019-03-01', 'test3'), ('2019-04-01', 'test4'), ('2019-05-01', 'test1'), ('2019-06-01', 'test2'), ('2019-07-01', 'test3'), ('2019-08-01', 'test4'), ('2019-09-01', 'test1'), ('2019-10-01', 'test2'), ('2019-11-01', 'test3'), ('2019-12-01', 'test4'); + +ALTER TABLE mutations_and_quorum1 DELETE WHERE something = 'test1' SETTINGS mutations_sync=2; + +SELECT COUNT() FROM mutations_and_quorum1; +SELECT COUNT() FROM mutations_and_quorum2; + +SELECT COUNT() FROM system.mutations WHERE table like 'mutations_and_quorum%' and is_done = 0; + +DROP TABLE IF EXISTS mutations_and_quorum1; +DROP TABLE IF EXISTS mutations_and_quorum2; diff --git a/dbms/tests/queries/0_stateless/01091_insert_with_default_json.reference b/dbms/tests/queries/0_stateless/01091_insert_with_default_json.reference new file mode 100644 index 00000000000..8e9a3516771 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01091_insert_with_default_json.reference @@ -0,0 +1,2 @@ +0 0 test0 +ClickHouse is great ClickHouse is fast diff --git a/dbms/tests/queries/0_stateless/01091_insert_with_default_json.sql b/dbms/tests/queries/0_stateless/01091_insert_with_default_json.sql new file mode 100644 index 00000000000..d8b998cdd5e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01091_insert_with_default_json.sql @@ -0,0 +1,26 @@ +DROP TABLE IF EXISTS table_with_complex_default; + +CREATE TABLE table_with_complex_default (i Int8, n UInt8 DEFAULT 42, s String DEFAULT concat('test', CAST(n, 'String'))) ENGINE=TinyLog; + +INSERT INTO table_with_complex_default FORMAT JSONEachRow {"i":0, "n": 0} + +SELECT * FROM table_with_complex_default; + +DROP TABLE IF EXISTS table_with_complex_default; + +DROP TABLE IF EXISTS test_default_using_alias; + +CREATE TABLE test_default_using_alias +( + what String, + a String DEFAULT concat(c, ' is great'), + b String DEFAULT concat(c, ' is fast'), + c String ALIAS concat(what, 'House') +) +ENGINE = TinyLog; + +INSERT INTO test_default_using_alias(what) VALUES ('Click'); + +SELECT a, b FROM test_default_using_alias; + +DROP TABLE IF EXISTS test_default_using_alias; diff --git a/dbms/tests/queries/0_stateless/01091_num_threads.reference b/dbms/tests/queries/0_stateless/01091_num_threads.reference new file mode 100644 index 00000000000..af1b5994f02 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01091_num_threads.reference @@ -0,0 +1,6 @@ +1 +1 +499999500000 +1 +499999500000 +1 diff --git a/dbms/tests/queries/0_stateless/01091_num_threads.sql b/dbms/tests/queries/0_stateless/01091_num_threads.sql new file mode 100644 index 00000000000..876a2d15d1a --- /dev/null +++ b/dbms/tests/queries/0_stateless/01091_num_threads.sql @@ -0,0 +1,47 @@ +set log_queries=1; +set log_query_threads=1; + +SELECT 1; +SYSTEM FLUSH LOGS; + +WITH + ( + SELECT query_id + FROM system.query_log + WHERE (query = 'SELECT 1') AND (event_date >= (today() - 1)) + ORDER BY event_time DESC + LIMIT 1 + ) AS id +SELECT uniqExact(thread_id) +FROM system.query_thread_log +WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); + +select sum(number) from numbers(1000000); +SYSTEM FLUSH LOGS; + +WITH + ( + SELECT query_id + FROM system.query_log + WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') AND (event_date >= (today() - 1)) + ORDER BY event_time DESC + LIMIT 1 + ) AS id +SELECT uniqExact(thread_id) +FROM system.query_thread_log +WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); + +select sum(number) from numbers_mt(1000000); +SYSTEM FLUSH LOGS; + +WITH + ( + SELECT query_id + FROM system.query_log + WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') AND (event_date >= (today() - 1)) + ORDER BY event_time DESC + LIMIT 1 + ) AS id +SELECT uniqExact(thread_id) > 2 +FROM system.query_thread_log +WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master_thread_id); diff --git a/dbms/tests/queries/0_stateless/01091_query_profiler_does_not_hang.reference b/dbms/tests/queries/0_stateless/01091_query_profiler_does_not_hang.reference new file mode 100644 index 00000000000..770fdcfb4b7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01091_query_profiler_does_not_hang.reference @@ -0,0 +1 @@ +1000000000 diff --git a/dbms/tests/queries/0_stateless/01091_query_profiler_does_not_hang.sql b/dbms/tests/queries/0_stateless/01091_query_profiler_does_not_hang.sql new file mode 100644 index 00000000000..010cbc6265a --- /dev/null +++ b/dbms/tests/queries/0_stateless/01091_query_profiler_does_not_hang.sql @@ -0,0 +1,2 @@ +SET query_profiler_cpu_time_period_ns = 1; +SELECT count() FROM numbers_mt(1000000000); diff --git a/dbms/tests/queries/0_stateless/01092_base64.reference b/dbms/tests/queries/0_stateless/01092_base64.reference new file mode 100644 index 00000000000..4c41ac31946 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01092_base64.reference @@ -0,0 +1 @@ +TEcgT3B0aW11cw== diff --git a/dbms/tests/queries/0_stateless/01092_base64.sql b/dbms/tests/queries/0_stateless/01092_base64.sql new file mode 100644 index 00000000000..476edfe31d7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01092_base64.sql @@ -0,0 +1,2 @@ +-- This query reproduces a bug in TurboBase64 library. +select distinct base64Encode(materialize('LG Optimus')) from numbers(100); diff --git a/dbms/tests/queries/0_stateless/01092_memory_profiler.reference b/dbms/tests/queries/0_stateless/01092_memory_profiler.reference new file mode 100644 index 00000000000..0d66ea1aee9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01092_memory_profiler.reference @@ -0,0 +1,2 @@ +0 +1 diff --git a/dbms/tests/queries/0_stateless/01092_memory_profiler.sql b/dbms/tests/queries/0_stateless/01092_memory_profiler.sql new file mode 100644 index 00000000000..c20b5c79cdb --- /dev/null +++ b/dbms/tests/queries/0_stateless/01092_memory_profiler.sql @@ -0,0 +1,6 @@ +SET allow_introspection_functions = 1; + +SET memory_profiler_step = 1000000; +SELECT ignore(groupArray(number), 'test memory profiler') FROM numbers(10000000); +SYSTEM FLUSH LOGS; +WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM system.trace_log t WHERE event_date >= yesterday() AND trace_type = 'Memory' AND query_id = (SELECT query_id FROM system.query_log WHERE event_date >= yesterday() AND query LIKE '%test memory profiler%' ORDER BY event_time DESC LIMIT 1); diff --git a/dbms/tests/queries/0_stateless/01093_cyclic_defaults_filimonov.reference b/dbms/tests/queries/0_stateless/01093_cyclic_defaults_filimonov.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01093_cyclic_defaults_filimonov.sql b/dbms/tests/queries/0_stateless/01093_cyclic_defaults_filimonov.sql new file mode 100644 index 00000000000..548cd794ba3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01093_cyclic_defaults_filimonov.sql @@ -0,0 +1,19 @@ +CREATE TABLE test +( + `a1` UInt64 DEFAULT a + 1, + `a1` UInt64 DEFAULT a + 1, + `a2` UInt64 DEFAULT a3 + a4, + `a3` UInt64 DEFAULT a2 + 1, + `a4` UInt64 ALIAS a3 + 1 +) +ENGINE = Log; -- { serverError 174 } + +CREATE TABLE pythagoras +( + `a` Float64 DEFAULT sqrt((c * c) - (b * b)), + `b` Float64 DEFAULT sqrt((c * c) - (a * a)), + `c` Float64 DEFAULT sqrt((a * a) + (b * b)) +) +ENGINE = Log; -- { serverError 174 } + +-- TODO: It works but should not: CREATE TABLE test (a DEFAULT b, b DEFAULT a) ENGINE = Memory diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index d5fa2db10ae..8fe0ae76260 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -43,9 +43,8 @@ RUN apt-get update -y \ rename \ wget -# This symlink required by gcc to find lld compiler but currently we have -# problems with query profiler when gcc use lld as linker -# RUN ln -s /usr/bin/lld-8 /usr/bin/ld.lld +# This symlink required by gcc to find lld compiler +RUN ln -s /usr/bin/lld-8 /usr/bin/ld.lld ENV CC=clang-8 ENV CXX=clang++-8 diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 494fc0f5ec8..3945fde4568 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -65,8 +65,6 @@ RUN apt-get --allow-unauthenticated update -y \ pigz \ moreutils - - # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able # to compress files using pigz (https://zlib.net/pigz/) instead of gzip. # Significantly increase deb packaging speed and compatible with old systems @@ -74,9 +72,8 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/dpkg-deb RUN chmod +x dpkg-deb RUN cp dpkg-deb /usr/bin -# This symlink required by gcc to find lld compiler but currently we have -# problems with query profiler when gcc use lld as linker -# RUN ln -s /usr/bin/lld-8 /usr/bin/ld.lld +# This symlink required by gcc to find lld compiler +RUN ln -s /usr/bin/lld-8 /usr/bin/ld.lld COPY build.sh / diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index eb5106996f5..cda7fecfbf2 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -206,6 +206,9 @@ function run_tests # the grep is to filter out set -x output and keep only time output { time "$script_dir/perf.py" "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" || continue + # The test completed with zero status, so we treat stderr as warnings + mv "$test_name-err.log" "$test_name-warn.log" + grep ^query "$test_name-raw.tsv" | cut -f2- > "$test_name-queries.tsv" grep ^client-time "$test_name-raw.tsv" | cut -f2- > "$test_name-client-time.tsv" skipped=$(grep ^skipped "$test_name-raw.tsv" | cut -f2-) @@ -220,16 +223,6 @@ function run_tests wait } -function analyze_queries -{ - # Build and analyze randomization distribution for all queries. - ls ./*-queries.tsv | xargs -n1 -I% basename % -queries.tsv | \ - parallel --verbose right/clickhouse local --file "{}-queries.tsv" \ - --structure "\"query text, run int, version UInt32, time float\"" \ - --query "\"$(cat "$script_dir/eqmed.sql")\"" \ - ">" {}-report.tsv -} - function get_profiles { # Collect the profiles @@ -251,6 +244,16 @@ function get_profiles wait } +# Build and analyze randomization distribution for all queries. +function analyze_queries +{ + ls ./*-queries.tsv | xargs -n1 -I% basename % -queries.tsv | \ + parallel --verbose right/clickhouse local --file "{}-queries.tsv" \ + --structure "\"query text, run int, version UInt32, time float\"" \ + --query "\"$(cat "$script_dir/eqmed.sql")\"" \ + ">" {}-report.tsv +} + # Analyze results function report { @@ -347,8 +350,8 @@ create table right_addresses_join engine Join(any, left, address) as create table unstable_query_runs engine File(TSVWithNamesAndTypes, 'unstable-query-runs.rep') as select query_id, query from right_query_log - join unstable_queries_tsv using query - where query_id not like 'prewarm %' + join queries using query + where query_id not like 'prewarm %' and (unstable or changed) ; create table unstable_query_log engine File(Vertical, 'unstable-query-log.rep') as @@ -423,8 +426,8 @@ wait unset IFS # Remember that grep sets error code when nothing is found, hence the bayan -# operator -grep -m2 'Exception:[^:]' ./*-err.log | sed 's/:/\t/' > run-errors.tsv ||: +# operator. +grep -H -m2 'Exception:[^:]' ./*-err.log | sed 's/:/\t/' > run-errors.tsv ||: "$script_dir/report.py" > report.html } @@ -450,6 +453,10 @@ case "$stage" in # to collect the logs. Prefer not to restart, because addresses might change # and we won't be able to process trace_log data. time get_profiles || restart || get_profiles + + # Stop the servers to free memory for the subsequent query analysis. + while killall clickhouse; do echo . ; sleep 1 ; done + echo Servers stopped. ;& "analyze_queries") time analyze_queries diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index ef8fd80c490..dae9ba5ef0e 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -10,10 +10,8 @@ cd workspace # We will compare to the most recent testing tag in master branch, let's find it. rm -rf ch ||: git clone --branch master --single-branch --depth 50 --bare https://github.com/ClickHouse/ClickHouse ch -(cd ch && git fetch origin "$SHA_TO_TEST:to-test") # fetch it so that we can show the commit message -# FIXME sometimes we have testing tags on commits without published builds -- these -# are documentation commits, normally. Loop to skip them. -start_ref=master +(cd ch && git fetch origin "$SHA_TO_TEST:to-test") +start_ref=to-test while : do ref_tag=$(cd ch && git describe --match='v*-testing' --abbrev=0 --first-parent $start_ref) @@ -22,6 +20,8 @@ do # dereference the tag to get the commit it points to, hence the '~0' thing. ref_sha=$(cd ch && git rev-parse "$ref_tag~0") + # FIXME sometimes we have testing tags on commits without published builds -- + # normally these are documentation commits. Loop to skip them. if curl --fail --head "https://clickhouse-builds.s3.yandex.net/0/$ref_sha/performance/performance.tgz" then break @@ -70,5 +70,5 @@ done dmesg -T > dmesg.log -7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/db/preprocessed_configs +7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs} cp compare.log /output diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index dbf0329f907..667537f8782 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -7,6 +7,7 @@ import clickhouse_driver import xml.etree.ElementTree as et import argparse import pprint +import string import time import traceback @@ -22,14 +23,21 @@ report_stage_end('start') parser = argparse.ArgumentParser(description='Run performance test.') # Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') -parser.add_argument('--host', nargs='*', default=['127.0.0.1', '127.0.0.1']) -parser.add_argument('--port', nargs='*', default=[9001, 9002]) -parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7))) +parser.add_argument('--host', nargs='*', default=['127.0.0.1', '127.0.0.1'], help="Server hostname. Parallel to '--port'.") +parser.add_argument('--port', nargs='*', default=[9001, 9002], help="Server port. Parallel to '--host'.") +parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.') +parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.') args = parser.parse_args() tree = et.parse(args.file[0]) root = tree.getroot() +# Skip long tests +for tag in root.findall('.//tag'): + if tag.text == 'long': + print('skipped\tTest is tagged as long.') + sys.exit(0) + # Check main metric main_metric_element = root.find('main_metric/*') if main_metric_element is not None and main_metric_element.tag != 'min_time': @@ -44,14 +52,54 @@ if infinite_sign is not None: servers = [{'host': host, 'port': port} for (host, port) in zip(args.host, args.port)] connections = [clickhouse_driver.Client(**server) for server in servers] +for s in servers: + print('server\t{}\t{}'.format(s['host'], s['port'])) + report_stage_end('connect') +# Process query parameters +subst_elems = root.findall('substitutions/substitution') +available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... } +for e in subst_elems: + available_parameters[e.find('name').text] = [v.text for v in e.findall('values/value')] + +# Take care to keep the order of queries -- sometimes we have DROP IF EXISTS +# followed by CREATE in create queries section, so the order matters. +def substitute_parameters(query_templates): + result = [] + for q in query_templates: + keys = set(n for _, n, _, _ in string.Formatter().parse(q) if n) + values = [available_parameters[k] for k in keys] + result.extend([ + q.format(**dict(zip(keys, values_combo))) + for values_combo in itertools.product(*values)]) + return result + +report_stage_end('substitute') + +# Run drop queries, ignoring errors. Do this before all other activity, because +# clickhouse_driver disconnects on error (this is not configurable), and the new +# connection loses the changes in settings. +drop_query_templates = [q.text for q in root.findall('drop_query')] +drop_queries = substitute_parameters(drop_query_templates) +for c in connections: + for q in drop_queries: + try: + c.execute(q) + except: + traceback.print_exc() + pass + +report_stage_end('drop1') + # Apply settings settings = root.findall('settings/*') for c in connections: for s in settings: c.execute("set {} = '{}'".format(s.tag, s.text)) +report_stage_end('settings') + # Check tables that should exist. If they don't exist, just skip this test. tables = [e.text for e in root.findall('preconditions/table_exists')] for t in tables: @@ -65,46 +113,16 @@ for t in tables: report_stage_end('preconditions') -# Process substitutions -subst_elems = root.findall('substitutions/substitution') - -parameter_keys = [] # ['table', 'limit' ] -parameter_value_arrays = [] # [['hits_100m', 'hits_10m'], ['1', '10']] -parameter_combinations = [] # [{table: hits_100m, limit: 1}, ...] -for se in subst_elems: - parameter_keys.append(se.find('name').text) - parameter_value_arrays.append([v.text for v in se.findall('values/value')]) -parameter_combinations = [dict(zip(parameter_keys, parameter_combination)) for parameter_combination in itertools.product(*parameter_value_arrays)] - -# Take care to keep the order of queries -- sometimes we have DROP IF EXISTS -# followed by CREATE in create queries section, so the order matters. -def substitute_parameters(query_templates, parameter_combinations): - return [template.format(**parameters) for template, parameters - in itertools.product(query_templates, parameter_combinations)] - -report_stage_end('substitute') - -# Run drop queries, ignoring errors -drop_query_templates = [q.text for q in root.findall('drop_query')] -drop_queries = substitute_parameters(drop_query_templates, parameter_combinations) -for c in connections: - for q in drop_queries: - try: - c.execute(q) - except: - traceback.print_exc() - pass - # Run create queries create_query_templates = [q.text for q in root.findall('create_query')] -create_queries = substitute_parameters(create_query_templates, parameter_combinations) +create_queries = substitute_parameters(create_query_templates) for c in connections: for q in create_queries: c.execute(q) # Run fill queries fill_query_templates = [q.text for q in root.findall('fill_query')] -fill_queries = substitute_parameters(fill_query_templates, parameter_combinations) +fill_queries = substitute_parameters(fill_query_templates) for c in connections: for q in fill_queries: c.execute(q) @@ -116,7 +134,7 @@ def tsv_escape(s): return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') test_query_templates = [q.text for q in root.findall('query')] -test_queries = substitute_parameters(test_query_templates, parameter_combinations) +test_queries = substitute_parameters(test_query_templates) report_stage_end('substitute2') @@ -146,9 +164,9 @@ report_stage_end('benchmark') # Run drop queries drop_query_templates = [q.text for q in root.findall('drop_query')] -drop_queries = substitute_parameters(drop_query_templates, parameter_combinations) +drop_queries = substitute_parameters(drop_query_templates) for c in connections: for q in drop_queries: c.execute(q) -report_stage_end('drop') +report_stage_end('drop2') diff --git a/docker/test/performance-comparison/performance_comparison.md b/docker/test/performance-comparison/performance_comparison.md index 6158ec11859..4a357fc9f3c 100644 --- a/docker/test/performance-comparison/performance_comparison.md +++ b/docker/test/performance-comparison/performance_comparison.md @@ -34,6 +34,17 @@ There are some environment variables that influence what the test does: * `-e CHPC_TEST_GLOB` -- the names of the tests (xml files) to run, interpreted as a shell glob. +#### Re-genarate report with your tweaks +From the workspace directory (extracted test output archive): +``` +stage=report compare.sh +``` +More stages are available, e.g. restart servers or run the tests. See the code. + +#### Run a single test on the already configured servers +``` +docker/test/performance-comparison/perf.py --host localhost --port 9000 --host localhost --port 9000 --runs=1 dbms/tests/performance/logical_functions_small.xml +``` ### References 1\. Box, Hunter, Hunter "Statictics for exprerimenters", p. 78: "A Randomized Design Used in the Comparison of Standard and Modified Fertilizer Mixtures for Tomato Plants." diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 3e8591bbc84..2c530c4273e 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -227,6 +227,13 @@ def print_test_times(): else: attrs[6] = '' + if float(r[5]) > 30: + # Just a hint for now. + # slow_average_tests += 1 + attrs[5] = 'style="background: #ffb0a0"' + else: + attrs[5] = '' + print(tableRow(r, attrs)) print(tableEnd()) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index aa545bfed97..0ce700bdc54 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -3,13 +3,20 @@ The HTTP interface lets you use ClickHouse on any platform from any programming language. We use it for working from Java and Perl, as well as shell scripts. In other departments, the HTTP interface is used from Perl, Python, and Go. The HTTP interface is more limited than the native interface, but it has better compatibility. By default, clickhouse-server listens for HTTP on port 8123 (this can be changed in the config). -If you make a GET / request without parameters, it returns the string "Ok." (with a line feed at the end). You can use this in health-check scripts. +If you make a GET / request without parameters, it returns 200 response code and the string which defined in [http_server_default_response](../operations/server_settings/settings.md#server_settings-http_server_default_response) default value "Ok." (with a line feed at the end) ```bash $ curl 'http://localhost:8123/' Ok. ``` +Use GET /ping request in health-check scripts. This handler always return "Ok." (with a line feed at the end). Available from version 18.12.13. +```bash +$ curl 'http://localhost:8123/ping' +Ok. +``` + + Send the request as a URL 'query' parameter, or as a POST. Or send the beginning of the query in the 'query' parameter, and the rest in the POST (we'll explain later why this is necessary). The size of the URL is limited to 16 KB, so keep this in mind when sending large queries. If successful, you receive the 200 response code and the result in the response body. diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 1cc85c3f881..d1c7a35cead 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -6,26 +6,71 @@ | Company | Industry | Usecase | Cluster Size | (Un)Compressed Data Size* | Reference | | --- | --- | --- | --- | --- | --- | | [2gis](https://2gis.ru) | Maps | Monitoring | — | — | [Talk in Russian, July 2019](https://youtu.be/58sPkXfq6nw) | +| [Aloha Browser](https://alohabrowser.com/) | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | | [Amadeus](https://amadeus.com/) | Travel | Analytics | — | — | [Press Release, April 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | | [Appsflyer](https://www.appsflyer.com) | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | +| [ArenaData](https://arenadata.tech/) | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | | [Badoo](https://badoo.com) | Dating | Timeseries | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | +| [Benocs](https://www.benocs.com/) | Network Telemetry and Analytics | Main Product | — | — | [Slides in English, October 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | | [Bloomberg](https://www.bloomberg.com/) | Finance, Media | Monitoring | 102 servers | — | [Slides, May 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| [Bloxy](https://bloxy.info) | Blockchain | Analytics | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | +| `Dataliance/UltraPower` | Telecom | Analytics | — | — | [Slides in Chinese, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | +| [CARTO](https://carto.com/) | Business Intelligence | Geo analytics | — | — | [Geospatial processing with Clickhouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | | [CERN](http://public.web.cern.ch/public/) | Research | Experiment | — | — | [Press release, April 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| [Cisco](http://public.web.cern.ch/public/) | Networking | Traffic analysis | — | — | [Lightning talk, October 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | +| [Cisco](http://cisco.com/) | Networking | Traffic analysis | — | — | [Lightning talk, October 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | | [Citadel Securities](https://www.citadelsecurities.com/) | Finance | — | — | — | [Contribution, March 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | +| [Citymobil](https://city-mobil.ru) | Taxi | Analytics | — | — | [Blog Post in Russian, March 2020](https://habr.com/en/company/citymobil/blog/490660/) | | [ContentSquare](https://contentsquare.com) | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | | [Cloudflare](https://cloudflare.com) | CDN | Traffic analysis | 36 servers | — | [Blog post, May 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog post, March 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | +| [Corunet](https://coru.net/) | Analytics | Main product | — | — | [Slides in English, April 2019 ](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | +| [CraiditX 氪信](https://creditx.com) | Finance AI | Analysis | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| [Criteo/Storetail](https://www.criteo.com/) | Retail | Main product | — | — | [Slides in English, October 2018 ](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | +| [Deutsche Bank](https://db.com) | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| [Diva-e](https://www.diva-e.com) | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | | [Exness](https://www.exness.com) | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | [Geniee](https://geniee.co.jp) | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | -| [LifeStreet](https://cloudflare.com) | Ad network | Main product | — | — | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | +| [HUYA](https://www.huya.com/) | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | +| [Idealista](https://www.idealista.com) | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Infovista](https://www.infovista.com/) | Networks | Analytics | — | — | [Slides in English, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | +| [InnoGames](https://www.innogames.com) | Games | Metrics, Logging | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | +| [Integros](https://integros.com) | Platform for video services | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| [Kodiak Data](https://www.kodiakdata.com/) | Clouds | Main product | — | — | [Slides in Engish, April 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | +| [Kontur](https://kontur.ru) | Software Development | Metrics | — | — | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | +| [LifeStreet](https://cloudflare.com) | Ad network | Main product | 60 servers in 3 replicas | 2-2.5 PiB | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | +| [Mail.ru Cloud Solutions](https://mcs.mail.ru/) | Cloud services | Main product | — | — | [Running ClickHouse Instance, in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) | +| [MessageBird](https://www.messagebird.com) | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | +| [MGID](https://www.mgid.com/) | Ad network | Web-analytics | — | — | [Our experience in implementing analytical DBMS ClickHouse, in Russian](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| [OneAPM](https://www.oneapm.com/) | Monitorings and Data Analysis | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | +| [Pragma Innovation](http://www.pragma-innovation.fr/) | Telemetry and Big Data Analysis | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | +| [QINGCLOUD](https://www.qingcloud.com/) | Cloud services | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | | [Qrator](https://qrator.net) | DDoS protection | Main product | — | — | [Blog Post, March 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | +| [Beijing PERCENT Information Technology Co., Ltd.](https://www.percent.cn/) | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| [Rambler](https://rambler.ru) | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | | [Tencent](https://www.tencent.com) | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | +| [Traffic Stars](https://trafficstars.com/) | AD network | — | — | — | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | | [S7 Airlines](https://www.s7.ru) | Airlines | Metrics, Logging | — | — | [Talk in Russian, March 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | +| [SEMrush](https://www.semrush.com/) | Marketing | Main product | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | +| [scireum GmbH](https://www.scireum.de/) | e-Commerce | Main product | — | — | [Talk in German, February 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | +| [Sentry](https://sentry.io/) | Software developer | Backend for product | — | — | [Blog Post in English, May 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | Goverment Social Security | Analytics | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | +| [seo.do](https://seo.do/) | Analytics | Main product | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | +| [Sina](http://english.sina.com/index.html) | News | — | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | +| [SMI2](https://smi2.ru/) | News | Analytics | — | — | [Blog Post in Russian, November 2017](https://habr.com/ru/company/smi2/blog/314558/) | +| [Splunk](https://www.splunk.com/) | Business Analytics | Main product | — | — | [Slides in English, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | | [Spotify](https://www.spotify.com) | Music | Experimentation | — | — | [Slides, July 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | +| [Tencent](https://www.tencent.com) | Big Data | Data processing | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | | [Uber](https://www.uber.com) | Taxi | Logging | — | — | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/ml.pdf) | +| [VKontakte](https://vk.com) | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| [Wisebits](https://wisebits.com/) | IT Solutions | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| [Xiaoxin Tech.](https://www.xiaoheiban.cn/) | Education | Common purpose | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| [Ximalaya](https://www.ximalaya.com/) | Audio sharing | OLAP | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | | [Yandex Cloud](https://cloud.yandex.ru/services/managed-clickhouse) | Public Cloud | Main product | — | — | [Talk in Russian, December 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | | [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | Business Intelligence | Main product | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | +| [Yandex Market](https://market.yandex.ru/) | e-Commerce | Metrics, Logging | — | — | [Talk in Russian, January 2019](https://youtu.be/_l1qP0DyBcA?t=478) | | [Yandex Metrica](https://metrica.yandex.com) | Web analytics | Main product | 360 servers in one cluster, 1862 servers in one department | 66.41 PiB / 5.68 PiB | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | +| [ЦВТ](https://htc-cs.ru/) | Software Development | Metrics, Logging | — | — | [Blog Post, March 2019, in Russian](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | +| [МКБ](https://mkb.ru/) | Bank | Web-system monitoring | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | +| [金数据](https://jinshuju.net) | BI Analytics | Main product | — | — | [Slides in Chinese, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/en/operations/monitoring.md b/docs/en/operations/monitoring.md index eef7d8c38bb..b9227cdfa1e 100644 --- a/docs/en/operations/monitoring.md +++ b/docs/en/operations/monitoring.md @@ -32,6 +32,6 @@ You can find metrics in the [system.metrics](system_tables.md#system_tables-metr You can configure ClickHouse to export metrics to [Graphite](https://github.com/graphite-project). See the [Graphite section](server_settings/settings.md#server_settings-graphite) in the ClickHouse server configuration file. Before configuring export of metrics, you should set up Graphite by following their official [guide](https://graphite.readthedocs.io/en/latest/install.html). -Additionally, you can monitor server availability through the HTTP API. Send the `HTTP GET` request to `/`. If the server is available, it responds with `200 OK`. +Additionally, you can monitor server availability through the HTTP API. Send the `HTTP GET` request to `/ping`. If the server is available, it responds with `200 OK`. To monitor servers in a cluster configuration, you should set the [max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) parameter and use the HTTP resource `/replicas_status`. A request to `/replicas_status` returns `200 OK` if the replica is available and is not delayed behind the other replicas. If a replica is delayed, it returns `503 HTTP_SERVICE_UNAVAILABLE` with information about the gap. diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index eae135796a0..1e48b374711 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -215,9 +215,10 @@ If `http_port` is specified, the openSSL configuration is ignored even if it is ``` -## http_server_default_response +## http_server_default_response {#server_settings-http_server_default_response} The page that is shown by default when you access the ClickHouse HTTP(s) server. +Default value is "Ok." (with a line feed at the end) **Example** diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md index e5f993540a7..e20d2fe926d 100644 --- a/docs/en/operations/settings/query_complexity.md +++ b/docs/en/operations/settings/query_complexity.md @@ -96,7 +96,7 @@ Maximum number of bytes before sorting. What to do if the number of rows received before sorting exceeds one of the limits: 'throw' or 'break'. By default, throw. -## max_result_rows +## max_result_rows {#setting-max_result_rows} Limit on the number of rows in the result. Also checked for subqueries, and on remote servers when running parts of a distributed query. @@ -107,7 +107,27 @@ Limit on the number of bytes in the result. The same as the previous setting. ## result_overflow_mode What to do if the volume of the result exceeds one of the limits: 'throw' or 'break'. By default, throw. -Using 'break' is similar to using LIMIT. + + +Using 'break' is similar to using LIMIT. Break interrupts execution only at the block level. This means that amount of returned rows is greater than [max_result_rows](#setting-max_result_rows), multiple of [max_block_size](settings.md#setting-max_block_size) and depends on [max_threads](settings.md#settings-max_threads). + +Пример: +```sql +SET max_threads = 3, max_block_size = 3333; +SET max_result_rows = 3334, result_overflow_mode = 'break'; + +SELECT * +FROM numbers_mt(100000) +FORMAT Null; +``` + +Результат: + +```text +6666 rows in set. ... +``` + + ## max_execution_time diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4928fba1ca8..6afba6f4fa4 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -406,7 +406,7 @@ Possible values: Default value: 1. -## max_block_size +## max_block_size {#setting-max_block_size} In ClickHouse, data is processed by blocks (sets of column parts). The internal processing cycles for a single block are efficient enough, but there are noticeable expenditures on each block. The `max_block_size` setting is a recommendation for what size of block (in number of rows) to load from tables. The block size shouldn't be too small, so that the expenditures on each block are still noticeable, but not too large, so that the query with LIMIT that is completed after the first block is processed quickly. The goal is to avoid consuming too much memory when extracting a large number of columns in multiple threads, and to preserve at least some cache locality. diff --git a/docs/en/operations/table_engines/generate.md b/docs/en/operations/table_engines/generate.md new file mode 100644 index 00000000000..126acb05626 --- /dev/null +++ b/docs/en/operations/table_engines/generate.md @@ -0,0 +1,53 @@ +# Generate {#table_engines-generate} + +The Generate table engine produces random data for given table schema. + +Usage examples: + +- Use in test to populate reproducible large table. +- Generate random input for fuzzing tests. + +## Usage in ClickHouse Server + +```sql +Generate(max_array_length, max_string_length, random_seed) +``` + +The `max_array_length` and `max_string_length` parameters specify maximum length of all +array columns and strings correspondingly in generated data. + +Generate table engine supports only `SELECT` queries. + +It supports all [DataTypes](../../data_types/index.md) that can be stored in a table except `LowCardinality` and `AggregateFunction`. + +**Example:** + +**1.** Set up the `generate_engine_table` table: + +```sql +CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE=Generate(3, 5, 1) +``` + +**2.** Query the data: + +```sql +SELECT * FROM generate_engine_table LIMIT 3 +``` + +```text +┌─name─┬──────value─┐ +│ c4xJ │ 1412771199 │ +│ r │ 1791099446 │ +│ 7#$ │ 124312908 │ +└──────┴────────────┘ +``` + +## Details of Implementation +- Not supported: + - `ALTER` + - `SELECT ... SAMPLE` + - `INSERT` + - Indices + - Replication + +[Original article](https://clickhouse.tech/docs/en/operations/table_engines/generate/) diff --git a/docs/en/query_language/agg_functions/reference.md b/docs/en/query_language/agg_functions/reference.md index 725ef3a4f62..7c099c26580 100644 --- a/docs/en/query_language/agg_functions/reference.md +++ b/docs/en/query_language/agg_functions/reference.md @@ -1034,7 +1034,7 @@ Alias: `medianExactWeighted`. - `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` value in the range of `[0.01, 0.99]`. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median). - `expr` — Expression over the column values resulting in numeric [data types](../../data_types/index.md#data_types), [Date](../../data_types/date.md) or [DateTime](../../data_types/datetime.md). -- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. +- `weight` — Column with weights of sequence members. Weight is a number of value occurrences. **Returned value** @@ -1300,7 +1300,7 @@ Result: ## quantileTDigestWeighted {#quantiletdigestweighted} -Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using the [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithm. The function takes into account the weight of each sequence number. The maximum error is 1%. Memory consumption is `log(n)`, where `n` is a number of values. +Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using the [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithm. The function takes into account the weight of each sequence member. The maximum error is 1%. Memory consumption is `log(n)`, where `n` is a number of values. The performance of the function is lower than performance of [quantile](#quantile) or [quantileTiming](#quantiletiming). In terms of the ratio of State size to precision, this function is much better than `quantile`. diff --git a/docs/en/query_language/functions/arithmetic_functions.md b/docs/en/query_language/functions/arithmetic_functions.md index ee6a122a008..55b62b404ca 100644 --- a/docs/en/query_language/functions/arithmetic_functions.md +++ b/docs/en/query_language/functions/arithmetic_functions.md @@ -55,6 +55,10 @@ If arguments are floating-point numbers, they are pre-converted to integers by d The remainder is taken in the same sense as in C++. Truncated division is used for negative numbers. An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. +## moduloOrZero(a, b) + +Differs from 'modulo' in that it returns zero when the divisor is zero. + ## negate(a), -a operator Calculates a number with the reverse sign. The result is always signed. diff --git a/docs/en/query_language/table_functions/generate.md b/docs/en/query_language/table_functions/generate.md new file mode 100644 index 00000000000..ed9e2150b03 --- /dev/null +++ b/docs/en/query_language/table_functions/generate.md @@ -0,0 +1,38 @@ +# generate + +Generates random data with given schema. +Allows to populate test tables with data. +Supports all data types that can be stored in table except `LowCardinality` and `AggregateFunction`. + +```sql +generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); +``` + +**Parameters** + +- `name` — Name of corresponding column. +- `TypeName` — Type of corresponding column. +- `limit` — Number of rows to generate. +- `max_array_length` — Maximum array length for all generated arrays. Defaults to `10`. +- `max_string_length` — Maximum string length for all generated strings. Defaults to `10`. +- `random_seed` — Specify random seed manually to produce stable results. Defaults to `0` — seed is randomly generated. + +**Returned Value** + +A table object with requested schema. + +## Usage Example + + +```sql +SELECT * FROM generate('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 3, 2, 10, 1); +``` +```text +┌─a────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ +│ [77] │ -124167.6723 │ ('2061-04-17 21:59:44.573','3f72f405-ec3e-13c8-44ca-66ef335f7835') │ +│ [32,110] │ -141397.7312 │ ('1979-02-09 03:43:48.526','982486d1-5a5d-a308-e525-7bd8b80ffa73') │ +│ [68] │ -67417.0770 │ ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') │ +└──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ +``` + +[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/generate/) diff --git a/docs/fa/operations/table_engines/generate.md b/docs/fa/operations/table_engines/generate.md new file mode 120000 index 00000000000..28cd09533e5 --- /dev/null +++ b/docs/fa/operations/table_engines/generate.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/generate.md \ No newline at end of file diff --git a/docs/fa/query_language/table_functions/generate.md b/docs/fa/query_language/table_functions/generate.md new file mode 120000 index 00000000000..141c05da1e3 --- /dev/null +++ b/docs/fa/query_language/table_functions/generate.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/docs/ja/introduction/features_considered_disadvantages.md b/docs/ja/introduction/features_considered_disadvantages.md deleted file mode 120000 index 45d3cdf563a..00000000000 --- a/docs/ja/introduction/features_considered_disadvantages.md +++ /dev/null @@ -1 +0,0 @@ -../../en/introduction/features_considered_disadvantages.md \ No newline at end of file diff --git a/docs/ja/introduction/features_considered_disadvantages.md b/docs/ja/introduction/features_considered_disadvantages.md new file mode 100644 index 00000000000..596fb9ba86f --- /dev/null +++ b/docs/ja/introduction/features_considered_disadvantages.md @@ -0,0 +1,8 @@ +# 欠点と考えられるClickHouseの機能 + +1. 本格的なトランザクションはありません。 +2. 既に挿入されたデータの変更または削除を、高頻度かつ低遅延に行う機能はありません。 [GDPR](https://gdpr-info.eu)に準拠するなど、データをクリーンアップまたは変更するために、バッチ削除およびバッチ更新が利用可能です。 +3. インデックスが疎であるため、ClickHouseは、キーで単一行を取得するようなクエリにはあまり適していません。 + +[Original article](https://clickhouse.yandex/docs/en/introduction/features_considered_disadvantages/) + diff --git a/docs/ja/introduction/history.md b/docs/ja/introduction/history.md deleted file mode 120000 index 7004e990a59..00000000000 --- a/docs/ja/introduction/history.md +++ /dev/null @@ -1 +0,0 @@ -../../en/introduction/history.md \ No newline at end of file diff --git a/docs/ja/introduction/history.md b/docs/ja/introduction/history.md new file mode 100644 index 00000000000..fbae277542d --- /dev/null +++ b/docs/ja/introduction/history.md @@ -0,0 +1,46 @@ +# ClickHouseの歴史 + +ClickHouseは元々、 [世界で2番目に大きなWeb分析プラットフォーム ](http://w3techs.com/technologies/overview/traffic_analysis/all) である [Yandex.Metrica](https://metrica.yandex.com/)を強化するために開発されたもので、このシステムのコアコンポーネントであり続けています。データベースには13兆を超えるレコードがあり、毎日200億を超えるイベントが発生しますが、ClickHouseでは集計されていないデータから直接カスタムレポートを生成できます。この記事では、ClickHouseの開発の初期段階におけるClickHouseの目標について簡単に説明します。 + +Yandex.Metricaは、ユーザーが定義した任意のセグメントを使用して、ヒットとセッションに基づいてカスタマイズされたレポートをその都度作成します。これには、多くの場合、一意のユーザー数などの複雑な集計を作成する必要があり、レポートを作成するための新しいデータがリアルタイムで受信されます。 + +2014年4月の時点で、Yandex.Metricaは毎日約120億のイベント(ページビューとクリック)を追跡していました。カスタムレポートを作成するには、これらすべてのイベントを保存する必要があります。単一のクエリで、数百ミリ秒以内に数百万行をスキャンしたり、わずか数秒で数億行をスキャンする必要があります。 + +## Yandex.Metricaおよびその他のYandexサービスでの用途 + +ClickHouseは、Yandex.Metricaで複数の目的に使用されます。その主なタスクは、非集計データを使用してオンラインでレポートを作成することです。 374台のサーバーからなるクラスターを使用し、20.3兆行をデータベースに保存します。圧縮されたデータの量は、重複データとレプリケーションを除いて約2PBです。非圧縮データ(TSV形式)の量は約17PBにもなります。 + +ClickHouseは以下の目的にも使用されます。 + +- Yandex.Metricaのデータをセッションリプレイのために保存する。 +- 中間データを処理する。 +- Analyticsを使用したグローバルレポートの作成。 +- Yandex.Metricaエンジンをデバッグするためのクエリの実行。 +- APIおよびユーザーインターフェイスからのログの分析。 + +ClickHouseは少なくとも、そのほか12のYandexのサービス(検索分野、Market、Direct、ビジネス分析、モバイル開発、AdFox、パーソナルサービスなど)で利用されています。 + +## 集約されたデータと非集約データ + +統計を効果的に計算するには、データの量を減らすため、データを集計する必要があるという一般的な意見があります。 + +ただし次の理由により、データ集約は非常に限られた解決策です。 + +- ユーザーが必要とするレポートの事前定義リストが必要です。 +- ユーザーはカスタムレポートを作成できません。 +- 大量のキーを集約する場合、データ量は削減されず、集約は役に立ちません。 +- 多数のレポートの場合、集計のバリエーションが多すぎます(組み合わせ爆発) +- カーディナリティの高いキー(URLなど)を集約する場合、データの量はそれほど減少しません(たかだか半分程度)。 +- このため、集約されたデータの量は減少するどころか増加する場合があります。 +- 生成した全てのレポートをユーザが見るわけではありません。計算の大部分は無駄になっています。 +- データの論理的な整合性は、さまざまな集計に対して違反する可能性があります。 + +何も集約せず、集約されていないデータを操作する場合、実際には計算量が減る可能性があります。 + +しかしながら集約は、その大部分がオフラインで実行され、比較的ゆったりと処理します。対照的に、オンライン計算では、ユーザーが結果を待っているため、できるだけ高速に計算する必要があります。 + +Yandex.Metricaには、Metrageと呼ばれるデータを集計するための特別なシステムがあり、これはほとんどのレポートで使用されています。 2009年以降、Yandex.Metricaは、以前にレポートビルダーで使用されていたOLAPServerと呼ばれる非集計データ用の特殊なOLAPデータベースも使用しました。 OLAPServerは非集計データに対してはうまく機能しましたが、多くの制限があり、必要に応じてすべてのレポートに使用することはできませんでした。制限とは、(数値のみしか扱えない)データ型サポートの欠如や、リアルタイムでデータを段階的に更新できないこと(毎日データを書き換えることによってのみ更新可能)がありました。 OLAPServerはDBMSではなく、特殊なDBだったのです。 + +OLAPServerの制限を取り除き、レポートのための非集計データを扱う問題を解決するために、私達は ClickHouse DBMSを開発しました。 + +[Original article](https://clickhouse.yandex/docs/en/introduction/history/) diff --git a/docs/ja/operations/table_engines/generate.md b/docs/ja/operations/table_engines/generate.md new file mode 120000 index 00000000000..28cd09533e5 --- /dev/null +++ b/docs/ja/operations/table_engines/generate.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/generate.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/generate.md b/docs/ja/query_language/table_functions/generate.md new file mode 120000 index 00000000000..141c05da1e3 --- /dev/null +++ b/docs/ja/query_language/table_functions/generate.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/docs/ru/getting_started/example_datasets/nyc_taxi.md b/docs/ru/getting_started/example_datasets/nyc_taxi.md index dfe82a4a679..da675887a54 100644 --- a/docs/ru/getting_started/example_datasets/nyc_taxi.md +++ b/docs/ru/getting_started/example_datasets/nyc_taxi.md @@ -331,7 +331,8 @@ ORDER BY year, count(*) DESC 128 GiB RAM, 8x6 TB HD на программном RAID-5 -Время выполнения — лучшее из трех запусков +Время выполнения — лучшее из трех запусков. + На самом деле начиная со второго запуска, запросы читают данные из кеша страниц файловой системы. Никакого дальнейшего кеширования не происходит: данные зачитываются и обрабатываются при каждом запуске. Создание таблицы на 3 серверах: diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index d1412bbd70e..f51689054e3 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -3,14 +3,20 @@ HTTP интерфейс позволяет использовать ClickHouse на любой платформе, из любого языка программирования. У нас он используется для работы из Java и Perl, а также из shell-скриптов. В других отделах, HTTP интерфейс используется из Perl, Python и Go. HTTP интерфейс более ограничен по сравнению с родным интерфейсом, но является более совместимым. По умолчанию, clickhouse-server слушает HTTP на порту 8123 (это можно изменить в конфиге). -Если запросить GET / без параметров, то вернётся строка "Ok." (с переводом строки на конце). Это может быть использовано в скриптах проверки доступности. - +Если запросить GET / без параметров, то вернётся строка заданная с помощью настройки [http_server_default_response](../operations/server_settings/settings.md#server_settings-http_server_default_response). Значение по умолчанию "Ok." (с переводом строки на конце). ```bash $ curl 'http://localhost:8123/' Ok. ``` -Запрос отправляется в виде параметра URL query. Или POST-ом. Или начало запроса в параметре query, а продолжение POST-ом (зачем это нужно, будет объяснено ниже). Размер URL ограничен 16KB, это следует учитывать при отправке больших запросов. +В скриптах проверки доступности вы можете использовать GET /ping без параметров. Если сервер доступен всегда возвращается "Ok." (с переводом строки на конце). +```bash +$ curl 'http://localhost:8123/ping' +Ok. +``` + +Запрос отправляется в виде URL параметра с именем query. Или как тело запроса при использовании метода POST. +Или начало запроса в URL параметре query, а продолжение POST-ом (зачем это нужно, будет объяснено ниже). Размер URL ограничен 16KB, это следует учитывать при отправке больших запросов. В случае успеха, вам вернётся код ответа 200 и результат обработки запроса в теле ответа. В случае ошибки, вам вернётся код ответа 500 и текст с описанием ошибки в теле ответа. diff --git a/docs/ru/operations/monitoring.md b/docs/ru/operations/monitoring.md index 7e7784b79b6..4467ef134d6 100644 --- a/docs/ru/operations/monitoring.md +++ b/docs/ru/operations/monitoring.md @@ -32,6 +32,6 @@ ClickHouse собирает: Можно настроить экспорт метрик из ClickHouse в [Graphite](https://github.com/graphite-project). Смотрите секцию [graphite](server_settings/settings.md#server_settings-graphite) конфигурационного файла ClickHouse. Перед настройкой экспорта метрик необходимо настроить Graphite, как указано в [официальном руководстве](https://graphite.readthedocs.io/en/latest/install.html). -Также, можно отслеживать доступность сервера через HTTP API. Отправьте `HTTP GET` к ресурсу `/`. Если сервер доступен, он отвечает `200 OK`. +Также, можно отслеживать доступность сервера через HTTP API. Отправьте `HTTP GET` к ресурсу `/ping`. Если сервер доступен, он отвечает `200 OK`. Для мониторинга серверов в кластерной конфигурации необходимо установить параметр [max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) и использовать HTTP ресурс `/replicas_status`. Если реплика доступна и не отстаёт от других реплик, то запрос к `/replicas_status` возвращает `200 OK`. Если реплика отстаёт, то запрос возвращает `503 HTTP_SERVICE_UNAVAILABLE`, включая информацию о размере отставания. diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_settings/settings.md index e72ae2c0054..47ec5ef7d9b 100644 --- a/docs/ru/operations/server_settings/settings.md +++ b/docs/ru/operations/server_settings/settings.md @@ -207,9 +207,10 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## http_server_default_response +## http_server_default_response {#server_settings-http_server_default_response} Страница, показываемая по умолчанию, при обращении к HTTP(s) серверу ClickHouse. +Значение по умолчанию "Ok." (с переводом строки на конце). **Пример** diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md index abdfd8dcc1f..01ab8745840 100644 --- a/docs/ru/operations/settings/query_complexity.md +++ b/docs/ru/operations/settings/query_complexity.md @@ -97,7 +97,7 @@ Что делать, если количество строк, полученное перед сортировкой, превысило одно из ограничений: throw или break. По умолчанию: throw. -## max_result_rows +## max_result_rows {#setting-max_result_rows} Ограничение на количество строк результата. Проверяются также для подзапросов и на удалённых серверах при выполнении части распределённого запроса. @@ -108,7 +108,24 @@ ## result_overflow_mode Что делать, если объём результата превысил одно из ограничений: throw или break. По умолчанию: throw. -Использование break по смыслу похоже на LIMIT. + +Использование break по смыслу похоже на LIMIT. Break прерывает выполнение только на уровне блока. Т.е. число строк которые вернет запрос будет больше чем ограничение [max_result_rows](#setting-max_result_rows), кратно [max_block_size](settings.md#setting-max_block_size) и зависит от [max_threads](settings.md#settings-max_threads). + +Пример: +```sql +SET max_threads = 3, max_block_size = 3333; +SET max_result_rows = 3334, result_overflow_mode = 'break'; + +SELECT * +FROM numbers_mt(100000) +FORMAT Null; +``` + +Результат: + +```text +6666 rows in set. ... +``` ## max_execution_time diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 298dd7364c3..80486502dcb 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -377,7 +377,7 @@ Ok. Значение по умолчанию: 0. -## max_block_size +## max_block_size {#setting-max_block_size} Данные в ClickHouse обрабатываются по блокам (наборам кусочков столбцов). Внутренние циклы обработки для одного блока достаточно эффективны, но есть заметные издержки на каждый блок. Настройка `max_block_size` — это рекомендация, какой размер блока (в количестве строк) загружать из таблиц. Размер блока не должен быть слишком маленьким, чтобы затраты на каждый блок были заметны, но не слишком велики, чтобы запрос с LIMIT, который завершается после первого блока, обрабатывался быстро. Цель состоит в том, чтобы не использовалось слишком много оперативки при вынимании большого количества столбцов в несколько потоков; чтобы оставалась хоть какая-нибудь кэш-локальность. diff --git a/docs/ru/operations/table_engines/generate.md b/docs/ru/operations/table_engines/generate.md new file mode 120000 index 00000000000..28cd09533e5 --- /dev/null +++ b/docs/ru/operations/table_engines/generate.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/generate.md \ No newline at end of file diff --git a/docs/ru/query_language/agg_functions/reference.md b/docs/ru/query_language/agg_functions/reference.md index 0367e15ba3a..f4ee9cffad2 100644 --- a/docs/ru/query_language/agg_functions/reference.md +++ b/docs/ru/query_language/agg_functions/reference.md @@ -860,7 +860,7 @@ FROM t quantile(level)(expr) ``` -Альяс: `median`. +Алиас: `median`. **Параметры** @@ -870,7 +870,7 @@ quantile(level)(expr) **Возвращаемое значение** -- Приблизительную квантиль заданного уровня. +- Приблизительный квантиль заданного уровня. Тип: @@ -926,7 +926,7 @@ SELECT quantile(val) FROM t quantileDeterministic(level)(expr, determinator) ``` -Альяс: `medianDeterministic`. +Алиас: `medianDeterministic`. **Параметры** @@ -936,7 +936,7 @@ quantileDeterministic(level)(expr, determinator) **Возвращаемое значение** -- Приблизительную квантиль заданного уровня. +- Приблизительный квантиль заданного уровня. Тип: @@ -993,7 +993,7 @@ SELECT quantileDeterministic(val, 1) FROM t quantileExact(level)(expr) ``` -Альяс: `medianExact`. +Алиас: `medianExact`. **Параметры** @@ -1046,7 +1046,7 @@ SELECT quantileExact(number) FROM numbers(10) quantileExactWeighted(level)(expr, weight) ``` -Альяс: `medianExactWeighted`. +Алиас: `medianExactWeighted`. **Параметры** @@ -1110,7 +1110,7 @@ SELECT quantileExactWeighted(n, val) FROM t quantileTiming(level)(expr) ``` -Альяс: `medianTiming`. +Алиас: `medianTiming`. **Параметры** @@ -1192,7 +1192,7 @@ SELECT quantileTiming(response_time) FROM t quantileTimingWeighted(level)(expr, weight) ``` -Альяс: `medianTimingWeighted`. +Алиас: `medianTimingWeighted`. **Параметры** @@ -1276,7 +1276,7 @@ SELECT quantileTimingWeighted(response_time, weight) FROM t quantileTDigest(level)(expr) ``` -Альяс: `medianTDigest`. +Алиас: `medianTDigest`. **Параметры** @@ -1333,7 +1333,7 @@ SELECT quantileTDigest(number) FROM numbers(10) quantileTDigestWeighted(level)(expr, weight) ``` -Альяс: `medianTDigest`. +Алиас: `medianTDigest`. **Параметры** @@ -1343,7 +1343,7 @@ quantileTDigestWeighted(level)(expr, weight) **Возвращаемое значение** -- Приблизительную квантиль заданного уровня. +- Приблизительный квантиль заданного уровня. Тип: diff --git a/docs/ru/query_language/table_functions/generate.md b/docs/ru/query_language/table_functions/generate.md new file mode 100644 index 00000000000..53544d16e7d --- /dev/null +++ b/docs/ru/query_language/table_functions/generate.md @@ -0,0 +1,37 @@ +# generate + +Генерирует случайные данные с заданной схемой. +Позволяет заполнять тестовые таблицы данными. +Поддерживает все типы данных, которые могут храниться в таблице, за исключением `LowCardinality` и `AggregateFunction`. + +```sql +generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); +``` + +**Входные параметры** +- `name` — название соответствующего столбца. +- `TypeName` — тип соответствующего столбца. +- `limit` — количество строк для генерации. +- `max_array_length` — максимальная длина массива для всех сгенерированных массивов. По умолчанию `10`. +- `max_string_length` — максимальная длина строки для всех генерируемых строк. По умолчанию `10`. +- `random_seed` — укажите состояние генератора случайных чисел вручную, чтобы получить стабильные результаты. По умолчанию `0` - генератор инициализируется случайным состоянием. + +**Возвращаемое значение** + +Объект таблицы с запрошенной схемой. + +## Пример + + +```sql +SELECT * FROM generate('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 3, 2, 10, 1); +``` +```text +┌─a────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ +│ [77] │ -124167.6723 │ ('2061-04-17 21:59:44.573','3f72f405-ec3e-13c8-44ca-66ef335f7835') │ +│ [32,110] │ -141397.7312 │ ('1979-02-09 03:43:48.526','982486d1-5a5d-a308-e525-7bd8b80ffa73') │ +│ [68] │ -67417.0770 │ ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') │ +└──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/generate/) diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 563d5dc3564..98c3e113fb6 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -75,6 +75,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'Generate': 'operations/table_engines/generate.md' - 'Database Engines': - 'Introduction': 'database_engines/index.md' @@ -143,6 +144,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index 5cafefc6e32..47197de62d9 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -109,6 +109,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'Generate': 'operations/table_engines/generate.md' - 'SQL Reference': - 'hidden': 'query_language/index.md' @@ -170,6 +171,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index 645791e6959..916459b0e04 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -79,7 +79,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' - + - 'Generate': 'operations/table_engines/generate.md' - 'SQL Reference': - 'hidden': 'query_language/index.md' - 'Syntax': 'query_language/syntax.md' @@ -142,6 +142,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 7b59a2aba38..60d73e0d075 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -80,6 +80,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'Generate': 'operations/table_engines/generate.md' - 'Справка по SQL': - 'hidden': 'query_language/index.md' @@ -143,6 +144,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Словари': - 'Введение': 'query_language/dicts/index.md' - 'Внешние словари': diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index de7cbb8ec5e..2fe4e9763b6 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -109,6 +109,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'Generate': 'operations/table_engines/generate.md' - 'SQL语法': - 'hidden': 'query_language/index.md' @@ -170,6 +171,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - '字典': - '介绍': 'query_language/dicts/index.md' - '外部字典': diff --git a/docs/zh/operations/monitoring.md b/docs/zh/operations/monitoring.md index cef3baf169b..bd177fe15b2 100644 --- a/docs/zh/operations/monitoring.md +++ b/docs/zh/operations/monitoring.md @@ -32,6 +32,6 @@ ClickHouse 收集的指标项： 可以配置ClickHouse 往 [Graphite](https://github.com/graphite-project)导入指标。 参考 [Graphite section](server_settings/settings.md#server_settings-graphite) 配置文件。在配置指标导出之前，需要参考Graphite[官方教程](https://graphite.readthedocs.io/en/latest/install.html)搭建服务。 -此外，您可以通过HTTP API监视服务器可用性。 将HTTP GET请求发送到 `/`。 如果服务器可用，它将以 `200 OK` 响应。 +此外，您可以通过HTTP API监视服务器可用性。 将HTTP GET请求发送到 `/ping`。 如果服务器可用，它将以 `200 OK` 响应。 要监视服务器集群的配置中，应设置[max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries)参数并使用HTTP资源`/replicas_status`。 如果副本可用，并且不延迟在其他副本之后，则对`/replicas_status`的请求将返回200 OK。 如果副本滞后，请求将返回 `503 HTTP_SERVICE_UNAVAILABLE`，包括有关待办事项大小的信息。 diff --git a/docs/zh/operations/table_engines/generate.md b/docs/zh/operations/table_engines/generate.md new file mode 120000 index 00000000000..28cd09533e5 --- /dev/null +++ b/docs/zh/operations/table_engines/generate.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/generate.md \ No newline at end of file diff --git a/docs/zh/query_language/table_functions/generate.md b/docs/zh/query_language/table_functions/generate.md new file mode 120000 index 00000000000..141c05da1e3 --- /dev/null +++ b/docs/zh/query_language/table_functions/generate.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/utils/check-style/check-style b/utils/check-style/check-style index c00d1134c3e..4734788b90e 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -45,3 +45,6 @@ find $ROOT_PATH/{dbms,base} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'Er # Duplicate ErrorCodes find $ROOT_PATH/{dbms,base} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'ErrorCodes::[_A-Z]+' | while read file; do grep -P 'extern const int [_A-Z]+;' $file | sort | uniq -c | grep -v -P ' +1 ' && echo "Duplicate ErrorCode in file $file"; done + +# Three or more consecutive empty lines +find $ROOT_PATH/{dbms,base} -name '*.h' -or -name '*.cpp' | while read file; do awk '/^$/ { ++i; if (i > 2) { print "More than two consecutive empty lines in file '$file'" } } /./ { i = 0 }' $file; done diff --git a/website/benchmark_hardware.html b/website/benchmark_hardware.html index b3b3a3e6d57..c5801e289f2 100644 --- a/website/benchmark_hardware.html +++ b/website/benchmark_hardware.html @@ -2172,6 +2172,57 @@ var results = [0.033, 0.025, 0.011] ] }, + + { + "system": "AMD EPYC 7502P / 128G DDR4 / 2NVME SAMSUNG MZQLB960HAJR", + "time": "2020-03-05 00:00:00", + "result": + [ +[0.012, 0.019, 0.009], +[0.042, 0.026, 0.038], +[0.026, 0.032, 0.017], +[0.058, 0.025, 0.027], +[0.095, 0.080, 0.087], +[0.143, 0.125, 0.124], +[0.018, 0.010, 0.016], +[0.013, 0.012, 0.013], +[0.201, 0.182, 0.182], +[0.228, 0.204, 0.204], +[0.093, 0.078, 0.077], +[0.100, 0.080, 0.081], +[0.241, 0.222, 0.218], +[0.291, 0.265, 0.270], +[0.268, 0.254, 0.256], +[0.255, 0.241, 0.242], +[0.623, 0.593, 0.599], +[0.373, 0.343, 0.339], +[1.354, 1.318, 1.311], +[0.054, 0.020, 0.022], +[0.495, 0.247, 0.242], +[0.520, 0.258, 0.248], +[0.957, 0.646, 0.652], +[null, null, null], +[0.149, 0.105, 0.099], +[0.091, 0.070, 0.069], +[0.150, 0.096, 0.094], +[0.499, 0.315, 0.309], +[0.437, 0.354, 0.357], +[1.002, 0.996, 0.991], +[0.234, 0.205, 0.207], +[0.380, 0.305, 0.305], +[1.733, 1.651, 1.655], +[1.230, 1.134, 1.132], +[1.217, 1.130, 1.114], +[0.396, 0.385, 0.383], +[0.156, 0.148, 0.160], +[0.065, 0.062, 0.063], +[0.057, 0.052, 0.052], +[0.368, 0.342, 0.336], +[0.030, 0.025, 0.027], +[0.022, 0.017, 0.019], +[0.005, 0.004, 0.004] + ] + }, ]; @@ -2602,6 +2653,7 @@ Results for AWS are from Wolf Kreuzerkrieg.
Results for Huawei Taishan are from Peng Gao in sina.com.
Results for Selectel and AMD EPYC 7402P are from Andrey Dudin.
Results for ProLiant are from Denis Ustinov.
+Results for AMD EPYC 7502P are from Kostiantyn Velychkovskyi.
Xeon Gold 6230 server is using 4 x SAMSUNG datacenter class SSD in RAID-10.
Results for Yandex Managed ClickHouse for "cold cache" are biased and should not be compared, because cache was not flushed for every next query.