diff --git a/cmake/tools.cmake b/cmake/tools.cmake index a6ea573a59d..723a14c6584 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -28,7 +28,7 @@ elseif (COMPILER_CLANG) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fchar8_t") endif () else () - set (CLANG_MINIMUM_VERSION 8) + set (CLANG_MINIMUM_VERSION 9) if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.") endif () diff --git a/docs/en/getting-started/playground.md b/docs/en/getting-started/playground.md index 7dd49e7d9ad..26fb105099b 100644 --- a/docs/en/getting-started/playground.md +++ b/docs/en/getting-started/playground.md @@ -6,11 +6,11 @@ toc_title: Playground # ClickHouse Playground {#clickhouse-playground} [ClickHouse Playground](https://play.clickhouse.tech) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. -Several example datasets are available in the Playground as well as sample queries that show ClickHouse features. There’s also a selection of ClickHouse LTS releases to experiment with. +Several example datasets are available in Playground as well as sample queries that show ClickHouse features. There’s also a selection of ClickHouse LTS releases to experiment with. ClickHouse Playground gives the experience of m2.small [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) instance (4 vCPU, 32 GB RAM) hosted in [Yandex.Cloud](https://cloud.yandex.com/). More information about [cloud providers](../commercial/cloud.md). -You can make queries to playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). +You can make queries to Playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). ## Credentials {#credentials} @@ -60,7 +60,7 @@ clickhouse client --secure -h play-api.clickhouse.tech --port 9440 -u playground ## Implementation Details {#implementation-details} ClickHouse Playground web interface makes requests via ClickHouse [HTTP API](../interfaces/http.md). -The Playground backend is just a ClickHouse cluster without any additional server-side application. As mentioned above, ClickHouse HTTPS and TCP/TLS endpoints are also publicly available as a part of the Playground, both are proxied through [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) to add extra layer of protection and improved global connectivity. +The Playground backend is just a ClickHouse cluster without any additional server-side application. As mentioned above, ClickHouse HTTPS and TCP/TLS endpoints are also publicly available as a part of the Playground, both are proxied through [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) to add an extra layer of protection and improved global connectivity. !!! warning "Warning" - Exposing ClickHouse server to public internet in any other situation is **strongly not recommended**. Make sure it listens only on private network and is covered by properly configured firewall. + Exposing the ClickHouse server to the public internet in any other situation is **strongly not recommended**. Make sure it listens only on a private network and is covered by a properly configured firewall. diff --git a/docs/en/sql-reference/ansi.md b/docs/en/sql-reference/ansi.md index 805741ba9d5..2cd9142c2f9 100644 --- a/docs/en/sql-reference/ansi.md +++ b/docs/en/sql-reference/ansi.md @@ -6,7 +6,7 @@ toc_title: ANSI Compatibility # ANSI SQL Compatibility of ClickHouse SQL Dialect {#ansi-sql-compatibility-of-clickhouse-sql-dialect} !!! note "Note" - This article relies on Table 38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013. + This article relies on Table 38, “Feature taxonomy and definition for mandatory features”, Annex F of [ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8). ## Differences in Behaviour {#differences-in-behaviour} @@ -77,6 +77,16 @@ The following table lists cases when query feature works in ClickHouse, but beha | E071-05 | Columns combined via table operators need not have exactly the same data type | Yes{.text-success} | | | E071-06 | Table operators in subqueries | Yes{.text-success} | | | **E081** | **Basic privileges** | **Partial**{.text-warning} | Work in progress | +| E081-01 | SELECT privilege at the table level | | | +| E081-02 | DELETE privilege | | | +| E081-03 | INSERT privilege at the table level | | | +| E081-04 | UPDATE privilege at the table level | | | +| E081-05 | UPDATE privilege at the column level | | | +| E081-06 | REFERENCES privilege at the table level | | | +| E081-07 | REFERENCES privilege at the column level | | | +| E081-08 | WITH GRANT OPTION | | | +| E081-09 | USAGE privilege | | | +| E081-10 | EXECUTE privilege | | | | **E091** | **Set functions** | **Yes**{.text-success} | | | E091-01 | AVG | Yes{.text-success} | | | E091-02 | COUNT | Yes{.text-success} | | @@ -169,6 +179,7 @@ The following table lists cases when query feature works in ClickHouse, but beha | **F471** | **Scalar subquery values** | **Yes**{.text-success} | | | **F481** | **Expanded NULL predicate** | **Yes**{.text-success} | | | **F812** | **Basic flagging** | **No**{.text-danger} | | +| **S011** | **Distinct data types** | | | | **T321** | **Basic SQL-invoked routines** | **No**{.text-danger} | | | T321-01 | User-defined functions with no overloading | No{.text-danger} | | | T321-02 | User-defined stored procedures with no overloading | No{.text-danger} | | diff --git a/docs/ru/getting-started/playground.md b/docs/ru/getting-started/playground.md index 3ddd066b2ed..5cb0612dfc7 100644 --- a/docs/ru/getting-started/playground.md +++ b/docs/ru/getting-started/playground.md @@ -1,38 +1,59 @@ # ClickHouse Playground {#clickhouse-playground} -ClickHouse Playground позволяет моментально выполнить запросы к ClickHouse из бразуера. -В Playground доступны несколько тестовых массивов данных и примеры запросов, которые показывают некоторые отличительные черты ClickHouse. +[ClickHouse Playground](https://play.clickhouse.tech) позволяет пользователям экспериментировать с ClickHouse, мгновенно выполняя запросы без настройки своего сервера или кластера. +В Playground доступны несколько тестовых массивов данных, а также примеры запросов, которые показывают возможности ClickHouse. Кроме того, вы можете выбрать LTS релиз ClickHouse, который хотите протестировать. -Запросы выполняются под пользователем с правами `readonly` для которого есть следующие ограничения: +ClickHouse Playground дает возможность поработать с [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) в конфигурации m2.small (4 vCPU, 32 ГБ ОЗУ), которую предосталяет [Яндекс.Облако](https://cloud.yandex.com/). Дополнительную информацию об облачных провайдерах читайте в разделе [Поставщики облачных услуг ClickHouse](../commercial/cloud.md). + +Вы можете отправлять запросы к Playground с помощью любого HTTP-клиента, например [curl](https://curl.haxx.se) или [wget](https://www.gnu.org/software/wget/), также можно установить соединение с помощью драйверов [JDBC](../interfaces/jdbc.md) или [ODBC](../interfaces/odbc.md). Более подробная информация о программных продуктах, поддерживающих ClickHouse, доступна [здесь](../interfaces/index.md). + +## Параметры доступа {#credentials} + +| Параметр | Значение | +|:--------------------|:----------------------------------------| +| Конечная точка HTTPS| `https://play-api.clickhouse.tech:8443` | +| Конечная точка TCP | `play-api.clickhouse.tech:9440` | +| Пользователь | `playground` | +| Пароль | `clickhouse` | + +Также можно подключаться к ClickHouse определённых релизов, чтобы протестировать их различия (порты и пользователь / пароль остаются неизменными): + +- 20.3 LTS: `play-api-v20-3.clickhouse.tech` +- 19.14 LTS: `play-api-v19-14.clickhouse.tech` + +!!! note "Примечание" + Для всех этих конечных точек требуется безопасное соединение TLS. + +## Ограничения {#limitations} + +Запросы выполняются под пользователем с правами `readonly`, для которого есть следующие ограничения: - запрещены DDL запросы - запрещены INSERT запросы Также установлены следующие опции: -- [`max_result_bytes=10485760`](../operations/settings/query_complexity/#max-result-bytes) -- [`max_result_rows=2000`](../operations/settings/query_complexity/#setting-max_result_rows) -- [`result_overflow_mode=break`](../operations/settings/query_complexity/#result-overflow-mode) -- [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time) +- [max\_result\_bytes=10485760](../operations/settings/query_complexity/#max-result-bytes) +- [max\_result\_rows=2000](../operations/settings/query_complexity/#setting-max_result_rows) +- [result\_overflow\_mode=break](../operations/settings/query_complexity/#result-overflow-mode) +- [max\_execution\_time=60000](../operations/settings/query_complexity/#max-execution-time) -ClickHouse Playground соответствует конфигурации m2.small хосту -[Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) -запущеному в [Яндекс.Облаке](https://cloud.yandex.com/). -Больше информации про [облачных провайдерах](../commercial/cloud.md). +## Примеры {#examples} -Веб интерфейс ClickHouse Playground делает запросы через ClickHouse HTTP API. -Бекендом служит обычный кластер ClickHouse. -ClickHouse HTTP интерфейс также доступен как часть Playground. - -Запросы к Playground могут быть выполнены с помощью curl/wget, а также через соединеие JDBC/ODBC драйвера -Больше информации про приложения с поддержкой ClickHouse доступно в разделе [Интерфейсы](../interfaces/index.md). - -| Параметр | Значение | -|:-----------------|:--------------------------------------| -| Адрес | https://play-api.clickhouse.tech:8443 | -| Имя пользователя | `playground` | -| Пароль | `clickhouse` | - -Требуется SSL соединение. +Пример конечной точки HTTPS с `curl`: ``` bash -curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse!';&user=playground&password=clickhouse&database=datasets" +curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse\!';&user=playground&password=clickhouse&database=datasets" ``` + +Пример конечной точки TCP с [CLI](../interfaces/cli.md): + +``` bash +clickhouse client --secure -h play-api.clickhouse.tech --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse\!'" +``` + +## Детали реализации {#implementation-details} + +Веб-интерфейс ClickHouse Playground выполняет запросы через ClickHouse [HTTP API](../interfaces/http.md). +Бэкэнд Playground - это кластер ClickHouse без дополнительных серверных приложений. Как упоминалось выше, способы подключения по HTTPS и TCP/TLS общедоступны как часть Playground. Они проксируются через [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) для добавления дополнительного уровня защиты и улучшенного глобального подключения. + +!!! warning "Предупреждение" +Открывать сервер ClickHouse для публичного доступа в любой другой ситуации **настоятельно не рекомендуется**. Убедитесь, что он настроен только на частную сеть и защищен брандмауэром. diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index ecb7c36b205..a10d2589576 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes CompressionCodecDelta::CompressionCodecDelta(UInt8 delta_bytes_size_) : delta_bytes_size(delta_bytes_size_) { + setCodecDescription("Delta", {std::make_shared(static_cast(delta_bytes_size))}); } uint8_t CompressionCodecDelta::getMethodByte() const @@ -30,12 +31,6 @@ uint8_t CompressionCodecDelta::getMethodByte() const return static_cast(CompressionMethodByte::Delta); } -ASTPtr CompressionCodecDelta::getCodecDesc() const -{ - auto literal = std::make_shared(static_cast(delta_bytes_size)); - return makeASTFunction("Delta", literal); -} - void CompressionCodecDelta::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); diff --git a/src/Compression/CompressionCodecDelta.h b/src/Compression/CompressionCodecDelta.h index a192fab051a..e892aa04242 100644 --- a/src/Compression/CompressionCodecDelta.h +++ b/src/Compression/CompressionCodecDelta.h @@ -12,8 +12,6 @@ public: uint8_t getMethodByte() const override; - ASTPtr getCodecDesc() const override; - void updateHash(SipHash & hash) const override; protected: diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index dd2e95a916d..96fd29fe356 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -327,6 +327,7 @@ UInt8 getDataBytesSize(DataTypePtr column_type) CompressionCodecDoubleDelta::CompressionCodecDoubleDelta(UInt8 data_bytes_size_) : data_bytes_size(data_bytes_size_) { + setCodecDescription("DoubleDelta"); } uint8_t CompressionCodecDoubleDelta::getMethodByte() const @@ -334,11 +335,6 @@ uint8_t CompressionCodecDoubleDelta::getMethodByte() const return static_cast(CompressionMethodByte::DoubleDelta); } -ASTPtr CompressionCodecDoubleDelta::getCodecDesc() const -{ - return std::make_shared("DoubleDelta"); -} - void CompressionCodecDoubleDelta::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); diff --git a/src/Compression/CompressionCodecDoubleDelta.h b/src/Compression/CompressionCodecDoubleDelta.h index 30ef086077d..11140ded61e 100644 --- a/src/Compression/CompressionCodecDoubleDelta.h +++ b/src/Compression/CompressionCodecDoubleDelta.h @@ -98,8 +98,6 @@ public: uint8_t getMethodByte() const override; - ASTPtr getCodecDesc() const override; - void updateHash(SipHash & hash) const override; protected: diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 3d08734fe91..d739623a94b 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -242,6 +242,7 @@ UInt8 getDataBytesSize(DataTypePtr column_type) CompressionCodecGorilla::CompressionCodecGorilla(UInt8 data_bytes_size_) : data_bytes_size(data_bytes_size_) { + setCodecDescription("Gorilla"); } uint8_t CompressionCodecGorilla::getMethodByte() const @@ -249,11 +250,6 @@ uint8_t CompressionCodecGorilla::getMethodByte() const return static_cast(CompressionMethodByte::Gorilla); } -ASTPtr CompressionCodecGorilla::getCodecDesc() const -{ - return std::make_shared("Gorilla"); -} - void CompressionCodecGorilla::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); diff --git a/src/Compression/CompressionCodecGorilla.h b/src/Compression/CompressionCodecGorilla.h index df0f329dc31..3613ab2a96f 100644 --- a/src/Compression/CompressionCodecGorilla.h +++ b/src/Compression/CompressionCodecGorilla.h @@ -95,8 +95,6 @@ public: uint8_t getMethodByte() const override; - ASTPtr getCodecDesc() const override; - void updateHash(SipHash & hash) const override; protected: diff --git a/src/Compression/CompressionCodecLZ4.cpp b/src/Compression/CompressionCodecLZ4.cpp index 1370349d68d..5f43b49706f 100644 --- a/src/Compression/CompressionCodecLZ4.cpp +++ b/src/Compression/CompressionCodecLZ4.cpp @@ -24,17 +24,16 @@ extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; extern const int ILLEGAL_CODEC_PARAMETER; } +CompressionCodecLZ4::CompressionCodecLZ4() +{ + setCodecDescription("LZ4"); +} uint8_t CompressionCodecLZ4::getMethodByte() const { return static_cast(CompressionMethodByte::LZ4); } -ASTPtr CompressionCodecLZ4::getCodecDesc() const -{ - return std::make_shared("LZ4"); -} - void CompressionCodecLZ4::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); @@ -63,12 +62,6 @@ void registerCodecLZ4(CompressionCodecFactory & factory) }); } -ASTPtr CompressionCodecLZ4HC::getCodecDesc() const -{ - auto literal = std::make_shared(static_cast(level)); - return makeASTFunction("LZ4HC", literal); -} - UInt32 CompressionCodecLZ4HC::doCompressData(const char * source, UInt32 source_size, char * dest) const { auto success = LZ4_compress_HC(source, dest, source_size, LZ4_COMPRESSBOUND(source_size), level); @@ -105,6 +98,7 @@ void registerCodecLZ4HC(CompressionCodecFactory & factory) CompressionCodecLZ4HC::CompressionCodecLZ4HC(int level_) : level(level_) { + setCodecDescription("LZ4HC", {std::make_shared(static_cast(level))}); } } diff --git a/src/Compression/CompressionCodecLZ4.h b/src/Compression/CompressionCodecLZ4.h index 229e25481e6..bf8b4e2dd1f 100644 --- a/src/Compression/CompressionCodecLZ4.h +++ b/src/Compression/CompressionCodecLZ4.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -12,9 +13,9 @@ namespace DB class CompressionCodecLZ4 : public ICompressionCodec { public: - uint8_t getMethodByte() const override; + CompressionCodecLZ4(); - ASTPtr getCodecDesc() const override; + uint8_t getMethodByte() const override; UInt32 getAdditionalSizeAtTheEndOfBuffer() const override { return LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER; } @@ -32,17 +33,15 @@ private: UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; mutable LZ4::PerformanceStatistics lz4_stat; + ASTPtr codec_desc; }; class CompressionCodecLZ4HC : public CompressionCodecLZ4 { public: - CompressionCodecLZ4HC(int level_); - ASTPtr getCodecDesc() const override; - protected: UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; diff --git a/src/Compression/CompressionCodecMultiple.cpp b/src/Compression/CompressionCodecMultiple.cpp index 77f0fc132fe..a0336d66a05 100644 --- a/src/Compression/CompressionCodecMultiple.cpp +++ b/src/Compression/CompressionCodecMultiple.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,11 @@ namespace ErrorCodes CompressionCodecMultiple::CompressionCodecMultiple(Codecs codecs_) : codecs(codecs_) { + ASTs arguments; + for (const auto & codec : codecs) + arguments.push_back(codec->getCodecDesc()); + /// Special case, codec doesn't have name and contain list of codecs. + setCodecDescription("", arguments); } uint8_t CompressionCodecMultiple::getMethodByte() const @@ -29,14 +35,6 @@ uint8_t CompressionCodecMultiple::getMethodByte() const return static_cast(CompressionMethodByte::Multiple); } -ASTPtr CompressionCodecMultiple::getCodecDesc() const -{ - auto result = std::make_shared(); - for (const auto & codec : codecs) - result->children.push_back(codec->getCodecDesc()); - return result; -} - void CompressionCodecMultiple::updateHash(SipHash & hash) const { for (const auto & codec : codecs) diff --git a/src/Compression/CompressionCodecMultiple.h b/src/Compression/CompressionCodecMultiple.h index 6bac189bdf7..1eb61842048 100644 --- a/src/Compression/CompressionCodecMultiple.h +++ b/src/Compression/CompressionCodecMultiple.h @@ -13,8 +13,6 @@ public: uint8_t getMethodByte() const override; - ASTPtr getCodecDesc() const override; - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; static std::vector getCodecsBytesFromData(const char * source); diff --git a/src/Compression/CompressionCodecNone.cpp b/src/Compression/CompressionCodecNone.cpp index f727c4b4860..84bcb5bd841 100644 --- a/src/Compression/CompressionCodecNone.cpp +++ b/src/Compression/CompressionCodecNone.cpp @@ -7,16 +7,16 @@ namespace DB { +CompressionCodecNone::CompressionCodecNone() +{ + setCodecDescription("NONE"); +} + uint8_t CompressionCodecNone::getMethodByte() const { return static_cast(CompressionMethodByte::NONE); } -ASTPtr CompressionCodecNone::getCodecDesc() const -{ - return std::make_shared("NONE"); -} - void CompressionCodecNone::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); diff --git a/src/Compression/CompressionCodecNone.h b/src/Compression/CompressionCodecNone.h index 370ef301694..bf6bb6de4e2 100644 --- a/src/Compression/CompressionCodecNone.h +++ b/src/Compression/CompressionCodecNone.h @@ -11,9 +11,9 @@ namespace DB class CompressionCodecNone : public ICompressionCodec { public: - uint8_t getMethodByte() const override; + CompressionCodecNone(); - ASTPtr getCodecDesc() const override; + uint8_t getMethodByte() const override; void updateHash(SipHash & hash) const override; diff --git a/src/Compression/CompressionCodecT64.cpp b/src/Compression/CompressionCodecT64.cpp index 30972a5fe1f..f081652f613 100644 --- a/src/Compression/CompressionCodecT64.cpp +++ b/src/Compression/CompressionCodecT64.cpp @@ -637,13 +637,14 @@ uint8_t CompressionCodecT64::getMethodByte() const return codecId(); } -ASTPtr CompressionCodecT64::getCodecDesc() const +CompressionCodecT64::CompressionCodecT64(TypeIndex type_idx_, Variant variant_) + : type_idx(type_idx_) + , variant(variant_) { if (variant == Variant::Byte) - return std::make_shared("T64"); - - auto literal = std::make_shared("bit"); - return makeASTFunction("T64", literal); + setCodecDescription("T64"); + else + setCodecDescription("T64", {std::make_shared("bit")}); } void CompressionCodecT64::updateHash(SipHash & hash) const diff --git a/src/Compression/CompressionCodecT64.h b/src/Compression/CompressionCodecT64.h index 06c34ba0a4a..d930ea353c4 100644 --- a/src/Compression/CompressionCodecT64.h +++ b/src/Compression/CompressionCodecT64.h @@ -26,15 +26,10 @@ public: Bit }; - CompressionCodecT64(TypeIndex type_idx_, Variant variant_) - : type_idx(type_idx_) - , variant(variant_) - {} + CompressionCodecT64(TypeIndex type_idx_, Variant variant_); uint8_t getMethodByte() const override; - ASTPtr getCodecDesc() const override; - void updateHash(SipHash & hash) const override; protected: diff --git a/src/Compression/CompressionCodecZSTD.cpp b/src/Compression/CompressionCodecZSTD.cpp index 3b317884fec..f236c4bf460 100644 --- a/src/Compression/CompressionCodecZSTD.cpp +++ b/src/Compression/CompressionCodecZSTD.cpp @@ -25,13 +25,6 @@ uint8_t CompressionCodecZSTD::getMethodByte() const return static_cast(CompressionMethodByte::ZSTD); } - -ASTPtr CompressionCodecZSTD::getCodecDesc() const -{ - auto literal = std::make_shared(static_cast(level)); - return makeASTFunction("ZSTD", literal); -} - void CompressionCodecZSTD::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); @@ -65,6 +58,7 @@ void CompressionCodecZSTD::doDecompressData(const char * source, UInt32 source_s CompressionCodecZSTD::CompressionCodecZSTD(int level_) : level(level_) { + setCodecDescription("ZSTD", {std::make_shared(static_cast(level))}); } void registerCodecZSTD(CompressionCodecFactory & factory) diff --git a/src/Compression/CompressionCodecZSTD.h b/src/Compression/CompressionCodecZSTD.h index 3bfb6bb1d4d..903af6d6c1b 100644 --- a/src/Compression/CompressionCodecZSTD.h +++ b/src/Compression/CompressionCodecZSTD.h @@ -17,8 +17,6 @@ public: uint8_t getMethodByte() const override; - ASTPtr getCodecDesc() const override; - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; void updateHash(SipHash & hash) const override; diff --git a/src/Compression/ICompressionCodec.cpp b/src/Compression/ICompressionCodec.cpp index 5de015b2680..1b2c90e5163 100644 --- a/src/Compression/ICompressionCodec.cpp +++ b/src/Compression/ICompressionCodec.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -15,24 +16,59 @@ namespace ErrorCodes { extern const int CANNOT_DECOMPRESS; extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; } -ASTPtr ICompressionCodec::getFullCodecDesc() const + +void ICompressionCodec::setCodecDescription(const String & codec_name, const ASTs & arguments) { std::shared_ptr result = std::make_shared(); result->name = "CODEC"; - ASTPtr codec_desc = getCodecDesc(); - if (codec_desc->as()) + + /// Special case for codec Multiple, which doens't have name. It's just list + /// of other codecs. + if (codec_name.empty()) { + ASTPtr codec_desc = std::make_shared(); + for (const auto & argument : arguments) + codec_desc->children.push_back(argument); result->arguments = codec_desc; } else { + ASTPtr codec_desc; + if (arguments.empty()) /// Codec without arguments is just ASTIdentifier + codec_desc = std::make_shared(codec_name); + else /// Codec with arguments represented as ASTFunction + codec_desc = makeASTFunction(codec_name, arguments); + result->arguments = std::make_shared(); result->arguments->children.push_back(codec_desc); } + result->children.push_back(result->arguments); - return result; + full_codec_desc = result; +} + + +ASTPtr ICompressionCodec::getFullCodecDesc() const +{ + if (full_codec_desc == nullptr) + throw Exception("Codec description is not prepared", ErrorCodes::LOGICAL_ERROR); + + return full_codec_desc; +} + + +ASTPtr ICompressionCodec::getCodecDesc() const +{ + + auto arguments = getFullCodecDesc()->as()->arguments; + /// If it has exactly one argument, than it's single codec, return it + if (arguments->children.size() == 1) + return arguments->children[0]; + else /// Otherwise we have multiple codecs and return them as expression list + return arguments; } UInt64 ICompressionCodec::getHash() const diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index 8d7d3fc800c..fa143af8b9c 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -31,7 +31,7 @@ public: virtual uint8_t getMethodByte() const = 0; /// Codec description, for example "ZSTD(2)" or "LZ4,LZ4HC(5)" - virtual ASTPtr getCodecDesc() const = 0; + virtual ASTPtr getCodecDesc() const; /// Codec description with "CODEC" prefix, for example "CODEC(ZSTD(2))" or /// "CODEC(LZ4,LZ4HC(5))" @@ -87,6 +87,12 @@ protected: /// Actually decompress data without header virtual void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const = 0; + + /// Construct and set codec description from codec name and arguments. Must be called in codec constructor. + void setCodecDescription(const String & name, const ASTs & arguments = {}); + +private: + ASTPtr full_codec_desc; }; } diff --git a/src/Functions/GatherUtils/Algorithms.h b/src/Functions/GatherUtils/Algorithms.h index 769d23b66dc..32501beebf0 100644 --- a/src/Functions/GatherUtils/Algorithms.h +++ b/src/Functions/GatherUtils/Algorithms.h @@ -21,7 +21,6 @@ namespace DB::GatherUtils inline constexpr size_t MAX_ARRAY_SIZE = 1 << 30; - /// Methods to copy Slice to Sink, overloaded for various combinations of types. template diff --git a/src/Functions/GatherUtils/GatherUtils.h b/src/Functions/GatherUtils/GatherUtils.h index 6699cc655e4..c2513214a79 100644 --- a/src/Functions/GatherUtils/GatherUtils.h +++ b/src/Functions/GatherUtils/GatherUtils.h @@ -41,18 +41,37 @@ std::unique_ptr createArraySource(const ColumnArray & col, bool is std::unique_ptr createValueSource(const IColumn & col, bool is_const, size_t total_rows); std::unique_ptr createArraySink(ColumnArray & col, size_t column_size); -void concat(const std::vector> & sources, IArraySink & sink); +ColumnArray::MutablePtr concat(const std::vector> & sources); -void sliceFromLeftConstantOffsetUnbounded(IArraySource & src, IArraySink & sink, size_t offset); -void sliceFromLeftConstantOffsetBounded(IArraySource & src, IArraySink & sink, size_t offset, ssize_t length); +ColumnArray::MutablePtr sliceFromLeftConstantOffsetUnbounded(IArraySource & src, size_t offset); +ColumnArray::MutablePtr sliceFromLeftConstantOffsetBounded(IArraySource & src, size_t offset, ssize_t length); -void sliceFromRightConstantOffsetUnbounded(IArraySource & src, IArraySink & sink, size_t offset); -void sliceFromRightConstantOffsetBounded(IArraySource & src, IArraySink & sink, size_t offset, ssize_t length); +ColumnArray::MutablePtr sliceFromRightConstantOffsetUnbounded(IArraySource & src, size_t offset); +ColumnArray::MutablePtr sliceFromRightConstantOffsetBounded(IArraySource & src, size_t offset, ssize_t length); -void sliceDynamicOffsetUnbounded(IArraySource & src, IArraySink & sink, const IColumn & offset_column); -void sliceDynamicOffsetBounded(IArraySource & src, IArraySink & sink, const IColumn & offset_column, const IColumn & length_column); +ColumnArray::MutablePtr sliceDynamicOffsetUnbounded(IArraySource & src, const IColumn & offset_column); +ColumnArray::MutablePtr sliceDynamicOffsetBounded(IArraySource & src, const IColumn & offset_column, const IColumn & length_column); -void sliceHas(IArraySource & first, IArraySource & second, ArraySearchType search_type, ColumnUInt8 & result); +void sliceHasAny(IArraySource & first, IArraySource & second, ColumnUInt8 & result); +void sliceHasAll(IArraySource & first, IArraySource & second, ColumnUInt8 & result); +void sliceHasSubstr(IArraySource & first, IArraySource & second, ColumnUInt8 & result); + +inline void sliceHas(IArraySource & first, IArraySource & second, ArraySearchType search_type, ColumnUInt8 & result) +{ + switch (search_type) + { + case ArraySearchType::All: + sliceHasAll(first, second, result); + break; + case ArraySearchType::Any: + sliceHasAny(first, second, result); + break; + case ArraySearchType::Substr: + sliceHasSubstr(first, second, result); + break; + + } +} void push(IArraySource & array_source, IValueSource & value_source, IArraySink & sink, bool push_front); diff --git a/src/Functions/GatherUtils/Sinks.h b/src/Functions/GatherUtils/Sinks.h index 5fd943ae78b..cdf4408b544 100644 --- a/src/Functions/GatherUtils/Sinks.h +++ b/src/Functions/GatherUtils/Sinks.h @@ -11,6 +11,7 @@ #include #include +#include namespace DB::GatherUtils { @@ -44,8 +45,8 @@ struct NumericArraySink : public ArraySinkImpl> size_t row_num = 0; ColumnArray::Offset current_offset = 0; - NumericArraySink(ColumnArray & arr, size_t column_size) - : elements(typeid_cast(arr.getData()).getData()), offsets(arr.getOffsets()) + NumericArraySink(IColumn & elements_, ColumnArray::Offsets & offsets_, size_t column_size) + : elements(assert_cast(elements_).getData()), offsets(offsets_) { offsets.resize(column_size); } @@ -161,8 +162,8 @@ struct GenericArraySink : public ArraySinkImpl size_t row_num = 0; ColumnArray::Offset current_offset = 0; - GenericArraySink(ColumnArray & arr, size_t column_size) - : elements(arr.getData()), offsets(arr.getOffsets()) + GenericArraySink(IColumn & elements_, ColumnArray::Offsets & offsets_, size_t column_size) + : elements(elements_), offsets(offsets_) { offsets.resize(column_size); } @@ -198,8 +199,9 @@ struct NullableArraySink : public ArraySink NullMap & null_map; - NullableArraySink(ColumnArray & arr, NullMap & null_map_, size_t column_size) - : ArraySink(arr, column_size), null_map(null_map_) + NullableArraySink(IColumn & elements_, ColumnArray::Offsets & offsets_, size_t column_size) + : ArraySink(assert_cast(elements_).getNestedColumn(), offsets_, column_size) + , null_map(assert_cast(elements_).getNullMapData()) { } diff --git a/src/Functions/GatherUtils/Sources.h b/src/Functions/GatherUtils/Sources.h index fe71a1f8be3..0cf37d159a8 100644 --- a/src/Functions/GatherUtils/Sources.h +++ b/src/Functions/GatherUtils/Sources.h @@ -29,6 +29,12 @@ namespace ErrorCodes namespace GatherUtils { +template struct NumericArraySink; +struct StringSink; +struct FixedStringSink; +struct GenericArraySink; +template struct NullableArraySink; + template struct NumericArraySource : public ArraySourceImpl> { @@ -36,14 +42,23 @@ struct NumericArraySource : public ArraySourceImpl> using Slice = NumericArraySlice; using Column = ColumnArray; + using SinkType = NumericArraySink; + + const ColVecType & column; const typename ColVecType::Container & elements; const typename ColumnArray::Offsets & offsets; size_t row_num = 0; ColumnArray::Offset prev_offset = 0; + MutableColumnPtr createValuesColumn() + { + return column.cloneEmpty(); + } + explicit NumericArraySource(const ColumnArray & arr) - : elements(typeid_cast(arr.getData()).getData()), offsets(arr.getOffsets()) + : column(typeid_cast(arr.getData())) + , elements(typeid_cast(arr.getData()).getData()), offsets(arr.getOffsets()) { } @@ -143,6 +158,8 @@ struct ConstSource : public Base using Slice = typename Base::Slice; using Column = ColumnConst; + using SinkType = typename Base::SinkType; + size_t total_rows; size_t row_num = 0; @@ -224,6 +241,8 @@ struct StringSource using Slice = NumericArraySlice; using Column = ColumnString; + using SinkType = StringSink; + const typename ColumnString::Chars & elements; const typename ColumnString::Offsets & offsets; @@ -391,6 +410,8 @@ struct FixedStringSource using Slice = NumericArraySlice; using Column = ColumnFixedString; + using SinkType = FixedStringSink; + const UInt8 * pos; const UInt8 * end; size_t string_size; @@ -511,12 +532,19 @@ struct GenericArraySource : public ArraySourceImpl using Slice = GenericArraySlice; using Column = ColumnArray; + using SinkType = GenericArraySink; + const IColumn & elements; const typename ColumnArray::Offsets & offsets; size_t row_num = 0; ColumnArray::Offset prev_offset = 0; + MutableColumnPtr createValuesColumn() + { + return elements.cloneEmpty(); + } + explicit GenericArraySource(const ColumnArray & arr) : elements(arr.getData()), offsets(arr.getOffsets()) { @@ -550,7 +578,7 @@ struct GenericArraySource : public ArraySourceImpl size_t getColumnSize() const override { - return elements.size(); + return offsets.size(); } size_t getElementSize() const @@ -605,6 +633,8 @@ struct NullableArraySource : public ArraySource using ArraySource::row_num; using ArraySource::offsets; + using SinkType = NullableArraySink; + const NullMap & null_map; NullableArraySource(const ColumnArray & arr, const NullMap & null_map_) @@ -612,6 +642,11 @@ struct NullableArraySource : public ArraySource { } + MutableColumnPtr createValuesColumn() + { + return ColumnNullable::create(static_cast(this)->createValuesColumn(), ColumnUInt8::create()); + } + void accept(ArraySourceVisitor & visitor) override { visitor.visit(*this); } Slice getWhole() const @@ -674,6 +709,8 @@ struct NumericValueSource : ValueSourceImpl> using Slice = NumericValueSlice; using Column = std::conditional_t, ColumnDecimal, ColumnVector>; + using SinkType = NumericArraySink; + const T * begin; size_t total_rows; size_t row_num = 0; @@ -716,6 +753,7 @@ struct NumericValueSource : ValueSourceImpl> struct GenericValueSource : public ValueSourceImpl { using Slice = GenericValueSlice; + using SinkType = GenericArraySink; const IColumn * column; size_t total_rows; @@ -759,6 +797,8 @@ struct GenericValueSource : public ValueSourceImpl template struct NullableValueSource : public ValueSource { + using SinkType = NullableArraySink; + using Slice = NullableSlice; using ValueSource::row_num; diff --git a/src/Functions/GatherUtils/concat.cpp b/src/Functions/GatherUtils/concat.cpp index 0962baed603..1329320938d 100644 --- a/src/Functions/GatherUtils/concat.cpp +++ b/src/Functions/GatherUtils/concat.cpp @@ -16,37 +16,52 @@ namespace ErrorCodes namespace GatherUtils { -struct ArrayConcat : public ArraySinkSourceSelector +struct ArrayConcat : public ArraySourceSelector { using Sources = std::vector>; - template - static void selectSourceSink(Source &&, Sink && sink, const Sources & sources) + template + static void selectImpl(Source && source, const Sources & sources, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; - concat(sources, sink); + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + concat(sources, std::move(sink)); } - template - static void selectSourceSink(ConstSource &&, Sink && sink, const Sources & sources) + template + static void selectImpl(ConstSource && source, const Sources & sources, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; - concat(sources, sink); + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + concat(sources, std::move(sink)); } - template - static void selectSourceSink(ConstSource &, Sink && sink, const Sources & sources) + template + static void selectImpl(ConstSource & source, const Sources & sources, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; - concat(sources, sink); + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + concat(sources, std::move(sink)); } }; -void concat(const std::vector> & sources, IArraySink & sink) +ColumnArray::MutablePtr concat(const std::vector> & sources) { if (sources.empty()) throw Exception("Concat function should get at least 1 ArraySource", ErrorCodes::LOGICAL_ERROR); - return ArrayConcat::select(*sources.front(), sink, sources); + + ColumnArray::MutablePtr res; + ArrayConcat::select(*sources.front(), sources, res); + return res; } } diff --git a/src/Functions/GatherUtils/createArraySink.cpp b/src/Functions/GatherUtils/createArraySink.cpp index 513fe112608..85779aba9d2 100644 --- a/src/Functions/GatherUtils/createArraySink.cpp +++ b/src/Functions/GatherUtils/createArraySink.cpp @@ -13,40 +13,44 @@ struct ArraySinkCreator; template struct ArraySinkCreator { - static std::unique_ptr create(ColumnArray & col, NullMap * null_map, size_t column_size) + static std::unique_ptr create(IColumn & values, ColumnArray::Offsets & offsets, size_t column_size) { using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; - if (typeid_cast(&col.getData())) + IColumn * not_null_values = &values; + bool is_nullable = false; + + if (auto * nullable = typeid_cast(&values)) { - if (null_map) - return std::make_unique>>(col, *null_map, column_size); - return std::make_unique>(col, column_size); + not_null_values = &nullable->getNestedColumn(); + is_nullable = true; } - return ArraySinkCreator::create(col, null_map, column_size); + if (typeid_cast(not_null_values)) + { + if (is_nullable) + return std::make_unique>>(values, offsets, column_size); + return std::make_unique>(values, offsets, column_size); + } + + return ArraySinkCreator::create(values, offsets, column_size); } }; template <> struct ArraySinkCreator<> { - static std::unique_ptr create(ColumnArray & col, NullMap * null_map, size_t column_size) + static std::unique_ptr create(IColumn & values, ColumnArray::Offsets & offsets, size_t column_size) { - if (null_map) - return std::make_unique>(col, *null_map, column_size); - return std::make_unique(col, column_size); + if (typeid_cast(&values)) + return std::make_unique>(values, offsets, column_size); + return std::make_unique(values, offsets, column_size); } }; std::unique_ptr createArraySink(ColumnArray & col, size_t column_size) { using Creator = ApplyTypeListForClass::Type; - if (auto * column_nullable = typeid_cast(&col.getData())) - { - auto column = ColumnArray::create(column_nullable->getNestedColumnPtr()->assumeMutable(), col.getOffsetsPtr()->assumeMutable()); - return Creator::create(*column, &column_nullable->getNullMapData(), column_size); - } - return Creator::create(col, nullptr, column_size); + return Creator::create(col.getData(), col.getOffsets(), column_size); } } diff --git a/src/Functions/GatherUtils/has.cpp b/src/Functions/GatherUtils/has.cpp deleted file mode 100644 index d996a155e76..00000000000 --- a/src/Functions/GatherUtils/has.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "GatherUtils.h" -#include "Selectors.h" -#include "Algorithms.h" - -namespace DB::GatherUtils -{ - -struct ArrayHasSelectArraySourcePair : public ArraySourcePairSelector -{ - template - static void selectSourcePair(FirstSource && first, SecondSource && second, ArraySearchType search_type, ColumnUInt8 & result) - { - switch (search_type) - { - case ArraySearchType::All: - arrayAllAny(first, second, result); - break; - case ArraySearchType::Any: - arrayAllAny(first, second, result); - break; - case ArraySearchType::Substr: - arrayAllAny(first, second, result); - break; - - } - } -}; - - -void sliceHas(IArraySource & first, IArraySource & second, ArraySearchType search_type, ColumnUInt8 & result) -{ - ArrayHasSelectArraySourcePair::select(first, second, search_type, result); -} - -} diff --git a/src/Functions/GatherUtils/has_all.cpp b/src/Functions/GatherUtils/has_all.cpp new file mode 100644 index 00000000000..ae2e6c7af42 --- /dev/null +++ b/src/Functions/GatherUtils/has_all.cpp @@ -0,0 +1,23 @@ +#include "GatherUtils.h" +#include "Selectors.h" +#include "Algorithms.h" + +namespace DB::GatherUtils +{ + +struct ArrayHasAllSelectArraySourcePair : public ArraySourcePairSelector +{ + template + static void selectSourcePair(FirstSource && first, SecondSource && second, ColumnUInt8 & result) + { + arrayAllAny(first, second, result); + } +}; + + +void sliceHasAll(IArraySource & first, IArraySource & second, ColumnUInt8 & result) +{ + ArrayHasAllSelectArraySourcePair::select(first, second, result); +} + +} diff --git a/src/Functions/GatherUtils/has_any.cpp b/src/Functions/GatherUtils/has_any.cpp new file mode 100644 index 00000000000..a25acdd666a --- /dev/null +++ b/src/Functions/GatherUtils/has_any.cpp @@ -0,0 +1,23 @@ +#include "GatherUtils.h" +#include "Selectors.h" +#include "Algorithms.h" + +namespace DB::GatherUtils +{ + +struct ArrayHasAnySelectArraySourcePair : public ArraySourcePairSelector +{ + template + static void selectSourcePair(FirstSource && first, SecondSource && second, ColumnUInt8 & result) + { + arrayAllAny(first, second, result); + } +}; + + +void sliceHasAny(IArraySource & first, IArraySource & second, ColumnUInt8 & result) +{ + ArrayHasAnySelectArraySourcePair::select(first, second, result); +} + +} diff --git a/src/Functions/GatherUtils/has_substr.cpp b/src/Functions/GatherUtils/has_substr.cpp new file mode 100644 index 00000000000..27c15c5b786 --- /dev/null +++ b/src/Functions/GatherUtils/has_substr.cpp @@ -0,0 +1,23 @@ +#include "GatherUtils.h" +#include "Selectors.h" +#include "Algorithms.h" + +namespace DB::GatherUtils +{ + +struct ArrayHasSubstrSelectArraySourcePair : public ArraySourcePairSelector +{ + template + static void selectSourcePair(FirstSource && first, SecondSource && second, ColumnUInt8 & result) + { + arrayAllAny(first, second, result); + } +}; + + +void sliceHasSubstr(IArraySource & first, IArraySource & second, ColumnUInt8 & result) +{ + ArrayHasSubstrSelectArraySourcePair::select(first, second, result); +} + +} diff --git a/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp b/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp index e4ea70dd09e..8354b16a1e2 100644 --- a/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp @@ -6,18 +6,24 @@ namespace DB::GatherUtils { -struct SliceDynamicOffsetBoundedSelectArraySource : public ArraySinkSourceSelector +struct SliceDynamicOffsetBoundedSelectArraySource : public ArraySourceSelector { - template - static void selectSourceSink(Source && source, Sink && sink, const IColumn & offset_column, const IColumn & length_column) + template + static void selectImpl(Source && source, const IColumn & offset_column, const IColumn & length_column, ColumnArray::MutablePtr & result) { + using SourceType = typename std::decay::type; + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); sliceDynamicOffsetBounded(source, sink, offset_column, length_column); } }; -void sliceDynamicOffsetBounded(IArraySource & src, IArraySink & sink, const IColumn & offset_column, const IColumn & length_column) +ColumnArray::MutablePtr sliceDynamicOffsetBounded(IArraySource & src, const IColumn & offset_column, const IColumn & length_column) { - SliceDynamicOffsetBoundedSelectArraySource::select(src, sink, offset_column, length_column); + ColumnArray::MutablePtr res; + SliceDynamicOffsetBoundedSelectArraySource::select(src, offset_column, length_column, res); + return res; } } diff --git a/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp index ba7d6835830..a0e1ea98642 100644 --- a/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp @@ -6,19 +6,25 @@ namespace DB::GatherUtils { -struct SliceDynamicOffsetUnboundedSelectArraySource : public ArraySinkSourceSelector +struct SliceDynamicOffsetUnboundedSelectArraySource : public ArraySourceSelector { - template - static void selectSourceSink(Source && source, Sink && sink, const IColumn & offset_column) + template + static void selectImpl(Source && source, const IColumn & offset_column, ColumnArray::MutablePtr & result) { + using SourceType = typename std::decay::type; + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); sliceDynamicOffsetUnbounded(source, sink, offset_column); } }; -void sliceDynamicOffsetUnbounded(IArraySource & src, IArraySink & sink, const IColumn & offset_column) +ColumnArray::MutablePtr sliceDynamicOffsetUnbounded(IArraySource & src, const IColumn & offset_column) { - SliceDynamicOffsetUnboundedSelectArraySource::select(src, sink, offset_column); + ColumnArray::MutablePtr res; + SliceDynamicOffsetUnboundedSelectArraySource::select(src, offset_column, res); + return res; } } diff --git a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp index d2f5082ad55..2b8ffe13f06 100644 --- a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp @@ -7,18 +7,24 @@ namespace DB::GatherUtils { struct SliceFromLeftConstantOffsetBoundedSelectArraySource - : public ArraySinkSourceSelector + : public ArraySourceSelector { - template - static void selectSourceSink(Source && source, Sink && sink, size_t & offset, ssize_t & length) + template + static void selectImpl(Source && source, size_t & offset, ssize_t & length, ColumnArray::MutablePtr & result) { + using SourceType = typename std::decay::type; + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); sliceFromLeftConstantOffsetBounded(source, sink, offset, length); } }; -void sliceFromLeftConstantOffsetBounded(IArraySource & src, IArraySink & sink, size_t offset, ssize_t length) +ColumnArray::MutablePtr sliceFromLeftConstantOffsetBounded(IArraySource & src, size_t offset, ssize_t length) { - SliceFromLeftConstantOffsetBoundedSelectArraySource::select(src, sink, offset, length); + ColumnArray::MutablePtr res; + SliceFromLeftConstantOffsetBoundedSelectArraySource::select(src, offset, length, res); + return res; } } diff --git a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp index 6f283d0dfec..d62d7477c24 100644 --- a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp @@ -7,18 +7,24 @@ namespace DB::GatherUtils { struct SliceFromLeftConstantOffsetUnboundedSelectArraySource - : public ArraySinkSourceSelector + : public ArraySourceSelector { - template - static void selectSourceSink(Source && source, Sink && sink, size_t & offset) + template + static void selectImpl(Source && source, size_t & offset, ColumnArray::MutablePtr & result) { + using SourceType = typename std::decay::type; + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); sliceFromLeftConstantOffsetUnbounded(source, sink, offset); } }; -void sliceFromLeftConstantOffsetUnbounded(IArraySource & src, IArraySink & sink, size_t offset) +ColumnArray::MutablePtr sliceFromLeftConstantOffsetUnbounded(IArraySource & src, size_t offset) { - SliceFromLeftConstantOffsetUnboundedSelectArraySource::select(src, sink, offset); + ColumnArray::MutablePtr res; + SliceFromLeftConstantOffsetUnboundedSelectArraySource::select(src, offset, res); + return res; } } diff --git a/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp b/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp index 1a6385924f4..52396084b9b 100644 --- a/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp @@ -7,18 +7,24 @@ namespace DB::GatherUtils { struct SliceFromRightConstantOffsetBoundedSelectArraySource - : public ArraySinkSourceSelector + : public ArraySourceSelector { - template - static void selectSourceSink(Source && source, Sink && sink, size_t & offset, ssize_t & length) + template + static void selectImpl(Source && source, size_t & offset, ssize_t & length, ColumnArray::MutablePtr & result) { + using SourceType = typename std::decay::type; + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); sliceFromRightConstantOffsetBounded(source, sink, offset, length); } }; -void sliceFromRightConstantOffsetBounded(IArraySource & src, IArraySink & sink, size_t offset, ssize_t length) +ColumnArray::MutablePtr sliceFromRightConstantOffsetBounded(IArraySource & src, size_t offset, ssize_t length) { - SliceFromRightConstantOffsetBoundedSelectArraySource::select(src, sink, offset, length); + ColumnArray::MutablePtr res; + SliceFromRightConstantOffsetBoundedSelectArraySource::select(src, offset, length, res); + return res; } } diff --git a/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp index e669c5d50b8..95a6ea6cedb 100644 --- a/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp @@ -7,18 +7,24 @@ namespace DB::GatherUtils { struct SliceFromRightConstantOffsetUnboundedSelectArraySource - : public ArraySinkSourceSelector + : public ArraySourceSelector { - template - static void selectSourceSink(Source && source, Sink && sink, size_t & offset) + template + static void selectImpl(Source && source, size_t & offset, ColumnArray::MutablePtr & result) { + using SourceType = typename std::decay::type; + using Sink = typename SourceType::SinkType; + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); sliceFromRightConstantOffsetUnbounded(source, sink, offset); } }; -void sliceFromRightConstantOffsetUnbounded(IArraySource & src, IArraySink & sink, size_t offset) +ColumnArray::MutablePtr sliceFromRightConstantOffsetUnbounded(IArraySource & src, size_t offset) { - SliceFromRightConstantOffsetUnboundedSelectArraySource::select(src, sink, offset); + ColumnArray::MutablePtr res; + SliceFromRightConstantOffsetUnboundedSelectArraySource::select(src, offset, res); + return res; } } diff --git a/src/Functions/array/arrayConcat.cpp b/src/Functions/array/arrayConcat.cpp index 23e6c7a28a8..bea797c1979 100644 --- a/src/Functions/array/arrayConcat.cpp +++ b/src/Functions/array/arrayConcat.cpp @@ -59,8 +59,6 @@ public: return; } - auto result_column = return_type->createColumn(); - size_t rows = input_rows_count; size_t num_args = arguments.size(); @@ -95,10 +93,9 @@ public: throw Exception{"Arguments for function " + getName() + " must be arrays.", ErrorCodes::LOGICAL_ERROR}; } - auto sink = GatherUtils::createArraySink(typeid_cast(*result_column), rows); - GatherUtils::concat(sources, *sink); + auto sink = GatherUtils::concat(sources); - block.getByPosition(result).column = std::move(result_column); + block.getByPosition(result).column = std::move(sink); } bool useDefaultImplementationForConstants() const override { return true; } diff --git a/src/Functions/array/arrayPop.h b/src/Functions/array/arrayPop.h index 45cbd091ddf..7c32acb78b4 100644 --- a/src/Functions/array/arrayPop.h +++ b/src/Functions/array/arrayPop.h @@ -48,8 +48,6 @@ public: return; } - auto result_column = return_type->createColumn(); - const auto & array_column = block.getByPosition(arguments[0]).column; std::unique_ptr source; @@ -61,14 +59,14 @@ public: else throw Exception{"First arguments for function " + getName() + " must be array.", ErrorCodes::LOGICAL_ERROR}; - auto sink = GatherUtils::createArraySink(typeid_cast(*result_column), size); + ColumnArray::MutablePtr sink; if (pop_front) - GatherUtils::sliceFromLeftConstantOffsetUnbounded(*source, *sink, 1); + sink = GatherUtils::sliceFromLeftConstantOffsetUnbounded(*source, 1); else - GatherUtils::sliceFromLeftConstantOffsetBounded(*source, *sink, 0, -1); + sink = GatherUtils::sliceFromLeftConstantOffsetBounded(*source, 0, -1); - block.getByPosition(result).column = std::move(result_column); + block.getByPosition(result).column = std::move(sink); } bool useDefaultImplementationForConstants() const override { return true; } diff --git a/src/Functions/array/arraySlice.cpp b/src/Functions/array/arraySlice.cpp index 463e20845cf..4cf7f62d991 100644 --- a/src/Functions/array/arraySlice.cpp +++ b/src/Functions/array/arraySlice.cpp @@ -79,8 +79,6 @@ public: return; } - auto result_column = return_type->createColumn(); - auto & array_column = block.getByPosition(arguments[0]).column; const auto & offset_column = block.getByPosition(arguments[1]).column; const auto & length_column = arguments.size() > 2 ? block.getByPosition(arguments[2]).column : nullptr; @@ -101,7 +99,7 @@ public: else throw Exception{"First arguments for function " + getName() + " must be array.", ErrorCodes::LOGICAL_ERROR}; - auto sink = GatherUtils::createArraySink(typeid_cast(*result_column), size); + ColumnArray::MutablePtr sink; if (offset_column->onlyNull()) { @@ -111,11 +109,11 @@ public: return; } else if (isColumnConst(*length_column)) - GatherUtils::sliceFromLeftConstantOffsetBounded(*source, *sink, 0, length_column->getInt(0)); + sink = GatherUtils::sliceFromLeftConstantOffsetBounded(*source, 0, length_column->getInt(0)); else { auto const_offset_column = ColumnConst::create(ColumnInt8::create(1, 1), size); - GatherUtils::sliceDynamicOffsetBounded(*source, *sink, *const_offset_column, *length_column); + sink = GatherUtils::sliceDynamicOffsetBounded(*source, *const_offset_column, *length_column); } } else if (isColumnConst(*offset_column)) @@ -125,30 +123,30 @@ public: if (!length_column || length_column->onlyNull()) { if (offset > 0) - GatherUtils::sliceFromLeftConstantOffsetUnbounded(*source, *sink, static_cast(offset - 1)); + sink = GatherUtils::sliceFromLeftConstantOffsetUnbounded(*source, static_cast(offset - 1)); else - GatherUtils::sliceFromRightConstantOffsetUnbounded(*source, *sink, static_cast(-offset)); + sink = GatherUtils::sliceFromRightConstantOffsetUnbounded(*source, static_cast(-offset)); } else if (isColumnConst(*length_column)) { ssize_t length = length_column->getInt(0); if (offset > 0) - GatherUtils::sliceFromLeftConstantOffsetBounded(*source, *sink, static_cast(offset - 1), length); + sink = GatherUtils::sliceFromLeftConstantOffsetBounded(*source, static_cast(offset - 1), length); else - GatherUtils::sliceFromRightConstantOffsetBounded(*source, *sink, static_cast(-offset), length); + sink = GatherUtils::sliceFromRightConstantOffsetBounded(*source, static_cast(-offset), length); } else - GatherUtils::sliceDynamicOffsetBounded(*source, *sink, *offset_column, *length_column); + sink = GatherUtils::sliceDynamicOffsetBounded(*source, *offset_column, *length_column); } else { if (!length_column || length_column->onlyNull()) - GatherUtils::sliceDynamicOffsetUnbounded(*source, *sink, *offset_column); + sink = GatherUtils::sliceDynamicOffsetUnbounded(*source, *offset_column); else - GatherUtils::sliceDynamicOffsetBounded(*source, *sink, *offset_column, *length_column); + sink = GatherUtils::sliceDynamicOffsetBounded(*source, *offset_column, *length_column); } - block.getByPosition(result).column = std::move(result_column); + block.getByPosition(result).column = std::move(sink); } bool useDefaultImplementationForConstants() const override { return true; } diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index 584bed3f8c5..9afa5e8240d 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -272,11 +272,12 @@ private: return false; auto res = block.getByPosition(result).type->createColumn(); + auto & arr_res = assert_cast(*res); conditional( NumericArraySource(*col_left_array), NumericArraySource(*col_right_array), - NumericArraySink(assert_cast(*res), input_rows_count), + NumericArraySink(arr_res.getData(), arr_res.getOffsets(), input_rows_count), cond_col->getData()); block.getByPosition(result).column = std::move(res); @@ -289,11 +290,12 @@ private: return false; auto res = block.getByPosition(result).type->createColumn(); + auto & arr_res = assert_cast(*res); conditional( NumericArraySource(*col_left_array), ConstSource>(*col_right_const_array), - NumericArraySink(assert_cast(*res), input_rows_count), + NumericArraySink(arr_res.getData(), arr_res.getOffsets(), input_rows_count), cond_col->getData()); block.getByPosition(result).column = std::move(res); @@ -329,11 +331,12 @@ private: return false; auto res = block.getByPosition(result).type->createColumn(); + auto & arr_res = assert_cast(*res); conditional( ConstSource>(*col_left_const_array), NumericArraySource(*col_right_array), - NumericArraySink(assert_cast(*res), input_rows_count), + NumericArraySink(arr_res.getData(), arr_res.getOffsets(), input_rows_count), cond_col->getData()); block.getByPosition(result).column = std::move(res); @@ -346,11 +349,12 @@ private: return false; auto res = block.getByPosition(result).type->createColumn(); + auto & arr_res = assert_cast(*res); conditional( ConstSource>(*col_left_const_array), ConstSource>(*col_right_const_array), - NumericArraySink(assert_cast(*res), input_rows_count), + NumericArraySink(arr_res.getData(), arr_res.getOffsets(), input_rows_count), cond_col->getData()); block.getByPosition(result).column = std::move(res); @@ -527,13 +531,13 @@ private: auto * col_res = assert_cast(res.get()); if (col_arr_then && col_arr_else) - conditional(GenericArraySource(*col_arr_then), GenericArraySource(*col_arr_else), GenericArraySink(*col_res, rows), cond_data); + conditional(GenericArraySource(*col_arr_then), GenericArraySource(*col_arr_else), GenericArraySink(col_res->getData(), col_res->getOffsets(), rows), cond_data); else if (col_arr_then && col_arr_else_const) - conditional(GenericArraySource(*col_arr_then), ConstSource(*col_arr_else_const), GenericArraySink(*col_res, rows), cond_data); + conditional(GenericArraySource(*col_arr_then), ConstSource(*col_arr_else_const), GenericArraySink(col_res->getData(), col_res->getOffsets(), rows), cond_data); else if (col_arr_then_const && col_arr_else) - conditional(ConstSource(*col_arr_then_const), GenericArraySource(*col_arr_else), GenericArraySink(*col_res, rows), cond_data); + conditional(ConstSource(*col_arr_then_const), GenericArraySource(*col_arr_else), GenericArraySink(col_res->getData(), col_res->getOffsets(), rows), cond_data); else if (col_arr_then_const && col_arr_else_const) - conditional(ConstSource(*col_arr_then_const), ConstSource(*col_arr_else_const), GenericArraySink(*col_res, rows), cond_data); + conditional(ConstSource(*col_arr_then_const), ConstSource(*col_arr_else_const), GenericArraySink(col_res->getData(), col_res->getOffsets(), rows), cond_data); else return false; diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 388b140bf11..97847b99bf6 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -190,7 +190,9 @@ SRCS( GatherUtils/createArraySink.cpp GatherUtils/createArraySource.cpp GatherUtils/createValueSource.cpp - GatherUtils/has.cpp + GatherUtils/has_all.cpp + GatherUtils/has_any.cpp + GatherUtils/has_substr.cpp GatherUtils/push.cpp GatherUtils/resizeConstantSize.cpp GatherUtils/resizeDynamicSize.cpp diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 34efdda9097..0f610d9f6d2 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -136,6 +136,22 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool setupState(thread_group_); } +inline UInt64 time_in_nanoseconds(std::chrono::time_point timepoint) +{ + return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); +} + +inline UInt64 time_in_microseconds(std::chrono::time_point timepoint) +{ + return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); +} + + +inline UInt64 time_in_seconds(std::chrono::time_point timepoint) +{ + return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); +} + void ThreadStatus::initPerformanceCounters() { performance_counters_finalized = false; @@ -146,9 +162,13 @@ void ThreadStatus::initPerformanceCounters() memory_tracker.resetCounters(); memory_tracker.setDescription("(for thread)"); - query_start_time_nanoseconds = getCurrentTimeNanoseconds(); - query_start_time = time(nullptr); - query_start_time_microseconds = getCurrentTimeMicroseconds(); + // query_start_time_{microseconds, nanoseconds} are all constructed from the same time point + // to ensure that they are all equal upto the precision of a second. + const auto now = std::chrono::system_clock::now(); + + query_start_time_nanoseconds = time_in_nanoseconds(now); + query_start_time = time_in_seconds(now); + query_start_time_microseconds = time_in_microseconds(now); ++queries_started; *last_rusage = RUsageCounters::current(query_start_time_nanoseconds); diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index e28997f0ad6..74622c72865 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -250,18 +250,26 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt } else if (const auto * asterisk_pattern = child->as()) { - bool first_table = true; - for (const auto & table : tables_with_columns) + if (asterisk_pattern->column_list) { - for (const auto & column : table.columns) + for (const auto & ident : asterisk_pattern->column_list->children) + node.children.emplace_back(ident->clone()); + } + else + { + bool first_table = true; + for (const auto & table : tables_with_columns) { - if (asterisk_pattern->isColumnMatching(column.name) && (first_table || !data.join_using_columns.count(column.name))) + for (const auto & column : table.columns) { - addIdentifier(node.children, table.table, column.name, AsteriskSemantic::getAliases(*asterisk_pattern)); + if (asterisk_pattern->isColumnMatching(column.name) && (first_table || !data.join_using_columns.count(column.name))) + { + addIdentifier(node.children, table.table, column.name, AsteriskSemantic::getAliases(*asterisk_pattern)); + } } - } - first_table = false; + first_table = false; + } } // ColumnsMatcher's transformers start to appear at child 1 for (auto it = asterisk_pattern->children.begin() + 1; it != asterisk_pattern->children.end(); ++it) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 5097e517707..59896065a87 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -182,6 +182,16 @@ static void logException(Context & context, QueryLogElement & elem) elem.exception, context.getClientInfo().current_address.toString(), joinLines(elem.query), elem.stack_trace); } +inline UInt64 time_in_microseconds(std::chrono::time_point timepoint) +{ + return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); +} + + +inline UInt64 time_in_seconds(std::chrono::time_point timepoint) +{ + return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); +} static void onExceptionBeforeStart(const String & query_for_logging, Context & context, time_t current_time, UInt64 current_time_microseconds, ASTPtr ast) { @@ -196,6 +206,9 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c elem.type = QueryLogElementType::EXCEPTION_BEFORE_START; + // all callers to onExceptionBeforeStart upstream construct the timespec for event_time and + // event_time_microseconds from the same timespec. So it can be assumed that both of these + // times are equal upto the precision of a second. elem.event_time = current_time; elem.query_start_time = current_time; elem.query_start_time_microseconds = current_time_microseconds; @@ -251,8 +264,12 @@ static std::tuple executeQueryImpl( bool has_query_tail, ReadBuffer * istr) { - time_t current_time = time(nullptr); - UInt64 current_time_microseconds = getCurrentTimeMicroseconds(); + // current_time and current_time_microseconds are both constructed from the same time point + // to ensure that both the times are equal upto the precision of a second. + const auto now = std::chrono::system_clock::now(); + + auto current_time = time_in_seconds(now); + auto current_time_microseconds = time_in_microseconds(now); /// If we already executing query and it requires to execute internal query, than /// don't replace thread context with given (it can be temporary). Otherwise, attach context to thread. diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index 191ca52c0e8..e9b2c4cc562 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -30,8 +30,12 @@ void ASTColumnsMatcher::updateTreeHashImpl(SipHash & hash_state) const void ASTColumnsMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(" - << quoteString(original_pattern) << ")"; + settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("; + if (column_list) + column_list->formatImpl(settings, state, frame); + else + settings.ostr << quoteString(original_pattern); + settings.ostr << ")"; for (ASTs::const_iterator it = children.begin() + 1; it != children.end(); ++it) { settings.ostr << ' '; diff --git a/src/Parsers/ASTColumnsMatcher.h b/src/Parsers/ASTColumnsMatcher.h index 47a9b86a519..76ece9c95cc 100644 --- a/src/Parsers/ASTColumnsMatcher.h +++ b/src/Parsers/ASTColumnsMatcher.h @@ -36,6 +36,8 @@ public: bool isColumnMatching(const String & column_name) const; void updateTreeHashImpl(SipHash & hash_state) const override; + ASTPtr column_list; + protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 19b6ff487cc..1d861c6d78a 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1188,6 +1188,7 @@ bool ParserAlias::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool ParserColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword columns("COLUMNS"); + ParserList columns_p(std::make_unique(), std::make_unique(TokenType::Comma), false); ParserStringLiteral regex; if (!columns.ignore(pos, expected)) @@ -1197,8 +1198,9 @@ bool ParserColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expect return false; ++pos; + ASTPtr column_list; ASTPtr regex_node; - if (!regex.parse(pos, regex_node, expected)) + if (!columns_p.parse(pos, column_list, expected) && !regex.parse(pos, regex_node, expected)) return false; if (pos->type != TokenType::ClosingRoundBracket) @@ -1206,8 +1208,17 @@ bool ParserColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expect ++pos; auto res = std::make_shared(); - res->setPattern(regex_node->as().value.get()); - res->children.push_back(regex_node); + if (column_list) + { + res->column_list = column_list; + res->children.push_back(res->column_list); + } + else + { + res->setPattern(regex_node->as().value.get()); + res->children.push_back(regex_node); + } + ParserColumnsTransformers transformers_p; ASTPtr transformer; while (transformers_p.parse(pos, transformer, expected)) diff --git a/src/Storages/MergeTree/MergeSelector.h b/src/Storages/MergeTree/MergeSelector.h index 9c043005312..e460b8ae06a 100644 --- a/src/Storages/MergeTree/MergeSelector.h +++ b/src/Storages/MergeTree/MergeSelector.h @@ -44,7 +44,7 @@ public: /// Information about different TTLs for part. Can be used by /// TTLSelector to assign merges with TTL. - MergeTreeDataPartTTLInfos ttl_infos; + const MergeTreeDataPartTTLInfos * ttl_infos; /// Part compression codec definition. ASTPtr compression_codec_desc; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 03235742a68..739dfedfde4 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -10,61 +11,89 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int NO_SUCH_COLUMN_IN_TABLE; } +namespace +{ + +/// Columns absent in part may depend on other absent columns so we are +/// searching all required physical columns recursively. Return true if found at +/// least one existing (physical) column in part. +bool injectRequiredColumnsRecursively( + const String & column_name, + const ColumnsDescription & storage_columns, + const MergeTreeData::AlterConversions & alter_conversions, + const MergeTreeData::DataPartPtr & part, + Names & columns, + NameSet & required_columns, + NameSet & injected_columns) +{ + /// This is needed to prevent stack overflow in case of cyclic defaults or + /// huge AST which for some reason was not validated on parsing/interpreter + /// stages. + checkStackSize(); + String column_name_in_part = column_name; + if (alter_conversions.isColumnRenamed(column_name_in_part)) + column_name_in_part = alter_conversions.getColumnOldName(column_name_in_part); + + /// column has files and hence does not require evaluation + if (storage_columns.hasPhysical(column_name) && part->hasColumnFiles(column_name_in_part, *storage_columns.getPhysical(column_name).type)) + { + /// ensure each column is added only once + if (required_columns.count(column_name) == 0) + { + columns.emplace_back(column_name); + required_columns.emplace(column_name); + injected_columns.emplace(column_name); + } + return true; + } + + /// Column doesn't have default value and don't exist in part + /// don't need to add to required set. + const auto column_default = storage_columns.getDefault(column_name); + if (!column_default) + return false; + + /// collect identifiers required for evaluation + IdentifierNameSet identifiers; + column_default->expression->collectIdentifierNames(identifiers); + + bool result = false; + for (const auto & identifier : identifiers) + result |= injectRequiredColumnsRecursively(identifier, storage_columns, alter_conversions, part, columns, required_columns, injected_columns); + + return result; +} + +} NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetadataPtr & metadata_snapshot, const MergeTreeData::DataPartPtr & part, Names & columns) { NameSet required_columns{std::begin(columns), std::end(columns)}; NameSet injected_columns; - auto all_column_files_missing = true; + bool have_at_least_one_physical_column = false; const auto & storage_columns = metadata_snapshot->getColumns(); auto alter_conversions = storage.getAlterConversionsForPart(part); for (size_t i = 0; i < columns.size(); ++i) { - /// possibly renamed - auto column_name_in_part = columns[i]; + /// We are going to fetch only physical columns + if (!storage_columns.hasPhysical(columns[i])) + throw Exception("There is no physical column " + columns[i] + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); - if (alter_conversions.isColumnRenamed(column_name_in_part)) - column_name_in_part = alter_conversions.getColumnOldName(column_name_in_part); - - /// column has files and hence does not require evaluation - if (part->hasColumnFiles(column_name_in_part, *storage_columns.getPhysical(columns[i]).type)) - { - all_column_files_missing = false; - continue; - } - - const auto column_default = storage_columns.getDefault(columns[i]); - if (!column_default) - continue; - - /// collect identifiers required for evaluation - IdentifierNameSet identifiers; - column_default->expression->collectIdentifierNames(identifiers); - - for (const auto & identifier : identifiers) - { - if (storage_columns.hasPhysical(identifier)) - { - /// ensure each column is added only once - if (required_columns.count(identifier) == 0) - { - columns.emplace_back(identifier); - required_columns.emplace(identifier); - injected_columns.emplace(identifier); - } - } - } + have_at_least_one_physical_column |= injectRequiredColumnsRecursively( + columns[i], storage_columns, alter_conversions, + part, columns, required_columns, injected_columns); } /** Add a column of the minimum size. * Used in case when no column is needed or files are missing, but at least you need to know number of rows. * Adds to the columns. */ - if (all_column_files_missing) + if (!have_at_least_one_physical_column) { const auto minimum_size_column_name = part->getColumnNameWithMinumumCompressedSize(metadata_snapshot); columns.push_back(minimum_size_column_name); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 3fe7738a9a4..89631b713ed 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -273,7 +273,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( part_info.age = current_time - part->modification_time; part_info.level = part->info.level; part_info.data = ∂ - part_info.ttl_infos = part->ttl_infos; + part_info.ttl_infos = &part->ttl_infos; part_info.compression_codec_desc = part->default_codec->getFullCodecDesc(); parts_ranges.back().emplace_back(part_info); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index a06feca4033..1ea40989dfc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -340,7 +340,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( if (multi_code == Coordination::Error::ZNODEEXISTS && deduplicate_block && failed_op_path == block_id_path) { - /// Block with the same id have just appeared in table (or other replica), rollback thee insertion. + /// Block with the same id have just appeared in table (or other replica), rollback the insertion. LOG_INFO(log, "Block with ID {} already exists; ignoring it (removing part {})", block_id, part->name); part->is_duplicate = true; diff --git a/src/Storages/MergeTree/TTLMergeSelector.cpp b/src/Storages/MergeTree/TTLMergeSelector.cpp index 1defc60d8bc..7f76da085c9 100644 --- a/src/Storages/MergeTree/TTLMergeSelector.cpp +++ b/src/Storages/MergeTree/TTLMergeSelector.cpp @@ -94,12 +94,12 @@ IMergeSelector::PartsRange ITTLMergeSelector::select( time_t TTLDeleteMergeSelector::getTTLForPart(const IMergeSelector::Part & part) const { - return only_drop_parts ? part.ttl_infos.part_max_ttl : part.ttl_infos.part_min_ttl; + return only_drop_parts ? part.ttl_infos->part_max_ttl : part.ttl_infos->part_min_ttl; } time_t TTLRecompressMergeSelector::getTTLForPart(const IMergeSelector::Part & part) const { - return part.ttl_infos.getMinimalMaxRecompressionTTL(); + return part.ttl_infos->getMinimalMaxRecompressionTTL(); } bool TTLRecompressMergeSelector::isTTLAlreadySatisfied(const IMergeSelector::Part & part) const @@ -107,7 +107,7 @@ bool TTLRecompressMergeSelector::isTTLAlreadySatisfied(const IMergeSelector::Par if (recompression_ttls.empty()) return false; - auto ttl_description = selectTTLDescriptionForTTLInfos(recompression_ttls, part.ttl_infos.recompression_ttl, current_time, true); + auto ttl_description = selectTTLDescriptionForTTLInfos(recompression_ttls, part.ttl_infos->recompression_ttl, current_time, true); if (!ttl_description) return true; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 558216a6216..9e60d5bad15 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -502,8 +502,12 @@ BlockOutputStreamPtr StorageFile::write( if (format_name == "Distributed") throw Exception("Method write is not implemented for Distributed format", ErrorCodes::NOT_IMPLEMENTED); + std::string path; + if (!paths.empty()) + path = paths[0]; + return std::make_shared(*this, metadata_snapshot, - chooseCompressionMethod(paths[0], compression_method), context); + chooseCompressionMethod(path, compression_method), context); } Strings StorageFile::getDataPaths() const diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 167524b9582..baa04134c7d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,6 @@ from helpers.test_tools import TSV + def pytest_assertrepr_compare(op, left, right): if isinstance(left, TSV) and isinstance(right, TSV) and op == '==': return ['TabSeparated values differ: '] + left.diff(right) diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 78c5bba09f3..deffa20753f 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -1,8 +1,7 @@ -import errno -import subprocess as sp -from threading import Timer -import tempfile import os +import subprocess as sp +import tempfile +from threading import Timer class Client: @@ -16,12 +15,13 @@ class Client: self.command += ['--host', self.host, '--port', str(self.port), '--stacktrace'] + def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, + ignore_error=False): + return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, + password=password, database=database, ignore_error=ignore_error).get_answer() - def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False): - return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error).get_answer() - - - def get_query_request(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False): + def get_query_request(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, + ignore_error=False): command = self.command[:] if stdin is None: @@ -45,14 +45,17 @@ class Client: return CommandRequest(command, stdin, timeout, ignore_error) + def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, + database=None): + return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, + password=password, database=database).get_error() - def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): - return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database).get_error() + def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, + database=None): + return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, + password=password, database=database).get_answer_and_error() - def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): - return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database).get_answer_and_error() - class QueryTimeoutExceedException(Exception): pass @@ -90,7 +93,6 @@ class CommandRequest: self.timer = Timer(timeout, kill_process) self.timer.start() - def get_answer(self): self.process.wait() self.stdout_file.seek(0) @@ -103,11 +105,11 @@ class CommandRequest: raise QueryTimeoutExceedException('Client timed out!') if (self.process.returncode != 0 or stderr) and not self.ignore_error: - raise QueryRuntimeException('Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr)) + raise QueryRuntimeException( + 'Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr)) return stdout - def get_error(self): self.process.wait() self.stdout_file.seek(0) @@ -124,7 +126,6 @@ class CommandRequest: return stderr - def get_answer_and_error(self): self.process.wait() self.stdout_file.seek(0) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 4d336838eb7..f5d9290a17e 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1,30 +1,31 @@ import base64 -import cassandra.cluster -import docker import errno import httplib import logging import os import os.path as p import pprint -import psycopg2 import pwd -import pymongo -import pymysql import re -import requests import shutil import socket import subprocess import time -import urllib import traceback +import urllib + +import cassandra.cluster +import docker +import psycopg2 +import pymongo +import pymysql +import requests import xml.dom.minidom +from confluent.schemaregistry.client import CachedSchemaRegistryClient from dicttoxml import dicttoxml from kazoo.client import KazooClient from kazoo.exceptions import KazooException from minio import Minio -from confluent.schemaregistry.client import CachedSchemaRegistryClient from .client import Client from .hdfs_api import HDFSApi @@ -67,13 +68,14 @@ def get_odbc_bridge_path(): return '/usr/bin/clickhouse-odbc-bridge' return path + def get_docker_compose_path(): compose_path = os.environ.get('DOCKER_COMPOSE_DIR') if compose_path is not None: return os.path.dirname(compose_path) else: if os.path.exists(os.path.dirname('/compose/')): - return os.path.dirname('/compose/') #default in docker runner container + return os.path.dirname('/compose/') # default in docker runner container else: print("Fallback docker_compose_path to LOCAL_DOCKER_COMPOSE_DIR: {}".format(LOCAL_DOCKER_COMPOSE_DIR)) return LOCAL_DOCKER_COMPOSE_DIR @@ -91,12 +93,12 @@ class ClickHouseCluster: def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None, odbc_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None): for param in os.environ.keys(): - print "ENV %40s %s" % (param,os.environ[param]) + print "ENV %40s %s" % (param, os.environ[param]) self.base_dir = p.dirname(base_path) self.name = name if name is not None else '' self.base_config_dir = base_config_dir or os.environ.get('CLICKHOUSE_TESTS_BASE_CONFIG_DIR', - '/etc/clickhouse-server/') + '/etc/clickhouse-server/') self.server_bin_path = p.realpath( server_bin_path or os.environ.get('CLICKHOUSE_TESTS_SERVER_BIN_PATH', '/usr/bin/clickhouse')) self.odbc_bridge_bin_path = p.realpath(odbc_bridge_bin_path or get_odbc_bridge_path()) @@ -165,8 +167,10 @@ class ClickHouseCluster: cmd += " client" return cmd - def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries = None, macros=None, - with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None, + def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries=None, + macros=None, + with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False, + clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_redis=False, with_minio=False, with_cassandra=False, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, @@ -247,7 +251,8 @@ class ClickHouseCluster: self.with_mysql = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]) self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')] cmds.append(self.base_mysql_cmd) @@ -255,7 +260,8 @@ class ClickHouseCluster: self.with_postgres = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')]) self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')] cmds.append(self.base_postgres_cmd) if with_odbc_drivers and not self.with_odbc_drivers: @@ -264,7 +270,8 @@ class ClickHouseCluster: self.with_mysql = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]) self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')] cmds.append(self.base_mysql_cmd) if not self.with_postgres: @@ -279,28 +286,32 @@ class ClickHouseCluster: self.with_kafka = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')]) self.base_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')] cmds.append(self.base_kafka_cmd) if with_rabbitmq and not self.with_rabbitmq: self.with_rabbitmq = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')]) self.base_rabbitmq_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')] cmds.append(self.base_rabbitmq_cmd) if with_hdfs and not self.with_hdfs: self.with_hdfs = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')]) self.base_hdfs_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')] cmds.append(self.base_hdfs_cmd) if with_mongo and not self.with_mongo: self.with_mongo = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')]) self.base_mongo_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')] cmds.append(self.base_mongo_cmd) if self.with_net_trics: @@ -311,21 +322,24 @@ class ClickHouseCluster: self.with_redis = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')]) self.base_redis_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')] if with_minio and not self.with_minio: self.with_minio = True self.minio_certs_dir = minio_certs_dir self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')]) self.base_minio_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')] cmds.append(self.base_minio_cmd) if with_cassandra and not self.with_cassandra: self.with_cassandra = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')]) self.base_cassandra_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')] + self.project_name, '--file', + p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')] return instance @@ -390,7 +404,8 @@ class ClickHouseCluster: print("Container {} uses image {}: ".format(container_id, image_id)) pprint.pprint(image_info) print("") - message = 'Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container_id, exit_code, output) + message = 'Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container_id, + exit_code, output) if nothrow: print(message) else: @@ -401,7 +416,8 @@ class ClickHouseCluster: with open(local_path, 'r') as fdata: data = fdata.read() encoded_data = base64.b64encode(data) - self.exec_in_container(container_id, ["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)], + self.exec_in_container(container_id, + ["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)], user='root') def wait_mysql_to_start(self, timeout=60): @@ -650,7 +666,6 @@ class ClickHouseCluster: subprocess_check_call(clickhouse_start_cmd) print("ClickHouse instance created") - start_deadline = time.time() + 20.0 # seconds for instance in self.instances.itervalues(): instance.docker_client = self.docker_client @@ -692,7 +707,7 @@ class ClickHouseCluster: try: subprocess_check_call(self.base_cmd + ['down', '--volumes', '--remove-orphans']) except Exception as e: - print "Down + remove orphans failed durung shutdown. {}".format(repr(e)) + print "Down + remove orphans failed durung shutdown. {}".format(repr(e)) self.is_up = False @@ -704,23 +719,26 @@ class ClickHouseCluster: instance.client = None if not self.zookeeper_use_tmpfs: - for i in range(1, 4): - zk_data_path = self.instances_dir + '/zkdata' + str(i) - zk_log_data_path = self.instances_dir + '/zklog' + str(i) - if os.path.exists(zk_data_path): - shutil.rmtree(zk_data_path) - if os.path.exists(zk_log_data_path): - shutil.rmtree(zk_log_data_path) + for i in range(1, 4): + zk_data_path = self.instances_dir + '/zkdata' + str(i) + zk_log_data_path = self.instances_dir + '/zklog' + str(i) + if os.path.exists(zk_data_path): + shutil.rmtree(zk_data_path) + if os.path.exists(zk_log_data_path): + shutil.rmtree(zk_log_data_path) if sanitizer_assert_instance is not None: - raise Exception("Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance)) + raise Exception( + "Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance)) def pause_container(self, instance_name): subprocess_check_call(self.base_cmd + ['pause', instance_name]) + # subprocess_check_call(self.base_cmd + ['kill', '-s SIGSTOP', instance_name]) def unpause_container(self, instance_name): subprocess_check_call(self.base_cmd + ['unpause', instance_name]) + # subprocess_check_call(self.base_cmd + ['kill', '-s SIGCONT', instance_name]) def open_bash_shell(self, instance_name): @@ -790,9 +808,12 @@ services: class ClickHouseInstance: def __init__( - self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs, custom_dictionaries, - macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, - with_cassandra, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None, + self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs, + custom_dictionaries, + macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, + with_redis, with_minio, + with_cassandra, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, + hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag="latest", stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None): @@ -848,15 +869,19 @@ class ClickHouseInstance: return "-fsanitize=thread" in build_opts # Connects to the instance via clickhouse-client, sends a query (1st argument) and returns the answer - def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False): - return self.client.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error) + def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, + ignore_error=False): + return self.client.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, + database=database, ignore_error=ignore_error) - def query_with_retry(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False, + def query_with_retry(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, + ignore_error=False, retry_count=20, sleep_time=0.5, check_callback=lambda x: True): result = None for i in range(retry_count): try: - result = self.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error) + result = self.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, + database=database, ignore_error=ignore_error) if check_callback(result): return result time.sleep(sleep_time) @@ -873,12 +898,16 @@ class ClickHouseInstance: return self.client.get_query_request(*args, **kwargs) # Connects to the instance via clickhouse-client, sends a query (1st argument), expects an error and return its code - def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): - return self.client.query_and_get_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database) + def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, + database=None): + return self.client.query_and_get_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, + password=password, database=database) # The same as query_and_get_error but ignores successful query. - def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): - return self.client.query_and_get_answer_with_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database) + def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, + database=None): + return self.client.query_and_get_answer_with_error(sql, stdin=stdin, timeout=timeout, settings=settings, + user=user, password=password, database=database) # Connects to the instance via HTTP interface, sends a query and returns the answer def http_query(self, sql, data=None, params=None, user=None, password=None, expect_fail_and_get_error=False): @@ -900,7 +929,8 @@ class ClickHouseInstance: open_result = urllib.urlopen(url, data) def http_code_and_message(): - return str(open_result.getcode()) + " " + httplib.responses[open_result.getcode()] + ": " + open_result.read() + return str(open_result.getcode()) + " " + httplib.responses[ + open_result.getcode()] + ": " + open_result.read() if expect_fail_and_get_error: if open_result.getcode() == 200: @@ -913,18 +943,19 @@ class ClickHouseInstance: # Connects to the instance via HTTP interface, sends a query and returns the answer def http_request(self, url, method='GET', params=None, data=None, headers=None): - url = "http://" + self.ip_address + ":8123/"+url + url = "http://" + self.ip_address + ":8123/" + url return requests.request(method=method, url=url, params=params, data=data, headers=headers) # Connects to the instance via HTTP interface, sends a query, expects an error and return the error message def http_query_and_get_error(self, sql, data=None, params=None, user=None, password=None): - return self.http_query(sql=sql, data=data, params=params, user=user, password=password, expect_fail_and_get_error=True) + return self.http_query(sql=sql, data=data, params=params, user=user, password=password, + expect_fail_and_get_error=True) def kill_clickhouse(self, stop_start_wait_sec=5): pid = self.get_process_pid("clickhouse") if not pid: raise Exception("No clickhouse found") - self.exec_in_container(["bash", "-c", "kill -9 {}".format(pid)], user='root') + self.exec_in_container(["bash", "-c", "kill -9 {}".format(pid)], user='root') time.sleep(stop_start_wait_sec) def restore_clickhouse(self, retries=100): @@ -1030,7 +1061,8 @@ class ClickHouseInstance: time_left = deadline - current_time if deadline is not None and current_time >= deadline: raise Exception("Timed out while waiting for instance `{}' with ip address {} to start. " - "Container status: {}, logs: {}".format(self.name, self.ip_address, status, handle.logs())) + "Container status: {}, logs: {}".format(self.name, self.ip_address, status, + handle.logs())) # Repeatedly poll the instance address until there is something that listens there. # Usually it means that ClickHouse is ready to accept queries. diff --git a/tests/integration/helpers/dictionary.py b/tests/integration/helpers/dictionary.py index 805726a0d47..b3f7a729777 100644 --- a/tests/integration/helpers/dictionary.py +++ b/tests/integration/helpers/dictionary.py @@ -59,7 +59,8 @@ class Row(object): class Field(object): - def __init__(self, name, field_type, is_key=False, is_range_key=False, default=None, hierarchical=False, range_hash_type=None, default_value_for_get=None): + def __init__(self, name, field_type, is_key=False, is_range_key=False, default=None, hierarchical=False, + range_hash_type=None, default_value_for_get=None): self.name = name self.field_type = field_type self.is_key = is_key @@ -123,7 +124,8 @@ class DictionaryStructure(object): self.range_key = field if not self.layout.is_complex and len(self.keys) > 1: - raise Exception("More than one key {} field in non complex layout {}".format(len(self.keys), self.layout.name)) + raise Exception( + "More than one key {} field in non complex layout {}".format(len(self.keys), self.layout.name)) if self.layout.is_ranged and (not self.range_key or len(self.range_fields) != 2): raise Exception("Inconsistent configuration of ranged dictionary") @@ -213,7 +215,8 @@ class DictionaryStructure(object): if or_default: or_default_expr = 'OrDefault' if field.default_value_for_get is None: - raise Exception("Can create 'dictGetOrDefault' query for field {} without default_value_for_get".format(field.name)) + raise Exception( + "Can create 'dictGetOrDefault' query for field {} without default_value_for_get".format(field.name)) val = field.default_value_for_get if isinstance(val, str): @@ -259,15 +262,16 @@ class DictionaryStructure(object): def get_get_or_default_expressions(self, dict_name, field, row): if not self.layout.is_ranged: return [ - self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=False, has=False), + self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=False, + has=False), self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=True, has=False), ] return [] - def get_has_expressions(self, dict_name, field, row): if not self.layout.is_ranged: - return [self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=False, has=True)] + return [self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=False, + has=True)] return [] def get_hierarchical_expressions(self, dict_name, row): @@ -290,7 +294,7 @@ class DictionaryStructure(object): "dictIsIn('{dict_name}', {child_key}, {parent_key})".format( dict_name=dict_name, child_key=child_key_expr, - parent_key=parent_key_expr,) + parent_key=parent_key_expr, ) ] return [] @@ -364,7 +368,8 @@ class Dictionary(object): return ['select {}'.format(expr) for expr in self.structure.get_get_expressions(self.name, field, row)] def get_select_get_or_default_queries(self, field, row): - return ['select {}'.format(expr) for expr in self.structure.get_get_or_default_expressions(self.name, field, row)] + return ['select {}'.format(expr) for expr in + self.structure.get_get_or_default_expressions(self.name, field, row)] def get_select_has_queries(self, field, row): return ['select {}'.format(expr) for expr in self.structure.get_has_expressions(self.name, field, row)] diff --git a/tests/integration/helpers/external_sources.py b/tests/integration/helpers/external_sources.py index 5e8d420ff94..0d01a1bcbfd 100644 --- a/tests/integration/helpers/external_sources.py +++ b/tests/integration/helpers/external_sources.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- -import warnings -import pymysql.cursors -import pymongo -import cassandra.cluster -import redis -import aerospike -from tzlocal import get_localzone import datetime import os import uuid +import warnings + +import aerospike +import cassandra.cluster +import pymongo +import pymysql.cursors +import redis +from tzlocal import get_localzone class ExternalSource(object): @@ -89,12 +90,12 @@ class SourceMySQL(ExternalSource): test {tbl}
'''.format( - hostname=self.docker_hostname, - port=self.docker_port, - user=self.user, - password=self.password, - tbl=table_name, - ) + hostname=self.docker_hostname, + port=self.docker_port, + user=self.user, + password=self.password, + tbl=table_name, + ) def prepare(self, structure, table_name, cluster): self.create_mysql_conn() @@ -160,7 +161,8 @@ class SourceMongo(ExternalSource): if field.field_type == "Date": self.converters[field.name] = lambda x: datetime.datetime.strptime(x, "%Y-%m-%d") elif field.field_type == "DateTime": - self.converters[field.name] = lambda x: get_localzone().localize(datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")) + self.converters[field.name] = lambda x: get_localzone().localize( + datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")) else: self.converters[field.name] = lambda x: x @@ -180,6 +182,7 @@ class SourceMongo(ExternalSource): result = tbl.insert_many(to_insert) + class SourceMongoURI(SourceMongo): def compatible_with_layout(self, layout): # It is enough to test one layout for this dictionary, since we're @@ -200,6 +203,7 @@ class SourceMongoURI(SourceMongo): tbl=table_name, ) + class SourceClickHouse(ExternalSource): def get_source_str(self, table_name): @@ -284,7 +288,8 @@ class SourceFile(ExternalSource): sorted_row.append(str(row.data[name])) str_data = '\t'.join(sorted_row) - self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user="root") + self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], + user="root") def compatible_with_layout(self, layout): return 'cache' not in layout.name and 'direct' not in layout.name @@ -324,7 +329,8 @@ class _SourceExecutableBase(ExternalSource): sorted_row.append(str(row.data[name])) str_data = '\t'.join(sorted_row) - self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user='root') + self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], + user='root') class SourceExecutableCache(_SourceExecutableBase): @@ -344,12 +350,14 @@ class SourceExecutableHashed(_SourceExecutableBase): def compatible_with_layout(self, layout): return 'cache' in layout.name -class SourceHTTPBase(ExternalSource): +class SourceHTTPBase(ExternalSource): PORT_COUNTER = 5555 + def get_source_str(self, table_name): self.http_port = SourceHTTPBase.PORT_COUNTER - url = "{schema}://{host}:{port}/".format(schema=self._get_schema(), host=self.docker_hostname, port=self.http_port) + url = "{schema}://{host}:{port}/".format(schema=self._get_schema(), host=self.docker_hostname, + port=self.http_port) SourceHTTPBase.PORT_COUNTER += 1 return ''' @@ -395,7 +403,8 @@ class SourceHTTPBase(ExternalSource): sorted_row.append(str(row.data[name])) str_data = '\t'.join(sorted_row) - self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user='root') + self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], + user='root') class SourceHTTP(SourceHTTPBase): @@ -407,6 +416,7 @@ class SourceHTTPS(SourceHTTPBase): def _get_schema(self): return "https" + class SourceCassandra(ExternalSource): TYPE_MAPPING = { 'UInt8': 'tinyint', @@ -426,7 +436,8 @@ class SourceCassandra(ExternalSource): } def __init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password): - ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password) + ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, + password) self.structure = dict() def get_source_str(self, table_name): @@ -448,13 +459,14 @@ class SourceCassandra(ExternalSource): def prepare(self, structure, table_name, cluster): self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port) self.session = self.client.connect() - self.session.execute("create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};") + self.session.execute( + "create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};") self.session.execute('drop table if exists test."{}"'.format(table_name)) self.structure[table_name] = structure columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()] keys = ['"' + col.name + '"' for col in structure.keys] query = 'create table test."{name}" ({columns}, primary key ({pk}));'.format( - name=table_name, columns=', '.join(columns), pk=', '.join(keys)) + name=table_name, columns=', '.join(columns), pk=', '.join(keys)) self.session.execute(query) self.prepared = True @@ -470,14 +482,16 @@ class SourceCassandra(ExternalSource): names_and_types = [(field.name, field.field_type) for field in self.structure[table_name].get_all_fields()] columns = ['"' + col[0] + '"' for col in names_and_types] insert = 'insert into test."{table}" ({columns}) values ({args})'.format( - table=table_name, columns=','.join(columns), args=','.join(['%s']*len(columns))) + table=table_name, columns=','.join(columns), args=','.join(['%s'] * len(columns))) for row in data: values = [self.get_value_to_insert(row.get_value_by_name(col[0]), col[1]) for col in names_and_types] self.session.execute(insert, values) + class SourceRedis(ExternalSource): def __init__( - self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password, db_index, storage_type + self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password, db_index, + storage_type ): super(SourceRedis, self).__init__( name, internal_hostname, internal_port, docker_hostname, docker_port, user, password @@ -503,7 +517,8 @@ class SourceRedis(ExternalSource): ) def prepare(self, structure, table_name, cluster): - self.client = redis.StrictRedis(host=self.internal_hostname, port=self.internal_port, db=self.db_index, password=self.password or None) + self.client = redis.StrictRedis(host=self.internal_hostname, port=self.internal_port, db=self.db_index, + password=self.password or None) self.prepared = True self.ordered_names = structure.get_ordered_names() @@ -521,11 +536,12 @@ class SourceRedis(ExternalSource): def compatible_with_layout(self, layout): return layout.is_simple and self.storage_type == "simple" or layout.is_complex and self.storage_type == "hash_map" + class SourceAerospike(ExternalSource): def __init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password): ExternalSource.__init__(self, name, internal_hostname, internal_port, - docker_hostname, docker_port, user, password) + docker_hostname, docker_port, user, password) self.namespace = "test" self.set = "test_set" @@ -543,7 +559,7 @@ class SourceAerospike(ExternalSource): def prepare(self, structure, table_name, cluster): config = { - 'hosts': [ (self.internal_hostname, self.internal_port) ] + 'hosts': [(self.internal_hostname, self.internal_port)] } self.client = aerospike.client(config).connect() self.prepared = True @@ -580,7 +596,7 @@ class SourceAerospike(ExternalSource): self.client.put(key, {"bin_value": value[1]}, policy={"key": aerospike.POLICY_KEY_SEND}) assert self.client.exists(key) else: - assert("VALUES SIZE != 2") + assert ("VALUES SIZE != 2") # print(values) diff --git a/tests/integration/helpers/hdfs_api.py b/tests/integration/helpers/hdfs_api.py index 97df8a13aeb..70111045ad2 100644 --- a/tests/integration/helpers/hdfs_api.py +++ b/tests/integration/helpers/hdfs_api.py @@ -1,10 +1,12 @@ -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- import StringIO import gzip -import requests import subprocess from tempfile import NamedTemporaryFile +import requests + + class HDFSApi(object): def __init__(self, user): self.host = "localhost" @@ -13,11 +15,15 @@ class HDFSApi(object): self.user = user def read_data(self, path): - response = requests.get("http://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(host=self.host, port=self.http_proxy_port, path=path), allow_redirects=False) + response = requests.get( + "http://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(host=self.host, port=self.http_proxy_port, + path=path), allow_redirects=False) if response.status_code != 307: response.raise_for_status() additional_params = '&'.join(response.headers['Location'].split('&')[1:2]) - response_data = requests.get("http://{host}:{port}/webhdfs/v1{path}?op=OPEN&{params}".format(host=self.host, port=self.http_data_port, path=path, params=additional_params)) + response_data = requests.get( + "http://{host}:{port}/webhdfs/v1{path}?op=OPEN&{params}".format(host=self.host, port=self.http_data_port, + path=path, params=additional_params)) if response_data.status_code != 200: response_data.raise_for_status() @@ -25,7 +31,9 @@ class HDFSApi(object): # Requests can't put file def _curl_to_put(self, filename, path, params): - url = "http://{host}:{port}/webhdfs/v1{path}?op=CREATE&{params}".format(host=self.host, port=self.http_data_port, path=path, params=params) + url = "http://{host}:{port}/webhdfs/v1{path}?op=CREATE&{params}".format(host=self.host, + port=self.http_data_port, path=path, + params=params) cmd = "curl -s -i -X PUT -T {fname} '{url}'".format(fname=filename, url=url) output = subprocess.check_output(cmd, shell=True) return output @@ -36,13 +44,15 @@ class HDFSApi(object): named_file.write(content) named_file.flush() response = requests.put( - "http://{host}:{port}/webhdfs/v1{path}?op=CREATE".format(host=self.host, port=self.http_proxy_port, path=path, user=self.user), + "http://{host}:{port}/webhdfs/v1{path}?op=CREATE".format(host=self.host, port=self.http_proxy_port, + path=path, user=self.user), allow_redirects=False ) if response.status_code != 307: response.raise_for_status() - additional_params = '&'.join(response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"]) + additional_params = '&'.join( + response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"]) output = self._curl_to_put(fpath, path, additional_params) if "201 Created" not in output: raise Exception("Can't create file on hdfs:\n {}".format(output)) diff --git a/tests/integration/helpers/http_server.py b/tests/integration/helpers/http_server.py index dd268b3a417..83e134606e3 100644 --- a/tests/integration/helpers/http_server.py +++ b/tests/integration/helpers/http_server.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import argparse -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +import csv import socket import ssl -import csv +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer # Decorator used to see if authentication works for external dictionary who use a HTTP source. @@ -15,6 +15,7 @@ def check_auth(fn): req.send_response(401) else: fn(req) + return wrapper @@ -37,7 +38,7 @@ def start_server(server_address, data_path, schema, cert_path, address_family): self.send_header('Content-type', 'text/tsv') self.end_headers() - def __send_data(self, only_ids = None): + def __send_data(self, only_ids=None): with open(data_path, 'r') as fl: reader = csv.reader(fl, delimiter='\t') for row in reader: diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index f5c2b4f8d19..f6505e81c91 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -1,12 +1,9 @@ -import os.path as p +import os import subprocess import time -import os import docker -from .cluster import CLICKHOUSE_ROOT_DIR - class PartitionManager: """Allows introducing failures in the network between docker containers. @@ -23,21 +20,18 @@ class PartitionManager: def __init__(self): self._iptables_rules = [] - def drop_instance_zk_connections(self, instance, action='DROP'): self._check_instance(instance) self._add_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action}) self._add_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action}) - def restore_instance_zk_connections(self, instance, action='DROP'): self._check_instance(instance) self._delete_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action}) self._delete_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action}) - def partition_instances(self, left, right, port=None, action='DROP'): self._check_instance(left) self._check_instance(right) @@ -51,7 +45,6 @@ class PartitionManager: self._add_rule(create_rule(left, right)) self._add_rule(create_rule(right, left)) - def heal_all(self): while self._iptables_rules: rule = self._iptables_rules.pop() @@ -66,7 +59,6 @@ class PartitionManager: for rule in rules: self._add_rule(rule) - @staticmethod def _check_instance(instance): if instance.ip_address is None: @@ -152,7 +144,6 @@ class _NetworkManager: ret.extend(['-j'] + action.split()) return ret - def __init__( self, container_expire_timeout=50, container_exit_timeout=60): @@ -175,7 +166,8 @@ class _NetworkManager: except docker.errors.NotFound: pass - self._container = self._docker_client.containers.run('yandex/clickhouse-integration-helper', auto_remove=True, + self._container = self._docker_client.containers.run('yandex/clickhouse-integration-helper', + auto_remove=True, command=('sleep %s' % self.container_exit_timeout), detach=True, network_mode='host') container_id = self._container.id diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index 67ca025c58a..d196142c518 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -1,6 +1,7 @@ import difflib import time + class TSV: """Helper to get pretty diffs between expected and actual tab-separated value files""" @@ -40,17 +41,22 @@ class TSV: def toMat(contents): return [line.split("\t") for line in contents.split("\n") if line.strip()] -def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_time=0.5, stdin=None, timeout=None, settings=None, user=None, ignore_error=False): + +def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_time=0.5, stdin=None, timeout=None, + settings=None, user=None, ignore_error=False): expectation_tsv = TSV(expectation) for i in xrange(retry_count): try: - if TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, ignore_error=ignore_error)) == expectation_tsv: + if TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, + ignore_error=ignore_error)) == expectation_tsv: break time.sleep(sleep_time) except Exception as ex: print "assert_eq_with_retry retry {} exception {}".format(i + 1, ex) time.sleep(sleep_time) else: - val = TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, ignore_error=ignore_error)) + val = TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, + ignore_error=ignore_error)) if expectation_tsv != val: - raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join(expectation_tsv.diff(val, n1="expectation", n2="query")))) + raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join( + expectation_tsv.diff(val, n1="expectation", n2="query")))) diff --git a/tests/integration/helpers/uclient.py b/tests/integration/helpers/uclient.py index 6318802d81a..098e17a38da 100644 --- a/tests/integration/helpers/uclient.py +++ b/tests/integration/helpers/uclient.py @@ -11,9 +11,10 @@ import uexpect prompt = ':\) ' end_of_block = r'.*\r\n.*\r\n' + class client(object): def __init__(self, command=None, name='', log=None): - self.client = uexpect.spawn(['/bin/bash','--noediting']) + self.client = uexpect.spawn(['/bin/bash', '--noediting']) if command is None: command = '/usr/bin/clickhouse-client' self.client.command = command diff --git a/tests/integration/helpers/uexpect.py b/tests/integration/helpers/uexpect.py index f71b32a53e1..873d9a749e0 100644 --- a/tests/integration/helpers/uexpect.py +++ b/tests/integration/helpers/uexpect.py @@ -13,13 +13,12 @@ # limitations under the License. import os import pty -import time -import sys import re - -from threading import Thread, Event -from subprocess import Popen +import time from Queue import Queue, Empty +from subprocess import Popen +from threading import Thread, Event + class TimeoutError(Exception): def __init__(self, timeout): @@ -28,6 +27,7 @@ class TimeoutError(Exception): def __str__(self): return 'Timeout %.3fs' % float(self.timeout) + class ExpectTimeoutError(Exception): def __init__(self, pattern, timeout, buffer): self.pattern = pattern @@ -43,6 +43,7 @@ class ExpectTimeoutError(Exception): s += 'or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]]) return s + class IO(object): class EOF(object): pass @@ -59,7 +60,7 @@ class IO(object): self._prefix = prefix def write(self, data): - self._logger.write(('\n' + data).replace('\n','\n' + self._prefix)) + self._logger.write(('\n' + data).replace('\n', '\n' + self._prefix)) def flush(self): self._logger.flush() @@ -165,7 +166,7 @@ class IO(object): data = '' timeleft = timeout try: - while timeleft >= 0 : + while timeleft >= 0: start_time = time.time() data += self.queue.get(timeout=timeleft) if data: @@ -182,6 +183,7 @@ class IO(object): return data + def spawn(command): master, slave = pty.openpty() process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1) @@ -193,7 +195,8 @@ def spawn(command): thread.daemon = True thread.start() - return IO(process, master, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) + return IO(process, master, queue, reader={'thread': thread, 'kill_event': reader_kill_event}) + def reader(process, out, queue, kill_event): while True: diff --git a/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py b/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py index 7e8f2000bca..6dc843e101a 100644 --- a/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py +++ b/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py @@ -24,6 +24,7 @@ system_logs = [ # decrease timeout for the test to show possible issues. timeout = pytest.mark.timeout(30) + @pytest.fixture(scope='module', autouse=True) def start_cluster(): try: @@ -32,10 +33,12 @@ def start_cluster(): finally: cluster.shutdown() + @pytest.fixture(scope='function') def flush_logs(): node.query('SYSTEM FLUSH LOGS') + @timeout @pytest.mark.parametrize('table,exists', system_logs) def test_system_logs(flush_logs, table, exists): @@ -45,6 +48,7 @@ def test_system_logs(flush_logs, table, exists): else: assert "Table {} doesn't exist".format(table) in node.query_and_get_error(q) + # Logic is tricky, let's check that there is no hang in case of message queue # is not empty (this is another code path in the code). @timeout diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index 9f053afb607..e804be2c94e 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -1,13 +1,12 @@ -import time import pytest from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) + @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: diff --git a/tests/integration/test_adaptive_granularity/test.py b/tests/integration/test_adaptive_granularity/test.py index d5ac91671e0..0c5d7bcb63c 100644 --- a/tests/integration/test_adaptive_granularity/test.py +++ b/tests/integration/test_adaptive_granularity/test.py @@ -1,31 +1,50 @@ import time + import pytest - -from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool from helpers.client import QueryRuntimeException, QueryTimeoutExceedException - +from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', with_installed_binary=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', + with_installed_binary=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True) -node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', with_installed_binary=True) -node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', + with_installed_binary=True) +node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True) -node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True) -node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) +node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, + with_installed_binary=True) +node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, + with_installed_binary=True) -node9 = cluster.add_instance('node9', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) -node10 = cluster.add_instance('node10', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True) +node9 = cluster.add_instance('node9', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', + 'configs/merge_tree_settings.xml'], with_zookeeper=True, + image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, + with_installed_binary=True) +node10 = cluster.add_instance('node10', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', + 'configs/merge_tree_settings.xml'], with_zookeeper=True, + image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, + with_installed_binary=True) -node11 = cluster.add_instance('node11', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) -node12 = cluster.add_instance('node12', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) +node11 = cluster.add_instance('node11', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, + with_installed_binary=True) +node12 = cluster.add_instance('node12', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], + with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, + with_installed_binary=True) def prepare_single_pair_with_setting(first_node, second_node, group): @@ -34,80 +53,80 @@ def prepare_single_pair_with_setting(first_node, second_node, group): # Two tables with adaptive granularity first_node.query( - ''' - CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity_bytes = 10485760 - '''.format(g=group)) - - second_node.query( - ''' - CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '2') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity_bytes = 10485760 - '''.format(g=group)) - - # Two tables with fixed granularity - first_node.query( - ''' - CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity_bytes = 0 - '''.format(g=group)) - - second_node.query( - ''' - CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity_bytes = 0 - '''.format(g=group)) - - # Two tables with different granularity - with pytest.raises(QueryRuntimeException): - first_node.query( ''' - CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '1') + CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '1') PARTITION BY toYYYYMM(date) ORDER BY id SETTINGS index_granularity_bytes = 10485760 '''.format(g=group)) - second_node.query( + second_node.query( ''' - CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '2') + CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '2') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity_bytes = 10485760 + '''.format(g=group)) + + # Two tables with fixed granularity + first_node.query( + ''' + CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1') PARTITION BY toYYYYMM(date) ORDER BY id SETTINGS index_granularity_bytes = 0 '''.format(g=group)) - # Two tables with different granularity, but enabled mixed parts - first_node.query( + second_node.query( ''' - CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '1') + CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2') PARTITION BY toYYYYMM(date) ORDER BY id - SETTINGS index_granularity_bytes = 10485760, enable_mixed_granularity_parts=1 + SETTINGS index_granularity_bytes = 0 '''.format(g=group)) + # Two tables with different granularity + with pytest.raises(QueryRuntimeException): + first_node.query( + ''' + CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity_bytes = 10485760 + '''.format(g=group)) + second_node.query( - ''' - CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '2') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts=1 - '''.format(g=group)) + ''' + CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '2') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity_bytes = 0 + '''.format(g=group)) + + # Two tables with different granularity, but enabled mixed parts + first_node.query( + ''' + CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity_bytes = 10485760, enable_mixed_granularity_parts=1 + '''.format(g=group)) + + second_node.query( + ''' + CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '2') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts=1 + '''.format(g=group)) def prepare_single_pair_without_setting(first_node, second_node, group): @@ -116,21 +135,21 @@ def prepare_single_pair_without_setting(first_node, second_node, group): # Two tables with fixed granularity first_node.query( - ''' - CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - '''.format(g=group)) + ''' + CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + '''.format(g=group)) second_node.query( - ''' - CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity_bytes = 0 - '''.format(g=group)) + ''' + CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity_bytes = 0 + '''.format(g=group)) @pytest.fixture(scope="module") @@ -160,7 +179,8 @@ def start_static_cluster(): def test_different_versions_cluster(start_static_cluster, first_node, second_node, table): counter = 1 for n1, n2 in ((first_node, second_node), (second_node, first_node)): - n1.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), {c1}, 333), (toDate('2018-10-02'), {c2}, 444)".format(tbl=table, c1=counter * 2, c2=counter * 2 + 1)) + n1.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), {c1}, 333), (toDate('2018-10-02'), {c2}, 444)".format( + tbl=table, c1=counter * 2, c2=counter * 2 + 1)) n2.query("SYSTEM SYNC REPLICA {tbl}".format(tbl=table)) assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2)) n1.query("DETACH TABLE {tbl}".format(tbl=table)) @@ -175,73 +195,74 @@ def test_different_versions_cluster(start_static_cluster, first_node, second_nod assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2)) counter += 1 + @pytest.fixture(scope="module") def start_dynamic_cluster(): try: cluster.start() node7.query( - ''' - CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_default_granularity', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - ''') + ''' + CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_default_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + ''') node7.query( - ''' - CREATE TABLE table_with_adaptive_default_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_adaptive_default_granularity', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity_bytes=10485760 - ''') + ''' + CREATE TABLE table_with_adaptive_default_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_adaptive_default_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity_bytes=10485760 + ''') node8.query( - ''' - CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/8/table_with_default_granularity', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - ''') + ''' + CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/8/table_with_default_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + ''') node9.query( - ''' - CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/9/table_with_default_granularity', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - ''') + ''' + CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/9/table_with_default_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + ''') node10.query( - ''' - CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/10/table_with_default_granularity', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - ''') + ''' + CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/10/table_with_default_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + ''') node11.query( - ''' - CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - ''') + ''' + CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + ''') node12.query( - ''' - CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '2') - PARTITION BY toYYYYMM(date) - ORDER BY id - ''') - + ''' + CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '2') + PARTITION BY toYYYYMM(date) + ORDER BY id + ''') yield cluster finally: cluster.shutdown() + @pytest.mark.parametrize( ('n', 'tables'), [ @@ -251,13 +272,16 @@ def start_dynamic_cluster(): ) def test_version_single_node_update(start_dynamic_cluster, n, tables): for table in tables: - n.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)".format(tbl=table)) + n.query( + "INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)".format(tbl=table)) n.restart_with_latest_version() for table in tables: assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '2\n' - n.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)".format(tbl=table)) + n.query( + "INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)".format(tbl=table)) assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '4\n' + @pytest.mark.parametrize( ('node',), [ @@ -266,27 +290,38 @@ def test_version_single_node_update(start_dynamic_cluster, n, tables): ] ) def test_mixed_granularity_single_node(start_dynamic_cluster, node): - node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") - node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)") + node.query( + "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") + node.query( + "INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)") def callback(n): - n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "1") - n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "1") + n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", + "1") + n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", + "1") node.restart_with_latest_version(callback_onstop=callback) node.query("SYSTEM RELOAD CONFIG") - assert_eq_with_retry(node, "SELECT value FROM system.merge_tree_settings WHERE name='enable_mixed_granularity_parts'", '1') + assert_eq_with_retry(node, + "SELECT value FROM system.merge_tree_settings WHERE name='enable_mixed_granularity_parts'", + '1') assert node.query("SELECT count() from table_with_default_granularity") == '4\n' - node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") + node.query( + "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") assert node.query("SELECT count() from table_with_default_granularity") == '6\n' node.query("OPTIMIZE TABLE table_with_default_granularity PARTITION 201810 FINAL") assert node.query("SELECT count() from table_with_default_granularity") == '6\n' - path_to_merged_part = node.query("SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition DESC LIMIT 1").strip() - node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_to_merged_part)]) # check that we have adaptive files + path_to_merged_part = node.query( + "SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition DESC LIMIT 1").strip() + node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format( + p=path_to_merged_part)]) # check that we have adaptive files - path_to_old_part = node.query("SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition ASC LIMIT 1").strip() + path_to_old_part = node.query( + "SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition ASC LIMIT 1").strip() - node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk' | grep '.*'".format(p=path_to_old_part)]) # check that we have non adaptive files + node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk' | grep '.*'".format( + p=path_to_old_part)]) # check that we have non adaptive files node.query("ALTER TABLE table_with_default_granularity UPDATE dummy = dummy + 1 WHERE 1") # still works @@ -295,46 +330,54 @@ def test_mixed_granularity_single_node(start_dynamic_cluster, node): node.query("ALTER TABLE table_with_default_granularity MODIFY COLUMN dummy String") node.query("ALTER TABLE table_with_default_granularity ADD COLUMN dummy2 Float64") - #still works + # still works assert node.query("SELECT count() from table_with_default_granularity") == '6\n' + @pytest.mark.skip(reason="flaky") def test_version_update_two_nodes(start_dynamic_cluster): - node11.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") + node11.query( + "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20) assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '2\n' + def callback(n): - n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "0") - n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "0") + n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", + "0") + n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", + "0") node12.restart_with_latest_version(callback_onstop=callback) - node12.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") + node12.query( + "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") node11.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20) assert node11.query("SELECT COUNT() FROM table_with_default_granularity") == '4\n' node12.query( - ''' - CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '2') - PARTITION BY toYYYYMM(date) - ORDER BY id - ''') + ''' + CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '2') + PARTITION BY toYYYYMM(date) + ORDER BY id + ''') node11.query( - ''' - CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - ''') + ''' + CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + ''') - node12.query("INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") + node12.query( + "INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") with pytest.raises(QueryTimeoutExceedException): node11.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=20) - node12.query("INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") + node12.query( + "INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") - node11.restart_with_latest_version(callback_onstop=callback) # just to be sure + node11.restart_with_latest_version(callback_onstop=callback) # just to be sure for i in range(3): try: @@ -350,7 +393,8 @@ def test_version_update_two_nodes(start_dynamic_cluster): assert node11.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n" assert node12.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n" - node11.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 5, 333), (toDate('2018-10-02'), 6, 444)") + node11.query( + "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 5, 333), (toDate('2018-10-02'), 6, 444)") for i in range(3): try: node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=120) diff --git a/tests/integration/test_adaptive_granularity_different_settings/test.py b/tests/integration/test_adaptive_granularity_different_settings/test.py index d84b438f77f..55b7e1c91b8 100644 --- a/tests/integration/test_adaptive_granularity_different_settings/test.py +++ b/tests/integration/test_adaptive_granularity_different_settings/test.py @@ -7,7 +7,9 @@ node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) # no adaptive granularity by default -node3 = cluster.add_instance('node3', image='yandex/clickhouse-server', tag='19.9.5.36', with_installed_binary=True, stay_alive=True) +node3 = cluster.add_instance('node3', image='yandex/clickhouse-server', tag='19.9.5.36', with_installed_binary=True, + stay_alive=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -20,7 +22,6 @@ def start_cluster(): def test_attach_detach(start_cluster): - node1.query(""" CREATE TABLE test (key UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test', '1') @@ -58,7 +59,8 @@ def test_mutate_with_mixed_granularity(start_cluster): ENGINE = MergeTree ORDER BY key PARTITION BY date""") - node3.query("INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500)") + node3.query( + "INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500)") assert node3.query("SELECT COUNT() FROM test") == "500\n" @@ -68,7 +70,8 @@ def test_mutate_with_mixed_granularity(start_cluster): node3.query("ALTER TABLE test MODIFY SETTING enable_mixed_granularity_parts = 1") - node3.query("INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500, 500)") + node3.query( + "INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500, 500)") assert node3.query("SELECT COUNT() FROM test") == "1000\n" assert node3.query("SELECT COUNT() FROM test WHERE key % 100 == 0") == "10\n" diff --git a/tests/integration/test_adaptive_granularity_replicated/test.py b/tests/integration/test_adaptive_granularity_replicated/test.py index 87956c82661..5903cb85603 100644 --- a/tests/integration/test_adaptive_granularity_replicated/test.py +++ b/tests/integration/test_adaptive_granularity_replicated/test.py @@ -1,17 +1,15 @@ import time + import pytest - from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool -from helpers.client import QueryRuntimeException, QueryTimeoutExceedException - -from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', with_installed_binary=True) +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', + with_installed_binary=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -23,11 +21,14 @@ def start_cluster(): finally: cluster.shutdown() + def test_creating_table_different_setting(start_cluster): - node1.query("CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '1') ORDER BY tuple(c1) SETTINGS index_granularity_bytes = 0") + node1.query( + "CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '1') ORDER BY tuple(c1) SETTINGS index_granularity_bytes = 0") node1.query("INSERT INTO t1 VALUES('x', 'y')") - node2.query("CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0") + node2.query( + "CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0") node1.query("INSERT INTO t1 VALUES('a', 'b')") node2.query("SYSTEM SYNC REPLICA t1", timeout=5) @@ -49,22 +50,26 @@ def test_creating_table_different_setting(start_cluster): node1.query("SELECT count() FROM t1") == "3\n" node2.query("SELECT count() FROM t1") == "2\n" - path_part = node1.query("SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip() + path_part = node1.query( + "SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip() - with pytest.raises(Exception): # check that we have no adaptive files + with pytest.raises(Exception): # check that we have no adaptive files node1.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)]) - path_part = node2.query("SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip() + path_part = node2.query( + "SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip() - with pytest.raises(Exception): # check that we have no adaptive files + with pytest.raises(Exception): # check that we have no adaptive files node2.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)]) def test_old_node_with_new_node(start_cluster): - node3.query("CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '3') ORDER BY tuple(c1)") + node3.query( + "CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '3') ORDER BY tuple(c1)") node3.query("INSERT INTO t2 VALUES('x', 'y')") - node2.query("CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0") + node2.query( + "CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0") node3.query("INSERT INTO t2 VALUES('a', 'b')") node2.query("SYSTEM SYNC REPLICA t2", timeout=5) @@ -86,12 +91,14 @@ def test_old_node_with_new_node(start_cluster): node3.query("SELECT count() FROM t2") == "3\n" node2.query("SELECT count() FROM t2") == "2\n" - path_part = node3.query("SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip() + path_part = node3.query( + "SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip() - with pytest.raises(Exception): # check that we have no adaptive files + with pytest.raises(Exception): # check that we have no adaptive files node3.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)]) - path_part = node2.query("SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip() + path_part = node2.query( + "SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip() - with pytest.raises(Exception): # check that we have no adaptive files + with pytest.raises(Exception): # check that we have no adaptive files node2.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)]) diff --git a/tests/integration/test_aggregation_memory_efficient/test.py b/tests/integration/test_aggregation_memory_efficient/test.py index 3a7ada5f02e..db0449173ca 100644 --- a/tests/integration/test_aggregation_memory_efficient/test.py +++ b/tests/integration/test_aggregation_memory_efficient/test.py @@ -1,22 +1,20 @@ -import time import pytest from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException, QueryTimeoutExceedException - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1') node2 = cluster.add_instance('node2') + @pytest.fixture(scope="module") def start_cluster(): try: cluster.start() for node in [node1, node2]: - node.query("create table da_memory_efficient_shard(A Int64, B Int64) Engine=MergeTree order by A partition by B % 2;") - + node.query( + "create table da_memory_efficient_shard(A Int64, B Int64) Engine=MergeTree order by A partition by B % 2;") node1.query("insert into da_memory_efficient_shard select number, number from numbers(100000);") node2.query("insert into da_memory_efficient_shard select number + 100000, number from numbers(100000);") @@ -28,19 +26,24 @@ def start_cluster(): def test_remote(start_cluster): - - node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") - res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") + node1.query( + "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") + res = node1.query( + "select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") assert res == '200000\n' node1.query("set distributed_aggregation_memory_efficient = 0") - res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") + res = node1.query( + "select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") assert res == '200000\n' - node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") - res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") + node1.query( + "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") + res = node1.query( + "SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") assert res == 'node1\t100000\nnode2\t100000\n' node1.query("set distributed_aggregation_memory_efficient = 0") - res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") + res = node1.query( + "SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") assert res == 'node1\t100000\nnode2\t100000\n' diff --git a/tests/integration/test_allowed_client_hosts/test.py b/tests/integration/test_allowed_client_hosts/test.py index f187b6d889c..e60e488b3ae 100644 --- a/tests/integration/test_allowed_client_hosts/test.py +++ b/tests/integration/test_allowed_client_hosts/test.py @@ -1,32 +1,32 @@ -import os import pytest from helpers.cluster import ClickHouseCluster - cluster = ClickHouseCluster(__file__) server = cluster.add_instance('server', user_configs=["configs/users.d/network.xml"]) -clientA1 = cluster.add_instance('clientA1', hostname = 'clientA1.com') -clientA2 = cluster.add_instance('clientA2', hostname = 'clientA2.com') -clientA3 = cluster.add_instance('clientA3', hostname = 'clientA3.com') -clientB1 = cluster.add_instance('clientB1', hostname = 'clientB001.ru') -clientB2 = cluster.add_instance('clientB2', hostname = 'clientB002.ru') -clientB3 = cluster.add_instance('clientB3', hostname = 'xxx.clientB003.rutracker.com') -clientC1 = cluster.add_instance('clientC1', hostname = 'clientC01.ru') -clientC2 = cluster.add_instance('clientC2', hostname = 'xxx.clientC02.ru') -clientC3 = cluster.add_instance('clientC3', hostname = 'xxx.clientC03.rutracker.com') -clientD1 = cluster.add_instance('clientD1', hostname = 'clientD0001.ru') -clientD2 = cluster.add_instance('clientD2', hostname = 'xxx.clientD0002.ru') -clientD3 = cluster.add_instance('clientD3', hostname = 'clientD0003.ru') +clientA1 = cluster.add_instance('clientA1', hostname='clientA1.com') +clientA2 = cluster.add_instance('clientA2', hostname='clientA2.com') +clientA3 = cluster.add_instance('clientA3', hostname='clientA3.com') +clientB1 = cluster.add_instance('clientB1', hostname='clientB001.ru') +clientB2 = cluster.add_instance('clientB2', hostname='clientB002.ru') +clientB3 = cluster.add_instance('clientB3', hostname='xxx.clientB003.rutracker.com') +clientC1 = cluster.add_instance('clientC1', hostname='clientC01.ru') +clientC2 = cluster.add_instance('clientC2', hostname='xxx.clientC02.ru') +clientC3 = cluster.add_instance('clientC3', hostname='xxx.clientC03.rutracker.com') +clientD1 = cluster.add_instance('clientD1', hostname='clientD0001.ru') +clientD2 = cluster.add_instance('clientD2', hostname='xxx.clientD0002.ru') +clientD3 = cluster.add_instance('clientD3', hostname='clientD0003.ru') def check_clickhouse_is_ok(client_node, server_node): - assert client_node.exec_in_container(["bash", "-c", "/usr/bin/curl -s {}:8123 ".format(server_node.hostname)]) == "Ok.\n" + assert client_node.exec_in_container( + ["bash", "-c", "/usr/bin/curl -s {}:8123 ".format(server_node.hostname)]) == "Ok.\n" def query_from_one_node_to_another(client_node, server_node, query): check_clickhouse_is_ok(client_node, server_node) - return client_node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.hostname, query)]) + return client_node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.hostname, query)]) def query(node, query): @@ -53,8 +53,8 @@ def test_allowed_host(): # Reverse DNS lookup currently isn't working as expected in this test. # For example, it gives something like "vitbartestallowedclienthosts_clientB1_1.vitbartestallowedclienthosts_default" instead of "clientB001.ru". # Maybe we should setup the test network better. - #expected_to_pass.extend([clientB1, clientB2, clientB3, clientC1, clientC2, clientD1, clientD3]) - #expected_to_fail.extend([clientC3, clientD2]) + # expected_to_pass.extend([clientB1, clientB2, clientB3, clientC1, clientC2, clientD1, clientD3]) + # expected_to_fail.extend([clientC3, clientD2]) for client_node in expected_to_pass: assert query_from_one_node_to_another(client_node, server, "SELECT * FROM test_table") == "5\n" diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 2a666e4e2ec..44715d92121 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -1,8 +1,6 @@ -import time import pytest - -from helpers.hdfs_api import HDFSApi from helpers.cluster import ClickHouseCluster +from helpers.hdfs_api import HDFSApi cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.xml']) @@ -21,26 +19,37 @@ def start_cluster(): finally: cluster.shutdown() + def test_config_with_hosts(start_cluster): assert node1.query("CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)") == "" assert node1.query("CREATE TABLE table_test_1_2 (word String) Engine=URL('https://yandex.ru', CSV)") == "" - assert "not allowed" in node1.query_and_get_error("CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)") - assert "not allowed" in node1.query_and_get_error("CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)") + assert "not allowed" in node1.query_and_get_error( + "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)") + assert "not allowed" in node1.query_and_get_error( + "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)") + def test_config_with_only_primary_hosts(start_cluster): assert node2.query("CREATE TABLE table_test_2_1 (word String) Engine=URL('https://host:80', CSV)") == "" assert node2.query("CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)") == "" assert node2.query("CREATE TABLE table_test_2_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" assert node2.query("CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)") == "" - assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)") - assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)") - assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)") + assert "not allowed" in node2.query_and_get_error( + "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)") + assert "not allowed" in node2.query_and_get_error( + "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)") + assert "not allowed" in node2.query_and_get_error( + "CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)") + def test_config_with_only_regexp_hosts(start_cluster): assert node3.query("CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)") == "" assert node3.query("CREATE TABLE table_test_3_2 (word String) Engine=URL('https://yandex.ru', CSV)") == "" - assert "not allowed" in node3.query_and_get_error("CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)") - assert "not allowed" in node3.query_and_get_error("CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)") + assert "not allowed" in node3.query_and_get_error( + "CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)") + assert "not allowed" in node3.query_and_get_error( + "CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)") + def test_config_without_allowed_hosts(start_cluster): assert node4.query("CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") == "" @@ -48,27 +57,60 @@ def test_config_without_allowed_hosts(start_cluster): assert node4.query("CREATE TABLE table_test_4_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" assert node4.query("CREATE TABLE table_test_4_4 (word String) Engine=URL('ftp://something.com', S3)") == "" + def test_table_function_remote(start_cluster): - assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) - assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) - assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remote('example01-0{1,2}-1', system, events", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) - assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) - assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) - assert "not allowed" in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) - assert "not allowed" in node6.query_and_get_error("SELECT * FROM remote('example01-01-{1|3}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) - assert "not allowed" in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) + assert "not allowed in config.xml" not in node6.query_and_get_error( + "SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)", + settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + assert "not allowed in config.xml" not in node6.query_and_get_error( + "SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)", + settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + assert "not allowed in config.xml" not in node6.query_and_get_error( + "SELECT * FROM remote('example01-0{1,2}-1', system, events", + settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + assert "not allowed in config.xml" not in node6.query_and_get_error( + "SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)", + settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + assert "not allowed in config.xml" not in node6.query_and_get_error( + "SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)", + settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + assert "not allowed" in node6.query_and_get_error( + "SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)", + settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + assert "not allowed" in node6.query_and_get_error("SELECT * FROM remote('example01-01-{1|3}', system, events)", + settings={"connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, "send_timeout": 1}) + assert "not allowed" in node6.query_and_get_error( + "SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)", + settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) assert node6.query("SELECT * FROM remote('localhost', system, events)") != "" assert node6.query("SELECT * FROM remoteSecure('localhost', system, metrics)") != "" - assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error("SELECT * FROM remoteSecure('localhost:800', system, events)") - assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error("SELECT * FROM remote('localhost:800', system, metrics)") + assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error( + "SELECT * FROM remoteSecure('localhost:800', system, events)") + assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error( + "SELECT * FROM remote('localhost:800', system, metrics)") + def test_redirect(start_cluster): hdfs_api = HDFSApi("root") hdfs_api.write_data("/simple_storage", "1\t\n") assert hdfs_api.read_data("/simple_storage") == "1\t\n" - node7.query("CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)") + node7.query( + "CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)") assert "not allowed" in node7.query_and_get_error("SET max_http_get_redirects=1; SELECT * from table_test_7_1") + def test_HDFS(start_cluster): - assert "not allowed" in node7.query_and_get_error("CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')") - assert "not allowed" in node7.query_and_get_error("SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')") + assert "not allowed" in node7.query_and_get_error( + "CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')") + assert "not allowed" in node7.query_and_get_error( + "SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')") diff --git a/tests/integration/test_alter_codec/test.py b/tests/integration/test_alter_codec/test.py index 4d251f60b16..f51dc9a54ff 100644 --- a/tests/integration/test_alter_codec/test.py +++ b/tests/integration/test_alter_codec/test.py @@ -2,14 +2,13 @@ import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml']) + main_configs=['configs/logs_config.xml']) node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml']) + main_configs=['configs/logs_config.xml']) @pytest.fixture(scope="module") @@ -39,7 +38,6 @@ def test_alter_codec_pk(started_cluster): with pytest.raises(QueryRuntimeException): node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format(name=name)) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format(name=name)) node1.query("INSERT INTO {name} (value) VALUES (1)".format(name=name)) diff --git a/tests/integration/test_always_fetch_merged/test.py b/tests/integration/test_always_fetch_merged/test.py index f471ec78eac..7ba8e05129b 100644 --- a/tests/integration/test_always_fetch_merged/test.py +++ b/tests/integration/test_always_fetch_merged/test.py @@ -1,14 +1,15 @@ -import pytest import time + +import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: diff --git a/tests/integration/test_asynchronous_metric_log_table/test.py b/tests/integration/test_asynchronous_metric_log_table/test.py index 0eb72c20376..0091832aa7c 100644 --- a/tests/integration/test_asynchronous_metric_log_table/test.py +++ b/tests/integration/test_asynchronous_metric_log_table/test.py @@ -39,7 +39,7 @@ def test_event_time_microseconds_field(started_cluster): node1.query(query_create) node1.query('''INSERT INTO replica.test VALUES (1, now())''') node1.query("SYSTEM FLUSH LOGS;") - #query assumes that the event_time field is accurate + # query assumes that the event_time field is accurate equals_query = '''WITH ( ( SELECT event_time_microseconds diff --git a/tests/integration/test_atomic_drop_table/test.py b/tests/integration/test_atomic_drop_table/test.py index ee79a3ff080..7ff06c7f369 100644 --- a/tests/integration/test_atomic_drop_table/test.py +++ b/tests/integration/test_atomic_drop_table/test.py @@ -1,12 +1,12 @@ import time + import pytest - -from helpers.network import PartitionManager from helpers.cluster import ClickHouseCluster - +from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/config.d/zookeeper_session_timeout.xml", "configs/remote_servers.xml"], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=["configs/config.d/zookeeper_session_timeout.xml", + "configs/remote_servers.xml"], with_zookeeper=True) @pytest.fixture(scope="module") @@ -25,12 +25,13 @@ def start_cluster(): finally: cluster.shutdown() + def test_atomic_delete_with_stopped_zookeeper(start_cluster): node1.query("insert into zktest.atomic_drop_table values (8192)") with PartitionManager() as pm: pm.drop_instance_zk_connections(node1) - error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") #Table won't drop + error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") # Table won't drop assert error != "" time.sleep(5) diff --git a/tests/integration/test_attach_without_checksums/test.py b/tests/integration/test_attach_without_checksums/test.py index 85dbc59e6f3..536ab4467ea 100644 --- a/tests/integration/test_attach_without_checksums/test.py +++ b/tests/integration/test_attach_without_checksums/test.py @@ -17,7 +17,8 @@ def start_cluster(): def test_attach_without_checksums(start_cluster): - node1.query("CREATE TABLE test (date Date, key Int32, value String) Engine=MergeTree ORDER BY key PARTITION by date") + node1.query( + "CREATE TABLE test (date Date, key Int32, value String) Engine=MergeTree ORDER BY key PARTITION by date") node1.query("INSERT INTO test SELECT toDate('2019-10-01'), number, toString(number) FROM numbers(100)") @@ -29,9 +30,13 @@ def test_attach_without_checksums(start_cluster): assert node1.query("SELECT COUNT() FROM test") == "0\n" # to be sure output not empty - node1.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" | grep -e ".*" '], privileged=True, user='root') + node1.exec_in_container( + ['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" | grep -e ".*" '], + privileged=True, user='root') - node1.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" -delete'], privileged=True, user='root') + node1.exec_in_container( + ['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" -delete'], + privileged=True, user='root') node1.query("ALTER TABLE test ATTACH PARTITION '2019-10-01'") diff --git a/tests/integration/test_authentication/test.py b/tests/integration/test_authentication/test.py index dedd5410188..0651efa11b4 100644 --- a/tests/integration/test_authentication/test.py +++ b/tests/integration/test_authentication/test.py @@ -30,7 +30,8 @@ def test_authentication_pass(): def test_authentication_fail(): # User doesn't exist. - assert "vasya: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user = 'vasya') - + assert "vasya: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user='vasya') + # Wrong password. - assert "masha: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user = 'masha', password = '123') + assert "masha: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user='masha', + password='123') diff --git a/tests/integration/test_backup_restore/test.py b/tests/integration/test_backup_restore/test.py index 46d687f7019..111dc6d24f8 100644 --- a/tests/integration/test_backup_restore/test.py +++ b/tests/integration/test_backup_restore/test.py @@ -1,10 +1,9 @@ import os.path -import pytest +import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance') q = instance.query @@ -47,7 +46,7 @@ def backup_restore(started_cluster): expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') res = q("SELECT * FROM test.tbl ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) q("ALTER TABLE test.tbl FREEZE") @@ -69,7 +68,7 @@ def test_restore(backup_restore): # Validate the attached parts are identical to the backup. expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') res = q("SELECT * FROM test.tbl1 ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) q("ALTER TABLE test.tbl1 UPDATE k=10 WHERE 1") q("SELECT sleep(2)") @@ -77,7 +76,7 @@ def test_restore(backup_restore): # Validate mutation has been applied to all attached parts. expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10') res = q("SELECT * FROM test.tbl1 ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) q("DROP TABLE IF EXISTS test.tbl1") @@ -91,7 +90,7 @@ def test_attach_partition(backup_restore): expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34') res = q("SELECT * FROM test.tbl2 ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) copy_backup_to_detached('test', 'tbl', 'tbl2') @@ -102,17 +101,19 @@ def test_attach_partition(backup_restore): q("ALTER TABLE test.tbl2 ATTACH PARTITION 197002") q("SELECT sleep(2)") - expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34') + expected = TSV( + '1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34') res = q("SELECT * FROM test.tbl2 ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) q("ALTER TABLE test.tbl2 UPDATE k=10 WHERE 1") q("SELECT sleep(2)") # Validate mutation has been applied to all attached parts. - expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10') + expected = TSV( + '1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10') res = q("SELECT * FROM test.tbl2 ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) q("DROP TABLE IF EXISTS test.tbl2") @@ -126,7 +127,7 @@ def test_replace_partition(backup_restore): expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34') res = q("SELECT * FROM test.tbl3 ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) copy_backup_to_detached('test', 'tbl', 'tbl3') @@ -138,7 +139,7 @@ def test_replace_partition(backup_restore): expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') res = q("SELECT * FROM test.tbl3 ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) q("ALTER TABLE test.tbl3 UPDATE k=10 WHERE 1") q("SELECT sleep(2)") @@ -146,6 +147,6 @@ def test_replace_partition(backup_restore): # Validate mutation has been applied to all copied parts. expected = TSV('1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10') res = q("SELECT * FROM test.tbl3 ORDER BY p") - assert(TSV(res) == expected) + assert (TSV(res) == expected) q("DROP TABLE IF EXISTS test.tbl3") diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index 503544cbfc5..df8bd6ab56f 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -1,13 +1,15 @@ import pytest - from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) - -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', + stay_alive=True, with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', + stay_alive=True, with_installed_binary=True) +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', + stay_alive=True, with_installed_binary=True) node4 = cluster.add_instance('node4') @@ -33,13 +35,15 @@ def test_backup_from_old_version(started_cluster): node1.restart_with_latest_version() - node1.query("CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1') ORDER BY tuple()") + node1.query( + "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1') ORDER BY tuple()") node1.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" - node1.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node1.exec_in_container(['bash', '-c', + 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -69,13 +73,15 @@ def test_backup_from_old_version_setting(started_cluster): node2.restart_with_latest_version() - node2.query("CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1") + node2.query( + "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1") node2.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" - node2.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node2.exec_in_container(['bash', '-c', + 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -104,17 +110,20 @@ def test_backup_from_old_version_config(started_cluster): node3.query("ALTER TABLE source_table FREEZE PARTITION tuple();") def callback(n): - n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "1") + n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", + "1") node3.restart_with_latest_version(callback_onstop=callback) - node3.query("CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1") + node3.query( + "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1") node3.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" - node3.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node3.exec_in_container(['bash', '-c', + 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -144,7 +153,8 @@ def test_backup_and_alter(started_cluster): node4.query("ALTER TABLE backup_table DROP PARTITION tuple()") - node4.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/backup_table/all_1_1_0/ /var/lib/clickhouse/data/default/backup_table/detached']) + node4.exec_in_container(['bash', '-c', + 'cp -r /var/lib/clickhouse/shadow/1/data/default/backup_table/all_1_1_0/ /var/lib/clickhouse/data/default/backup_table/detached']) node4.query("ALTER TABLE backup_table ATTACH PARTITION tuple()") diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 5b51823d361..bc6d534c50f 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -1,22 +1,23 @@ import pytest -import helpers.client as client from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', + stay_alive=True, with_installed_binary=True) node2 = cluster.add_instance('node2', with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): try: cluster.start() for i, node in enumerate([node1, node2]): node.query( - '''CREATE TABLE t(date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/t', '{}') - PARTITION BY toYYYYMM(date) - ORDER BY id'''.format(i)) + '''CREATE TABLE t(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/t', '{}') + PARTITION BY toYYYYMM(date) + ORDER BY id'''.format(i)) yield cluster diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py index b1b9fecf54e..3b35c112887 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py @@ -1,16 +1,18 @@ import pytest -import helpers.client as client from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', - with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) + with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, + with_installed_binary=True) node2 = cluster.add_instance('node2', - with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) + with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, + with_installed_binary=True) node3 = cluster.add_instance('node3', with_zookeeper=False) node4 = cluster.add_instance('node4', with_zookeeper=False) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -20,6 +22,7 @@ def start_cluster(): finally: cluster.shutdown() + # We will test that serialization of internal state of "avg" function is compatible between different versions. # TODO Implement versioning of serialization format for aggregate function states. # NOTE This test is too ad-hoc. @@ -35,18 +38,18 @@ def test_backward_compatability(start_cluster): node3.query("INSERT INTO tab VALUES (3)") node4.query("INSERT INTO tab VALUES (4)") - assert(node1.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') - assert(node2.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') - assert(node3.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') - assert(node4.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') + assert (node1.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') + assert (node2.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') + assert (node3.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') + assert (node4.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') # Also check with persisted aggregate function state node1.query("create table state (x AggregateFunction(avg, UInt64)) engine = Log") node1.query("INSERT INTO state SELECT avgState(arrayJoin(CAST([1, 2, 3, 4] AS Array(UInt64))))") - assert(node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') + assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') node1.restart_with_latest_version() - assert(node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') + assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index 5cf78b481b9..91a0a87b6e2 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -1,13 +1,15 @@ import pytest -import helpers.client as client from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) +node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', + stay_alive=True, with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', + stay_alive=True, with_installed_binary=True) node3 = cluster.add_instance('node3', with_zookeeper=False) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -23,6 +25,7 @@ def test_backward_compatability(start_cluster): node2.query("create table tab (s String) engine = MergeTree order by s") node1.query("insert into tab select number from numbers(50)") node2.query("insert into tab select number from numbers(1000000)") - res = node3.query("select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50") + res = node3.query( + "select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50") print(res) assert res == ''.join('{}\t2\n'.format(i) for i in range(50)) diff --git a/tests/integration/test_block_structure_mismatch/test.py b/tests/integration/test_block_structure_mismatch/test.py index fa9272b8401..12f9bd090a3 100644 --- a/tests/integration/test_block_structure_mismatch/test.py +++ b/tests/integration/test_block_structure_mismatch/test.py @@ -1,7 +1,5 @@ -import time import pytest -from contextlib import contextmanager from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) @@ -9,7 +7,8 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) -#test reproducing issue https://github.com/ClickHouse/ClickHouse/issues/3162 + +# test reproducing issue https://github.com/ClickHouse/ClickHouse/issues/3162 @pytest.fixture(scope="module") def started_cluster(): try: @@ -44,7 +43,9 @@ CREATE TABLE dist_test ( finally: cluster.shutdown() + def test(started_cluster): node1.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 0, 'x', 'y')") node2.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 1, 'foo', 'bar')") - assert node1.query("SELECT col1, col2 FROM dist_test WHERE (t < 3600000) AND (col1 = 'foo') ORDER BY t ASC") == "foo\tbar\n" + assert node1.query( + "SELECT col1, col2 FROM dist_test WHERE (t < 3600000) AND (col1 = 'foo') ORDER BY t ASC") == "foo\tbar\n" diff --git a/tests/integration/test_check_table/test.py b/tests/integration/test_check_table/test.py index 83df59b44a0..e57235502d3 100644 --- a/tests/integration/test_check_table/test.py +++ b/tests/integration/test_check_table/test.py @@ -1,9 +1,6 @@ -import time - import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) @@ -34,15 +31,22 @@ def started_cluster(): def corrupt_data_part_on_disk(node, table, part_name): - part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() - node.exec_in_container(['bash', '-c', 'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format(p=part_path)], privileged=True) + part_path = node.query( + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + node.exec_in_container(['bash', '-c', + 'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format( + p=part_path)], privileged=True) + def remove_checksums_on_disk(node, table, part_name): - part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + part_path = node.query( + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() node.exec_in_container(['bash', '-c', 'rm -r {p}/checksums.txt'.format(p=part_path)], privileged=True) + def remove_part_from_disk(node, table, part_name): - part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + part_path = node.query( + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() if not part_path: raise Exception("Part " + part_name + "doesn't exist") node.exec_in_container(['bash', '-c', 'rm -r {p}/*'.format(p=part_path)], privileged=True) @@ -50,35 +54,42 @@ def remove_part_from_disk(node, table, part_name): def test_check_normal_table_corruption(started_cluster): node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)") - assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n" + assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", + settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n" remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0") - assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." + assert node1.query("CHECK TABLE non_replicated_mt", settings={ + "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n" remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0") - assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." + assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={ + "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n" corrupt_data_part_on_disk(node1, "non_replicated_mt", "201902_1_1_0") - assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16." + assert node1.query("CHECK TABLE non_replicated_mt", settings={ + "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16." - assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16." + assert node1.query("CHECK TABLE non_replicated_mt", settings={ + "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16." node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-01-01'), 1, 10), (toDate('2019-01-01'), 2, 12)") - assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n" + assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n" corrupt_data_part_on_disk(node1, "non_replicated_mt", "201901_2_2_0") remove_checksums_on_disk(node1, "non_replicated_mt", "201901_2_2_0") - assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n" + assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={ + "check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n" def test_check_replicated_table_simple(started_cluster): @@ -90,16 +101,20 @@ def test_check_replicated_table_simple(started_cluster): assert node1.query("SELECT count() from replicated_mt") == "2\n" assert node2.query("SELECT count() from replicated_mt") == "2\n" - assert node1.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" - assert node2.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" + assert node1.query("CHECK TABLE replicated_mt", + settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" + assert node2.query("CHECK TABLE replicated_mt", + settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" node2.query("INSERT INTO replicated_mt VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)") node1.query("SYSTEM SYNC REPLICA replicated_mt") assert node1.query("SELECT count() from replicated_mt") == "4\n" assert node2.query("SELECT count() from replicated_mt") == "4\n" - assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" - assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" + assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" + assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" def test_check_replicated_table_corruption(started_cluster): @@ -112,18 +127,25 @@ def test_check_replicated_table_corruption(started_cluster): assert node1.query("SELECT count() from replicated_mt") == "4\n" assert node2.query("SELECT count() from replicated_mt") == "4\n" - part_name = node1.query("SELECT name from system.parts where table = 'replicated_mt' and partition_id = '201901' and active = 1").strip() + part_name = node1.query( + "SELECT name from system.parts where table = 'replicated_mt' and partition_id = '201901' and active = 1").strip() corrupt_data_part_on_disk(node1, "replicated_mt", part_name) - assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(p=part_name) + assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={ + "check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format( + p=part_name) node1.query("SYSTEM SYNC REPLICA replicated_mt") - assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) + assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) assert node1.query("SELECT count() from replicated_mt") == "4\n" remove_part_from_disk(node2, "replicated_mt", part_name) - assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(p=part_name) + assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={ + "check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format( + p=part_name) node1.query("SYSTEM SYNC REPLICA replicated_mt") - assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) + assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) assert node1.query("SELECT count() from replicated_mt") == "4\n" diff --git a/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py b/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py index ab2db469157..71850ee7318 100644 --- a/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py +++ b/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py @@ -1,13 +1,13 @@ import time -import pytest +import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -16,6 +16,7 @@ def start_cluster(): finally: cluster.shutdown() + # This tests if the data directory for a table is cleaned up if there is a Zookeeper # connection exception during a CreateQuery operation involving ReplicatedMergeTree tables. # Test flow is as follows: @@ -48,20 +49,30 @@ def test_cleanup_dir_after_bad_zk_conn(start_cluster): node1.query('''INSERT INTO replica.test VALUES (1, now())''') assert "1\n" in node1.query('''SELECT count() from replica.test FORMAT TSV''') + def test_cleanup_dir_after_wrong_replica_name(start_cluster): - node1.query("CREATE TABLE test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n") - error = node1.query_and_get_error("CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n") + node1.query( + "CREATE TABLE test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n") + error = node1.query_and_get_error( + "CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n") assert "already exists" in error - node1.query("CREATE TABLE test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n") + node1.query( + "CREATE TABLE test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n") + def test_cleanup_dir_after_wrong_zk_path(start_cluster): - node1.query("CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n") - error = node1.query_and_get_error("CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n") + node1.query( + "CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n") + error = node1.query_and_get_error( + "CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n") assert "Cannot create" in error - node1.query("CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n") + node1.query( + "CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n") + def test_attach_without_zk(start_cluster): - node1.query("CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n") + node1.query( + "CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n") node1.query("DETACH TABLE test4_r1") with PartitionManager() as pm: pm._add_rule({'probability': 0.5, 'source': node1.ip_address, 'destination_port': 2181, 'action': 'DROP'}) diff --git a/tests/integration/test_cluster_all_replicas/test.py b/tests/integration/test_cluster_all_replicas/test.py index 0af5693fc75..7cb170ce52a 100644 --- a/tests/integration/test_cluster_all_replicas/test.py +++ b/tests/integration/test_cluster_all_replicas/test.py @@ -7,6 +7,7 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): try: diff --git a/tests/integration/test_cluster_copier/test.py b/tests/integration/test_cluster_copier/test.py index 3f9ca8a053c..2a9e696ca46 100644 --- a/tests/integration/test_cluster_copier/test.py +++ b/tests/integration/test_cluster_copier/test.py @@ -1,15 +1,15 @@ import os +import random import sys import time +from contextlib import contextmanager + +import docker import kazoo import pytest -import docker -import random -from contextlib import contextmanager from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) @@ -18,9 +18,10 @@ MOVING_FAIL_PROBABILITY = 0.2 cluster = ClickHouseCluster(__file__) + def check_all_hosts_sucesfully_executed(tsv_content, num_hosts): M = TSV.toMat(tsv_content) - hosts = [(l[0], l[1]) for l in M] # (host, port) + hosts = [(l[0], l[1]) for l in M] # (host, port) codes = [l[2] for l in M] assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content @@ -39,14 +40,14 @@ def started_cluster(): global cluster try: clusters_schema = { - "0" : { - "0" : ["0", "1"], - "1" : ["0"] - }, - "1" : { - "0" : ["0", "1"], - "1" : ["0"] - } + "0": { + "0": ["0", "1"], + "1": ["0"] + }, + "1": { + "0": ["0", "1"], + "1": ["0"] + } } for cluster_name, shards in clusters_schema.iteritems(): @@ -54,10 +55,11 @@ def started_cluster(): for replica_name in replicas: name = "s{}_{}_{}".format(cluster_name, shard_name, replica_name) cluster.add_instance(name, - main_configs=["configs/conf.d/query_log.xml", "configs/conf.d/ddl.xml", "configs/conf.d/clusters.xml"], - user_configs=["configs/users.xml"], - macros={"cluster": cluster_name, "shard": shard_name, "replica": replica_name}, - with_zookeeper=True) + main_configs=["configs/conf.d/query_log.xml", "configs/conf.d/ddl.xml", + "configs/conf.d/clusters.xml"], + user_configs=["configs/users.xml"], + macros={"cluster": cluster_name, "shard": shard_name, "replica": replica_name}, + with_zookeeper=True) cluster.start() yield cluster @@ -70,24 +72,27 @@ class Task1: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path="/clickhouse-copier/task_simple" + self.zk_task_path = "/clickhouse-copier/task_simple" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task0_description.xml'), 'r').read() - def start(self): instance = cluster.instances['s0_0_0'] for cluster_num in ["0", "1"]: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) - ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(cluster_num)) + ddl_check_query(instance, + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format( + cluster_num)) ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " + - "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " + - "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") - ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)") - ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)") - instance.query("INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002", settings={"insert_distributed_sync": 1}) - + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " + + "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") + ddl_check_query(instance, + "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)") + ddl_check_query(instance, + "CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)") + instance.query("INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002", + settings={"insert_distributed_sync": 1}) def check(self): assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM hits_all")) == TSV("1002\n") @@ -107,31 +112,44 @@ class Task2: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path="/clickhouse-copier/task_month_to_week_partition" + self.zk_task_path = "/clickhouse-copier/task_month_to_week_partition" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_month_to_week_description.xml'), 'r').read() - def start(self): instance = cluster.instances['s0_0_0'] for cluster_num in ["0", "1"]: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) - ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(cluster_num)) + ddl_check_query(instance, + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format( + cluster_num)) - ddl_check_query(instance, "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") - ddl_check_query(instance, "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)") + ddl_check_query(instance, + "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") + ddl_check_query(instance, + "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)") - instance.query("INSERT INTO a_all SELECT toDate(17581 + number) AS date, number AS d FROM system.numbers LIMIT 85", settings={"insert_distributed_sync": 1}) + instance.query( + "INSERT INTO a_all SELECT toDate(17581 + number) AS date, number AS d FROM system.numbers LIMIT 85", + settings={"insert_distributed_sync": 1}) def check(self): - assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM cluster(cluster0, default, a)")) == TSV("85\n") - assert TSV(self.cluster.instances['s1_0_0'].query("SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("85\t85\n") + assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM cluster(cluster0, default, a)")) == TSV( + "85\n") + assert TSV(self.cluster.instances['s1_0_0'].query( + "SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("85\t85\n") - assert TSV(self.cluster.instances['s1_0_0'].query("SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("0\n") - assert TSV(self.cluster.instances['s1_1_0'].query("SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("1\n") + assert TSV(self.cluster.instances['s1_0_0'].query( + "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("0\n") + assert TSV(self.cluster.instances['s1_1_0'].query( + "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("1\n") - assert TSV(self.cluster.instances['s1_0_0'].query("SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV("1\n") - assert TSV(self.cluster.instances['s1_1_0'].query("SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV("1\n") + assert TSV(self.cluster.instances['s1_0_0'].query( + "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV( + "1\n") + assert TSV(self.cluster.instances['s1_1_0'].query( + "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV( + "1\n") instance = cluster.instances['s0_0_0'] ddl_check_query(instance, "DROP TABLE a ON CLUSTER cluster0") @@ -142,11 +160,10 @@ class Task_test_block_size: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path="/clickhouse-copier/task_test_block_size" + self.zk_task_path = "/clickhouse-copier/task_test_block_size" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_test_block_size.xml'), 'r').read() self.rows = 1000000 - def start(self): instance = cluster.instances['s0_0_0'] @@ -155,11 +172,13 @@ class Task_test_block_size: ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/test_block_size', '{replica}') ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2) - instance.query("INSERT INTO test_block_size SELECT toDate(0) AS partition, number as d FROM system.numbers LIMIT {}".format(self.rows)) - + instance.query( + "INSERT INTO test_block_size SELECT toDate(0) AS partition, number as d FROM system.numbers LIMIT {}".format( + self.rows)) def check(self): - assert TSV(self.cluster.instances['s1_0_0'].query("SELECT count() FROM cluster(cluster1, default, test_block_size)")) == TSV("{}\n".format(self.rows)) + assert TSV(self.cluster.instances['s1_0_0'].query( + "SELECT count() FROM cluster(cluster1, default, test_block_size)")) == TSV("{}\n".format(self.rows)) instance = cluster.instances['s0_0_0'] ddl_check_query(instance, "DROP TABLE test_block_size ON CLUSTER shard_0_0", 2) @@ -170,17 +189,15 @@ class Task_no_index: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path="/clickhouse-copier/task_no_index" + self.zk_task_path = "/clickhouse-copier/task_no_index" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_no_index.xml'), 'r').read() self.rows = 1000000 - def start(self): instance = cluster.instances['s0_0_0'] instance.query("create table ontime (Year UInt16, FlightDate String) ENGINE = Memory") instance.query("insert into ontime values (2016, 'test6'), (2017, 'test7'), (2018, 'test8')") - def check(self): assert TSV(self.cluster.instances['s1_1_0'].query("SELECT Year FROM ontime22")) == TSV("2017\n") instance = cluster.instances['s0_0_0'] @@ -193,17 +210,16 @@ class Task_no_arg: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path="/clickhouse-copier/task_no_arg" + self.zk_task_path = "/clickhouse-copier/task_no_arg" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_no_arg.xml'), 'r').read() self.rows = 1000000 - def start(self): instance = cluster.instances['s0_0_0'] - instance.query("create table copier_test1 (date Date, id UInt32) engine = MergeTree PARTITION BY date ORDER BY date SETTINGS index_granularity = 8192") + instance.query( + "create table copier_test1 (date Date, id UInt32) engine = MergeTree PARTITION BY date ORDER BY date SETTINGS index_granularity = 8192") instance.query("insert into copier_test1 values ('2016-01-01', 10);") - def check(self): assert TSV(self.cluster.instances['s1_1_0'].query("SELECT date FROM copier_test1_1")) == TSV("2016-01-01\n") instance = cluster.instances['s0_0_0'] @@ -227,15 +243,14 @@ def execute_task(task, cmd_options): zk.ensure_path(zk_task_path) zk.create(zk_task_path + "/description", task.copier_task_config) - # Run cluster-copier processes on each node docker_api = docker.from_env().api copiers_exec_ids = [] cmd = ['/usr/bin/clickhouse', 'copier', - '--config', '/etc/clickhouse-server/config-copier.xml', - '--task-path', zk_task_path, - '--base-dir', '/var/log/clickhouse-server/copier'] + '--config', '/etc/clickhouse-server/config-copier.xml', + '--task-path', zk_task_path, + '--base-dir', '/var/log/clickhouse-server/copier'] cmd += cmd_options copiers = random.sample(cluster.instances.keys(), 3) @@ -243,7 +258,8 @@ def execute_task(task, cmd_options): for instance_name in copiers: instance = cluster.instances[instance_name] container = instance.get_docker_handle() - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml") + instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), + "/etc/clickhouse-server/config-copier.xml") print "Copied copier config to {}".format(instance.name) exec_id = docker_api.exec_create(container.id, cmd, stderr=True) output = docker_api.exec_start(exec_id).decode('utf8') @@ -277,7 +293,6 @@ def execute_task(task, cmd_options): True ] ) - def test_copy_simple(started_cluster, use_sample_offset): if use_sample_offset: execute_task(Task1(started_cluster), ['--experimental-use-sample-offset', '1']) @@ -292,7 +307,6 @@ def test_copy_simple(started_cluster, use_sample_offset): True ] ) - def test_copy_with_recovering(started_cluster, use_sample_offset): if use_sample_offset: execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), @@ -300,6 +314,7 @@ def test_copy_with_recovering(started_cluster, use_sample_offset): else: execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + @pytest.mark.parametrize( ('use_sample_offset'), [ @@ -307,7 +322,6 @@ def test_copy_with_recovering(started_cluster, use_sample_offset): True ] ) - def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offset): if use_sample_offset: execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), @@ -315,29 +329,36 @@ def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offs else: execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) + @pytest.mark.timeout(600) def test_copy_month_to_week_partition(started_cluster): execute_task(Task2(started_cluster), []) + @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering(started_cluster): execute_task(Task2(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering_after_move_faults(started_cluster): execute_task(Task2(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) + def test_block_size(started_cluster): execute_task(Task_test_block_size(started_cluster), []) + def test_no_index(started_cluster): execute_task(Task_no_index(started_cluster), []) + def test_no_arg(started_cluster): execute_task(Task_no_arg(started_cluster), []) + if __name__ == '__main__': with contextmanager(started_cluster)() as cluster: - for name, instance in cluster.instances.items(): - print name, instance.ip_address - raw_input("Cluster created, press any key to destroy...") + for name, instance in cluster.instances.items(): + print name, instance.ip_address + raw_input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_cluster_copier/trivial_test.py b/tests/integration/test_cluster_copier/trivial_test.py index 1697f8bbdfa..3d0c5d0f5b0 100644 --- a/tests/integration/test_cluster_copier/trivial_test.py +++ b/tests/integration/test_cluster_copier/trivial_test.py @@ -1,13 +1,10 @@ import os -import os.path as p import sys import time -import datetime -import pytest from contextlib import contextmanager -import docker -from kazoo.client import KazooClient +import docker +import pytest CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) @@ -18,13 +15,14 @@ COPYING_FAIL_PROBABILITY = 0.33 MOVING_FAIL_PROBABILITY = 0.1 cluster = None + @pytest.fixture(scope="function") def started_cluster(): global cluster try: clusters_schema = { - "0" : {"0" : ["0"]}, - "1" : {"0" : ["0"]} + "0": {"0": ["0"]}, + "1": {"0": ["0"]} } cluster = ClickHouseCluster(__file__) @@ -50,12 +48,11 @@ class TaskTrivial: def __init__(self, cluster, use_sample_offset): self.cluster = cluster if use_sample_offset: - self.zk_task_path="/clickhouse-copier/task_trivial_use_sample_offset" + self.zk_task_path = "/clickhouse-copier/task_trivial_use_sample_offset" else: - self.zk_task_path="/clickhouse-copier/task_trivial" + self.zk_task_path = "/clickhouse-copier/task_trivial" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_trivial.xml'), 'r').read() - def start(self): source = cluster.instances['s0_0_0'] destination = cluster.instances['s1_0_0'] @@ -68,8 +65,8 @@ class TaskTrivial: "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial', '1') " "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") - source.query("INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002", settings={"insert_distributed_sync": 1}) - + source.query("INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002", + settings={"insert_distributed_sync": 1}) def check(self): source = cluster.instances['s0_0_0'] @@ -138,7 +135,6 @@ def execute_task(task, cmd_options): True ] ) - def test_trivial_copy(started_cluster, use_sample_offset): if use_sample_offset: execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--experimental-use-sample-offset', '1']) @@ -146,6 +142,7 @@ def test_trivial_copy(started_cluster, use_sample_offset): print("AAAAA") execute_task(TaskTrivial(started_cluster, use_sample_offset), []) + @pytest.mark.parametrize( ('use_sample_offset'), [ @@ -153,7 +150,6 @@ def test_trivial_copy(started_cluster, use_sample_offset): True ] ) - def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset): if use_sample_offset: execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), @@ -161,6 +157,7 @@ def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset): else: execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + @pytest.mark.parametrize( ('use_sample_offset'), [ @@ -168,7 +165,6 @@ def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset): True ] ) - def test_trivial_copy_with_move_fault(started_cluster, use_sample_offset): if use_sample_offset: execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), diff --git a/tests/integration/test_concurrent_queries_for_user_restriction/test.py b/tests/integration/test_concurrent_queries_for_user_restriction/test.py index 4b7cc87c15a..e287eb763ce 100644 --- a/tests/integration/test_concurrent_queries_for_user_restriction/test.py +++ b/tests/integration/test_concurrent_queries_for_user_restriction/test.py @@ -1,8 +1,7 @@ import time +from multiprocessing.dummy import Pool import pytest - -from multiprocessing.dummy import Pool from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) @@ -10,6 +9,7 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', user_configs=['configs/user_restrictions.xml']) node2 = cluster.add_instance('node2', user_configs=['configs/user_restrictions.xml']) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -21,6 +21,7 @@ def started_cluster(): finally: cluster.shutdown() + def test_exception_message(started_cluster): assert node1.query("select number from nums order by number") == "0\n1\n" @@ -30,7 +31,7 @@ def test_exception_message(started_cluster): busy_pool = Pool(3) busy_pool.map_async(node_busy, xrange(3)) - time.sleep(1) # wait a little until polling starts + time.sleep(1) # wait a little until polling starts try: assert node2.query("select number from remote('node1', 'default', 'nums')", user='good') == "0\n1\n" except Exception as ex: diff --git a/tests/integration/test_concurrent_ttl_merges/test.py b/tests/integration/test_concurrent_ttl_merges/test.py index f1704a80710..f77ae5996d1 100644 --- a/tests/integration/test_concurrent_ttl_merges/test.py +++ b/tests/integration/test_concurrent_ttl_merges/test.py @@ -1,12 +1,9 @@ import time + import pytest - -import helpers.client as client from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) @@ -24,14 +21,16 @@ def started_cluster(): def count_ttl_merges_in_queue(node, table): - result = node.query("SELECT count() FROM system.replication_queue WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table)) + result = node.query( + "SELECT count() FROM system.replication_queue WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table)) if not result: return 0 return int(result.strip()) def count_ttl_merges_in_background_pool(node, table): - result = node.query("SELECT count() FROM system.merges WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table)) + result = node.query( + "SELECT count() FROM system.merges WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table)) if not result: return 0 return int(result.strip()) @@ -55,12 +54,14 @@ def count_running_mutations(node, table): # but it revealed a bug when we assign different merges to the same part # on the borders of partitions. def test_no_ttl_merges_in_busy_pool(started_cluster): - node1.query("CREATE TABLE test_ttl (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0, number_of_free_entries_in_pool_to_execute_mutation = 0") + node1.query( + "CREATE TABLE test_ttl (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0, number_of_free_entries_in_pool_to_execute_mutation = 0") node1.query("SYSTEM STOP TTL MERGES") for i in range(1, 7): - node1.query("INSERT INTO test_ttl SELECT now() - INTERVAL 1 MONTH + number - 1, {}, number FROM numbers(5)".format(i)) + node1.query( + "INSERT INTO test_ttl SELECT now() - INTERVAL 1 MONTH + number - 1, {}, number FROM numbers(5)".format(i)) node1.query("ALTER TABLE test_ttl UPDATE data = data + 1 WHERE sleepEachRow(1) = 0") @@ -85,7 +86,8 @@ def test_no_ttl_merges_in_busy_pool(started_cluster): def test_limited_ttl_merges_in_empty_pool(started_cluster): - node1.query("CREATE TABLE test_ttl_v2 (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") + node1.query( + "CREATE TABLE test_ttl_v2 (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") node1.query("SYSTEM STOP TTL MERGES") @@ -107,7 +109,8 @@ def test_limited_ttl_merges_in_empty_pool(started_cluster): def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster): - node1.query("CREATE TABLE replicated_ttl (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") + node1.query( + "CREATE TABLE replicated_ttl (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") node1.query("SYSTEM STOP TTL MERGES") @@ -134,14 +137,17 @@ def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster): def test_limited_ttl_merges_two_replicas(started_cluster): # Actually this test quite fast and often we cannot catch any merges. # To check for sure just add some sleeps in mergePartsToTemporaryPart - node1.query("CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") - node2.query("CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '2') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") + node1.query( + "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") + node2.query( + "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '2') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") node1.query("SYSTEM STOP TTL MERGES") node2.query("SYSTEM STOP TTL MERGES") for i in range(100): - node1.query("INSERT INTO replicated_ttl_2 SELECT now() - INTERVAL 1 MONTH, {}, number FROM numbers(10000)".format(i)) + node1.query( + "INSERT INTO replicated_ttl_2 SELECT now() - INTERVAL 1 MONTH, {}, number FROM numbers(10000)".format(i)) node2.query("SYSTEM SYNC REPLICA replicated_ttl_2", timeout=10) assert node1.query("SELECT COUNT() FROM replicated_ttl_2") == "1000000\n" @@ -155,7 +161,8 @@ def test_limited_ttl_merges_two_replicas(started_cluster): while True: merges_with_ttl_count_node1.add(count_ttl_merges_in_background_pool(node1, "replicated_ttl_2")) merges_with_ttl_count_node2.add(count_ttl_merges_in_background_pool(node2, "replicated_ttl_2")) - if node1.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" and node2.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n": + if node1.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" and node2.query( + "SELECT COUNT() FROM replicated_ttl_2") == "0\n": break # Both replicas can assign merges with TTL. If one will perform better than diff --git a/tests/integration/test_config_corresponding_root/test.py b/tests/integration/test_config_corresponding_root/test.py index 1c714654820..da6af7d11ef 100644 --- a/tests/integration/test_config_corresponding_root/test.py +++ b/tests/integration/test_config_corresponding_root/test.py @@ -1,6 +1,6 @@ import os -import pytest +import pytest from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -9,6 +9,7 @@ cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=["configs/config.d/bad.xml"]) caught_exception = "" + @pytest.fixture(scope="module") def start_cluster(): global caught_exception @@ -17,6 +18,7 @@ def start_cluster(): except Exception as e: caught_exception = str(e) + def test_work(start_cluster): print(caught_exception) assert caught_exception.find("Root element doesn't have the corresponding root element as the config file.") != -1 diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index 8472f85a285..3a2d0d98281 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -1,21 +1,26 @@ -import time import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', user_configs=['configs/config_no_substs.xml']) # hardcoded value 33333 -node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'], env_variables={"MAX_QUERY_SIZE": "55555"}) +node1 = cluster.add_instance('node1', user_configs=['configs/config_no_substs.xml']) # hardcoded value 33333 +node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'], + env_variables={"MAX_QUERY_SIZE": "55555"}) node3 = cluster.add_instance('node3', user_configs=['configs/config_zk.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'], main_configs=['configs/max_query_size.xml']) # include value 77777 +node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'], + main_configs=['configs/max_query_size.xml']) # include value 77777 node5 = cluster.add_instance('node5', user_configs=['configs/config_allow_databases.xml']) -node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'], env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"}, main_configs=['configs/max_query_size.xml']) +node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'], + env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"}, + main_configs=['configs/max_query_size.xml']) + @pytest.fixture(scope="module") def start_cluster(): try: def create_zk_roots(zk): zk.create(path="/setting/max_query_size", value="77777", makepath=True) + cluster.add_zookeeper_startup_command(create_zk_roots) cluster.start() @@ -23,25 +28,36 @@ def start_cluster(): finally: cluster.shutdown() + def test_config(start_cluster): - assert node1.query("select value from system.settings where name = 'max_query_size'") == "33333\n" - assert node2.query("select value from system.settings where name = 'max_query_size'") == "55555\n" - assert node3.query("select value from system.settings where name = 'max_query_size'") == "77777\n" - assert node4.query("select value from system.settings where name = 'max_query_size'") == "99999\n" - assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n" + assert node1.query("select value from system.settings where name = 'max_query_size'") == "33333\n" + assert node2.query("select value from system.settings where name = 'max_query_size'") == "55555\n" + assert node3.query("select value from system.settings where name = 'max_query_size'") == "77777\n" + assert node4.query("select value from system.settings where name = 'max_query_size'") == "99999\n" + assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n" + def test_allow_databases(start_cluster): node5.query("CREATE DATABASE db1") - node5.query("CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)") + node5.query( + "CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)") node5.query("INSERT INTO db1.test_table VALUES('2000-01-01', 'test_key', 1)") assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'") == "db1\n" - assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' ") == "test_table\n" - assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'") == "date\nk1\nv1\n" - assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n" - assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n" + assert node5.query( + "SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' ") == "test_table\n" + assert node5.query( + "SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'") == "date\nk1\nv1\n" + assert node5.query( + "SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n" + assert node5.query( + "SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n" assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'", user="test_allow").strip() == "" - assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'", user="test_allow").strip() == "" - assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == "" - assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == "" - assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == "" + assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'", + user="test_allow").strip() == "" + assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'", + user="test_allow").strip() == "" + assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'", + user="test_allow").strip() == "" + assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'", + user="test_allow").strip() == "" diff --git a/tests/integration/test_consistant_parts_after_move_partition/test.py b/tests/integration/test_consistant_parts_after_move_partition/test.py index 312f5dc30af..05e721ee5ea 100644 --- a/tests/integration/test_consistant_parts_after_move_partition/test.py +++ b/tests/integration/test_consistant_parts_after_move_partition/test.py @@ -1,12 +1,8 @@ -import os - import pytest from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry - CLICKHOUSE_DATABASE = 'test' @@ -46,13 +42,14 @@ def test_consistent_part_after_move_partition(start_cluster): # insert into all replicas for i in range(100): node1.query('INSERT INTO `{database}`.src VALUES ({value} % 2, {value})'.format(database=CLICKHOUSE_DATABASE, - value=i)) + value=i)) query_source = 'SELECT COUNT(*) FROM `{database}`.src'.format(database=CLICKHOUSE_DATABASE) query_dest = 'SELECT COUNT(*) FROM `{database}`.dest'.format(database=CLICKHOUSE_DATABASE) assert_eq_with_retry(node2, query_source, node1.query(query_source)) assert_eq_with_retry(node2, query_dest, node1.query(query_dest)) - node1.query('ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest'.format(database=CLICKHOUSE_DATABASE)) + node1.query( + 'ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest'.format(database=CLICKHOUSE_DATABASE)) assert_eq_with_retry(node2, query_source, node1.query(query_source)) assert_eq_with_retry(node2, query_dest, node1.query(query_dest)) diff --git a/tests/integration/test_consistent_parts_after_clone_replica/test.py b/tests/integration/test_consistent_parts_after_clone_replica/test.py index b8a58242ad1..60b91bcb282 100644 --- a/tests/integration/test_consistent_parts_after_clone_replica/test.py +++ b/tests/integration/test_consistent_parts_after_clone_replica/test.py @@ -8,19 +8,20 @@ from helpers.test_tools import assert_eq_with_retry def fill_nodes(nodes, shard): for node in nodes: node.query( - ''' - CREATE DATABASE test; - CREATE TABLE test_table(date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') - ORDER BY id PARTITION BY toYYYYMM(date) - SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + ''' + CREATE DATABASE test; + CREATE TABLE test_table(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') + ORDER BY id PARTITION BY toYYYYMM(date) + SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + '''.format(shard=shard, replica=node.name)) cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -56,5 +57,3 @@ def test_inconsistent_parts_if_drop_while_replica_not_active(start_cluster): # the first replica will be cloned from the second pm.heal_all() assert_eq_with_retry(node1, "SELECT count(*) FROM test_table", node2.query("SELECT count(*) FROM test_table")) - - diff --git a/tests/integration/test_cross_replication/test.py b/tests/integration/test_cross_replication/test.py index 6c1172912ed..9171fea5547 100644 --- a/tests/integration/test_cross_replication/test.py +++ b/tests/integration/test_cross_replication/test.py @@ -2,18 +2,17 @@ import time from contextlib import contextmanager import pytest - from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: diff --git a/tests/integration/test_custom_settings/test.py b/tests/integration/test_custom_settings/test.py index 62c765a6ba0..32df79ec1e9 100644 --- a/tests/integration/test_custom_settings/test.py +++ b/tests/integration/test_custom_settings/test.py @@ -2,7 +2,8 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"], user_configs=["configs/users.d/custom_settings.xml"]) +node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"], + user_configs=["configs/users.d/custom_settings.xml"]) @pytest.fixture(scope="module", autouse=True) @@ -22,9 +23,10 @@ def test(): assert node.query("SELECT getSetting('custom_d')") == "some text\n" assert "custom_a = -5, custom_b = 10000000000, custom_c = -4.325, custom_d = \\'some text\\'" \ - in node.query("SHOW CREATE SETTINGS PROFILE default") + in node.query("SHOW CREATE SETTINGS PROFILE default") - assert "no settings profile" in node.query_and_get_error("SHOW CREATE SETTINGS PROFILE profile_with_unknown_setting") + assert "no settings profile" in node.query_and_get_error( + "SHOW CREATE SETTINGS PROFILE profile_with_unknown_setting") assert "no settings profile" in node.query_and_get_error("SHOW CREATE SETTINGS PROFILE profile_illformed_setting") @@ -33,9 +35,9 @@ def test_invalid_settings(): node.query("SYSTEM FLUSH LOGS") assert node.query("SELECT COUNT() FROM system.text_log WHERE" - " message LIKE '%Could not parse profile `profile_illformed_setting`%'" - " AND message LIKE '%Couldn\\'t restore Field from dump%'") == "1\n" + " message LIKE '%Could not parse profile `profile_illformed_setting`%'" + " AND message LIKE '%Couldn\\'t restore Field from dump%'") == "1\n" assert node.query("SELECT COUNT() FROM system.text_log WHERE" - " message LIKE '%Could not parse profile `profile_with_unknown_setting`%'" - " AND message LIKE '%Setting x is neither a builtin setting nor started with the prefix \\'custom_\\'%'") == "1\n" + " message LIKE '%Could not parse profile `profile_with_unknown_setting`%'" + " AND message LIKE '%Setting x is neither a builtin setting nor started with the prefix \\'custom_\\'%'") == "1\n" diff --git a/tests/integration/test_ddl_alter_query/test.py b/tests/integration/test_ddl_alter_query/test.py index 2a16c58c9e6..d65e40084f6 100644 --- a/tests/integration/test_ddl_alter_query/test.py +++ b/tests/integration/test_ddl_alter_query/test.py @@ -10,7 +10,6 @@ node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml' node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) - @pytest.fixture(scope="module") def started_cluster(): try: @@ -18,10 +17,14 @@ def started_cluster(): for i, node in enumerate([node1, node2]): node.query("CREATE DATABASE testdb") - node.query('''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format(i)) + node.query( + '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format( + i)) for i, node in enumerate([node3, node4]): node.query("CREATE DATABASE testdb") - node.query('''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format(i)) + node.query( + '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format( + i)) yield cluster finally: @@ -34,7 +37,8 @@ def test_alter(started_cluster): node2.query("SYSTEM SYNC REPLICA testdb.test_table") node4.query("SYSTEM SYNC REPLICA testdb.test_table") - node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", settings={"replication_alter_partitions_sync": "2"}) + node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", + settings={"replication_alter_partitions_sync": "2"}) node1.query("SYSTEM SYNC REPLICA testdb.test_table") node2.query("SYSTEM SYNC REPLICA testdb.test_table") diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index d312a93ba01..3eef292018c 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -1,14 +1,17 @@ -import string import random -import pytest +import string +import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/default_compression.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/default_compression.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml'], image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True, with_installed_binary=True) +node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml'], + image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True, + with_installed_binary=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -21,12 +24,14 @@ def start_cluster(): def get_compression_codec_byte(node, table_name, part_name): - cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -N 1 | head -n 1 | awk '{{print $2}}'".format(table_name, part_name) + cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -N 1 | head -n 1 | awk '{{print $2}}'".format( + table_name, part_name) return node.exec_in_container(["bash", "-c", cmd]).strip() def get_second_multiple_codec_byte(node, table_name, part_name): - cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -j 11 -N 1 | head -n 1 | awk '{{print $2}}'".format(table_name, part_name) + cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -j 11 -N 1 | head -n 1 | awk '{{print $2}}'".format( + table_name, part_name) return node.exec_in_container(["bash", "-c", cmd]).strip() @@ -74,16 +79,22 @@ def test_default_codec_single(start_cluster): # Same codec for all assert get_compression_codec_byte(node1, "compression_table", "1_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n" assert get_compression_codec_byte(node1, "compression_table", "2_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n" assert get_compression_codec_byte(node1, "compression_table", "3_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n" # just to be sure that replication works node1.query("OPTIMIZE TABLE compression_table FINAL") @@ -101,16 +112,22 @@ def test_default_codec_single(start_cluster): node2.query("SYSTEM FLUSH LOGS") assert get_compression_codec_byte(node1, "compression_table", "1_0_0_1") == CODECS_MAPPING['ZSTD'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n" assert get_compression_codec_byte(node1, "compression_table", "2_0_0_1") == CODECS_MAPPING['LZ4HC'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n" assert get_compression_codec_byte(node1, "compression_table", "3_0_0_1") == CODECS_MAPPING['LZ4'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n" assert node1.query("SELECT COUNT() FROM compression_table") == "3\n" assert node2.query("SELECT COUNT() FROM compression_table") == "3\n" @@ -137,18 +154,24 @@ def test_default_codec_multiple(start_cluster): # Same codec for all assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['Multiple'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n" assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['Multiple'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n" assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['Multiple'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n" node2.query("SYSTEM SYNC REPLICA compression_table_multiple", timeout=15) @@ -156,18 +179,24 @@ def test_default_codec_multiple(start_cluster): assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['Multiple'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['ZSTD'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n" assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n" assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['LZ4'] - assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n" - assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n" assert node1.query("SELECT COUNT() FROM compression_table_multiple") == "3\n" assert node2.query("SELECT COUNT() FROM compression_table_multiple") == "3\n" @@ -187,15 +216,21 @@ def test_default_codec_version_update(start_cluster): node3.restart_with_latest_version() - assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'") == "ZSTD(1)\n" - assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_0'") == "ZSTD(1)\n" - assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_0'") == "ZSTD(1)\n" + assert node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'") == "ZSTD(1)\n" + assert node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_0'") == "ZSTD(1)\n" + assert node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_0'") == "ZSTD(1)\n" node3.query("OPTIMIZE TABLE compression_table FINAL") - assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_1'") == "ZSTD(10)\n" - assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_1'") == "LZ4HC(5)\n" - assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'") == "LZ4\n" + assert node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_1'") == "ZSTD(10)\n" + assert node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_1'") == "LZ4HC(5)\n" + assert node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'") == "LZ4\n" assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC'] assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple'] diff --git a/tests/integration/test_default_database_on_cluster/test.py b/tests/integration/test_default_database_on_cluster/test.py index cfe11c34660..28a3cfad1d1 100644 --- a/tests/integration/test_default_database_on_cluster/test.py +++ b/tests/integration/test_default_database_on_cluster/test.py @@ -1,12 +1,20 @@ -import time import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True) -ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True) -ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True) -ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True) +ch1 = cluster.add_instance('ch1', + main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], + with_zookeeper=True) +ch2 = cluster.add_instance('ch2', + main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], + with_zookeeper=True) +ch3 = cluster.add_instance('ch3', + main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], + with_zookeeper=True) +ch4 = cluster.add_instance('ch4', + main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], + with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): @@ -20,13 +28,17 @@ def started_cluster(): def test_default_database_on_cluster(started_cluster): - ch1.query(database='test_default_database', sql="CREATE TABLE test_local_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Memory;") + ch1.query(database='test_default_database', + sql="CREATE TABLE test_local_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Memory;") for node in [ch1, ch2, ch3, ch4]: assert node.query("SHOW TABLES FROM test_default_database FORMAT TSV") == "test_local_table\n" - ch1.query(database='test_default_database', sql="CREATE TABLE test_distributed_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Distributed(cluster, currentDatabase(), 'test_local_table');") + ch1.query(database='test_default_database', + sql="CREATE TABLE test_distributed_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Distributed(cluster, currentDatabase(), 'test_local_table');") for node in [ch1, ch2, ch3, ch4]: - assert node.query("SHOW TABLES FROM test_default_database FORMAT TSV") == "test_distributed_table\ntest_local_table\n" - assert node.query("SHOW CREATE TABLE test_default_database.test_distributed_table FORMAT TSV") == "CREATE TABLE test_default_database.test_distributed_table\\n(\\n `column` UInt8\\n)\\nENGINE = Distributed(\\'cluster\\', \\'test_default_database\\', \\'test_local_table\\')\n" + assert node.query( + "SHOW TABLES FROM test_default_database FORMAT TSV") == "test_distributed_table\ntest_local_table\n" + assert node.query( + "SHOW CREATE TABLE test_default_database.test_distributed_table FORMAT TSV") == "CREATE TABLE test_default_database.test_distributed_table\\n(\\n `column` UInt8\\n)\\nENGINE = Distributed(\\'cluster\\', \\'test_default_database\\', \\'test_local_table\\')\n" diff --git a/tests/integration/test_default_role/test.py b/tests/integration/test_default_role/test.py index 2b6b4698b20..2f00fb603a8 100644 --- a/tests/integration/test_default_role/test.py +++ b/tests/integration/test_default_role/test.py @@ -1,7 +1,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -import re cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance') @@ -11,7 +10,7 @@ instance = cluster.add_instance('instance') def started_cluster(): try: cluster.start() - + instance.query("CREATE USER john") instance.query("CREATE ROLE rx") instance.query("CREATE ROLE ry") @@ -32,41 +31,41 @@ def test_set_default_roles(): assert instance.query("SHOW CURRENT ROLES", user="john") == "" instance.query("GRANT rx, ry TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]]) instance.query("SET DEFAULT ROLE NONE TO john") assert instance.query("SHOW CURRENT ROLES", user="john") == "" instance.query("SET DEFAULT ROLE rx TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1]]) instance.query("SET DEFAULT ROLE ry TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]]) instance.query("SET DEFAULT ROLE ALL TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]]) instance.query("SET DEFAULT ROLE ALL EXCEPT rx TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]]) def test_alter_user(): assert instance.query("SHOW CURRENT ROLES", user="john") == "" instance.query("GRANT rx, ry TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]]) instance.query("ALTER USER john DEFAULT ROLE NONE") assert instance.query("SHOW CURRENT ROLES", user="john") == "" instance.query("ALTER USER john DEFAULT ROLE rx") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1]]) instance.query("ALTER USER john DEFAULT ROLE ALL") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]]) instance.query("ALTER USER john DEFAULT ROLE ALL EXCEPT rx") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]]) def test_wrong_set_default_role(): diff --git a/tests/integration/test_delayed_replica_failover/test.py b/tests/integration/test_delayed_replica_failover/test.py index 882cd566472..f657edae6fb 100644 --- a/tests/integration/test_delayed_replica_failover/test.py +++ b/tests/integration/test_delayed_replica_failover/test.py @@ -1,14 +1,14 @@ -import pytest +import os +import sys import time -import os, sys + +import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import helpers from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager - cluster = ClickHouseCluster(__file__) # Cluster with 2 shards of 2 replicas each. node_1_1 is the instance with Distributed table. @@ -19,6 +19,7 @@ node_1_2 = cluster.add_instance('node_1_2', with_zookeeper=True) node_2_1 = cluster.add_instance('node_2_1', with_zookeeper=True) node_2_2 = cluster.add_instance('node_2_2', with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -30,7 +31,7 @@ def started_cluster(): node.query(''' CREATE TABLE replicated (d Date, x UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}', d, d, 8192)''' - .format(shard=shard, instance=node.name)) + .format(shard=shard, instance=node.name)) node_1_1.query( "CREATE TABLE distributed (d Date, x UInt32) ENGINE = " @@ -51,7 +52,7 @@ def test(started_cluster): node_1_2.query("INSERT INTO replicated VALUES ('2017-05-08', 1)") node_2_2.query("INSERT INTO replicated VALUES ('2017-05-08', 2)") - time.sleep(1) # accrue replica delay + time.sleep(1) # accrue replica delay assert node_1_1.query("SELECT sum(x) FROM replicated").strip() == '0' assert node_1_2.query("SELECT sum(x) FROM replicated").strip() == '1' @@ -78,7 +79,7 @@ SELECT sum(x) FROM distributed WITH TOTALS SETTINGS pm.drop_instance_zk_connections(node_1_2) pm.drop_instance_zk_connections(node_2_2) - time.sleep(4) # allow pings to zookeeper to timeout (must be greater than ZK session timeout). + time.sleep(4) # allow pings to zookeeper to timeout (must be greater than ZK session timeout). # At this point all replicas are stale, but the query must still go to second replicas which are the least stale ones. assert instance_with_dist_table.query(''' diff --git a/tests/integration/test_dictionaries_access/test.py b/tests/integration/test_dictionaries_access/test.py index cbba651a3b7..1b64b0de1fb 100644 --- a/tests/integration/test_dictionaries_access/test.py +++ b/tests/integration/test_dictionaries_access/test.py @@ -9,7 +9,7 @@ instance = cluster.add_instance('instance') def started_cluster(): try: cluster.start() - + instance.query("CREATE USER mira") instance.query("CREATE TABLE test_table(x Int32, y Int32) ENGINE=Log") instance.query("INSERT INTO test_table VALUES (5,6)") diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index 5a46498ce08..5880ead7c5a 100644 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -1,11 +1,12 @@ -import pytest +import math import os +import pytest from helpers.cluster import ClickHouseCluster from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout -from helpers.external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, SourceExecutableHashed -from helpers.external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra -import math +from helpers.external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceCassandra +from helpers.external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, \ + SourceExecutableHashed SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries') @@ -103,8 +104,6 @@ VALUES = { ] } - - LAYOUTS = [ Layout("flat"), Layout("hashed"), @@ -135,6 +134,7 @@ DICTIONARIES = [] cluster = None node = None + def get_dict(source, layout, fields, suffix_name=''): global dict_configs_path @@ -173,7 +173,8 @@ def setup_module(module): for fname in os.listdir(dict_configs_path): dictionaries.append(os.path.join(dict_configs_path, fname)) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True, with_mongo=True, with_redis=True, with_cassandra=True) + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True, + with_mongo=True, with_redis=True, with_cassandra=True) @pytest.fixture(scope="module") @@ -195,7 +196,7 @@ def get_dictionaries(fold, total_folds, all_dicts): chunk_len = int(math.ceil(len(all_dicts) / float(total_folds))) if chunk_len * fold >= len(all_dicts): return [] - return all_dicts[fold * chunk_len : (fold + 1) * chunk_len] + return all_dicts[fold * chunk_len: (fold + 1) * chunk_len] def remove_mysql_dicts(): @@ -225,8 +226,8 @@ def remove_mysql_dicts(): TODO remove this when open ssl will be fixed or thread sanitizer will be suppressed """ - #global DICTIONARIES - #DICTIONARIES = [d for d in DICTIONARIES if not d.name.startswith("MySQL")] + # global DICTIONARIES + # DICTIONARIES = [d for d in DICTIONARIES if not d.name.startswith("MySQL")] @pytest.mark.parametrize("fold", list(range(10))) @@ -281,7 +282,6 @@ def test_simple_dictionaries(started_cluster, fold): @pytest.mark.parametrize("fold", list(range(10))) def test_complex_dictionaries(started_cluster, fold): - if node.is_built_with_thread_sanitizer(): remove_mysql_dicts() diff --git a/tests/integration/test_dictionaries_complex_key_cache_string/test.py b/tests/integration/test_dictionaries_complex_key_cache_string/test.py index 8c676841f16..c8969aee63e 100644 --- a/tests/integration/test_dictionaries_complex_key_cache_string/test.py +++ b/tests/integration/test_dictionaries_complex_key_cache_string/test.py @@ -1,30 +1,40 @@ -import pytest import os + +import pytest from helpers.cluster import ClickHouseCluster + @pytest.fixture(scope="function") def cluster(request): SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) try: if request.param == "memory": - node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/complex_key_cache_string.xml']) + node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml', + 'configs/dictionaries/complex_key_cache_string.xml']) if request.param == "ssd": - node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/ssd_complex_key_cache_string.xml']) + node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml', + 'configs/dictionaries/ssd_complex_key_cache_string.xml']) cluster.start() - node.query("create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id") + node.query( + "create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id") yield cluster finally: cluster.shutdown() + @pytest.mark.parametrize("cluster", ["memory", "ssd"], indirect=True) def test_memory_consumption(cluster): node = cluster.instances['node'] - node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('w' * 8)) - node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('x' * 16)) - node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('y' * 32)) - node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('z' * 64)) + node.query( + "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('w' * 8)) + node.query( + "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('x' * 16)) + node.query( + "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('y' * 32)) + node.query( + "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('z' * 64)) # Fill dictionary node.query("select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)") diff --git a/tests/integration/test_dictionaries_ddl/test.py b/tests/integration/test_dictionaries_ddl/test.py index 6f52dba7deb..3ea64383fbf 100644 --- a/tests/integration/test_dictionaries_ddl/test.py +++ b/tests/integration/test_dictionaries_ddl/test.py @@ -1,17 +1,24 @@ -import pytest import os -from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException -import pymysql import warnings import time +import pymysql +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], user_configs=['configs/user_admin.xml', 'configs/user_default.xml']) -node2 = cluster.add_instance('node2', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], main_configs=['configs/dictionaries/lazy_load.xml', 'configs/allow_remote_node.xml'], user_configs=['configs/user_admin.xml', 'configs/user_default.xml']) -node3 = cluster.add_instance('node3', main_configs=['configs/allow_remote_node.xml'], dictionaries=['configs/dictionaries/dictionary_with_conflict_name.xml', 'configs/dictionaries/conflict_name_dictionary.xml'], user_configs=['configs/user_admin.xml']) +node1 = cluster.add_instance('node1', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], + user_configs=['configs/user_admin.xml', 'configs/user_default.xml']) +node2 = cluster.add_instance('node2', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], + main_configs=['configs/dictionaries/lazy_load.xml', 'configs/allow_remote_node.xml'], + user_configs=['configs/user_admin.xml', 'configs/user_default.xml']) +node3 = cluster.add_instance('node3', main_configs=['configs/allow_remote_node.xml'], + dictionaries=['configs/dictionaries/dictionary_with_conflict_name.xml', + 'configs/dictionaries/conflict_name_dictionary.xml'], + user_configs=['configs/user_admin.xml']) node4 = cluster.add_instance('node4', user_configs=['configs/user_admin.xml', 'configs/config_password.xml']) @@ -22,6 +29,7 @@ def create_mysql_conn(user, password, hostname, port): host=hostname, port=port) + def execute_mysql_query(connection, query): with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -36,15 +44,18 @@ def started_cluster(): cluster.start() for clickhouse in [node1, node2, node3, node4]: clickhouse.query("CREATE DATABASE test", user="admin") - clickhouse.query("CREATE TABLE test.xml_dictionary_table (id UInt64, SomeValue1 UInt8, SomeValue2 String) ENGINE = MergeTree() ORDER BY id", user="admin") - clickhouse.query("INSERT INTO test.xml_dictionary_table SELECT number, number % 23, hex(number) from numbers(1000)", user="admin") + clickhouse.query( + "CREATE TABLE test.xml_dictionary_table (id UInt64, SomeValue1 UInt8, SomeValue2 String) ENGINE = MergeTree() ORDER BY id", + user="admin") + clickhouse.query( + "INSERT INTO test.xml_dictionary_table SELECT number, number % 23, hex(number) from numbers(1000)", + user="admin") yield cluster finally: cluster.shutdown() - @pytest.mark.parametrize("clickhouse,name,layout", [ (node1, 'complex_node1_hashed', 'LAYOUT(COMPLEX_KEY_HASHED())'), (node1, 'complex_node1_cache', 'LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10))'), @@ -54,7 +65,9 @@ def started_cluster(): def test_create_and_select_mysql(started_cluster, clickhouse, name, layout): mysql_conn = create_mysql_conn("root", "clickhouse", "localhost", 3308) execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS clickhouse") - execute_mysql_query(mysql_conn, "CREATE TABLE clickhouse.{} (key_field1 int, key_field2 bigint, value1 text, value2 float, PRIMARY KEY (key_field1, key_field2))".format(name)) + execute_mysql_query(mysql_conn, + "CREATE TABLE clickhouse.{} (key_field1 int, key_field2 bigint, value1 text, value2 float, PRIMARY KEY (key_field1, key_field2))".format( + name)) values = [] for i in range(1000): values.append('(' + ','.join([str(i), str(i * i), str(i) * 5, str(i * 3.14)]) + ')') @@ -81,12 +94,16 @@ def test_create_and_select_mysql(started_cluster, clickhouse, name, layout): """.format(name, name, layout)) for i in range(172, 200): - assert clickhouse.query("SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)) == str(i) * 5 + '\n' - stroka = clickhouse.query("SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)).strip() + assert clickhouse.query( + "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, + i * i)) == str( + i) * 5 + '\n' + stroka = clickhouse.query( + "SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, + i * i)).strip() value = float(stroka) assert int(value) == int(i * 3.14) - for i in range(1000): values.append('(' + ','.join([str(i), str(i * i), str(i) * 3, str(i * 2.718)]) + ')') execute_mysql_query(mysql_conn, "REPLACE INTO clickhouse.{} VALUES ".format(name) + ','.join(values)) @@ -94,8 +111,13 @@ def test_create_and_select_mysql(started_cluster, clickhouse, name, layout): clickhouse.query("SYSTEM RELOAD DICTIONARY 'default.{}'".format(name)) for i in range(172, 200): - assert clickhouse.query("SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)) == str(i) * 3 + '\n' - string = clickhouse.query("SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)).strip() + assert clickhouse.query( + "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, + i * i)) == str( + i) * 3 + '\n' + string = clickhouse.query( + "SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, + i * i)).strip() value = float(string) assert int(value) == int(i * 2.718) @@ -183,6 +205,7 @@ def test_conflicting_name(started_cluster): # old version still works node3.query("select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))") == '17\n' + def test_http_dictionary_restrictions(started_cluster): try: node3.query(""" @@ -199,6 +222,7 @@ def test_http_dictionary_restrictions(started_cluster): except QueryRuntimeException as ex: assert 'is not allowed in config.xml' in str(ex) + def test_file_dictionary_restrictions(started_cluster): try: node3.query(""" @@ -219,7 +243,8 @@ def test_file_dictionary_restrictions(started_cluster): def test_dictionary_with_where(started_cluster): mysql_conn = create_mysql_conn("root", "clickhouse", "localhost", 3308) execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS clickhouse") - execute_mysql_query(mysql_conn, "CREATE TABLE clickhouse.special_table (key_field1 int, value1 text, PRIMARY KEY (key_field1))") + execute_mysql_query(mysql_conn, + "CREATE TABLE clickhouse.special_table (key_field1 int, value1 text, PRIMARY KEY (key_field1))") execute_mysql_query(mysql_conn, "INSERT INTO clickhouse.special_table VALUES (1, 'abcabc'), (2, 'qweqwe')") node1.query(""" diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py index 4505bf73a7c..119bd7c6863 100644 --- a/tests/integration/test_dictionaries_dependency/test.py +++ b/tests/integration/test_dictionaries_dependency/test.py @@ -17,10 +17,10 @@ def start_cluster(): node.query("CREATE DATABASE IF NOT EXISTS ztest") node.query("CREATE TABLE test.source(x UInt64, y UInt64) ENGINE=Log") node.query("INSERT INTO test.source VALUES (5,6)") - - node.query("CREATE DICTIONARY test.dict(x UInt64, y UInt64) PRIMARY KEY x "\ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) "\ - "LAYOUT(FLAT()) LIFETIME(0)") + + node.query("CREATE DICTIONARY test.dict(x UInt64, y UInt64) PRIMARY KEY x " \ + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) " \ + "LAYOUT(FLAT()) LIFETIME(0)") yield cluster finally: @@ -48,14 +48,14 @@ def cleanup_after_test(): def test_dependency_via_implicit_table(node): d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"] for d_name in d_names: - node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict' DB 'test')) "\ + node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \ + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict' DB 'test')) " \ "LAYOUT(FLAT()) LIFETIME(0)".format(d_name)) - + def check(): for d_name in d_names: assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" - + check() # Restart must not break anything. @@ -72,14 +72,14 @@ def test_dependency_via_explicit_table(node): tbl_database, tbl_shortname = tbl_name.split('.') d_name = d_names[i] node.query("CREATE TABLE {}(x UInt64, y UInt64) ENGINE=Dictionary('test.dict')".format(tbl_name)) - node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE '{}' DB '{}')) "\ + node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \ + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE '{}' DB '{}')) " \ "LAYOUT(FLAT()) LIFETIME(0)".format(d_name, tbl_shortname, tbl_database)) - + def check(): for d_name in d_names: assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" - + check() # Restart must not break anything. @@ -93,14 +93,14 @@ def test_dependency_via_dictionary_database(node): d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"] for d_name in d_names: - node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test.dict' DB 'dict_db')) "\ + node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \ + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test.dict' DB 'dict_db')) " \ "LAYOUT(FLAT()) LIFETIME(0)".format(d_name)) - + def check(): for d_name in d_names: assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" - + check() # Restart must not break anything. diff --git a/tests/integration/test_dictionaries_dependency_xml/test.py b/tests/integration/test_dictionaries_dependency_xml/test.py index da1146cd54c..d5453bb4814 100644 --- a/tests/integration/test_dictionaries_dependency_xml/test.py +++ b/tests/integration/test_dictionaries_dependency_xml/test.py @@ -1,13 +1,13 @@ import pytest -import os from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml'] -DICTIONARY_FILES = ['configs/dictionaries/dep_x.xml', 'configs/dictionaries/dep_y.xml', 'configs/dictionaries/dep_z.xml'] +DICTIONARY_FILES = ['configs/dictionaries/dep_x.xml', 'configs/dictionaries/dep_y.xml', + 'configs/dictionaries/dep_z.xml'] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES,) +instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES, ) @pytest.fixture(scope="module") @@ -60,14 +60,14 @@ def test_get_data(started_cluster): query("INSERT INTO test.elements VALUES (3, 'fire', 30, 8)") # Wait for dictionaries to be reloaded. - assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time = 2, retry_count = 10) + assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time=2, retry_count=10) assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" # dep_x and dep_z are updated only when there `intDiv(count(), 4)` is changed. query("INSERT INTO test.elements VALUES (4, 'ether', 404, 0.001)") - assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time = 2, retry_count = 10) + assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time=2, retry_count=10) assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n" diff --git a/tests/integration/test_dictionaries_mysql/test.py b/tests/integration/test_dictionaries_mysql/test.py index 4d2a063e91d..a8e91c94d00 100644 --- a/tests/integration/test_dictionaries_mysql/test.py +++ b/tests/integration/test_dictionaries_mysql/test.py @@ -1,17 +1,13 @@ -import pytest -import os -import time - ## sudo -H pip install PyMySQL import pymysql.cursors - +import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry -CONFIG_FILES = ['configs/dictionaries/mysql_dict1.xml', 'configs/dictionaries/mysql_dict2.xml', 'configs/remote_servers.xml'] +CONFIG_FILES = ['configs/dictionaries/mysql_dict1.xml', 'configs/dictionaries/mysql_dict2.xml', + 'configs/remote_servers.xml'] CONFIG_FILES += ['configs/enable_dictionaries.xml'] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=CONFIG_FILES, with_mysql = True) +instance = cluster.add_instance('instance', main_configs=CONFIG_FILES, with_mysql=True) create_table_mysql_template = """ CREATE TABLE IF NOT EXISTS `test`.`{}` ( @@ -25,10 +21,11 @@ create_clickhouse_dictionary_table_template = """ CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` (`id` UInt64, `value` String) ENGINE = Dictionary({}) """ + @pytest.fixture(scope="module") def started_cluster(): try: - #time.sleep(30) + # time.sleep(30) cluster.start() # Create a MySQL database @@ -66,10 +63,12 @@ def test_load_mysql_dictionaries(started_cluster): # Check number of row assert query("SELECT count() FROM `test`.`dict_table_{}`".format('test' + str(n % 5))).rstrip() == '10000' + def create_mysql_db(mysql_connection, name): with mysql_connection.cursor() as cursor: cursor.execute("CREATE DATABASE IF NOT EXISTS {} DEFAULT CHARACTER SET 'utf8'".format(name)) + def prepare_mysql_table(table_name, index): mysql_connection = get_mysql_conn() @@ -78,17 +77,21 @@ def prepare_mysql_table(table_name, index): # Insert rows using CH query = instance.query - query("INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format(table_name + str(index), table_name + str(index))) + query( + "INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format( + table_name + str(index), table_name + str(index))) assert query("SELECT count() FROM `clickhouse_mysql`.{}".format(table_name + str(index))).rstrip() == '10000' mysql_connection.close() - #Create CH Dictionary tables based on MySQL tables + # Create CH Dictionary tables based on MySQL tables query(create_clickhouse_dictionary_table_template.format(table_name + str(index), 'dict' + str(index))) + def get_mysql_conn(): conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.10', port=3308) return conn + def create_mysql_table(conn, table_name): with conn.cursor() as cursor: cursor.execute(create_table_mysql_template.format(table_name)) diff --git a/tests/integration/test_dictionaries_null_value/test.py b/tests/integration/test_dictionaries_null_value/test.py index c4ad3782498..3c3ef59902f 100644 --- a/tests/integration/test_dictionaries_null_value/test.py +++ b/tests/integration/test_dictionaries_null_value/test.py @@ -1,13 +1,11 @@ import pytest -import os from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV, assert_eq_with_retry ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml'] DICTIONARY_FILES = ['configs/dictionaries/cache.xml'] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES) +instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES) @pytest.fixture(scope="module") @@ -42,4 +40,5 @@ def test_null_value(started_cluster): # Check, that empty null_value interprets as default value assert query("select dictGetUInt64('cache', 'UInt64_', toUInt64(12121212))") == "0\n" - assert query("select toTimeZone(dictGetDateTime('cache', 'DateTime_', toUInt64(12121212)), 'UTC')") == "1970-01-01 00:00:00\n" + assert query( + "select toTimeZone(dictGetDateTime('cache', 'DateTime_', toUInt64(12121212)), 'UTC')") == "1970-01-01 00:00:00\n" diff --git a/tests/integration/test_dictionaries_redis/test.py b/tests/integration/test_dictionaries_redis/test.py index 1588efa3426..385580816e0 100644 --- a/tests/integration/test_dictionaries_redis/test.py +++ b/tests/integration/test_dictionaries_redis/test.py @@ -1,7 +1,6 @@ import os -import pytest -import redis +import pytest from helpers.cluster import ClickHouseCluster from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.external_sources import SourceRedis @@ -22,10 +21,10 @@ KEY_FIELDS = { } KEY_VALUES = { - "simple" : [ + "simple": [ [1], [2] ], - "complex" : [ + "complex": [ [1, 'world'], [2, 'qwerty2'] ] } @@ -76,6 +75,7 @@ LAYOUTS = [ DICTIONARIES = [] + def get_dict(source, layout, fields, suffix_name=''): global dict_configs_path @@ -99,8 +99,10 @@ def setup_module(module): for i, field in enumerate(FIELDS): DICTIONARIES.append([]) sources = [] - sources.append(SourceRedis("RedisSimple", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2, storage_type="simple")) - sources.append(SourceRedis("RedisHash", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2 + 1, storage_type="hash_map")) + sources.append(SourceRedis("RedisSimple", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2, + storage_type="simple")) + sources.append(SourceRedis("RedisHash", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2 + 1, + storage_type="hash_map")) for source in sources: for layout in LAYOUTS: if not source.compatible_with_layout(layout): @@ -118,6 +120,7 @@ def setup_module(module): cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_redis=True) + @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: @@ -134,6 +137,7 @@ def started_cluster(): finally: cluster.shutdown() + @pytest.mark.parametrize("id", range(len(FIELDS))) def test_redis_dictionaries(started_cluster, id): print 'id:', id diff --git a/tests/integration/test_dictionaries_select_all/generate_dictionaries.py b/tests/integration/test_dictionaries_select_all/generate_dictionaries.py index 30a5648fdbe..5c92d0d67e8 100644 --- a/tests/integration/test_dictionaries_select_all/generate_dictionaries.py +++ b/tests/integration/test_dictionaries_select_all/generate_dictionaries.py @@ -1,6 +1,5 @@ -import os -import glob import difflib +import os files = ['key_simple.tsv', 'key_complex_integers.tsv', 'key_complex_mixed.tsv'] @@ -12,7 +11,6 @@ types = [ 'Date', 'DateTime' ] - implicit_defaults = [ '1', '1', '1', '', '-1', '-1', '-1', '-1', diff --git a/tests/integration/test_dictionaries_select_all/test.py b/tests/integration/test_dictionaries_select_all/test.py index 7dc93b2df44..5b8d39a7a63 100644 --- a/tests/integration/test_dictionaries_select_all/test.py +++ b/tests/integration/test_dictionaries_select_all/test.py @@ -1,7 +1,9 @@ -import pytest import os + +import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV, assert_eq_with_retry +from helpers.test_tools import TSV + from generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -20,7 +22,7 @@ def setup_module(module): dictionary_files = generate_dictionaries(os.path.join(SCRIPT_DIR, 'configs/dictionaries'), structure) cluster = ClickHouseCluster(__file__) - instance = cluster.add_instance('instance', main_configs=dictionary_files+['configs/enable_dictionaries.xml']) + instance = cluster.add_instance('instance', main_configs=dictionary_files + ['configs/enable_dictionaries.xml']) test_table = DictionaryTestTable(os.path.join(SCRIPT_DIR, 'configs/dictionaries/source.tsv')) diff --git a/tests/integration/test_dictionaries_update_and_reload/test.py b/tests/integration/test_dictionaries_update_and_reload/test.py index 762fd3adc28..8dfd10da14d 100644 --- a/tests/integration/test_dictionaries_update_and_reload/test.py +++ b/tests/integration/test_dictionaries_update_and_reload/test.py @@ -1,16 +1,18 @@ -import pytest import os import time -from helpers.cluster import ClickHouseCluster + +import pytest from helpers.client import QueryTimeoutExceedException +from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml'] -DICTIONARY_FILES = ['configs/dictionaries/cache_xypairs.xml', 'configs/dictionaries/executable.xml', 'configs/dictionaries/file.xml', 'configs/dictionaries/file.txt', 'configs/dictionaries/slow.xml'] +DICTIONARY_FILES = ['configs/dictionaries/cache_xypairs.xml', 'configs/dictionaries/executable.xml', + 'configs/dictionaries/file.xml', 'configs/dictionaries/file.txt', 'configs/dictionaries/slow.xml'] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES) +instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES) @pytest.fixture(scope="module") @@ -30,24 +32,31 @@ def get_status(dictionary_name): def get_last_exception(dictionary_name): - return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n").replace("\\'", "'") + return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip( + "\n").replace("\\'", "'") def get_loading_start_time(dictionary_name): - s = instance.query("SELECT toTimeZone(loading_start_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") + s = instance.query( + "SELECT toTimeZone(loading_start_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip( + "\n") if s == "1970-01-01 00:00:00": return None return time.strptime(s, "%Y-%m-%d %H:%M:%S") + def get_last_successful_update_time(dictionary_name): - s = instance.query("SELECT toTimeZone(last_successful_update_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") + s = instance.query( + "SELECT toTimeZone(last_successful_update_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip( + "\n") if s == "1970-01-01 00:00:00": return None return time.strptime(s, "%Y-%m-%d %H:%M:%S") def get_loading_duration(dictionary_name): - return float(instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'")) + return float( + instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'")) def replace_in_file_in_container(file_name, what, replace_with): @@ -63,14 +72,14 @@ def test_reload_while_loading(started_cluster): # It's not possible to get a value from the dictionary within 0.5 second, so the following query fails by timeout. with pytest.raises(QueryTimeoutExceedException): - query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout = 0.5) + query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout=0.5) # The dictionary is now loading. assert get_status('slow') == "LOADING" start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') assert duration > 0 - time.sleep(0.5) # Still loading. + time.sleep(0.5) # Still loading. assert get_status('slow') == "LOADING" prev_start_time, prev_duration = start_time, duration start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') @@ -79,14 +88,14 @@ def test_reload_while_loading(started_cluster): # SYSTEM RELOAD DICTIONARY should restart loading. with pytest.raises(QueryTimeoutExceedException): - query("SYSTEM RELOAD DICTIONARY 'slow'", timeout = 0.5) + query("SYSTEM RELOAD DICTIONARY 'slow'", timeout=0.5) assert get_status('slow') == "LOADING" prev_start_time, prev_duration = start_time, duration start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') assert start_time > prev_start_time assert duration < prev_duration - time.sleep(0.5) # Still loading. + time.sleep(0.5) # Still loading. assert get_status('slow') == "LOADING" prev_start_time, prev_duration = start_time, duration start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') @@ -95,7 +104,7 @@ def test_reload_while_loading(started_cluster): # Changing the configuration file should restart loading again. replace_in_file_in_container('/etc/clickhouse-server/config.d/slow.xml', 'sleep 100', 'sleep 0') - time.sleep(5) # Configuration files are reloaded once in 5 seconds. + time.sleep(5) # Configuration files are reloaded once in 5 seconds. # This time loading should finish quickly. assert get_status('slow') == "LOADED" @@ -129,7 +138,7 @@ def test_reload_after_loading(started_cluster): assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "101\n" # SYSTEM RELOAD DICTIONARIES reloads all loaded dictionaries. - time.sleep(1) # see the comment above + time.sleep(1) # see the comment above replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '81', '82') replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '101', '102') query("SYSTEM RELOAD DICTIONARIES") @@ -138,7 +147,7 @@ def test_reload_after_loading(started_cluster): # Configuration files are reloaded and lifetimes are checked automatically once in 5 seconds. # Wait slightly more, to be sure it did reload. - time.sleep(1) # see the comment above + time.sleep(1) # see the comment above replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '82', '83') replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '102', '103') time.sleep(7) @@ -163,7 +172,8 @@ def test_reload_after_fail_by_system_reload(started_cluster): assert get_status("no_file") == "FAILED" # Creating the file source makes the dictionary able to load. - instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), "/etc/clickhouse-server/config.d/no_file.txt") + instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), + "/etc/clickhouse-server/config.d/no_file.txt") query("SYSTEM RELOAD DICTIONARY 'no_file'") query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" assert get_status("no_file") == "LOADED" @@ -192,7 +202,8 @@ def test_reload_after_fail_by_timer(started_cluster): assert get_status("no_file_2") == "FAILED" # Creating the file source makes the dictionary able to load. - instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), "/etc/clickhouse-server/config.d/no_file_2.txt") + instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), + "/etc/clickhouse-server/config.d/no_file_2.txt") time.sleep(6); query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" assert get_status("no_file_2") == "LOADED" diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py index b6b742c1de8..1266c37dcd3 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py @@ -1,9 +1,8 @@ from __future__ import print_function -import pytest -import time -import os -from contextlib import contextmanager +import time + +import pytest from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseKiller from helpers.network import PartitionManager @@ -14,6 +13,7 @@ dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/cache_ints_dictionary.xml']) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -32,6 +32,7 @@ def started_cluster(): finally: cluster.shutdown() + # @pytest.mark.skip(reason="debugging") def test_default_reading(started_cluster): assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" @@ -39,14 +40,22 @@ def test_default_reading(started_cluster): # Key 0 is not in dictionary, so default value will be returned def test_helper(): - assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i8', toUInt64(13), toInt8(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i16', toUInt64(13), toInt16(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i32', toUInt64(13), toInt32(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i64', toUInt64(13), toInt64(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u8', toUInt64(13), toUInt8(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u16', toUInt64(13), toUInt16(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u32', toUInt64(13), toUInt32(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u64', toUInt64(13), toUInt64(42));").rstrip() + assert '42' == main_node.query( + "select dictGetOrDefault('anime_dict', 'i8', toUInt64(13), toInt8(42));").rstrip() + assert '42' == main_node.query( + "select dictGetOrDefault('anime_dict', 'i16', toUInt64(13), toInt16(42));").rstrip() + assert '42' == main_node.query( + "select dictGetOrDefault('anime_dict', 'i32', toUInt64(13), toInt32(42));").rstrip() + assert '42' == main_node.query( + "select dictGetOrDefault('anime_dict', 'i64', toUInt64(13), toInt64(42));").rstrip() + assert '42' == main_node.query( + "select dictGetOrDefault('anime_dict', 'u8', toUInt64(13), toUInt8(42));").rstrip() + assert '42' == main_node.query( + "select dictGetOrDefault('anime_dict', 'u16', toUInt64(13), toUInt16(42));").rstrip() + assert '42' == main_node.query( + "select dictGetOrDefault('anime_dict', 'u32', toUInt64(13), toUInt32(42));").rstrip() + assert '42' == main_node.query( + "select dictGetOrDefault('anime_dict', 'u64', toUInt64(13), toUInt64(42));").rstrip() test_helper() @@ -61,4 +70,3 @@ def test_default_reading(started_cluster): time.sleep(3) test_helper() - diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py index d6517379086..2aa6fb448ca 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py @@ -1,10 +1,11 @@ from __future__ import print_function -import pytest + import os import random import string import time +import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV @@ -12,13 +13,16 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) -main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml','configs/dictionaries/cache_ints_dictionary.xml','configs/dictionaries/cache_strings_default_settings.xml']) +main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml', + 'configs/dictionaries/cache_ints_dictionary.xml', + 'configs/dictionaries/cache_strings_default_settings.xml']) def get_random_string(string_length=8): alphabet = string.ascii_letters + string.digits return ''.join((random.choice(alphabet) for _ in range(string_length))) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -31,13 +35,15 @@ def started_cluster(): ENGINE = Memory; """) - values_to_insert = ", ".join(["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)]) + values_to_insert = ", ".join( + ["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)]) dictionary_node.query("INSERT INTO test.strings VALUES {}".format(values_to_insert)) yield cluster finally: cluster.shutdown() + # @pytest.mark.skip(reason="debugging") def test_return_real_values(started_cluster): assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py index 44698b380e3..9de0b3be4eb 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py @@ -1,18 +1,18 @@ from __future__ import print_function -import pytest -import time -import os -from contextlib import contextmanager +import time + +import pytest from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseKiller from helpers.network import PartitionManager -from helpers.network import PartitionManagerDisabler cluster = ClickHouseCluster(__file__) dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) -main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/cache_ints_dictionary.xml']) +main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml', + 'configs/dictionaries/cache_ints_dictionary.xml']) + @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py index e0b546aae24..31f0e469555 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py @@ -1,9 +1,8 @@ from __future__ import print_function -import pytest -import time -import os -from contextlib import contextmanager +import time + +import pytest from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseKiller from helpers.network import PartitionManager @@ -11,7 +10,9 @@ from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) -main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml','configs/dictionaries/cache_ints_dictionary.xml']) +main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml', + 'configs/dictionaries/cache_ints_dictionary.xml']) + @pytest.fixture(scope="module") def started_cluster(): @@ -31,19 +32,28 @@ def started_cluster(): finally: cluster.shutdown() + # @pytest.mark.skip(reason="debugging") def test_simple_dict_get_or_default(started_cluster): assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" def test_helper(): - assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i8', toUInt64(5), toInt8(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i16', toUInt64(5), toInt16(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i32', toUInt64(5), toInt32(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i64', toUInt64(5), toInt64(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u8', toUInt64(5), toUInt8(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u16', toUInt64(5), toUInt16(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u32', toUInt64(5), toUInt32(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u64', toUInt64(5), toUInt64(42));").rstrip() + assert '5' == main_node.query( + "select dictGetOrDefault('anime_dict', 'i8', toUInt64(5), toInt8(42));").rstrip() + assert '5' == main_node.query( + "select dictGetOrDefault('anime_dict', 'i16', toUInt64(5), toInt16(42));").rstrip() + assert '5' == main_node.query( + "select dictGetOrDefault('anime_dict', 'i32', toUInt64(5), toInt32(42));").rstrip() + assert '5' == main_node.query( + "select dictGetOrDefault('anime_dict', 'i64', toUInt64(5), toInt64(42));").rstrip() + assert '5' == main_node.query( + "select dictGetOrDefault('anime_dict', 'u8', toUInt64(5), toUInt8(42));").rstrip() + assert '5' == main_node.query( + "select dictGetOrDefault('anime_dict', 'u16', toUInt64(5), toUInt16(42));").rstrip() + assert '5' == main_node.query( + "select dictGetOrDefault('anime_dict', 'u32', toUInt64(5), toUInt32(42));").rstrip() + assert '5' == main_node.query( + "select dictGetOrDefault('anime_dict', 'u64', toUInt64(5), toUInt64(42));").rstrip() test_helper() diff --git a/tests/integration/test_dictionary_custom_settings/http_server.py b/tests/integration/test_dictionary_custom_settings/http_server.py index c7920a9024d..20487ccf447 100644 --- a/tests/integration/test_dictionary_custom_settings/http_server.py +++ b/tests/integration/test_dictionary_custom_settings/http_server.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import argparse -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +import csv import socket import ssl -import csv +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer # Decorator used to see if authentication works for external dictionary who use a HTTP source. @@ -15,6 +15,7 @@ def check_auth(fn): req.send_response(401) else: fn(req) + return wrapper @@ -37,7 +38,7 @@ def start_server(server_address, data_path, schema, cert_path, address_family): self.send_header('Content-type', 'text/csv') self.end_headers() - def __send_data(self, only_ids = None): + def __send_data(self, only_ids=None): with open(data_path, 'r') as fl: reader = csv.reader(fl, delimiter='\t') for row in reader: diff --git a/tests/integration/test_dictionary_custom_settings/test.py b/tests/integration/test_dictionary_custom_settings/test.py index 4cff5cb3e4a..022822c8a80 100644 --- a/tests/integration/test_dictionary_custom_settings/test.py +++ b/tests/integration/test_dictionary_custom_settings/test.py @@ -1,6 +1,6 @@ import os -import pytest +import pytest from helpers.cluster import ClickHouseCluster ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml'] @@ -13,7 +13,8 @@ DICTIONARY_FILES = [ ] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('node', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES) +instance = cluster.add_instance('node', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES) + def prepare(): node = instance diff --git a/tests/integration/test_dictionary_ddl_on_cluster/test.py b/tests/integration/test_dictionary_ddl_on_cluster/test.py index 6239fda1752..feca1532974 100644 --- a/tests/integration/test_dictionary_ddl_on_cluster/test.py +++ b/tests/integration/test_dictionary_ddl_on_cluster/test.py @@ -1,19 +1,24 @@ -import time import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True) -ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True) -ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True) -ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True) +ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], + with_zookeeper=True) +ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], + with_zookeeper=True) +ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], + with_zookeeper=True) +ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], + with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - ch1.query("CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key") + ch1.query( + "CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key") yield cluster finally: @@ -27,7 +32,6 @@ def test_dictionary_ddl_on_cluster(started_cluster): for num, node in enumerate([ch1, ch2, ch3, ch4]): node.query("insert into sometbl values ({}, '{}')".format(num, node.name)) - ch1.query( """ CREATE DICTIONARY somedict ON CLUSTER 'cluster' ( @@ -42,7 +46,8 @@ def test_dictionary_ddl_on_cluster(started_cluster): for num, node in enumerate([ch1, ch2, ch3, ch4]): assert node.query("SELECT count() from sometbl") == "1\n" - assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' + assert node.query( + "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' ch1.query("DETACH DICTIONARY default.somedict ON CLUSTER 'cluster'") @@ -54,8 +59,8 @@ def test_dictionary_ddl_on_cluster(started_cluster): for num, node in enumerate([ch1, ch2, ch3, ch4]): assert node.query("SELECT count() from sometbl") == "1\n" - assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' - + assert node.query( + "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' for num, node in enumerate([ch1, ch2, ch3, ch4]): node.query("ALTER TABLE sometbl UPDATE value = 'new_key' WHERE 1") @@ -63,7 +68,8 @@ def test_dictionary_ddl_on_cluster(started_cluster): ch1.query("SYSTEM RELOAD DICTIONARY ON CLUSTER 'cluster' `default.somedict`") for num, node in enumerate([ch1, ch2, ch3, ch4]): - assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == 'new_key' + '\n' + assert node.query( + "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == 'new_key' + '\n' ch1.query("DROP DICTIONARY default.somedict ON CLUSTER 'cluster'") diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index dab6758cbd6..ad31be4284a 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -27,27 +27,32 @@ def create_entities(): @pytest.fixture(autouse=True) def drop_entities(): - instance.query("DROP USER IF EXISTS u1, u2") - instance.query("DROP ROLE IF EXISTS rx, ry") + instance.query("DROP USER IF EXISTS u1, u2") + instance.query("DROP ROLE IF EXISTS rx, ry") instance.query("DROP ROW POLICY IF EXISTS p ON mydb.mytable") instance.query("DROP QUOTA IF EXISTS q") instance.query("DROP SETTINGS PROFILE IF EXISTS s1, s2") - + def test_create(): create_entities() def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" - assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n" - assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" - assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n" + assert instance.query( + "SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n" + assert instance.query( + "SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" + assert instance.query( + "SHOW CREATE QUOTA q") == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n" assert instance.query("SHOW GRANTS FOR u1") == "" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx TO u2\n" assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s1\n" assert instance.query("SHOW GRANTS FOR rx") == "" - assert instance.query("SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n" - assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" + assert instance.query( + "SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n" + assert instance.query( + "SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" check() instance.restart_clickhouse() # Check persistency @@ -69,15 +74,18 @@ def test_alter(): def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" - assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n" + assert instance.query( + "SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n" assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n" assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n" assert instance.query("SHOW CREATE ROLE ry") == "CREATE ROLE ry\n" assert instance.query("SHOW GRANTS FOR rx") == "GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION\n" assert instance.query("SHOW GRANTS FOR ry") == "GRANT rx TO ry WITH ADMIN OPTION\n" - assert instance.query("SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n" - assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" + assert instance.query( + "SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n" + assert instance.query( + "SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" check() instance.restart_clickhouse() # Check persistency @@ -98,7 +106,8 @@ def test_drop(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n" assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2\n" assert "There is no user `u2`" in instance.query_and_get_error("SHOW CREATE USER u2") - assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error("SHOW CREATE ROW POLICY p ON mydb.mytable") + assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error( + "SHOW CREATE ROW POLICY p ON mydb.mytable") assert "There is no quota `q`" in instance.query_and_get_error("SHOW CREATE QUOTA q") check() diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index 3c65315a7e3..a97b90af27d 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -1,15 +1,14 @@ - import pytest from helpers.cluster import ClickHouseCluster disk_types = { - "default" : "local", - "disk_s3" : "s3", - "disk_memory" : "memory", + "default": "local", + "disk_s3": "s3", + "disk_memory": "memory", } -@pytest.fixture(scope="module") +@pytest.fixture(scope="module") def cluster(): try: cluster = ClickHouseCluster(__file__) @@ -19,19 +18,20 @@ def cluster(): finally: cluster.shutdown() + def test_different_types(cluster): node = cluster.instances["node"] responce = node.query("SELECT * FROM system.disks") disks = responce.split("\n") for disk in disks: - if disk == '': # skip empty line (after split at last position) + if disk == '': # skip empty line (after split at last position) continue fields = disk.split("\t") assert len(fields) >= 6 assert disk_types.get(fields[0], "UNKNOWN") == fields[5] + def test_select_by_type(cluster): node = cluster.instances["node"] for name, disk_type in disk_types.items(): assert node.query("SELECT name FROM system.disks WHERE type='" + disk_type + "'") == name + "\n" - diff --git a/tests/integration/test_distributed_backward_compatability/test.py b/tests/integration/test_distributed_backward_compatability/test.py index 7ce7edb2860..eb18019c8df 100644 --- a/tests/integration/test_distributed_backward_compatability/test.py +++ b/tests/integration/test_distributed_backward_compatability/test.py @@ -1,16 +1,14 @@ import pytest -import time from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager -from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) -node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) +node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server', + tag='19.17.8.54', stay_alive=True, with_installed_binary=True) node_new = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -22,14 +20,17 @@ def started_cluster(): node_old.query("INSERT INTO local_table VALUES (1, 'node1')") node_new.query("INSERT INTO local_table VALUES (2, 'node2')") - node_old.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") - node_new.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") + node_old.query( + "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") + node_new.query( + "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") yield cluster finally: cluster.shutdown() + def test_distributed_in_tuple(started_cluster): query1 = "SELECT count() FROM distributed WHERE (id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))" query2 = "SELECT sum((id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))) FROM distributed" diff --git a/tests/integration/test_distributed_ddl/cluster.py b/tests/integration/test_distributed_ddl/cluster.py index 258478de990..efd6ce7e65c 100644 --- a/tests/integration/test_distributed_ddl/cluster.py +++ b/tests/integration/test_distributed_ddl/cluster.py @@ -17,18 +17,21 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): def prepare(self, replace_hostnames_with_ips=True): try: - main_configs_files = ["clusters.xml", "zookeeper_session_timeout.xml", "macro.xml", "query_log.xml","ddl.xml"] + main_configs_files = ["clusters.xml", "zookeeper_session_timeout.xml", "macro.xml", "query_log.xml", + "ddl.xml"] main_configs = [os.path.join(self.test_config_dir, "config.d", f) for f in main_configs_files] - user_configs = [os.path.join(self.test_config_dir, "users.d", f) for f in ["restricted_user.xml", "query_log.xml"]] + user_configs = [os.path.join(self.test_config_dir, "users.d", f) for f in + ["restricted_user.xml", "query_log.xml"]] if self.test_config_dir == "configs_secure": - main_configs += [os.path.join(self.test_config_dir, f) for f in ["server.crt", "server.key", "dhparam.pem", "config.d/ssl_conf.xml"]] + main_configs += [os.path.join(self.test_config_dir, f) for f in + ["server.crt", "server.key", "dhparam.pem", "config.d/ssl_conf.xml"]] for i in xrange(4): self.add_instance( - 'ch{}'.format(i+1), + 'ch{}'.format(i + 1), main_configs=main_configs, user_configs=user_configs, - macros={"layer": 0, "shard": i/2 + 1, "replica": i%2 + 1}, + macros={"layer": 0, "shard": i / 2 + 1, "replica": i % 2 + 1}, with_zookeeper=True) self.start() @@ -40,8 +43,12 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): # Select sacrifice instance to test CONNECTION_LOSS and server fail on it sacrifice = self.instances['ch4'] self.pm_random_drops = PartitionManager() - self.pm_random_drops._add_rule({'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) - self.pm_random_drops._add_rule({'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) + self.pm_random_drops._add_rule( + {'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, + 'action': 'REJECT --reject-with tcp-reset'}) + self.pm_random_drops._add_rule( + {'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, + 'action': 'REJECT --reject-with tcp-reset'}) # Initialize databases and service tables instance = self.instances['ch1'] @@ -67,7 +74,7 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): num_hosts = len(self.instances) M = TSV.toMat(tsv_content) - hosts = [(l[0], l[1]) for l in M] # (host, port) + hosts = [(l[0], l[1]) for l in M] # (host, port) codes = [l[2] for l in M] messages = [l[3] for l in M] @@ -88,14 +95,17 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): for inst_name in instances_to_replace: inst = self.instances[inst_name] - self.instances[inst_name].exec_in_container(['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'], environment={"NEW_CONFIG": clusters_config}, privileged=True) + self.instances[inst_name].exec_in_container( + ['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'], + environment={"NEW_CONFIG": clusters_config}, privileged=True) # print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"]) @staticmethod def ddl_check_there_are_no_dublicates(instance): query = "SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)" rows = instance.query(query) - assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name, instance.ip_address, query) + assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name, + instance.ip_address, query) @staticmethod def insert_reliable(instance, query_insert): diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index f5dbe0ef8d2..08027fa13ca 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -41,7 +41,8 @@ def test_default_database(test_cluster): test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV") - test_cluster.ddl_check_query(instance, "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null") + test_cluster.ddl_check_query(instance, + "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null") contents = instance.query("SELECT hostName() AS h, database FROM all_tables WHERE name = 'null' ORDER BY h") assert TSV(contents) == TSV("ch1\tdefault\nch2\ttest2\nch3\tdefault\nch4\ttest2\n") @@ -52,13 +53,18 @@ def test_default_database(test_cluster): def test_create_view(test_cluster): instance = test_cluster.instances['ch3'] - test_cluster.ddl_check_query(instance, "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") - test_cluster.ddl_check_query(instance, "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV") + test_cluster.ddl_check_query(instance, + "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") + test_cluster.ddl_check_query(instance, + "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV") test_cluster.ddl_check_query(instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV") + test_cluster.ddl_check_query(instance, + "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV") - test_cluster.ddl_check_query(instance, "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory") - test_cluster.ddl_check_query(instance, "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV") + test_cluster.ddl_check_query(instance, + "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory") + test_cluster.ddl_check_query(instance, + "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV") test_cluster.ddl_check_query(instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'") @@ -69,7 +75,8 @@ def test_on_server_fail(test_cluster): test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'") kill_instance.get_docker_handle().stop() - request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", timeout=30) + request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", + timeout=30) kill_instance.get_docker_handle().start() test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'") @@ -78,7 +85,8 @@ def test_on_server_fail(test_cluster): test_cluster.check_all_hosts_successfully_executed(request.get_answer()) # And check query artefacts - contents = instance.query("SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h") + contents = instance.query( + "SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h") assert TSV(contents) == TSV("ch1\nch2\nch3\nch4\n") test_cluster.ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'") @@ -98,11 +106,11 @@ def _test_on_connection_losses(test_cluster, zk_timeout): def test_on_connection_loss(test_cluster): - _test_on_connection_losses(test_cluster, 1.5) # connection loss will occur only (3 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 1.5) # connection loss will occur only (3 sec ZK timeout in config) def test_on_session_expired(test_cluster): - _test_on_connection_losses(test_cluster, 4) # session should be expired (3 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 4) # session should be expired (3 sec ZK timeout in config) def test_simple_alters(test_cluster): @@ -127,28 +135,31 @@ ENGINE = Distributed('{cluster}', default, merge, i) for i in xrange(0, 4, 2): k = (i / 2) * 2 - test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k+1)) - - assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)])) + test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k + 1)) + assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV( + ''.join(['{}\n'.format(x) for x in xrange(4)])) time.sleep(5) test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' MODIFY COLUMN i Int64") time.sleep(5) - test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' ADD COLUMN s String DEFAULT toString(i) FORMAT TSV") - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) + test_cluster.ddl_check_query(instance, + "ALTER TABLE merge ON CLUSTER '{cluster}' ADD COLUMN s String DEFAULT toString(i) FORMAT TSV") + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( + ''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)])) for i in xrange(0, 4, 2): k = (i / 2) * 2 + 4 - test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k+1)) - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)])) + test_cluster.instances['ch{}'.format(i + 1)].query( + "INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k + 1)) + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( + ''.join(['{}\t{}\n'.format(x, x) for x in xrange(8)])) test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' DETACH PARTITION 197002") - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( + ''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)])) test_cluster.ddl_check_query(instance, "DROP TABLE merge ON CLUSTER '{cluster}'") test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER '{cluster}'") @@ -160,9 +171,11 @@ def test_macro(test_cluster): test_cluster.ddl_check_query(instance, "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory") for i in xrange(4): - test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO tab VALUES ({})".format(i)) + test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], + "INSERT INTO tab VALUES ({})".format(i)) - test_cluster.ddl_check_query(instance, "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)") + test_cluster.ddl_check_query(instance, + "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)") assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') assert TSV(test_cluster.instances['ch3'].query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') @@ -197,22 +210,27 @@ def test_allowed_databases(test_cluster): instance.query("CREATE DATABASE IF NOT EXISTS db1 ON CLUSTER cluster") instance.query("CREATE DATABASE IF NOT EXISTS db2 ON CLUSTER cluster") - instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) + instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory", + settings={"user": "restricted_user"}) with pytest.raises(Exception): - instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) + instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory", + settings={"user": "restricted_user"}) with pytest.raises(Exception): - instance.query("CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) + instance.query("CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory", + settings={"user": "restricted_user"}) with pytest.raises(Exception): - instance.query("DROP DATABASE db2 ON CLUSTER cluster", settings={"user" : "restricted_user"}) + instance.query("DROP DATABASE db2 ON CLUSTER cluster", settings={"user": "restricted_user"}) + + instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user": "restricted_user"}) - instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user" : "restricted_user"}) def test_kill_query(test_cluster): instance = test_cluster.instances['ch3'] test_cluster.ddl_check_query(instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV") + def test_detach_query(test_cluster): instance = test_cluster.instances['ch3'] @@ -226,21 +244,25 @@ def test_optimize_query(test_cluster): instance = test_cluster.instances['ch3'] test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV") - test_cluster.ddl_check_query(instance, "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)") + test_cluster.ddl_check_query(instance, + "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)") test_cluster.ddl_check_query(instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV") def test_create_as_select(test_cluster): instance = test_cluster.instances['ch2'] - test_cluster.ddl_check_query(instance, "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)") + test_cluster.ddl_check_query(instance, + "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)") assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV("1\n2\n") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster") def test_create_reserved(test_cluster): instance = test_cluster.instances['ch2'] - test_cluster.ddl_check_query(instance, "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)") - test_cluster.ddl_check_query(instance, "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)") + test_cluster.ddl_check_query(instance, + "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)") + test_cluster.ddl_check_query(instance, + "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_reserved ON CLUSTER cluster") @@ -248,11 +270,12 @@ def test_create_reserved(test_cluster): def test_rename(test_cluster): instance = test_cluster.instances['ch1'] rules = test_cluster.pm_random_drops.pop_rules() - test_cluster.ddl_check_query(instance, "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)") - test_cluster.ddl_check_query(instance, "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") + test_cluster.ddl_check_query(instance, + "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)") + test_cluster.ddl_check_query(instance, + "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") test_cluster.ddl_check_query(instance, "RENAME TABLE rename_new TO rename ON CLUSTER cluster;") - for i in range(10): instance.query("insert into rename (id) values ({})".format(i)) @@ -261,10 +284,13 @@ def test_rename(test_cluster): # so ddl query will always fail on some replicas even if query was actually executed by leader # Also such inconsistency in cluster configuration may lead to query duplication if leader suddenly changed # because path of lock in zk contains shard name, which is list of host names of replicas - instance.query("ALTER TABLE rename_shard ON CLUSTER cluster MODIFY COLUMN sid String DEFAULT concat('new', toString(id))", ignore_error=True) + instance.query( + "ALTER TABLE rename_shard ON CLUSTER cluster MODIFY COLUMN sid String DEFAULT concat('new', toString(id))", + ignore_error=True) time.sleep(1) - test_cluster.ddl_check_query(instance, "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") + test_cluster.ddl_check_query(instance, + "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") instance.query("system stop distributed sends rename") @@ -283,36 +309,43 @@ def test_rename(test_cluster): # system stop distributed sends does not affect inserts into local shard, # so some ids in range (10, 20) will be inserted into rename_shard assert instance.query("select count(id), sum(id) from rename").rstrip() == "25\t360" - #assert instance.query("select count(id), sum(id) from rename").rstrip() == "20\t290" + # assert instance.query("select count(id), sum(id) from rename").rstrip() == "20\t290" assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "15\t115" - #assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "10\t45" + # assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "10\t45" assert instance.query("select count(id), sum(id) from rename where sid like 'new%'").rstrip() == "10\t245" test_cluster.pm_random_drops.push_rules(rules) + def test_socket_timeout(test_cluster): instance = test_cluster.instances['ch1'] # queries should not fail with "Timeout exceeded while reading from socket" in case of EINTR caused by query profiler for i in range(0, 100): instance.query("select hostName() as host, count() from cluster('cluster', 'system', 'settings') group by host") + def test_replicated_without_arguments(test_cluster): rules = test_cluster.pm_random_drops.pop_rules() instance = test_cluster.instances['ch1'] test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic", settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) - test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", + test_cluster.ddl_check_query(instance, + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster") - test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", + test_cluster.ddl_check_query(instance, + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) test_cluster.ddl_check_query(instance, "RENAME TABLE test_atomic.rmt TO test_atomic.rmt_renamed ON CLUSTER cluster") - test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", + test_cluster.ddl_check_query(instance, + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) - test_cluster.ddl_check_query(instance, "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, + "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster") test_cluster.pm_random_drops.push_rules(rules) + if __name__ == '__main__': with contextmanager(test_cluster)() as ctx_cluster: - for name, instance in ctx_cluster.instances.items(): - print name, instance.ip_address - raw_input("Cluster created, press any key to destroy...") + for name, instance in ctx_cluster.instances.items(): + print name, instance.ip_address + raw_input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_distributed_ddl/test_replicated_alter.py b/tests/integration/test_distributed_ddl/test_replicated_alter.py index e66e731cbb1..840803f61ef 100644 --- a/tests/integration/test_distributed_ddl/test_replicated_alter.py +++ b/tests/integration/test_distributed_ddl/test_replicated_alter.py @@ -1,6 +1,7 @@ import os import sys import time + import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -60,29 +61,34 @@ ENGINE = Distributed(cluster, default, merge_for_alter, i) for i in xrange(4): k = (i / 2) * 2 - test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k+1)) + test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], + "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k + 1)) test_cluster.sync_replicas("merge_for_alter") - assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)])) - + assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV( + ''.join(['{}\n'.format(x) for x in xrange(4)])) test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64") - test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)") - - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) + test_cluster.ddl_check_query(instance, + "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)") + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( + ''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)])) for i in xrange(4): k = (i / 2) * 2 + 4 - test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k+1)) + test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], + "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k + 1)) test_cluster.sync_replicas("merge_for_alter") - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)])) + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( + ''.join(['{}\t{}\n'.format(x, x) for x in xrange(8)])) test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002") - assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) + assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( + ''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)])) test_cluster.ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster") diff --git a/tests/integration/test_distributed_ddl_on_cross_replication/test.py b/tests/integration/test_distributed_ddl_on_cross_replication/test.py index 4aa64362060..16238f0326d 100644 --- a/tests/integration/test_distributed_ddl_on_cross_replication/test.py +++ b/tests/integration/test_distributed_ddl_on_cross_replication/test.py @@ -1,4 +1,3 @@ -import time import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry @@ -12,6 +11,7 @@ node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml' node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, macros={"shard": 3, "replica": 1, "shard_bk": 2, "replica_bk": 2}) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -49,7 +49,8 @@ def started_cluster(): 2017-06-17 31 2 ''' - node1.query("INSERT INTO replica_1.replicated FORMAT TSV", stdin=to_insert, settings={"insert_distributed_sync" : 1}) + node1.query("INSERT INTO replica_1.replicated FORMAT TSV", stdin=to_insert, + settings={"insert_distributed_sync": 1}) yield cluster finally: @@ -64,7 +65,8 @@ def test_alter_ddl(started_cluster): WHERE part_key='2017-06-16'") node1.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) - assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", '3') + assert_eq_with_retry(node1, + "SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", '3') node1.query("ALTER TABLE replica_1.replicated_local \ ON CLUSTER cross_3shards_2replicas DELETE WHERE shard_id >=3;") @@ -75,4 +77,3 @@ def test_alter_ddl(started_cluster): node2.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated", '0') - diff --git a/tests/integration/test_distributed_ddl_password/test.py b/tests/integration/test_distributed_ddl_password/test.py index 961b60857dd..0c061914497 100644 --- a/tests/integration/test_distributed_ddl_password/test.py +++ b/tests/integration/test_distributed_ddl_password/test.py @@ -1,17 +1,21 @@ -import time import pytest +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry -from helpers.client import QueryRuntimeException - cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node5 = cluster.add_instance('node5', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node6 = cluster.add_instance('node6', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) +node5 = cluster.add_instance('node5', main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) +node6 = cluster.add_instance('node6', main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) @pytest.fixture(scope="module") @@ -21,27 +25,29 @@ def start_cluster(): for node, shard in [(node1, 1), (node2, 1), (node3, 2), (node4, 2), (node5, 3), (node6, 3)]: node.query( - ''' - CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') - PARTITION BY date - ORDER BY id - '''.format(shard=shard, replica=node.name), settings={"password": "clickhouse"}) + ''' + CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') + PARTITION BY date + ORDER BY id + '''.format(shard=shard, replica=node.name), settings={"password": "clickhouse"}) yield cluster finally: cluster.shutdown() + def test_truncate(start_cluster): - node1.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", settings={"password": "clickhouse"}) + node1.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", + settings={"password": "clickhouse"}) assert node1.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" node2.query("system sync replica test_table", settings={"password": "clickhouse"}) assert node2.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" - - node3.query("insert into test_table values ('2019-02-16', 1, 2), ('2019-02-16', 2, 3), ('2019-02-16', 3, 4)", settings={"password": "clickhouse"}) + node3.query("insert into test_table values ('2019-02-16', 1, 2), ('2019-02-16', 2, 3), ('2019-02-16', 3, 4)", + settings={"password": "clickhouse"}) assert node3.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" node4.query("system sync replica test_table", settings={"password": "clickhouse"}) @@ -55,11 +61,15 @@ def test_truncate(start_cluster): node2.query("drop table test_table on cluster 'awesome_cluster'", settings={"password": "clickhouse"}) for node in [node1, node2, node3, node4]: - assert_eq_with_retry(node, "select count(*) from system.tables where name='test_table'", "0", settings={"password": "clickhouse"}) + assert_eq_with_retry(node, "select count(*) from system.tables where name='test_table'", "0", + settings={"password": "clickhouse"}) + def test_alter(start_cluster): - node5.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", settings={"password": "clickhouse"}) - node6.query("insert into test_table values ('2019-02-15', 4, 2), ('2019-02-15', 5, 3), ('2019-02-15', 6, 4)", settings={"password": "clickhouse"}) + node5.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", + settings={"password": "clickhouse"}) + node6.query("insert into test_table values ('2019-02-15', 4, 2), ('2019-02-15', 5, 3), ('2019-02-15', 6, 4)", + settings={"password": "clickhouse"}) node5.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"}) node6.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"}) @@ -75,24 +85,30 @@ def test_alter(start_cluster): assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) - node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DETACH PARTITION '2019-02-15'", settings={"password": "clickhouse"}) + node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DETACH PARTITION '2019-02-15'", + settings={"password": "clickhouse"}) assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) with pytest.raises(QueryRuntimeException): - node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' ATTACH PARTITION '2019-02-15'", settings={"password": "clickhouse"}) + node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' ATTACH PARTITION '2019-02-15'", + settings={"password": "clickhouse"}) node5.query("ALTER TABLE test_table ATTACH PARTITION '2019-02-15'", settings={"password": "clickhouse"}) assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) - node5.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' MODIFY COLUMN dummy String", settings={"password": "clickhouse"}) + node5.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' MODIFY COLUMN dummy String", + settings={"password": "clickhouse"}) - assert_eq_with_retry(node5, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", settings={"password": "clickhouse"}) - assert_eq_with_retry(node6, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", settings={"password": "clickhouse"}) + assert_eq_with_retry(node5, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", + settings={"password": "clickhouse"}) + assert_eq_with_retry(node6, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", + settings={"password": "clickhouse"}) - node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DROP PARTITION '2019-02-15'", settings={"password": "clickhouse"}) + node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DROP PARTITION '2019-02-15'", + settings={"password": "clickhouse"}) assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) diff --git a/tests/integration/test_distributed_format/test.py b/tests/integration/test_distributed_format/test.py index 251ec766b74..7658814a720 100644 --- a/tests/integration/test_distributed_format/test.py +++ b/tests/integration/test_distributed_format/test.py @@ -1,12 +1,6 @@ -import time import pytest from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool -from helpers.client import QueryRuntimeException, QueryTimeoutExceedException - -from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=['configs/remote_servers.xml']) @@ -27,10 +21,13 @@ def started_cluster(): finally: cluster.shutdown() + @cluster_param def test_single_file(started_cluster, cluster): - node.query("create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) - node.query("insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"}) + node.query( + "create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) + node.query("insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')", + settings={"use_compact_format_in_distributed_parts_names": "1"}) query = "select * from file('/var/lib/clickhouse/data/test/distr_1/shard1_replica1/1.bin', 'Distributed')" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) @@ -48,9 +45,12 @@ def test_single_file(started_cluster, cluster): @cluster_param def test_two_files(started_cluster, cluster): - node.query("create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) - node.query("insert into test.distr_2 values (0, '_'), (1, 'a')", settings={"use_compact_format_in_distributed_parts_names": "1"}) - node.query("insert into test.distr_2 values (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"}) + node.query( + "create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) + node.query("insert into test.distr_2 values (0, '_'), (1, 'a')", + settings={"use_compact_format_in_distributed_parts_names": "1"}) + node.query("insert into test.distr_2 values (2, 'bb'), (3, 'ccc')", + settings={"use_compact_format_in_distributed_parts_names": "1"}) query = "select * from file('/var/lib/clickhouse/data/test/distr_2/shard1_replica1/{1,2,3,4}.bin', 'Distributed') order by x" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) @@ -68,7 +68,8 @@ def test_two_files(started_cluster, cluster): @cluster_param def test_single_file_old(started_cluster, cluster): - node.query("create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) + node.query( + "create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) node.query("insert into test.distr_3 values (1, 'a'), (2, 'bb'), (3, 'ccc')") query = "select * from file('/var/lib/clickhouse/data/test/distr_3/default@not_existing:9000/1.bin', 'Distributed')" diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py index c538dc7fb3a..b227c57fb04 100644 --- a/tests/integration/test_distributed_load_balancing/test.py +++ b/tests/integration/test_distributed_load_balancing/test.py @@ -3,8 +3,8 @@ # pylint: disable=line-too-long import uuid -import pytest +import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) @@ -14,7 +14,8 @@ n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml']) n3 = cluster.add_instance('n3', main_configs=['configs/remote_servers.xml']) nodes = len(cluster.instances) -queries = nodes*5 +queries = nodes * 5 + def bootstrap(): for n in cluster.instances.values(): @@ -58,9 +59,11 @@ def bootstrap(): data) """.format()) + def make_uuid(): return uuid.uuid4().hex + @pytest.fixture(scope='module', autouse=True) def start_cluster(): try: @@ -70,6 +73,7 @@ def start_cluster(): finally: cluster.shutdown() + def get_node(query_node, table='dist', *args, **kwargs): query_id = make_uuid() @@ -106,6 +110,7 @@ def get_node(query_node, table='dist', *args, **kwargs): """.format(query_id=query_id)) return rows.strip() + # TODO: right now random distribution looks bad, but works def test_load_balancing_default(): unique_nodes = set() @@ -113,6 +118,7 @@ def test_load_balancing_default(): unique_nodes.add(get_node(n1, settings={'load_balancing': 'random'})) assert len(unique_nodes) == nodes, unique_nodes + def test_load_balancing_nearest_hostname(): unique_nodes = set() for _ in range(0, queries): @@ -120,6 +126,7 @@ def test_load_balancing_nearest_hostname(): assert len(unique_nodes) == 1, unique_nodes assert unique_nodes == set(['n1']) + def test_load_balancing_in_order(): unique_nodes = set() for _ in range(0, queries): @@ -127,6 +134,7 @@ def test_load_balancing_in_order(): assert len(unique_nodes) == 1, unique_nodes assert unique_nodes == set(['n1']) + def test_load_balancing_first_or_random(): unique_nodes = set() for _ in range(0, queries): @@ -134,6 +142,7 @@ def test_load_balancing_first_or_random(): assert len(unique_nodes) == 1, unique_nodes assert unique_nodes == set(['n1']) + def test_load_balancing_round_robin(): unique_nodes = set() for _ in range(0, nodes): @@ -141,6 +150,7 @@ def test_load_balancing_round_robin(): assert len(unique_nodes) == nodes, unique_nodes assert unique_nodes == set(['n1', 'n2', 'n3']) + @pytest.mark.parametrize('dist_table', [ ('dist_priority'), ('dist_priority_negative'), @@ -153,6 +163,7 @@ def test_load_balancing_priority_round_robin(dist_table): # n2 has bigger priority in config assert unique_nodes == set(['n1', 'n3']) + def test_distributed_replica_max_ignored_errors(): settings = { 'load_balancing': 'in_order', diff --git a/tests/integration/test_distributed_over_distributed/test.py b/tests/integration/test_distributed_over_distributed/test.py index 31d6de55bea..716bc66d629 100644 --- a/tests/integration/test_distributed_over_distributed/test.py +++ b/tests/integration/test_distributed_over_distributed/test.py @@ -3,16 +3,8 @@ from __future__ import print_function -import itertools -import timeit -import logging - import pytest - from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager -from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) @@ -44,6 +36,7 @@ ENGINE = Distributed('test_cluster', default, distributed_table); INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" + @pytest.fixture(scope="session") def started_cluster(): try: @@ -59,11 +52,12 @@ def started_cluster(): @pytest.mark.parametrize("node", NODES.values()) -@pytest.mark.parametrize("source", ["distributed_over_distributed_table", "cluster('test_cluster', default, distributed_table)"]) +@pytest.mark.parametrize("source", + ["distributed_over_distributed_table", "cluster('test_cluster', default, distributed_table)"]) class TestDistributedOverDistributedSuite: def test_select_with_order_by_node(self, started_cluster, node, source): assert node.query("SELECT * FROM {source} ORDER BY node, key".format(source=source)) \ - == """node1 0 0 + == """node1 0 0 node1 0 0 node1 1 1 node1 1 1 @@ -75,7 +69,7 @@ node2 1 11 def test_select_with_order_by_key(self, started_cluster, node, source): assert node.query("SELECT * FROM {source} ORDER BY key, node".format(source=source)) \ - == """node1 0 0 + == """node1 0 0 node1 0 0 node2 0 10 node2 0 10 @@ -87,12 +81,12 @@ node2 1 11 def test_select_with_group_by_node(self, started_cluster, node, source): assert node.query("SELECT node, SUM(value) FROM {source} GROUP BY node ORDER BY node".format(source=source)) \ - == "node1 2\nnode2 42\n" + == "node1 2\nnode2 42\n" def test_select_with_group_by_key(self, started_cluster, node, source): assert node.query("SELECT key, SUM(value) FROM {source} GROUP BY key ORDER BY key".format(source=source)) \ - == "0 20\n1 24\n" + == "0 20\n1 24\n" def test_select_sum(self, started_cluster, node, source): assert node.query("SELECT SUM(value) FROM {source}".format(source=source)) \ - == "44\n" + == "44\n" diff --git a/tests/integration/test_distributed_over_live_view/test.py b/tests/integration/test_distributed_over_live_view/test.py index 46c0bada535..d01a8febd92 100644 --- a/tests/integration/test_distributed_over_live_view/test.py +++ b/tests/integration/test_distributed_over_live_view/test.py @@ -2,16 +2,10 @@ from __future__ import print_function import sys import time -import itertools -import timeit -import logging import pytest - -from helpers.uclient import client, prompt, end_of_block from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager -from helpers.test_tools import TSV +from helpers.uclient import client, prompt, end_of_block cluster = ClickHouseCluster(__file__) @@ -46,6 +40,7 @@ ENGINE = Distributed(test_cluster, default, base_table, rand()); INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" + @pytest.fixture(scope="function") def started_cluster(): try: @@ -77,7 +72,8 @@ class TestLiveViewOverDistributedSuite: client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.expect(prompt) - client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.send( + "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.expect(prompt) client1.send(select_query) @@ -115,7 +111,8 @@ class TestLiveViewOverDistributedSuite: client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.expect(prompt) - client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.send( + "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.expect(prompt) client1.send(select_query) @@ -153,7 +150,8 @@ class TestLiveViewOverDistributedSuite: client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.expect(prompt) - client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.send( + "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.expect(prompt) client1.send(select_query) @@ -192,7 +190,8 @@ class TestLiveViewOverDistributedSuite: client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.expect(prompt) - client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.send( + "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.expect(prompt) client1.send(select_query) @@ -230,7 +229,8 @@ class TestLiveViewOverDistributedSuite: client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.expect(prompt) - client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") + client1.send( + "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.expect(prompt) client1.send("SELECT sum(value) FROM distributed_over_lv") @@ -254,4 +254,3 @@ class TestLiveViewOverDistributedSuite: client1.send("SELECT sum(value) FROM distributed_over_lv") client1.expect(r"31" + end_of_block) client1.expect(prompt) - diff --git a/tests/integration/test_distributed_respect_user_timeouts/test.py b/tests/integration/test_distributed_respect_user_timeouts/test.py index dc5168bfdad..e5d9d0c1857 100644 --- a/tests/integration/test_distributed_respect_user_timeouts/test.py +++ b/tests/integration/test_distributed_respect_user_timeouts/test.py @@ -1,13 +1,12 @@ import itertools -import timeit import os.path -import pytest +import timeit +import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) NODES = {'node' + str(i): None for i in (1, 2)} @@ -62,6 +61,7 @@ TIMEOUT_DIFF_UPPER_BOUND = { }, } + def _check_exception(exception, expected_tries=3): lines = exception.split('\n') @@ -88,7 +88,6 @@ def _check_exception(exception, expected_tries=3): @pytest.fixture(scope="module", params=["configs", "configs_secure"]) def started_cluster(request): - cluster = ClickHouseCluster(__file__) cluster.__with_ssl_config = request.param == "configs_secure" main_configs = [] diff --git a/tests/integration/test_distributed_storage_configuration/test.py b/tests/integration/test_distributed_storage_configuration/test.py index 716dd3e3075..a932e9a55c5 100644 --- a/tests/integration/test_distributed_storage_configuration/test.py +++ b/tests/integration/test_distributed_storage_configuration/test.py @@ -9,8 +9,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', - main_configs=["configs/config.d/storage_configuration.xml"], - tmpfs=['/disk1:size=100M', '/disk2:size=100M']) + main_configs=["configs/config.d/storage_configuration.xml"], + tmpfs=['/disk1:size=100M', '/disk2:size=100M']) + @pytest.fixture(scope='module') def start_cluster(): @@ -21,14 +22,17 @@ def start_cluster(): finally: cluster.shutdown() + def _files_in_dist_mon(node, root, table): return int(node.exec_in_container([ 'bash', '-c', # `-maxdepth 1` to avoid /tmp/ subdirectory - 'find /{root}/data/test/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f 2>/dev/null | wc -l'.format(root=root, table=table) + 'find /{root}/data/test/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f 2>/dev/null | wc -l'.format( + root=root, table=table) ]).split('\n')[0]) + def test_insert(start_cluster): node.query('CREATE TABLE test.foo (key Int) Engine=Memory()') node.query(""" diff --git a/tests/integration/test_distributed_system_query/test.py b/tests/integration/test_distributed_system_query/test.py index c6e28c44034..bf643fabf86 100644 --- a/tests/integration/test_distributed_system_query/test.py +++ b/tests/integration/test_distributed_system_query/test.py @@ -1,5 +1,3 @@ -from contextlib import contextmanager - import pytest from helpers.cluster import ClickHouseCluster @@ -18,7 +16,8 @@ def started_cluster(): for node in (node1, node2): node.query('''CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id;''') - node1.query('''CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, id);''') + node1.query( + '''CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, id);''') yield cluster @@ -38,4 +37,3 @@ def test_start_and_stop_replica_send(started_cluster): node1.query("SYSTEM START DISTRIBUTED SENDS distributed_table;") node1.query("SYSTEM FLUSH DISTRIBUTED distributed_table;") assert node1.query("SELECT COUNT() FROM distributed_table").rstrip() == '2' - diff --git a/tests/integration/test_drop_replica/test.py b/tests/integration/test_drop_replica/test.py index 0c281fe9c9b..fac8802b2f9 100644 --- a/tests/integration/test_drop_replica/test.py +++ b/tests/integration/test_drop_replica/test.py @@ -1,53 +1,52 @@ import time -import pytest +import pytest from helpers.cluster import ClickHouseCluster -from helpers.cluster import ClickHouseKiller -from helpers.test_tools import assert_eq_with_retry from helpers.network import PartitionManager + def fill_nodes(nodes, shard): for node in nodes: node.query( - ''' - CREATE DATABASE test; - - CREATE TABLE test.test_table(date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + ''' + CREATE DATABASE test; + + CREATE TABLE test.test_table(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + '''.format(shard=shard, replica=node.name)) node.query( - ''' - CREATE DATABASE test1; - - CREATE TABLE test1.test_table(date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test1/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + ''' + CREATE DATABASE test1; + + CREATE TABLE test1.test_table(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test1/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + '''.format(shard=shard, replica=node.name)) node.query( - ''' - CREATE DATABASE test2; - - CREATE TABLE test2.test_table(date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test2/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) - + ''' + CREATE DATABASE test2; + + CREATE TABLE test2.test_table(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test2/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + '''.format(shard=shard, replica=node.name)) node.query( - ''' - CREATE DATABASE test3; - - CREATE TABLE test3.test_table(date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test3/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + ''' + CREATE DATABASE test3; + + CREATE TABLE test3.test_table(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test3/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + '''.format(shard=shard, replica=node.name)) node.query( - ''' - CREATE DATABASE test4; + ''' + CREATE DATABASE test4; + + CREATE TABLE test4.test_table(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test4/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + '''.format(shard=shard, replica=node.name)) - CREATE TABLE test4.test_table(date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test4/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) cluster = ClickHouseCluster(__file__) @@ -71,6 +70,7 @@ def start_cluster(): finally: cluster.shutdown() + def test_drop_replica(start_cluster): for i in range(100): node_1_1.query("INSERT INTO test.test_table VALUES (1, {})".format(i)) @@ -81,17 +81,25 @@ def test_drop_replica(start_cluster): zk = cluster.get_kazoo_client('zoo1') assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'") - assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test") - assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") + assert "can't drop local replica" in node_1_1.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test") + assert "can't drop local replica" in node_1_1.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'") assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test") assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") assert "it's active" in \ - node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1)) + node_1_3.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( + shard=1)) assert "There is a local table" in \ - node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1)) + node_1_2.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( + shard=1)) assert "There is a local table" in \ - node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1)) + node_1_1.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( + shard=1)) assert "does not look like a table path" in \ node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test'") @@ -100,31 +108,48 @@ def test_drop_replica(start_cluster): pm.drop_instance_zk_connections(node_1_1) time.sleep(10) - assert "doesn't exist" in node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") + assert "doesn't exist" in node_1_3.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") assert "doesn't exist" in node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1") node_1_3.query("SYSTEM DROP REPLICA 'node_1_1'") - exists_replica_1_1 = zk.exists("/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) + exists_replica_1_1 = zk.exists( + "/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, + replica='node_1_1')) assert (exists_replica_1_1 != None) ## If you want to drop a inactive/stale replicate table that does not have a local replica, you can following syntax(ZKPATH): - node_1_3.query("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test2/{shard}/replicated/test_table'".format(shard=1)) - exists_replica_1_1 = zk.exists("/clickhouse/tables/test2/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) + node_1_3.query( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test2/{shard}/replicated/test_table'".format( + shard=1)) + exists_replica_1_1 = zk.exists( + "/clickhouse/tables/test2/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, + replica='node_1_1')) assert (exists_replica_1_1 == None) node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") - exists_replica_1_1 = zk.exists("/clickhouse/tables/test/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) + exists_replica_1_1 = zk.exists( + "/clickhouse/tables/test/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, + replica='node_1_1')) assert (exists_replica_1_1 == None) node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1") - exists_replica_1_1 = zk.exists("/clickhouse/tables/test1/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) + exists_replica_1_1 = zk.exists( + "/clickhouse/tables/test1/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, + replica='node_1_1')) assert (exists_replica_1_1 == None) - node_1_3.query("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test3/{shard}/replicated/test_table'".format(shard=1)) - exists_replica_1_1 = zk.exists("/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) + node_1_3.query( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test3/{shard}/replicated/test_table'".format( + shard=1)) + exists_replica_1_1 = zk.exists( + "/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, + replica='node_1_1')) assert (exists_replica_1_1 == None) node_1_2.query("SYSTEM DROP REPLICA 'node_1_1'") - exists_replica_1_1 = zk.exists("/clickhouse/tables/test4/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) + exists_replica_1_1 = zk.exists( + "/clickhouse/tables/test4/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, + replica='node_1_1')) assert (exists_replica_1_1 == None) diff --git a/tests/integration/test_enabling_access_management/test.py b/tests/integration/test_enabling_access_management/test.py index 4a6ad59f0bb..e93a643cd16 100644 --- a/tests/integration/test_enabling_access_management/test.py +++ b/tests/integration/test_enabling_access_management/test.py @@ -4,6 +4,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', user_configs=["configs/users.d/extra_users.xml"]) + @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: diff --git a/tests/integration/test_extreme_deduplication/test.py b/tests/integration/test_extreme_deduplication/test.py index a7e6f10c1f6..d0d4b83d10f 100644 --- a/tests/integration/test_extreme_deduplication/test.py +++ b/tests/integration/test_extreme_deduplication/test.py @@ -1,21 +1,22 @@ import time -from contextlib import contextmanager import pytest - -from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager -from helpers.test_tools import TSV from helpers.client import CommandRequest from helpers.client import QueryTimeoutExceedException - +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1}) -node2 = cluster.add_instance('node2', main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2}) +node1 = cluster.add_instance('node1', + main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], + with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1}) +node2 = cluster.add_instance('node2', + main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], + with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2}) nodes = [node1, node2] + @pytest.fixture(scope="module") def started_cluster(): try: @@ -36,15 +37,18 @@ def test_deduplication_window_in_seconds(started_cluster): node.query("INSERT INTO simple VALUES (0, 0)") time.sleep(1) - node.query("INSERT INTO simple VALUES (0, 0)") # deduplication works here + node.query("INSERT INTO simple VALUES (0, 0)") # deduplication works here node.query("INSERT INTO simple VALUES (0, 1)") assert TSV(node.query("SELECT count() FROM simple")) == TSV("2\n") # wait clean thread time.sleep(2) - assert TSV.toMat(node.query("SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple/blocks'"))[0][0] == "1" - node.query("INSERT INTO simple VALUES (0, 0)") # deduplication doesn't works here, the first hash node was deleted + assert \ + TSV.toMat(node.query("SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple/blocks'"))[ + 0][ + 0] == "1" + node.query("INSERT INTO simple VALUES (0, 0)") # deduplication doesn't works here, the first hash node was deleted assert TSV.toMat(node.query("SELECT count() FROM simple"))[0][0] == "3" node1.query("""DROP TABLE simple ON CLUSTER test_cluster""") diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index f3ea2d4f152..9ad56d4fb17 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -1,15 +1,13 @@ from __future__ import print_function -from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException -import helpers -import pytest +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) - @pytest.fixture(scope="module") def start_cluster(): try: diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index 83389b3d9bd..93c8d3f7033 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -25,4 +25,5 @@ def test_file_path_escaping(started_cluster): node.query('''ALTER TABLE test.`T.a_b,l-e!` FREEZE;''') node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) - node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) + node.exec_in_container( + ["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) diff --git a/tests/integration/test_force_deduplication/test.py b/tests/integration/test_force_deduplication/test.py index 5ad964ecb27..991e289f912 100644 --- a/tests/integration/test_force_deduplication/test.py +++ b/tests/integration/test_force_deduplication/test.py @@ -2,14 +2,14 @@ # pylint: disable=redefined-outer-name import pytest - -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', with_zookeeper=True) + @pytest.fixture(scope='module') def start_cluster(): try: @@ -19,6 +19,7 @@ def start_cluster(): finally: cluster.shutdown() + def get_counts(): src = int(node.query("SELECT count() FROM test")) a = int(node.query("SELECT count() FROM test_mv_a")) @@ -69,7 +70,7 @@ def test_basic(start_cluster): ) src, a, b, c = get_counts() assert src == 11 - assert a == old_a + 10 # first insert could be succesfull with disabled dedup + assert a == old_a + 10 # first insert could be succesfull with disabled dedup assert b == 11 assert c == old_c + 10 @@ -81,7 +82,7 @@ def test_basic(start_cluster): INSERT INTO test SELECT number FROM numbers(100,10); ''' ) - + node.query( ''' SET deduplicate_blocks_in_dependent_materialized_views = 1; @@ -94,5 +95,3 @@ def test_basic(start_cluster): assert a == old_a + 20 assert b == 21 assert c == old_c + 20 - - diff --git a/tests/integration/test_format_avro_confluent/test.py b/tests/integration/test_format_avro_confluent/test.py index a93b5585f8d..67d15305333 100644 --- a/tests/integration/test_format_avro_confluent/test.py +++ b/tests/integration/test_format_avro_confluent/test.py @@ -1,20 +1,15 @@ -import json -import logging import io - -import pytest - -from helpers.cluster import ClickHouseCluster, ClickHouseInstance - -import helpers.client +import logging import avro.schema -from confluent.schemaregistry.client import CachedSchemaRegistryClient +import pytest from confluent.schemaregistry.serializers import MessageSerializer +from helpers.cluster import ClickHouseCluster, ClickHouseInstance logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) + @pytest.fixture(scope="module") def cluster(): try: @@ -42,7 +37,6 @@ def run_query(instance, query, data=None, settings=None): return result - def test_select(cluster): # type: (ClickHouseCluster) -> None @@ -55,7 +49,7 @@ def test_select(cluster): 'fields': [ { 'name': 'value', - 'type': 'long' + 'type': 'long' } ] }) @@ -73,7 +67,7 @@ def test_select(cluster): cluster.schema_registry_host, cluster.schema_registry_port ) - + run_query(instance, "create table avro_data(value Int64) engine = Memory()") settings = {'format_avro_schema_registry_url': schema_registry_url} run_query(instance, "insert into avro_data format AvroConfluent", data, settings) diff --git a/tests/integration/test_format_schema_on_server/test.py b/tests/integration/test_format_schema_on_server/test.py index 9d0f6948aef..3b53a897dc0 100644 --- a/tests/integration/test_format_schema_on_server/test.py +++ b/tests/integration/test_format_schema_on_server/test.py @@ -36,5 +36,6 @@ def test_protobuf_format_input(started_cluster): def test_protobuf_format_output(started_cluster): create_simple_table() instance.query("INSERT INTO test.simple VALUES (1, 'abc'), (2, 'def')"); - assert instance.http_query("SELECT * FROM test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'") == \ + assert instance.http_query( + "SELECT * FROM test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'") == \ "\x07\x08\x01\x12\x03abc\x07\x08\x02\x12\x03def" diff --git a/tests/integration/test_freeze_table/test.py b/tests/integration/test_freeze_table/test.py index ab1af3f371e..4d4aa22d4e2 100644 --- a/tests/integration/test_freeze_table/test.py +++ b/tests/integration/test_freeze_table/test.py @@ -39,7 +39,7 @@ def test_freeze_table(started_cluster): ''')) assert 11 == len(freeze_result) path_col_ix = freeze_result[0].index('part_backup_path') - for row in freeze_result[1:]: # skip header + for row in freeze_result[1:]: # skip header part_backup_path = row[path_col_ix] node.exec_in_container( ["bash", "-c", "test -d {}".format(part_backup_path)] @@ -55,7 +55,7 @@ def test_freeze_table(started_cluster): ''')) assert 2 == len(freeze_result) path_col_ix = freeze_result[0].index('part_backup_path') - for row in freeze_result[1:]: # skip header + for row in freeze_result[1:]: # skip header part_backup_path = row[path_col_ix] assert 'test_01417_single_part' in part_backup_path node.exec_in_container( diff --git a/tests/integration/test_globs_in_filepath/test.py b/tests/integration/test_globs_in_filepath/test.py index c85c39a8838..7e534dd69bc 100644 --- a/tests/integration/test_globs_in_filepath/test.py +++ b/tests/integration/test_globs_in_filepath/test.py @@ -4,7 +4,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node') -path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/" # should be the same as in config file +path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/" # should be the same as in config file + @pytest.fixture(scope="module") def start_cluster(): @@ -19,11 +20,13 @@ def start_cluster(): finally: cluster.shutdown() + def test_strange_filenames(start_cluster): # 2 rows data some_data = "\t111.222\nData\t333.444" - node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)], privileged=True, user='root') + node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)], + privileged=True, user='root') files = ["p.o.i.n.t.s", "b}{ra{ces", @@ -31,7 +34,10 @@ def test_strange_filenames(start_cluster): # filename inside testing data for debug simplicity for filename in files: - node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root') + node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data, + path_to_userfiles_from_defaut_config, + filename)], privileged=True, + user='root') test_requests = [("p.o.??n.t.s", "2"), ("p.o.*t.s", "2"), @@ -47,6 +53,7 @@ def test_strange_filenames(start_cluster): select count(*) from file('{}strange_names/{}', 'TSV', 'text String, number Float64') '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) + def test_linear_structure(start_cluster): # 2 rows data some_data = "\t123.456\nData\t789.012" @@ -58,7 +65,9 @@ def test_linear_structure(start_cluster): # filename inside testing data for debug simplicity for filename in files: - node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root') + node.exec_in_container(['bash', '-c', + 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, + filename)], privileged=True, user='root') test_requests = [("file{0..9}", "10"), ("file?", "10"), @@ -81,6 +90,7 @@ def test_linear_structure(start_cluster): select count(*) from file('{}{}', 'TSV', 'text String, number Float64') '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) + def test_deep_structure(start_cluster): # 2 rows data some_data = "\t135.791\nData\t246.802" @@ -92,7 +102,8 @@ def test_deep_structure(start_cluster): "we/need/", "we/need/to/", "we/need/to/go/", "we/need/to/go/deeper/"] for dir in dirs: - node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)], privileged=True, user='root') + node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)], + privileged=True, user='root') # all directories appeared in files must be listed in dirs files = [] @@ -102,13 +113,15 @@ def test_deep_structure(start_cluster): files.append("directory1/big_dir/file" + str(i) + str(j) + str(k)) for dir in dirs: - files.append(dir+"file") + files.append(dir + "file") # filename inside testing data for debug simplicity for filename in files: - node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root') + node.exec_in_container(['bash', '-c', + 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, + filename)], privileged=True, user='root') - test_requests = [ ("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"), + test_requests = [("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"), ("?", "0"), ("directory{0..5}/dir{1..3}/file", "10"), ("directory{0..5}/dir?/file", "10"), ("we/need/to/go/deeper/file", "2"), ("*/*/*/*/*/*", "2"), ("we/need/??/go/deeper/*?*?*?*?*", "2")] @@ -121,14 +134,17 @@ def test_deep_structure(start_cluster): select count(*) from file('{}{}', 'TSV', 'text String, number Float64') '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) + def test_table_function_and_virtual_columns(start_cluster): node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)]) node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)]) - node.query("insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)") - assert node.query("select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000' + node.query( + "insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)") + assert node.query( + "select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000' node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)") - assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()== '1' + assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() == '1' assert "nonexist.csv" in node.query("select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip() assert "nonexist.csv" in node.query("select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() assert "nonexist.csv" == node.query("select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip() - assert "nonexist.csv" == node.query("select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() \ No newline at end of file + assert "nonexist.csv" == node.query("select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 1557e81bce8..073578edaa5 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -1,7 +1,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -import re cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance') @@ -11,11 +10,11 @@ instance = cluster.add_instance('instance') def start_cluster(): try: cluster.start() - + instance.query("CREATE DATABASE test") instance.query("CREATE TABLE test.table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()") instance.query("INSERT INTO test.table VALUES (1,5), (2,10)") - + yield cluster finally: @@ -34,7 +33,7 @@ def cleanup_after_test(): def test_smoke(): instance.query("CREATE USER A") assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test.table", user='A') - + instance.query('GRANT SELECT ON test.table TO A') assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n" @@ -49,7 +48,7 @@ def test_grant_option(): instance.query('GRANT SELECT ON test.table TO A') assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n" assert "Not enough privileges" in instance.query_and_get_error("GRANT SELECT ON test.table TO B", user='A') - + instance.query('GRANT SELECT ON test.table TO A WITH GRANT OPTION') instance.query("GRANT SELECT ON test.table TO B", user='A') assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n" @@ -60,7 +59,7 @@ def test_grant_option(): def test_revoke_requires_grant_option(): instance.query("CREATE USER A") instance.query("CREATE USER B") - + instance.query("GRANT SELECT ON test.table TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" @@ -111,7 +110,8 @@ def test_grant_all_on_table(): instance.query("CREATE USER A, B") instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION") instance.query("GRANT ALL ON test.table TO B", user='A') - assert instance.query("SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n" + assert instance.query( + "SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n" instance.query("REVOKE ALL ON test.table FROM B", user='A') assert instance.query("SHOW GRANTS FOR B") == "" @@ -120,36 +120,42 @@ def test_implicit_show_grants(): instance.query("CREATE USER A") assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "0\n" + assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", + user="A") == "0\n" instance.query("GRANT SELECT(x) ON test.table TO A") assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT(x) ON test.table TO A\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "1\n" + assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", + user="A") == "1\n" instance.query("GRANT SELECT ON test.table TO A") assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.table TO A\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n" + assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", + user="A") == "2\n" instance.query("GRANT SELECT ON test.* TO A") assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.* TO A\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n" + assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", + user="A") == "2\n" instance.query("GRANT SELECT ON *.* TO A") assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON *.* TO A\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n" + assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", + user="A") == "2\n" instance.query("REVOKE ALL ON *.* FROM A") assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "0\n" + assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", + user="A") == "0\n" def test_implicit_create_view_grant(): @@ -184,46 +190,53 @@ def test_introspection(): instance.query('GRANT SELECT ON test.table TO A') instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') - assert instance.query("SHOW USERS") == TSV([ "A", "B", "default" ]) - assert instance.query("SHOW CREATE USERS A") == TSV([ "CREATE USER A" ]) - assert instance.query("SHOW CREATE USERS B") == TSV([ "CREATE USER B" ]) - assert instance.query("SHOW CREATE USERS A,B") == TSV([ "CREATE USER A", "CREATE USER B" ]) - assert instance.query("SHOW CREATE USERS") == TSV([ "CREATE USER A", "CREATE USER B", "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" ]) + assert instance.query("SHOW USERS") == TSV(["A", "B", "default"]) + assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A"]) + assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B"]) + assert instance.query("SHOW CREATE USERS A,B") == TSV(["CREATE USER A", "CREATE USER B"]) + assert instance.query("SHOW CREATE USERS") == TSV(["CREATE USER A", "CREATE USER B", + "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default"]) - assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A" ]) - assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) - assert instance.query("SHOW GRANTS FOR A,B") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) - assert instance.query("SHOW GRANTS FOR B,A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) - assert instance.query("SHOW GRANTS FOR ALL") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT ALL ON *.* TO default WITH GRANT OPTION" ]) + assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"]) + assert instance.query("SHOW GRANTS FOR B") == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS FOR A,B") == TSV( + ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS FOR B,A") == TSV( + ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS FOR ALL") == TSV( + ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + "GRANT ALL ON *.* TO default WITH GRANT OPTION"]) - assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A" ]) - assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) + assert instance.query("SHOW GRANTS", user='A') == TSV(["GRANT SELECT ON test.table TO A"]) + assert instance.query("SHOW GRANTS", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) - expected_access1 = "CREATE USER A\n"\ - "CREATE USER B\n"\ + expected_access1 = "CREATE USER A\n" \ + "CREATE USER B\n" \ "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" - expected_access2 = "GRANT SELECT ON test.table TO A\n"\ - "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n"\ + expected_access2 = "GRANT SELECT ON test.table TO A\n" \ + "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n" \ "GRANT ALL ON *.* TO default WITH GRANT OPTION\n" assert expected_access1 in instance.query("SHOW ACCESS") assert expected_access2 in instance.query("SHOW ACCESS") - assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ - TSV([[ "A", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], - [ "B", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) - - assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") ==\ - TSV([[ "A", "\N", "SELECT", "test", "table", "\N", 0, 0 ], - [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 1 ]]) + assert instance.query( + "SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") == \ + TSV([["A", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]"], + ["B", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]"]]) + + assert instance.query( + "SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") == \ + TSV([["A", "\N", "SELECT", "test", "table", "\N", 0, 0], + ["B", "\N", "CREATE", "\N", "\N", "\N", 0, 1]]) def test_current_database(): instance.query("CREATE USER A") instance.query("GRANT SELECT ON table TO A", database="test") - - assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A" ]) - assert instance.query("SHOW GRANTS FOR A", database="test") == TSV([ "GRANT SELECT ON test.table TO A" ]) - + + assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"]) + assert instance.query("SHOW GRANTS FOR A", database="test") == TSV(["GRANT SELECT ON test.table TO A"]) + assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n" assert instance.query("SELECT * FROM table", user='A', database='test') == "1\t5\n2\t10\n" diff --git a/tests/integration/test_graphite_merge_tree/test.py b/tests/integration/test_graphite_merge_tree/test.py index 509fbac97d0..319fdb816ff 100644 --- a/tests/integration/test_graphite_merge_tree/test.py +++ b/tests/integration/test_graphite_merge_tree/test.py @@ -1,12 +1,11 @@ +import datetime import os.path as p import time -import datetime -import pytest +import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', main_configs=['configs/graphite_rollup.xml']) @@ -317,20 +316,20 @@ def test_combined_rules(graphite_table): expected_unmerged = '' for i in range(384): to_insert += "('five_min.count', {v}, {t}, toDate({t}), 1), ".format( - v=1, t=1487970000+(i*300) + v=1, t=1487970000 + (i * 300) ) to_insert += "('five_min.max', {v}, {t}, toDate({t}), 1), ".format( - v=i, t=1487970000+(i*300) + v=i, t=1487970000 + (i * 300) ) expected_unmerged += ("five_min.count\t{v1}\t{t}\n" "five_min.max\t{v2}\t{t}\n").format( - v1=1, v2=i, - t=1487970000+(i*300) - ) + v1=1, v2=i, + t=1487970000 + (i * 300) + ) q(to_insert) assert TSV(q('SELECT metric, value, timestamp FROM test.graphite' - ' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged) + ' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged) q('OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL') expected_merged = ''' @@ -370,16 +369,16 @@ CREATE TABLE test.graphite expected_unmerged = '' for i in range(100): to_insert += "('top_level.count', {v}, {t}, toDate({t}), 1), ".format( - v=1, t=1487970000+(i*60) + v=1, t=1487970000 + (i * 60) ) to_insert += "('top_level.max', {v}, {t}, toDate({t}), 1), ".format( - v=i, t=1487970000+(i*60) + v=i, t=1487970000 + (i * 60) ) expected_unmerged += ("top_level.count\t{v1}\t{t}\n" "top_level.max\t{v2}\t{t}\n").format( - v1=1, v2=i, - t=1487970000+(i*60) - ) + v1=1, v2=i, + t=1487970000 + (i * 60) + ) q(to_insert) assert TSV(q('SELECT metric, value, timestamp FROM test.graphite' diff --git a/tests/integration/test_host_ip_change/test.py b/tests/integration/test_host_ip_change/test.py index a2a38158dc4..951af699a5f 100644 --- a/tests/integration/test_host_ip_change/test.py +++ b/tests/integration/test_host_ip_change/test.py @@ -1,15 +1,12 @@ -import time import pytest - -import subprocess -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - +from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) + def _fill_nodes(nodes, table_name): for node in nodes: node.query( @@ -21,9 +18,12 @@ def _fill_nodes(nodes, table_name): '''.format(table_name, node.name) ) -node1 = cluster.add_instance('node1', main_configs=['configs/listen_host.xml'], with_zookeeper=True, ipv6_address='2001:3984:3989::1:1111') + +node1 = cluster.add_instance('node1', main_configs=['configs/listen_host.xml'], with_zookeeper=True, + ipv6_address='2001:3984:3989::1:1111') node2 = cluster.add_instance('node2', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'], - with_zookeeper=True, ipv6_address='2001:3984:3989::1:1112') + with_zookeeper=True, ipv6_address='2001:3984:3989::1:1112') + @pytest.fixture(scope="module") def cluster_without_dns_cache_update(): @@ -41,6 +41,7 @@ def cluster_without_dns_cache_update(): cluster.shutdown() pass + # node1 is a source, node2 downloads data # node2 has long dns_cache_update_period, so dns cache update wouldn't work def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update): @@ -73,9 +74,11 @@ def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update): node3 = cluster.add_instance('node3', main_configs=['configs/listen_host.xml'], - with_zookeeper=True, ipv6_address='2001:3984:3989::1:1113') -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/listen_host.xml', 'configs/dns_update_short.xml'], - with_zookeeper=True, ipv6_address='2001:3984:3989::1:1114') + with_zookeeper=True, ipv6_address='2001:3984:3989::1:1113') +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/listen_host.xml', + 'configs/dns_update_short.xml'], + with_zookeeper=True, ipv6_address='2001:3984:3989::1:1114') + @pytest.fixture(scope="module") def cluster_with_dns_cache_update(): @@ -93,6 +96,7 @@ def cluster_with_dns_cache_update(): cluster.shutdown() pass + # node3 is a source, node4 downloads data # node4 has short dns_cache_update_period, so testing update of dns cache def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): @@ -107,7 +111,6 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): # Put some data to source node3 node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)") - # Check that data is placed on node3 assert node3.query("SELECT count(*) from test_table_update") == "6\n" @@ -126,9 +129,12 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): assert node3.query("SELECT count(*) from test_table_update") == "7\n" assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "7") + def set_hosts(node, hosts): new_content = '\\n'.join(['127.0.0.1 localhost', '::1 localhost'] + hosts) - node.exec_in_container(['bash', '-c', 'echo -e "{}" > /etc/hosts'.format(new_content)], privileged=True, user='root') + node.exec_in_container(['bash', '-c', 'echo -e "{}" > /etc/hosts'.format(new_content)], privileged=True, + user='root') + def test_dns_cache_update(cluster_with_dns_cache_update): set_hosts(node4, ['127.255.255.255 lost_host']) @@ -136,7 +142,8 @@ def test_dns_cache_update(cluster_with_dns_cache_update): with pytest.raises(QueryRuntimeException): node4.query("SELECT * FROM remote('lost_host', 'system', 'one')") - node4.query("CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')") + node4.query( + "CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')") with pytest.raises(QueryRuntimeException): node4.query("SELECT * FROM distributed_lost_host") @@ -146,21 +153,26 @@ def test_dns_cache_update(cluster_with_dns_cache_update): assert_eq_with_retry(node4, "SELECT * FROM remote('lost_host', 'system', 'one')", "0") assert_eq_with_retry(node4, "SELECT * FROM distributed_lost_host", "0") - assert TSV(node4.query("SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV("lost_host\t127.0.0.1\n") + assert TSV(node4.query( + "SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV( + "lost_host\t127.0.0.1\n") assert TSV(node4.query("SELECT hostName()")) == TSV("node4") + # Check SYSTEM DROP DNS CACHE on node5 and background cache update on node6 node5 = cluster.add_instance('node5', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'], user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1115') node6 = cluster.add_instance('node6', main_configs=['configs/listen_host.xml', 'configs/dns_update_short.xml'], user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1116') + @pytest.mark.parametrize("node", [node5, node6]) def test_user_access_ip_change(cluster_with_dns_cache_update, node): node_name = node.name node_num = node.name[-1] # getaddrinfo(...) may hang for a log time without this options - node.exec_in_container(['bash', '-c', 'echo -e "options timeout:1\noptions attempts:2" >> /etc/resolv.conf'], privileged=True, user='root') + node.exec_in_container(['bash', '-c', 'echo -e "options timeout:1\noptions attempts:2" >> /etc/resolv.conf'], + privileged=True, user='root') assert node3.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n" assert node4.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n" @@ -180,8 +192,11 @@ def test_user_access_ip_change(cluster_with_dns_cache_update, node): retry_count = 60 if node_name == 'node5': # client is not allowed to connect, so execute it directly in container to send query from localhost - node.exec_in_container(['bash', '-c', 'clickhouse client -q "SYSTEM DROP DNS CACHE"'], privileged=True, user='root') + node.exec_in_container(['bash', '-c', 'clickhouse client -q "SYSTEM DROP DNS CACHE"'], privileged=True, + user='root') retry_count = 1 - assert_eq_with_retry(node3, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", retry_count=retry_count, sleep_time=1) - assert_eq_with_retry(node4, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", retry_count=retry_count, sleep_time=1) + assert_eq_with_retry(node3, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", + retry_count=retry_count, sleep_time=1) + assert_eq_with_retry(node4, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", + retry_count=retry_count, sleep_time=1) diff --git a/tests/integration/test_http_and_readonly/test.py b/tests/integration/test_http_and_readonly/test.py index ea25b787a67..9929e34c9d2 100644 --- a/tests/integration/test_http_and_readonly/test.py +++ b/tests/integration/test_http_and_readonly/test.py @@ -16,5 +16,7 @@ def setup_nodes(): def test_http_get_is_readonly(): - assert "Cannot execute query in readonly mode" in instance.http_query_and_get_error("CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)") - assert "Cannot modify 'readonly' setting in readonly mode" in instance.http_query_and_get_error("CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)", params={"readonly": 0}) + assert "Cannot execute query in readonly mode" in instance.http_query_and_get_error( + "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)") + assert "Cannot modify 'readonly' setting in readonly mode" in instance.http_query_and_get_error( + "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)", params={"readonly": 0}) diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index 6b0ced11204..06602ba3ca3 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -1,6 +1,6 @@ +import contextlib import os import urllib -import contextlib from helpers.cluster import ClickHouseCluster @@ -20,101 +20,145 @@ class SimpleCluster: def test_dynamic_query_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "dynamic_handler", "test_dynamic_handler")) as cluster: + with contextlib.closing( + SimpleCluster(ClickHouseCluster(__file__), "dynamic_handler", "test_dynamic_handler")) as cluster: test_query = urllib.quote_plus('SELECT * FROM system.settings WHERE name = \'max_threads\'') assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code - assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='POST', headers={'XXX': 'xxx'}).status_code + assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='POST', + headers={'XXX': 'xxx'}).status_code - assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', headers={'XXX': 'bad'}).status_code + assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', + headers={'XXX': 'bad'}).status_code - assert 400 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code + assert 400 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', + headers={'XXX': 'xxx'}).status_code - assert 200 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1&get_dynamic_handler_query=' + test_query, - method='GET', headers={'XXX': 'xxx'}).status_code + assert 200 == cluster.instance.http_request( + 'test_dynamic_handler_get?max_threads=1&get_dynamic_handler_query=' + test_query, + method='GET', headers={'XXX': 'xxx'}).status_code def test_predefined_query_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "predefined_handler", "test_predefined_handler")) as cluster: + with contextlib.closing( + SimpleCluster(ClickHouseCluster(__file__), "predefined_handler", "test_predefined_handler")) as cluster: assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code - assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', headers={'XXX': 'bad'}).status_code + assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', + headers={'XXX': 'bad'}).status_code - assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='POST', headers={'XXX': 'xxx'}).status_code + assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='POST', + headers={'XXX': 'xxx'}).status_code - assert 500 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code + assert 500 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', + headers={'XXX': 'xxx'}).status_code - assert 'max_threads\t1\n' == cluster.instance.http_request('test_predefined_handler_get?max_threads=1&setting_name=max_threads', method='GET', headers={'XXX': 'xxx'}).content + assert 'max_threads\t1\n' == cluster.instance.http_request( + 'test_predefined_handler_get?max_threads=1&setting_name=max_threads', method='GET', + headers={'XXX': 'xxx'}).content assert 'max_threads\t1\nmax_alter_threads\t1\n' == cluster.instance.http_request( - 'query_param_with_url/max_threads?max_threads=1&max_alter_threads=1', headers={'XXX': 'max_alter_threads'}).content + 'query_param_with_url/max_threads?max_threads=1&max_alter_threads=1', + headers={'XXX': 'max_alter_threads'}).content def test_fixed_static_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: + with contextlib.closing( + SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code - assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'bad'}).status_code + assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', + headers={'XXX': 'bad'}).status_code - assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code + assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='POST', + headers={'XXX': 'xxx'}).status_code - assert 402 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code - assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type'] - assert 'Test get static handler and fix content' == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).content + assert 402 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', + headers={'XXX': 'xxx'}).status_code + assert 'text/html; charset=UTF-8' == \ + cluster.instance.http_request('test_get_fixed_static_handler', method='GET', + headers={'XXX': 'xxx'}).headers['Content-Type'] + assert 'Test get static handler and fix content' == cluster.instance.http_request( + 'test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).content def test_config_static_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: + with contextlib.closing( + SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code - assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'bad'}).status_code + assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='GET', + headers={'XXX': 'bad'}).status_code - assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code + assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='POST', + headers={'XXX': 'xxx'}).status_code # check default status code - assert 200 == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code - assert 'text/plain; charset=UTF-8' == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type'] - assert 'Test get static handler and config content' == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).content + assert 200 == cluster.instance.http_request('test_get_config_static_handler', method='GET', + headers={'XXX': 'xxx'}).status_code + assert 'text/plain; charset=UTF-8' == \ + cluster.instance.http_request('test_get_config_static_handler', method='GET', + headers={'XXX': 'xxx'}).headers['Content-Type'] + assert 'Test get static handler and config content' == cluster.instance.http_request( + 'test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).content def test_absolute_path_static_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: + with contextlib.closing( + SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: cluster.instance.exec_in_container( - ['bash', '-c', 'echo "Absolute Path File" > /var/lib/clickhouse/user_files/absolute_path_file.html'], + ['bash', '-c', + 'echo "Absolute Path File" > /var/lib/clickhouse/user_files/absolute_path_file.html'], privileged=True, user='root') assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code - assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'bad'}).status_code + assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', + headers={'XXX': 'bad'}).status_code - assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code + assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='POST', + headers={'XXX': 'xxx'}).status_code # check default status code - assert 200 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code - assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type'] - assert 'Absolute Path File\n' == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content + assert 200 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', + headers={'XXX': 'xxx'}).status_code + assert 'text/html; charset=UTF-8' == \ + cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', + headers={'XXX': 'xxx'}).headers['Content-Type'] + assert 'Absolute Path File\n' == cluster.instance.http_request( + 'test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content def test_relative_path_static_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: + with contextlib.closing( + SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: cluster.instance.exec_in_container( - ['bash', '-c', 'echo "Relative Path File" > /var/lib/clickhouse/user_files/relative_path_file.html'], + ['bash', '-c', + 'echo "Relative Path File" > /var/lib/clickhouse/user_files/relative_path_file.html'], privileged=True, user='root') assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code - assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'bad'}).status_code + assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', + headers={'XXX': 'bad'}).status_code - assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code + assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='POST', + headers={'XXX': 'xxx'}).status_code # check default status code - assert 200 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code - assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type'] - assert 'Relative Path File\n' == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content + assert 200 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', + headers={'XXX': 'xxx'}).status_code + assert 'text/html; charset=UTF-8' == \ + cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', + headers={'XXX': 'xxx'}).headers['Content-Type'] + assert 'Relative Path File\n' == cluster.instance.http_request( + 'test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content + def test_defaults_http_handlers(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers")) as cluster: + with contextlib.closing( + SimpleCluster(ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers")) as cluster: assert 200 == cluster.instance.http_request('', method='GET').status_code assert 'Default server response' == cluster.instance.http_request('', method='GET').content @@ -130,24 +174,34 @@ def test_defaults_http_handlers(): assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content + def test_prometheus_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster: + with contextlib.closing( + SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster: assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'bad'}).status_code - assert 404 == cluster.instance.http_request('test_prometheus', method='POST', headers={'XXX': 'xxx'}).status_code + assert 404 == cluster.instance.http_request('test_prometheus', method='POST', + headers={'XXX': 'xxx'}).status_code assert 200 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).status_code - assert 'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).content + assert 'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET', + headers={'XXX': 'xxx'}).content + def test_replicas_status_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler", "test_replicas_status_handler")) as cluster: + with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler", + "test_replicas_status_handler")) as cluster: assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code - assert 404 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'bad'}).status_code + assert 404 == cluster.instance.http_request('test_replicas_status', method='GET', + headers={'XXX': 'bad'}).status_code - assert 404 == cluster.instance.http_request('test_replicas_status', method='POST', headers={'XXX': 'xxx'}).status_code + assert 404 == cluster.instance.http_request('test_replicas_status', method='POST', + headers={'XXX': 'xxx'}).status_code - assert 200 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).status_code - assert 'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).content + assert 200 == cluster.instance.http_request('test_replicas_status', method='GET', + headers={'XXX': 'xxx'}).status_code + assert 'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET', + headers={'XXX': 'xxx'}).content diff --git a/tests/integration/test_https_replication/test.py b/tests/integration/test_https_replication/test.py index 4974da850b4..84c2744923d 100644 --- a/tests/integration/test_https_replication/test.py +++ b/tests/integration/test_https_replication/test.py @@ -1,30 +1,36 @@ -import time -import pytest - -from helpers.cluster import ClickHouseCluster - -from helpers.test_tools import assert_eq_with_retry -from helpers.network import PartitionManager -from multiprocessing.dummy import Pool import random +import time +from multiprocessing.dummy import Pool + +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry """ Both ssl_conf.xml and no_ssl_conf.xml have the same port """ + def _fill_nodes(nodes, shard): for node in nodes: node.query( - ''' - CREATE DATABASE test; + ''' + CREATE DATABASE test; + + CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + '''.format(shard=shard, replica=node.name)) - CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); - '''.format(shard=shard, replica=node.name)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) +node1 = cluster.add_instance('node1', + main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", + "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) +node2 = cluster.add_instance('node2', + main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", + "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) + @pytest.fixture(scope="module") def both_https_cluster(): @@ -38,6 +44,7 @@ def both_https_cluster(): finally: cluster.shutdown() + def test_both_https(both_https_cluster): node1.query("insert into test_table values ('2017-06-16', 111, 0)") @@ -77,9 +84,11 @@ def test_replication_after_partition(both_https_cluster): assert_eq_with_retry(node2, "SELECT count() FROM test_table", '100') +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], + with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], + with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) @pytest.fixture(scope="module") def both_http_cluster(): @@ -93,6 +102,7 @@ def both_http_cluster(): finally: cluster.shutdown() + def test_both_http(both_http_cluster): node3.query("insert into test_table values ('2017-06-16', 111, 0)") @@ -104,8 +114,13 @@ def test_both_http(both_http_cluster): assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", '111\n222') assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111\n222') -node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) -node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) + +node5 = cluster.add_instance('node5', + main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", + "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) +node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], + with_zookeeper=True) + @pytest.fixture(scope="module") def mixed_protocol_cluster(): @@ -119,6 +134,7 @@ def mixed_protocol_cluster(): finally: cluster.shutdown() + def test_mixed_protocol(mixed_protocol_cluster): node5.query("insert into test_table values ('2017-06-16', 111, 0)") diff --git a/tests/integration/test_inherit_multiple_profiles/test.py b/tests/integration/test_inherit_multiple_profiles/test.py index 1540196f9b6..658ccc3f51b 100644 --- a/tests/integration/test_inherit_multiple_profiles/test.py +++ b/tests/integration/test_inherit_multiple_profiles/test.py @@ -4,7 +4,6 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', user_configs=['configs/combined_profile.xml']) diff --git a/tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py b/tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py index 06a1a72162d..69c7a5821fd 100644 --- a/tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py +++ b/tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py @@ -10,6 +10,7 @@ cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', main_configs=['configs/conf.xml']) + @pytest.fixture(scope='module', autouse=True) def start_cluster(): try: @@ -18,6 +19,7 @@ def start_cluster(): finally: cluster.shutdown() + # max_memory_usage_for_user cannot be used, since the memory for user accounted # correctly, only total is not def test_memory_tracking_total(): @@ -25,9 +27,10 @@ def test_memory_tracking_total(): CREATE TABLE null (row String) ENGINE=Null; ''') instance.exec_in_container(['bash', '-c', - 'clickhouse client -q "SELECT arrayStringConcat(arrayMap(x->toString(cityHash64(x)), range(1000)), \' \') from numbers(10000)" > data.json']) + 'clickhouse client -q "SELECT arrayStringConcat(arrayMap(x->toString(cityHash64(x)), range(1000)), \' \') from numbers(10000)" > data.json']) for it in range(0, 20): # the problem can be triggered only via HTTP, # since clickhouse-client parses the data by itself. assert instance.exec_in_container(['curl', '--silent', '--show-error', '--data-binary', '@data.json', - 'http://127.1:8123/?query=INSERT%20INTO%20null%20FORMAT%20TSV']) == '', 'Failed on {} iteration'.format(it) + 'http://127.1:8123/?query=INSERT%20INTO%20null%20FORMAT%20TSV']) == '', 'Failed on {} iteration'.format( + it) diff --git a/tests/integration/test_insert_distributed_load_balancing/test.py b/tests/integration/test_insert_distributed_load_balancing/test.py index 52ee3ba1c4a..29cc953280f 100644 --- a/tests/integration/test_insert_distributed_load_balancing/test.py +++ b/tests/integration/test_insert_distributed_load_balancing/test.py @@ -16,6 +16,7 @@ params = pytest.mark.parametrize('cluster,q', [ ('no_internal_replication', 1), ]) + @pytest.fixture(scope='module', autouse=True) def start_cluster(): try: @@ -24,6 +25,7 @@ def start_cluster(): finally: cluster.shutdown() + def create_tables(cluster): n1.query('DROP TABLE IF EXISTS data') n2.query('DROP TABLE IF EXISTS data') @@ -41,37 +43,43 @@ def create_tables(cluster): ) """.format(cluster=cluster)) + def insert_data(cluster, **settings): create_tables(cluster) n1.query('INSERT INTO dist SELECT * FROM numbers(10)', settings=settings) n1.query('SYSTEM FLUSH DISTRIBUTED dist') + @params def test_prefer_localhost_replica_1(cluster, q): insert_data(cluster) assert int(n1.query('SELECT count() FROM data')) == 10 - assert int(n2.query('SELECT count() FROM data')) == 10*q + assert int(n2.query('SELECT count() FROM data')) == 10 * q + @params def test_prefer_localhost_replica_1_load_balancing_in_order(cluster, q): insert_data(cluster, load_balancing='in_order') assert int(n1.query('SELECT count() FROM data')) == 10 - assert int(n2.query('SELECT count() FROM data')) == 10*q + assert int(n2.query('SELECT count() FROM data')) == 10 * q + @params def test_prefer_localhost_replica_0_load_balancing_nearest_hostname(cluster, q): insert_data(cluster, load_balancing='nearest_hostname', prefer_localhost_replica=0) assert int(n1.query('SELECT count() FROM data')) == 10 - assert int(n2.query('SELECT count() FROM data')) == 10*q + assert int(n2.query('SELECT count() FROM data')) == 10 * q + @params def test_prefer_localhost_replica_0_load_balancing_in_order(cluster, q): insert_data(cluster, load_balancing='in_order', prefer_localhost_replica=0) - assert int(n1.query('SELECT count() FROM data')) == 10*q + assert int(n1.query('SELECT count() FROM data')) == 10 * q assert int(n2.query('SELECT count() FROM data')) == 10 + @params def test_prefer_localhost_replica_0_load_balancing_in_order_sync(cluster, q): insert_data(cluster, load_balancing='in_order', prefer_localhost_replica=0, insert_distributed_sync=1) - assert int(n1.query('SELECT count() FROM data')) == 10*q + assert int(n1.query('SELECT count() FROM data')) == 10 * q assert int(n2.query('SELECT count() FROM data')) == 10 diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index 731ffbbe2fd..52beaf06ec2 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -1,11 +1,10 @@ -import pytest import time +import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) instance_test_reconnect = cluster.add_instance('instance_test_reconnect', main_configs=['configs/remote_servers.xml']) @@ -24,6 +23,7 @@ node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml' shard1 = cluster.add_instance('shard1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) shard2 = cluster.add_instance('shard2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -39,7 +39,8 @@ CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'defaul CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local2') ''') - instance_test_inserts_local_cluster.query("CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)") + instance_test_inserts_local_cluster.query( + "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)") instance_test_inserts_local_cluster.query(''' CREATE TABLE distributed_on_local (d Date, x UInt32) ENGINE = Distributed('test_local_cluster', 'default', 'local') ''') @@ -74,8 +75,6 @@ CREATE TABLE table_function (n UInt8, s String) ENGINE = MergeTree() ORDER BY n' node2.query(''' CREATE TABLE table_function (n UInt8, s String) ENGINE = MergeTree() ORDER BY n''') - - yield cluster finally: @@ -162,6 +161,7 @@ def test_inserts_local(started_cluster): time.sleep(0.5) assert instance.query("SELECT count(*) FROM local").strip() == '1' + def test_prefer_localhost_replica(started_cluster): test_query = "SELECT * FROM distributed ORDER BY id" @@ -174,13 +174,13 @@ def test_prefer_localhost_replica(started_cluster): 2017-06-17\t22 ''' - expected_from_node2 = '''\ + expected_from_node2 = '''\ 2017-06-17\t11 2017-06-17\t22 2017-06-17\t44 ''' - expected_from_node1 = '''\ + expected_from_node1 = '''\ 2017-06-17\t11 2017-06-17\t22 2017-06-17\t33 @@ -204,7 +204,9 @@ def test_prefer_localhost_replica(started_cluster): assert TSV(node2.query(test_query)) == TSV(expected_from_node2) # Now query is sent to node1, as it higher in order - assert TSV(node2.query(test_query + " SETTINGS load_balancing='in_order', prefer_localhost_replica=0")) == TSV(expected_from_node1) + assert TSV(node2.query(test_query + " SETTINGS load_balancing='in_order', prefer_localhost_replica=0")) == TSV( + expected_from_node1) + def test_inserts_low_cardinality(started_cluster): instance = shard1 @@ -212,6 +214,9 @@ def test_inserts_low_cardinality(started_cluster): time.sleep(0.5) assert instance.query("SELECT count(*) FROM low_cardinality_all").strip() == '1' + def test_table_function(started_cluster): - node1.query("insert into table function cluster('shard_with_local_replica', 'default', 'table_function') select number, concat('str_', toString(number)) from numbers(100000)") - assert node1.query("select count() from cluster('shard_with_local_replica', 'default', 'table_function')").rstrip() == '100000' + node1.query( + "insert into table function cluster('shard_with_local_replica', 'default', 'table_function') select number, concat('str_', toString(number)) from numbers(100000)") + assert node1.query( + "select count() from cluster('shard_with_local_replica', 'default', 'table_function')").rstrip() == '100000' diff --git a/tests/integration/test_insert_into_distributed_sync_async/test.py b/tests/integration/test_insert_into_distributed_sync_async/test.py index 1b713f17660..30c80e50c43 100755 --- a/tests/integration/test_insert_into_distributed_sync_async/test.py +++ b/tests/integration/test_insert_into_distributed_sync_async/test.py @@ -1,11 +1,11 @@ #!/usr/bin/env python2 -import sys import os +import sys from contextlib import contextmanager + import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from helpers.network import PartitionManager from helpers.test_tools import TSV from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException, QueryTimeoutExceedException @@ -15,6 +15,7 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -25,7 +26,6 @@ def started_cluster(): CREATE TABLE local_table(date Date, val UInt64) ENGINE = MergeTree(date, (date, val), 8192); ''') - node1.query(''' CREATE TABLE distributed_table(date Date, val UInt64) ENGINE = Distributed(test_cluster, default, local_table) ''') @@ -37,7 +37,6 @@ CREATE TABLE distributed_table(date Date, val UInt64) ENGINE = Distributed(test_ def test_insertion_sync(started_cluster): - node1.query('''SET insert_distributed_sync = 1, insert_distributed_timeout = 0; INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers LIMIT 10000''') @@ -105,8 +104,9 @@ def test_insertion_sync_with_disabled_timeout(started_cluster): def test_async_inserts_into_local_shard(started_cluster): node1.query('''CREATE TABLE shard_local (i Int64) ENGINE = Memory''') - node1.query('''CREATE TABLE shard_distributed (i Int64) ENGINE = Distributed(local_shard_with_internal_replication, default, shard_local)''') - node1.query('''INSERT INTO shard_distributed VALUES (1)''', settings={ "insert_distributed_sync" : 0 }) + node1.query( + '''CREATE TABLE shard_distributed (i Int64) ENGINE = Distributed(local_shard_with_internal_replication, default, shard_local)''') + node1.query('''INSERT INTO shard_distributed VALUES (1)''', settings={"insert_distributed_sync": 0}) assert TSV(node1.query('''SELECT count() FROM shard_distributed''')) == TSV("1\n") node1.query('''DETACH TABLE shard_distributed''') diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py index 1df803920f1..32edb6829c8 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py @@ -1,11 +1,10 @@ -import pytest import time +import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) instance_test_reconnect = cluster.add_instance('instance_test_reconnect', main_configs=['configs/remote_servers.xml']) @@ -30,24 +29,26 @@ def started_cluster(): CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local1') ''') instance_test_reconnect.query("CREATE TABLE local1_source (x UInt32) ENGINE = Memory") - instance_test_reconnect.query("CREATE MATERIALIZED VIEW local1_view to distributed AS SELECT x FROM local1_source") + instance_test_reconnect.query( + "CREATE MATERIALIZED VIEW local1_view to distributed AS SELECT x FROM local1_source") remote.query("CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)") instance_test_inserts_batching.query(''' CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local2') ''') instance_test_inserts_batching.query("CREATE TABLE local2_source (d Date, x UInt32) ENGINE = Log") - instance_test_inserts_batching.query("CREATE MATERIALIZED VIEW local2_view to distributed AS SELECT d,x FROM local2_source") - + instance_test_inserts_batching.query( + "CREATE MATERIALIZED VIEW local2_view to distributed AS SELECT d,x FROM local2_source") instance_test_inserts_local_cluster.query("CREATE TABLE local_source (d Date, x UInt32) ENGINE = Memory") - instance_test_inserts_local_cluster.query("CREATE MATERIALIZED VIEW local_view to distributed_on_local AS SELECT d,x FROM local_source") - instance_test_inserts_local_cluster.query("CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)") + instance_test_inserts_local_cluster.query( + "CREATE MATERIALIZED VIEW local_view to distributed_on_local AS SELECT d,x FROM local_source") + instance_test_inserts_local_cluster.query( + "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)") instance_test_inserts_local_cluster.query(''' CREATE TABLE distributed_on_local (d Date, x UInt32) ENGINE = Distributed('test_local_cluster', 'default', 'local') ''') - yield cluster finally: @@ -78,6 +79,7 @@ def test_reconnect(started_cluster): assert remote.query("SELECT count(*) FROM local1").strip() == '3' + @pytest.mark.skip(reason="Flapping test") def test_inserts_batching(started_cluster): instance = instance_test_inserts_batching diff --git a/tests/integration/test_live_view_over_distributed/test.py b/tests/integration/test_live_view_over_distributed/test.py index 1ae9a4b5199..67ff4d8dfe7 100644 --- a/tests/integration/test_live_view_over_distributed/test.py +++ b/tests/integration/test_live_view_over_distributed/test.py @@ -1,16 +1,10 @@ from __future__ import print_function import sys -import itertools -import timeit -import logging import pytest - -from helpers.uclient import client, prompt, end_of_block from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager -from helpers.test_tools import TSV +from helpers.uclient import client, prompt, end_of_block cluster = ClickHouseCluster(__file__) @@ -45,6 +39,7 @@ CREATE LIVE VIEW lv_over_distributed_table AS SELECT * FROM distributed_table; INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" + @pytest.fixture(scope="function") def started_cluster(): try: @@ -73,7 +68,7 @@ node2\t1\t11 def test_select_with_order_by_key(self, started_cluster, node, source): assert node.query("SELECT * FROM {source} ORDER BY key, node".format(source=source)) \ - == """node1\t0\t0 + == """node1\t0\t0 node2\t0\t10 node1\t1\t1 node2\t1\t11 @@ -81,15 +76,15 @@ node2\t1\t11 def test_select_with_group_by_node(self, started_cluster, node, source): assert node.query("SELECT node, SUM(value) FROM {source} GROUP BY node ORDER BY node".format(source=source)) \ - == "node1\t1\nnode2\t21\n" + == "node1\t1\nnode2\t21\n" def test_select_with_group_by_key(self, started_cluster, node, source): assert node.query("SELECT key, SUM(value) FROM {source} GROUP BY key ORDER BY key".format(source=source)) \ - == "0\t10\n1\t12\n" + == "0\t10\n1\t12\n" def test_select_sum(self, started_cluster, node, source): assert node.query("SELECT SUM(value) FROM {source}".format(source=source)) \ - == "22\n" + == "22\n" def test_watch_live_view_order_by_node(self, started_cluster, node, source): log = sys.stdout @@ -193,7 +188,8 @@ node2\t1\t11 client1.send("DROP TABLE IF EXISTS lv") client1.expect(prompt) - client1.send("CREATE LIVE VIEW lv AS SELECT node, SUM(value) FROM distributed_table GROUP BY node ORDER BY node") + client1.send( + "CREATE LIVE VIEW lv AS SELECT node, SUM(value) FROM distributed_table GROUP BY node ORDER BY node") client1.expect(prompt) client1.send("WATCH lv FORMAT CSV") @@ -227,7 +223,8 @@ node2\t1\t11 client1.send("DROP TABLE IF EXISTS lv") client1.expect(prompt) - client1.send("CREATE LIVE VIEW lv AS SELECT key, SUM(value) FROM distributed_table GROUP BY key ORDER BY key") + client1.send( + "CREATE LIVE VIEW lv AS SELECT key, SUM(value) FROM distributed_table GROUP BY key ORDER BY key") client1.expect(prompt) client1.send("WATCH lv FORMAT CSV") @@ -247,7 +244,6 @@ node2\t1\t11 client1.expect('2,2,3') client1.expect('3,3,3') - def test_watch_live_view_sum(self, started_cluster, node, source): log = sys.stdout command = " ".join(node.client.command) diff --git a/tests/integration/test_log_family_s3/test.py b/tests/integration/test_log_family_s3/test.py index 3b0d847967b..40e263c9c69 100644 --- a/tests/integration/test_log_family_s3/test.py +++ b/tests/integration/test_log_family_s3/test.py @@ -11,7 +11,9 @@ logging.getLogger().addHandler(logging.StreamHandler()) def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/minio.xml", "configs/ssl.xml", "configs/config.d/log_conf.xml"], with_minio=True) + cluster.add_instance("node", + main_configs=["configs/minio.xml", "configs/ssl.xml", "configs/config.d/log_conf.xml"], + with_minio=True) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -36,11 +38,13 @@ def test_log_family_s3(cluster, log_engine, files_overhead, files_overhead_per_i node.query("INSERT INTO s3_test SELECT number + 5 FROM numbers(3)") assert node.query("SELECT * FROM s3_test order by id") == "0\n1\n2\n3\n4\n5\n6\n7\n" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == files_overhead_per_insert * 2 + files_overhead + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == files_overhead_per_insert * 2 + files_overhead node.query("INSERT INTO s3_test SELECT number + 8 FROM numbers(1)") assert node.query("SELECT * FROM s3_test order by id") == "0\n1\n2\n3\n4\n5\n6\n7\n8\n" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == files_overhead_per_insert * 3 + files_overhead + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == files_overhead_per_insert * 3 + files_overhead node.query("TRUNCATE TABLE s3_test") assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 0 diff --git a/tests/integration/test_logs_level/test.py b/tests/integration/test_logs_level/test.py index 302686b6fa0..9aa3f7ffd9a 100644 --- a/tests/integration/test_logs_level/test.py +++ b/tests/integration/test_logs_level/test.py @@ -5,6 +5,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=['configs/config_information.xml']) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -13,6 +14,7 @@ def start_cluster(): finally: cluster.shutdown() + def test_check_client_logs_level(start_cluster): logs = node.query_and_get_answer_with_error("SELECT 1", settings={"send_logs_level": 'trace'})[1] assert logs.count('Trace') != 0 diff --git a/tests/integration/test_match_process_uid_against_data_owner/test.py b/tests/integration/test_match_process_uid_against_data_owner/test.py index 77cbe25ff80..c9f87e683c8 100644 --- a/tests/integration/test_match_process_uid_against_data_owner/test.py +++ b/tests/integration/test_match_process_uid_against_data_owner/test.py @@ -1,9 +1,9 @@ -import docker import os import pwd -import pytest import re +import docker +import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_START_COMMAND @@ -27,11 +27,13 @@ def test_different_user(): container.exec_run('chown {} /var/lib/clickhouse'.format(other_user_id), privileged=True) container.exec_run(CLICKHOUSE_START_COMMAND) - cluster.shutdown() # cleanup + cluster.shutdown() # cleanup with open(os.path.join(node.path, 'logs/clickhouse-server.err.log')) as log: expected_message = "Effective user of the process \(.*\) does not match the owner of the data \(.*\)\. Run under 'sudo -u .*'\." last_message = log.readlines()[-1].strip() if re.search(expected_message, last_message) is None: - pytest.fail('Expected the server to fail with a message "{}", but the last message is "{}"'.format(expected_message, last_message)) + pytest.fail( + 'Expected the server to fail with a message "{}", but the last message is "{}"'.format(expected_message, + last_message)) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index dbf86044583..a0a3b0b1cb6 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -1,4 +1,5 @@ import time + import pymysql.cursors @@ -21,73 +22,75 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam # existed before the mapping was created mysql_node.query("CREATE TABLE test_database.test_table_1 (" - "`key` INT NOT NULL PRIMARY KEY, " - "unsigned_tiny_int TINYINT UNSIGNED, tiny_int TINYINT, " - "unsigned_small_int SMALLINT UNSIGNED, small_int SMALLINT, " - "unsigned_medium_int MEDIUMINT UNSIGNED, medium_int MEDIUMINT, " - "unsigned_int INT UNSIGNED, _int INT, " - "unsigned_integer INTEGER UNSIGNED, _integer INTEGER, " - "unsigned_bigint BIGINT UNSIGNED, _bigint BIGINT, " - "/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */" - "unsigned_float FLOAT UNSIGNED, _float FLOAT, " - "unsigned_double DOUBLE UNSIGNED, _double DOUBLE, " - "_varchar VARCHAR(10), _char CHAR(10), " - "/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */" - "_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;") + "`key` INT NOT NULL PRIMARY KEY, " + "unsigned_tiny_int TINYINT UNSIGNED, tiny_int TINYINT, " + "unsigned_small_int SMALLINT UNSIGNED, small_int SMALLINT, " + "unsigned_medium_int MEDIUMINT UNSIGNED, medium_int MEDIUMINT, " + "unsigned_int INT UNSIGNED, _int INT, " + "unsigned_integer INTEGER UNSIGNED, _integer INTEGER, " + "unsigned_bigint BIGINT UNSIGNED, _bigint BIGINT, " + "/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */" + "unsigned_float FLOAT UNSIGNED, _float FLOAT, " + "unsigned_double DOUBLE UNSIGNED, _double DOUBLE, " + "_varchar VARCHAR(10), _char CHAR(10), " + "/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */" + "_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;") # it already has some data mysql_node.query( "INSERT INTO test_database.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', " "'2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', true);") - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n") check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV", - "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" - "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n") + "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" + "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n") mysql_node.query( "INSERT INTO test_database.test_table_1 VALUES(2, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', " "'2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', false);") check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV", - "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" - "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n") + "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" + "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" + "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n") mysql_node.query("UPDATE test_database.test_table_1 SET unsigned_tiny_int = 2 WHERE `key` = 1") check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," - " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " - " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, " - " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " - " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV", - "1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" - "2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n") + " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " + " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, " + " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " + " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV", + "1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" + "2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" + "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n") # update primary key mysql_node.query("UPDATE test_database.test_table_1 SET `key` = 3 WHERE `unsigned_tiny_int` = 2") check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," - " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " - " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, " - " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " - " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV", - "2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t" - "4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t1\n") + " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " + " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, " + " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " + " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV", + "2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" + "varchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t" + "4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t2020-01-01 00:00:00\t1\n") mysql_node.query('DELETE FROM test_database.test_table_1 WHERE `key` = 2') check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," - " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " - " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, " - " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " - " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV", - "3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" - "2020-01-01 00:00:00\t1\n") + " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " + " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, " + " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " + " _bool FROM test_database.test_table_1 ORDER BY key FORMAT TSV", + "3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\t2020-01-01\t" + "2020-01-01 00:00:00\t1\n") mysql_node.query('DELETE FROM test_database.test_table_1 WHERE `unsigned_tiny_int` = 2') check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV", "") @@ -108,7 +111,8 @@ def drop_table_with_materialize_mysql_database(clickhouse_node, mysql_node, serv # create mapping clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_2\n") @@ -117,7 +121,8 @@ def drop_table_with_materialize_mysql_database(clickhouse_node, mysql_node, serv mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(1), (2), (3), (4), (5), (6)") mysql_node.query("CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", "1\n2\n3\n4\n5\n6\n") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", + "1\n2\n3\n4\n5\n6\n") mysql_node.query("DROP TABLE test_database.test_table_1;") mysql_node.query("TRUNCATE TABLE test_database.test_table_2;") @@ -136,17 +141,21 @@ def create_table_with_materialize_mysql_database(clickhouse_node, mysql_node, se mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(1), (2), (3), (5), (6), (7);") # create mapping - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) # Check for pre-existing status assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY id FORMAT TSV", "1\n2\n3\n5\n6\n7\n") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY id FORMAT TSV", + "1\n2\n3\n5\n6\n7\n") mysql_node.query("CREATE TABLE test_database.test_table_2 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(1), (2), (3), (4), (5), (6);") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", "1\n2\n3\n4\n5\n6\n") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", + "1\n2\n3\n4\n5\n6\n") clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") @@ -160,7 +169,8 @@ def rename_table_with_materialize_mysql_database(clickhouse_node, mysql_node, se # create mapping clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_2\n") @@ -178,30 +188,35 @@ def alter_add_column_with_materialize_mysql_database(clickhouse_node, mysql_node mysql_node.query("ALTER TABLE test_database.test_table_1 ADD COLUMN add_column_1 INT NOT NULL") mysql_node.query("ALTER TABLE test_database.test_table_1 ADD COLUMN add_column_2 INT NOT NULL FIRST") mysql_node.query("ALTER TABLE test_database.test_table_1 ADD COLUMN add_column_3 INT NOT NULL AFTER add_column_1") - mysql_node.query("ALTER TABLE test_database.test_table_1 ADD COLUMN add_column_4 INT NOT NULL DEFAULT " + ("0" if service_name == "mysql1" else "(id)")) + mysql_node.query("ALTER TABLE test_database.test_table_1 ADD COLUMN add_column_4 INT NOT NULL DEFAULT " + ( + "0" if service_name == "mysql1" else "(id)")) # create mapping clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n") check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", - "add_column_2\tInt32\t\t\t\t\t\nid\tInt32\t\t\t\t\t\nadd_column_1\tInt32\t\t\t\t\t\nadd_column_3\tInt32\t\t\t\t\t\nadd_column_4\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + "add_column_2\tInt32\t\t\t\t\t\nid\tInt32\t\t\t\t\t\nadd_column_1\tInt32\t\t\t\t\t\nadd_column_3\tInt32\t\t\t\t\t\nadd_column_4\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("CREATE TABLE test_database.test_table_2 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE test_database.test_table_2 ADD COLUMN add_column_1 INT NOT NULL, ADD COLUMN add_column_2 INT NOT NULL FIRST") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + mysql_node.query( + "ALTER TABLE test_database.test_table_2 ADD COLUMN add_column_1 INT NOT NULL, ADD COLUMN add_column_2 INT NOT NULL FIRST") mysql_node.query( "ALTER TABLE test_database.test_table_2 ADD COLUMN add_column_3 INT NOT NULL AFTER add_column_1, ADD COLUMN add_column_4 INT NOT NULL DEFAULT " + ( "0" if service_name == "mysql1" else "(id)")) default_expression = "DEFAULT\t0" if service_name == "mysql1" else "DEFAULT\tid" check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", - "add_column_2\tInt32\t\t\t\t\t\nid\tInt32\t\t\t\t\t\nadd_column_1\tInt32\t\t\t\t\t\nadd_column_3\tInt32\t\t\t\t\t\nadd_column_4\tInt32\t" + default_expression + "\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + "add_column_2\tInt32\t\t\t\t\t\nid\tInt32\t\t\t\t\t\nadd_column_1\tInt32\t\t\t\t\t\nadd_column_3\tInt32\t\t\t\t\t\nadd_column_4\tInt32\t" + default_expression + "\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(1, 2, 3, 4, 5), (6, 7, 8, 9, 10)") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", "1\t2\t3\t4\t5\n6\t7\t8\t9\t10\n") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", + "1\t2\t3\t4\t5\n6\t7\t8\t9\t10\n") clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") @@ -209,22 +224,28 @@ def alter_add_column_with_materialize_mysql_database(clickhouse_node, mysql_node def alter_drop_column_with_materialize_mysql_database(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT) ENGINE = InnoDB;") mysql_node.query("ALTER TABLE test_database.test_table_1 DROP COLUMN drop_column") # create mapping clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n") - check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("CREATE TABLE test_database.test_table_2 (id INT NOT NULL PRIMARY KEY, drop_column INT NOT NULL) ENGINE = InnoDB;") + check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + mysql_node.query( + "CREATE TABLE test_database.test_table_2 (id INT NOT NULL PRIMARY KEY, drop_column INT NOT NULL) ENGINE = InnoDB;") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "id\tInt32\t\t\t\t\t\ndrop_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\ndrop_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("ALTER TABLE test_database.test_table_2 DROP COLUMN drop_column") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(1), (2), (3), (4), (5)") check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", "1\n2\n3\n4\n5\n") @@ -237,25 +258,32 @@ def alter_rename_column_with_materialize_mysql_database(clickhouse_node, mysql_n mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") # maybe should test rename primary key? - mysql_node.query("CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, rename_column INT NOT NULL) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, rename_column INT NOT NULL) ENGINE = InnoDB;") mysql_node.query("ALTER TABLE test_database.test_table_1 RENAME COLUMN rename_column TO new_column_name") # create mapping clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n") - check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", "id\tInt32\t\t\t\t\t\nnew_column_name\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("CREATE TABLE test_database.test_table_2 (id INT NOT NULL PRIMARY KEY, rename_column INT NOT NULL) ENGINE = InnoDB;") + check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nnew_column_name\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + mysql_node.query( + "CREATE TABLE test_database.test_table_2 (id INT NOT NULL PRIMARY KEY, rename_column INT NOT NULL) ENGINE = InnoDB;") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "id\tInt32\t\t\t\t\t\nrename_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nrename_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("ALTER TABLE test_database.test_table_2 RENAME COLUMN rename_column TO new_column_name") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "id\tInt32\t\t\t\t\t\nnew_column_name\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nnew_column_name\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", "1\t2\n3\t4\n5\t6\n7\t8\n9\t10\n") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", + "1\t2\n3\t4\n5\t6\n7\t8\n9\t10\n") clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") @@ -265,26 +293,34 @@ def alter_modify_column_with_materialize_mysql_database(clickhouse_node, mysql_n mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") # maybe should test rename primary key? - mysql_node.query("CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, modify_column INT NOT NULL) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, modify_column INT NOT NULL) ENGINE = InnoDB;") mysql_node.query("ALTER TABLE test_database.test_table_1 MODIFY COLUMN modify_column INT") # create mapping clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n") - check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("CREATE TABLE test_database.test_table_2 (id INT NOT NULL PRIMARY KEY, modify_column INT NOT NULL) ENGINE = InnoDB;") + check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + mysql_node.query( + "CREATE TABLE test_database.test_table_2 (id INT NOT NULL PRIMARY KEY, modify_column INT NOT NULL) ENGINE = InnoDB;") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "id\tInt32\t\t\t\t\t\nmodify_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nmodify_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("ALTER TABLE test_database.test_table_2 MODIFY COLUMN modify_column INT") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("ALTER TABLE test_database.test_table_2 MODIFY COLUMN modify_column INT FIRST") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "modify_column\tNullable(Int32)\t\t\t\t\t\nid\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "modify_column\tNullable(Int32)\t\t\t\t\t\nid\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("ALTER TABLE test_database.test_table_2 MODIFY COLUMN modify_column INT AFTER id") - check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(1, 2), (3, NULL)") check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY id FORMAT TSV", "1\t2\n3\t\\N\n") @@ -299,23 +335,31 @@ def alter_modify_column_with_materialize_mysql_database(clickhouse_node, mysql_n def alter_rename_table_with_materialize_mysql_database(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT) ENGINE = InnoDB;") - mysql_node.query("ALTER TABLE test_database.test_table_1 DROP COLUMN drop_column, RENAME TO test_database.test_table_2, RENAME TO test_database.test_table_3") + mysql_node.query( + "ALTER TABLE test_database.test_table_1 DROP COLUMN drop_column, RENAME TO test_database.test_table_2, RENAME TO test_database.test_table_3") # create mapping clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name)) assert "test_database" in clickhouse_node.query("SHOW DATABASES") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_3\n") - check_query(clickhouse_node, "DESC test_database.test_table_3 FORMAT TSV", "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT NOT NULL) ENGINE = InnoDB;") + check_query(clickhouse_node, "DESC test_database.test_table_3 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + mysql_node.query( + "CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT NOT NULL) ENGINE = InnoDB;") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\ntest_table_3\n") - check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", "id\tInt32\t\t\t\t\t\ndrop_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE test_database.test_table_1 DROP COLUMN drop_column, RENAME TO test_database.test_table_2, RENAME TO test_database.test_table_4") + check_query(clickhouse_node, "DESC test_database.test_table_1 FORMAT TSV", + "id\tInt32\t\t\t\t\t\ndrop_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + mysql_node.query( + "ALTER TABLE test_database.test_table_1 DROP COLUMN drop_column, RENAME TO test_database.test_table_2, RENAME TO test_database.test_table_4") check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_3\ntest_table_4\n") - check_query(clickhouse_node, "DESC test_database.test_table_4 FORMAT TSV", "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query(clickhouse_node, "DESC test_database.test_table_4 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") mysql_node.query("INSERT INTO test_database.test_table_4 VALUES(1), (2), (3), (4), (5)") check_query(clickhouse_node, "SELECT * FROM test_database.test_table_4 ORDER BY id FORMAT TSV", "1\n2\n3\n4\n5\n") @@ -323,6 +367,7 @@ def alter_rename_table_with_materialize_mysql_database(clickhouse_node, mysql_no clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") + def query_event_with_empty_transaction(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE DATABASE test_database") diff --git a/tests/integration/test_materialize_mysql_database/test.py b/tests/integration/test_materialize_mysql_database/test.py index c00b310436d..c7314fb50d9 100644 --- a/tests/integration/test_materialize_mysql_database/test.py +++ b/tests/integration/test_materialize_mysql_database/test.py @@ -4,9 +4,9 @@ import time import pymysql.cursors import pytest +from helpers.cluster import ClickHouseCluster, get_docker_compose_path import materialize_with_ddl -from helpers.cluster import ClickHouseCluster, get_docker_compose_path DOCKER_COMPOSE_PATH = get_docker_compose_path() @@ -33,7 +33,8 @@ class MySQLNodeInstance: def alloc_connection(self): if self.mysql_connection is None: - self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, port=self.port, autocommit=True) + self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, + port=self.port, autocommit=True) return self.mysql_connection def query(self, execution_query): @@ -65,12 +66,14 @@ def started_mysql_5_7(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql.yml') try: - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d']) mysql_node.wait_mysql_to_start(120) yield mysql_node finally: mysql_node.close() - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'down', '--volumes', '--remove-orphans']) + subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'down', '--volumes', + '--remove-orphans']) @pytest.fixture(scope="module") @@ -79,12 +82,14 @@ def started_mysql_8_0(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_8_0.yml') try: - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d']) mysql_node.wait_mysql_to_start(120) yield mysql_node finally: mysql_node.close() - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'down', '--volumes', '--remove-orphans']) + subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'down', '--volumes', + '--remove-orphans']) def test_materialize_database_dml_with_mysql_5_7(started_cluster, started_mysql_5_7): @@ -94,19 +99,25 @@ def test_materialize_database_dml_with_mysql_5_7(started_cluster, started_mysql_ def test_materialize_database_dml_with_mysql_8_0(started_cluster, started_mysql_8_0): materialize_with_ddl.dml_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") + def test_materialize_database_ddl_with_mysql_5_7(started_cluster, started_mysql_5_7): try: materialize_with_ddl.drop_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") materialize_with_ddl.create_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") materialize_with_ddl.rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") - materialize_with_ddl.alter_add_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") - materialize_with_ddl.alter_drop_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") + materialize_with_ddl.alter_add_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, + "mysql1") + materialize_with_ddl.alter_drop_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, + "mysql1") # mysql 5.7 cannot support alter rename column # materialize_with_ddl.alter_rename_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") - materialize_with_ddl.alter_rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") - materialize_with_ddl.alter_modify_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") + materialize_with_ddl.alter_rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, + "mysql1") + materialize_with_ddl.alter_modify_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, + "mysql1") except: - print(clickhouse_node.query("select '\n', thread_id, query_id, arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym from system.stack_trace format TSVRaw")) + print(clickhouse_node.query( + "select '\n', thread_id, query_id, arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym from system.stack_trace format TSVRaw")) raise @@ -114,14 +125,21 @@ def test_materialize_database_ddl_with_mysql_8_0(started_cluster, started_mysql_ materialize_with_ddl.drop_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") materialize_with_ddl.create_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") materialize_with_ddl.rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") - materialize_with_ddl.alter_add_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") - materialize_with_ddl.alter_drop_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") - materialize_with_ddl.alter_rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") - materialize_with_ddl.alter_rename_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") - materialize_with_ddl.alter_modify_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") + materialize_with_ddl.alter_add_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, + "mysql8_0") + materialize_with_ddl.alter_drop_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, + "mysql8_0") + materialize_with_ddl.alter_rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, + "mysql8_0") + materialize_with_ddl.alter_rename_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, + "mysql8_0") + materialize_with_ddl.alter_modify_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, + "mysql8_0") + def test_materialize_database_ddl_with_empty_transaction_5_7(started_cluster, started_mysql_5_7): materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_5_7, "mysql1") + def test_materialize_database_ddl_with_empty_transaction_8_0(started_cluster, started_mysql_8_0): materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_8_0, "mysql8_0") diff --git a/tests/integration/test_max_http_connections_for_replication/test.py b/tests/integration/test_max_http_connections_for_replication/test.py index 0317aa19cc3..5ef45c2a893 100644 --- a/tests/integration/test_max_http_connections_for_replication/test.py +++ b/tests/integration/test_max_http_connections_for_replication/test.py @@ -1,29 +1,33 @@ import time -import pytest - -from helpers.cluster import ClickHouseCluster from multiprocessing.dummy import Pool +import pytest +from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry + def _fill_nodes(nodes, shard, connections_count): for node in nodes: node.query( - ''' - CREATE DATABASE test; + ''' + CREATE DATABASE test; + + CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') + PARTITION BY date + ORDER BY id + SETTINGS + replicated_max_parallel_fetches_for_host={connections}, + index_granularity=8192; + '''.format(shard=shard, replica=node.name, connections=connections_count)) - CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') - PARTITION BY date - ORDER BY id - SETTINGS - replicated_max_parallel_fetches_for_host={connections}, - index_granularity=8192; - '''.format(shard=shard, replica=node.name, connections=connections_count)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', user_configs=[], main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', user_configs=[], main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node1 = cluster.add_instance('node1', user_configs=[], + main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', user_configs=[], + main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def start_small_cluster(): @@ -37,8 +41,8 @@ def start_small_cluster(): finally: cluster.shutdown() -def test_single_endpoint_connections_count(start_small_cluster): +def test_single_endpoint_connections_count(start_small_cluster): def task(count): print("Inserting ten times from {}".format(count)) for i in xrange(count, count + 10): @@ -52,6 +56,7 @@ def test_single_endpoint_connections_count(start_small_cluster): assert node2.query("SELECT value FROM system.events where event='CreatedHTTPConnections'") == '1\n' + def test_keepalive_timeout(start_small_cluster): current_count = int(node1.query("select count() from test_table").strip()) node1.query("insert into test_table values ('2017-06-16', 777, 0)") @@ -68,9 +73,14 @@ def test_keepalive_timeout(start_small_cluster): assert not node2.contains_in_log("No message received"), "Found 'No message received' in clickhouse-server.log" -node3 = cluster.add_instance('node3', user_configs=[], main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', user_configs=[], main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) -node5 = cluster.add_instance('node5', user_configs=[], main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) + +node3 = cluster.add_instance('node3', user_configs=[], + main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', user_configs=[], + main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node5 = cluster.add_instance('node5', user_configs=[], + main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def start_big_cluster(): @@ -84,8 +94,8 @@ def start_big_cluster(): finally: cluster.shutdown() -def test_multiple_endpoint_connections_count(start_big_cluster): +def test_multiple_endpoint_connections_count(start_big_cluster): def task(count): print("Inserting ten times from {}".format(count)) if (count / 10) % 2 == 1: diff --git a/tests/integration/test_merge_table_over_distributed/test.py b/tests/integration/test_merge_table_over_distributed/test.py index b52a605943b..2e73bd09ded 100644 --- a/tests/integration/test_merge_table_over_distributed/test.py +++ b/tests/integration/test_merge_table_over_distributed/test.py @@ -1,7 +1,6 @@ from contextlib import contextmanager import pytest - from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) @@ -9,6 +8,7 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -36,28 +36,29 @@ CREATE TABLE merge_table(id UInt32, val String) ENGINE = Merge(default, '^distri def test_global_in(started_cluster): - assert node1.query("SELECT val FROM distributed_table WHERE id GLOBAL IN (SELECT toUInt32(3 - id) FROM local_table)").rstrip() \ - == 'node2' + assert node1.query( + "SELECT val FROM distributed_table WHERE id GLOBAL IN (SELECT toUInt32(3 - id) FROM local_table)").rstrip() \ + == 'node2' - assert node1.query("SELECT val FROM merge_table WHERE id GLOBAL IN (SELECT toUInt32(3 - id) FROM local_table)").rstrip() \ - == 'node2' + assert node1.query( + "SELECT val FROM merge_table WHERE id GLOBAL IN (SELECT toUInt32(3 - id) FROM local_table)").rstrip() \ + == 'node2' def test_filtering(started_cluster): - assert node1.query("SELECT id, val FROM merge_table WHERE id = 1").rstrip() == '1\tnode1' assert node1.query("SELECT id + 1, val FROM merge_table WHERE id = 1").rstrip() == '2\tnode1' assert node1.query("SELECT id + 1 FROM merge_table WHERE val = 'node1'").rstrip() == '2' - assert node1.query("SELECT id + 1, val FROM merge_table PREWHERE id = 1 WHERE _table != '_dummy'").rstrip() == '2\tnode1' + assert node1.query( + "SELECT id + 1, val FROM merge_table PREWHERE id = 1 WHERE _table != '_dummy'").rstrip() == '2\tnode1' assert node1.query("SELECT count() FROM merge_table PREWHERE id = 1").rstrip() == '1' def test_select_table_name_from_merge_over_distributed(started_cluster): - node1.query("INSERT INTO local_table_2 VALUES (1, 'node1')") node2.query("INSERT INTO local_table_2 VALUES (2, 'node2')") diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index c4e7bbefd87..7acb8c5fe00 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -14,7 +14,9 @@ logging.getLogger().addHandler(logging.StreamHandler()) def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", "configs/config.d/bg_processing_pool_conf.xml", "configs/config.d/log_conf.xml"], user_configs=[], with_minio=True) + cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", + "configs/config.d/bg_processing_pool_conf.xml", + "configs/config.d/log_conf.xml"], user_configs=[], with_minio=True) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -36,7 +38,7 @@ def random_string(length): def generate_values(date_str, count, sign=1): - data = [[date_str, sign*(i + 1), random_string(10)] for i in range(count)] + data = [[date_str, sign * (i + 1), random_string(10)] for i in range(count)] data.sort(key=lambda tup: tup[1]) return ",".join(["('{}',{},'{}')".format(x, y, z) for x, y, z in data]) @@ -103,7 +105,7 @@ def test_simple_insert_select(cluster, min_rows_for_wide_part, files_per_part): values2 = generate_values('2020-01-04', 4096) node.query("INSERT INTO s3_test VALUES {}".format(values2)) assert node.query("SELECT * FROM s3_test ORDER BY dt, id FORMAT Values") == values1 + "," + values2 - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + files_per_part*2 + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + files_per_part * 2 assert node.query("SELECT count(*) FROM s3_test where id = 1 FORMAT Values") == "(2)" @@ -132,7 +134,8 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical): node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD_PER_PART_WIDE*6 + FILES_OVERHEAD + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD_PER_PART_WIDE * 6 + FILES_OVERHEAD node.query("SYSTEM START MERGES s3_test") # Wait for merges and old parts deletion @@ -161,7 +164,8 @@ def test_alter_table_columns(cluster): assert node.query("SELECT sum(col1) FROM s3_test FORMAT Values") == "(8192)" assert node.query("SELECT sum(col1) FROM s3_test WHERE id > 0 FORMAT Values") == "(4096)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN + assert len(list(minio.list_objects(cluster.minio_bucket, + 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN node.query("ALTER TABLE s3_test MODIFY COLUMN col1 String", settings={"mutations_sync": 2}) @@ -170,7 +174,8 @@ def test_alter_table_columns(cluster): assert node.query("SELECT distinct(col1) FROM s3_test FORMAT Values") == "('1')" # and file with mutation - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == (FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN + 1) + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == ( + FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN + 1) node.query("ALTER TABLE s3_test DROP COLUMN col1", settings={"mutations_sync": 2}) @@ -178,7 +183,8 @@ def test_alter_table_columns(cluster): time.sleep(3) # and 2 files with mutations - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + 2 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + 2 def test_attach_detach_partition(cluster): @@ -190,15 +196,18 @@ def test_attach_detach_partition(cluster): node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096))) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*2 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("ALTER TABLE s3_test DETACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*2 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("ALTER TABLE s3_test ATTACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*2 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("ALTER TABLE s3_test DROP PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" @@ -219,7 +228,8 @@ def test_move_partition_to_another_disk(cluster): node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096))) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*2 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 'hdd'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" @@ -227,7 +237,8 @@ def test_move_partition_to_another_disk(cluster): node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 's3'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*2 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 def test_table_manipulations(cluster): @@ -241,7 +252,8 @@ def test_table_manipulations(cluster): node.query("RENAME TABLE s3_test TO s3_renamed") assert node.query("SELECT count(*) FROM s3_renamed FORMAT Values") == "(8192)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*2 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("RENAME TABLE s3_renamed TO s3_test") assert node.query("CHECK TABLE s3_test FORMAT Values") == "(1)" @@ -249,7 +261,8 @@ def test_table_manipulations(cluster): node.query("DETACH TABLE s3_test") node.query("ATTACH TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*2 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("TRUNCATE TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" @@ -268,7 +281,8 @@ def test_move_replace_partition_to_another_table(cluster): node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-06', 4096, -1))) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*4 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 create_table(cluster, "s3_clone") @@ -279,14 +293,16 @@ def test_move_replace_partition_to_another_table(cluster): assert node.query("SELECT sum(id) FROM s3_clone FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_clone FORMAT Values") == "(8192)" # Number of objects in S3 should be unchanged. - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD*2 + FILES_OVERHEAD_PER_PART_WIDE*4 + assert len(list( + minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 # Add new partitions to source table, but with different values and replace them from copied table. node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-05', 4096))) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD*2 + FILES_OVERHEAD_PER_PART_WIDE*6 + assert len(list( + minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 6 node.query("ALTER TABLE s3_test REPLACE PARTITION '2020-01-03' FROM s3_clone") node.query("ALTER TABLE s3_test REPLACE PARTITION '2020-01-05' FROM s3_clone") @@ -297,23 +313,26 @@ def test_move_replace_partition_to_another_table(cluster): # Wait for outdated partitions deletion. time.sleep(3) - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD*2 + FILES_OVERHEAD_PER_PART_WIDE*4 + assert len(list( + minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 node.query("DROP TABLE s3_clone NO DELAY") time.sleep(1) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" # Data should remain in S3 - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*4 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 node.query("ALTER TABLE s3_test FREEZE") # Number S3 objects should be unchanged. - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE*4 + assert len( + list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 node.query("DROP TABLE s3_test NO DELAY") time.sleep(1) # Backup data should remain in S3. - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD_PER_PART_WIDE*4 + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD_PER_PART_WIDE * 4 for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): minio.remove_object(cluster.minio_bucket, obj.object_name) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index 8a4c0a9c30f..d12c7f7b357 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -1,30 +1,32 @@ import json -import pytest import random import re import string import threading import time from multiprocessing.dummy import Pool + +import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml', 'configs/config.d/storage_configuration.xml', 'configs/config.d/cluster.xml'], - with_zookeeper=True, - stay_alive=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 1} ) + main_configs=['configs/logs_config.xml', 'configs/config.d/storage_configuration.xml', + 'configs/config.d/cluster.xml'], + with_zookeeper=True, + stay_alive=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], + macros={"shard": 0, "replica": 1}) node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml', 'configs/config.d/storage_configuration.xml', 'configs/config.d/cluster.xml'], - with_zookeeper=True, - stay_alive=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 2} ) + main_configs=['configs/logs_config.xml', 'configs/config.d/storage_configuration.xml', + 'configs/config.d/cluster.xml'], + with_zookeeper=True, + stay_alive=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], + macros={"shard": 0, "replica": 2}) @pytest.fixture(scope="module") @@ -38,7 +40,6 @@ def start_cluster(): def test_system_tables(start_cluster): - expected_disks_data = [ { "name": "default", @@ -62,7 +63,8 @@ def test_system_tables(start_cluster): } ] - click_disk_data = json.loads(node1.query("SELECT name, path, keep_free_space FROM system.disks FORMAT JSON"))["data"] + click_disk_data = json.loads(node1.query("SELECT name, path, keep_free_space FROM system.disks FORMAT JSON"))[ + "data"] assert sorted(click_disk_data, key=lambda x: x["name"]) == sorted(expected_disks_data, key=lambda x: x["name"]) expected_policies_data = [ @@ -194,7 +196,9 @@ def test_system_tables(start_cluster): }, ] - clickhouse_policies_data = json.loads(node1.query("SELECT * FROM system.storage_policies WHERE policy_name != 'default' FORMAT JSON"))["data"] + clickhouse_policies_data = \ + json.loads(node1.query("SELECT * FROM system.storage_policies WHERE policy_name != 'default' FORMAT JSON"))[ + "data"] def key(x): return (x["policy_name"], x["volume_name"], x["volume_priority"]) @@ -222,7 +226,6 @@ def test_query_parser(start_cluster): SETTINGS storage_policy='jbod1' """) - node1.query(""" CREATE TABLE table_with_normal_policy ( d UInt64 @@ -246,14 +249,15 @@ def test_query_parser(start_cluster): node1.query("ALTER TABLE table_with_normal_policy MOVE PARTITION 'yyyy' TO DISK 'jbod1'") with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE table_with_normal_policy MODIFY SETTING storage_policy='moving_jbod_with_external'") + node1.query( + "ALTER TABLE table_with_normal_policy MODIFY SETTING storage_policy='moving_jbod_with_external'") finally: node1.query("DROP TABLE IF EXISTS table_with_normal_policy") @pytest.mark.parametrize("name,engine", [ - ("test_alter_policy","MergeTree()"), - ("replicated_test_alter_policy","ReplicatedMergeTree('/clickhouse/test_alter_policy', '1')",), + ("test_alter_policy", "MergeTree()"), + ("replicated_test_alter_policy", "ReplicatedMergeTree('/clickhouse/test_alter_policy', '1')",), ]) def test_alter_policy(start_cluster, name, engine): try: @@ -265,21 +269,28 @@ def test_alter_policy(start_cluster, name, engine): SETTINGS storage_policy='small_jbod_with_external' """.format(name=name, engine=engine)) - assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format(name=name)) == "small_jbod_with_external\n" + assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( + name=name)) == "small_jbod_with_external\n" with pytest.raises(QueryRuntimeException): - node1.query("""ALTER TABLE {name} MODIFY SETTING storage_policy='one_more_small_jbod_with_external'""".format(name=name)) + node1.query( + """ALTER TABLE {name} MODIFY SETTING storage_policy='one_more_small_jbod_with_external'""".format( + name=name)) - assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format(name=name)) == "small_jbod_with_external\n" + assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( + name=name)) == "small_jbod_with_external\n" node1.query("""ALTER TABLE {name} MODIFY SETTING storage_policy='jbods_with_external'""".format(name=name)) - assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format(name=name)) == "jbods_with_external\n" + assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( + name=name)) == "jbods_with_external\n" with pytest.raises(QueryRuntimeException): - node1.query("""ALTER TABLE {name} MODIFY SETTING storage_policy='small_jbod_with_external'""".format(name=name)) + node1.query( + """ALTER TABLE {name} MODIFY SETTING storage_policy='small_jbod_with_external'""".format(name=name)) - assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format(name=name)) == "jbods_with_external\n" + assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( + name=name)) == "jbods_with_external\n" finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) @@ -288,8 +299,12 @@ def test_alter_policy(start_cluster, name, engine): def get_random_string(length): return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + def get_used_disks_for_table(node, table_name): - return node.query("select disk_name from system.parts where table == '{}' and active=1 order by modification_time".format(table_name)).strip().split('\n') + return node.query( + "select disk_name from system.parts where table == '{}' and active=1 order by modification_time".format( + table_name)).strip().split('\n') + def test_no_warning_about_zero_max_data_part_size(start_cluster): def get_log(node): @@ -314,8 +329,8 @@ def test_no_warning_about_zero_max_data_part_size(start_cluster): @pytest.mark.parametrize("name,engine", [ - ("mt_on_jbod","MergeTree()"), - ("replicated_mt_on_jbod","ReplicatedMergeTree('/clickhouse/replicated_mt_on_jbod', '1')",), + ("mt_on_jbod", "MergeTree()"), + ("replicated_mt_on_jbod", "ReplicatedMergeTree('/clickhouse/replicated_mt_on_jbod', '1')",), ]) def test_round_robin(start_cluster, name, engine): try: @@ -348,9 +363,10 @@ def test_round_robin(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) + @pytest.mark.parametrize("name,engine", [ - ("mt_with_huge_part","MergeTree()"), - ("replicated_mt_with_huge_part","ReplicatedMergeTree('/clickhouse/replicated_mt_with_huge_part', '1')",), + ("mt_with_huge_part", "MergeTree()"), + ("replicated_mt_with_huge_part", "ReplicatedMergeTree('/clickhouse/replicated_mt_with_huge_part', '1')",), ]) def test_max_data_part_size(start_cluster, name, engine): try: @@ -361,9 +377,9 @@ def test_max_data_part_size(start_cluster, name, engine): ORDER BY tuple() SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - data = [] # 10MB in total + data = [] # 10MB in total for i in range(10): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) used_disks = get_used_disks_for_table(node1, name) @@ -372,9 +388,10 @@ def test_max_data_part_size(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) + @pytest.mark.parametrize("name,engine", [ - ("mt_with_overflow","MergeTree()"), - ("replicated_mt_with_overflow","ReplicatedMergeTree('/clickhouse/replicated_mt_with_overflow', '1')",), + ("mt_with_overflow", "MergeTree()"), + ("replicated_mt_with_overflow", "ReplicatedMergeTree('/clickhouse/replicated_mt_with_overflow', '1')",), ]) def test_jbod_overflow(start_cluster, name, engine): try: @@ -390,18 +407,18 @@ def test_jbod_overflow(start_cluster, name, engine): # small jbod size is 40MB, so lets insert 5MB batch 7 times for i in range(7): - data = [] # 5MB in total + data = [] # 5MB in total for i in range(5): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) used_disks = get_used_disks_for_table(node1, name) assert all(disk == 'jbod1' for disk in used_disks) # should go to the external disk (jbod is overflown) - data = [] # 10MB in total + data = [] # 10MB in total for i in range(10): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) @@ -415,16 +432,19 @@ def test_jbod_overflow(start_cluster, name, engine): node1.query("OPTIMIZE TABLE {} FINAL".format(name)) time.sleep(2) - disks_for_merges = node1.query("SELECT disk_name FROM system.parts WHERE table == '{}' AND level >= 1 and active = 1 ORDER BY modification_time".format(name)).strip().split('\n') + disks_for_merges = node1.query( + "SELECT disk_name FROM system.parts WHERE table == '{}' AND level >= 1 and active = 1 ORDER BY modification_time".format( + name)).strip().split('\n') assert all(disk == 'external' for disk in disks_for_merges) finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) + @pytest.mark.parametrize("name,engine", [ - ("moving_mt","MergeTree()"), - ("moving_replicated_mt","ReplicatedMergeTree('/clickhouse/moving_replicated_mt', '1')",), + ("moving_mt", "MergeTree()"), + ("moving_replicated_mt", "ReplicatedMergeTree('/clickhouse/moving_replicated_mt', '1')",), ]) def test_background_move(start_cluster, name, engine): try: @@ -437,13 +457,12 @@ def test_background_move(start_cluster, name, engine): """.format(name=name, engine=engine)) for i in range(5): - data = [] # 5MB in total + data = [] # 5MB in total for i in range(5): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row # small jbod size is 40MB, so lets insert 5MB batch 5 times node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) - used_disks = get_used_disks_for_table(node1, name) retry = 20 @@ -458,7 +477,9 @@ def test_background_move(start_cluster, name, engine): # first (oldest) part was moved to external assert used_disks[0] == 'external' - path = node1.query("SELECT path_on_disk FROM system.part_log WHERE table = '{}' AND event_type='MovePart' ORDER BY event_time LIMIT 1".format(name)) + path = node1.query( + "SELECT path_on_disk FROM system.part_log WHERE table = '{}' AND event_type='MovePart' ORDER BY event_time LIMIT 1".format( + name)) # first (oldest) part was moved to external assert path.startswith("/external") @@ -466,9 +487,10 @@ def test_background_move(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + @pytest.mark.parametrize("name,engine", [ - ("stopped_moving_mt","MergeTree()"), - ("stopped_moving_replicated_mt","ReplicatedMergeTree('/clickhouse/stopped_moving_replicated_mt', '1')",), + ("stopped_moving_mt", "MergeTree()"), + ("stopped_moving_replicated_mt", "ReplicatedMergeTree('/clickhouse/stopped_moving_replicated_mt', '1')",), ]) def test_start_stop_moves(start_cluster, name, engine): try: @@ -486,7 +508,9 @@ def test_start_stop_moves(start_cluster, name, engine): used_disks = get_used_disks_for_table(node1, name) assert all(d == "jbod1" for d in used_disks), "All writes shoud go to jbods" - first_part = node1.query("SELECT name FROM system.parts WHERE table = '{}' and active = 1 ORDER BY modification_time LIMIT 1".format(name)).strip() + first_part = node1.query( + "SELECT name FROM system.parts WHERE table = '{}' and active = 1 ORDER BY modification_time LIMIT 1".format( + name)).strip() node1.query("SYSTEM STOP MOVES") @@ -500,7 +524,9 @@ def test_start_stop_moves(start_cluster, name, engine): node1.query("ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, first_part)) - disk = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, first_part)).strip() + disk = node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, + first_part)).strip() assert disk == "external" @@ -510,9 +536,9 @@ def test_start_stop_moves(start_cluster, name, engine): node1.query("SYSTEM STOP MERGES {}".format(name)) for i in range(5): - data = [] # 5MB in total + data = [] # 5MB in total for i in range(5): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row # jbod size is 40MB, so lets insert 5MB batch 7 times node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) @@ -548,20 +574,26 @@ def test_start_stop_moves(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + def get_path_for_part_from_part_log(node, table, part_name): node.query("SYSTEM FLUSH LOGS") - path = node.query("SELECT path_on_disk FROM system.part_log WHERE table = '{}' and part_name = '{}' ORDER BY event_time DESC LIMIT 1".format(table, part_name)) + path = node.query( + "SELECT path_on_disk FROM system.part_log WHERE table = '{}' and part_name = '{}' ORDER BY event_time DESC LIMIT 1".format( + table, part_name)) return path.strip() + def get_paths_for_partition_from_part_log(node, table, partition_id): node.query("SYSTEM FLUSH LOGS") - paths = node.query("SELECT path_on_disk FROM system.part_log WHERE table = '{}' and partition_id = '{}' ORDER BY event_time DESC".format(table, partition_id)) + paths = node.query( + "SELECT path_on_disk FROM system.part_log WHERE table = '{}' and partition_id = '{}' ORDER BY event_time DESC".format( + table, partition_id)) return paths.strip().split('\n') @pytest.mark.parametrize("name,engine", [ - ("altering_mt","MergeTree()"), - #("altering_replicated_mt","ReplicatedMergeTree('/clickhouse/altering_replicated_mt', '1')",), + ("altering_mt", "MergeTree()"), + # ("altering_replicated_mt","ReplicatedMergeTree('/clickhouse/altering_replicated_mt', '1')",), # SYSTEM STOP MERGES doesn't disable merges assignments ]) def test_alter_move(start_cluster, name, engine): @@ -576,7 +608,7 @@ def test_alter_move(start_cluster, name, engine): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - node1.query("SYSTEM STOP MERGES {}".format(name)) # to avoid conflicts + node1.query("SYSTEM STOP MERGES {}".format(name)) # to avoid conflicts node1.query("INSERT INTO {} VALUES(toDate('2019-03-15'), 65)".format(name)) node1.query("INSERT INTO {} VALUES(toDate('2019-03-16'), 66)".format(name)) @@ -585,34 +617,45 @@ def test_alter_move(start_cluster, name, engine): used_disks = get_used_disks_for_table(node1, name) assert all(d.startswith("jbod") for d in used_disks), "All writes should go to jbods" - first_part = node1.query("SELECT name FROM system.parts WHERE table = '{}' and active = 1 ORDER BY modification_time LIMIT 1".format(name)).strip() + first_part = node1.query( + "SELECT name FROM system.parts WHERE table = '{}' and active = 1 ORDER BY modification_time LIMIT 1".format( + name)).strip() time.sleep(1) node1.query("ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, first_part)) - disk = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, first_part)).strip() + disk = node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, + first_part)).strip() assert disk == 'external' assert get_path_for_part_from_part_log(node1, name, first_part).startswith("/external") - time.sleep(1) node1.query("ALTER TABLE {} MOVE PART '{}' TO DISK 'jbod1'".format(name, first_part)) - disk = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, first_part)).strip() + disk = node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, + first_part)).strip() assert disk == 'jbod1' assert get_path_for_part_from_part_log(node1, name, first_part).startswith("/jbod1") time.sleep(1) node1.query("ALTER TABLE {} MOVE PARTITION 201904 TO VOLUME 'external'".format(name)) - disks = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201904' and active = 1".format(name)).strip().split('\n') + disks = node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201904' and active = 1".format( + name)).strip().split('\n') assert len(disks) == 2 assert all(d == "external" for d in disks) - assert all(path.startswith("/external") for path in get_paths_for_partition_from_part_log(node1, name, '201904')[:2]) + assert all( + path.startswith("/external") for path in get_paths_for_partition_from_part_log(node1, name, '201904')[:2]) time.sleep(1) node1.query("ALTER TABLE {} MOVE PARTITION 201904 TO DISK 'jbod2'".format(name)) - disks = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201904' and active = 1".format(name)).strip().split('\n') + disks = node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201904' and active = 1".format( + name)).strip().split('\n') assert len(disks) == 2 assert all(d == "jbod2" for d in disks) - assert all(path.startswith("/jbod2") for path in get_paths_for_partition_from_part_log(node1, name, '201904')[:2]) + assert all( + path.startswith("/jbod2") for path in get_paths_for_partition_from_part_log(node1, name, '201904')[:2]) assert node1.query("SELECT COUNT() FROM {}".format(name)) == "4\n" @@ -650,13 +693,18 @@ def test_alter_move_half_of_partition(start_cluster, volume_or_disk): assert len(parts) == 2 node1.query("ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, parts[0])) - disks = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, parts[0])).splitlines() + disks = node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, parts[ + 0])).splitlines() assert disks == ["external"] time.sleep(1) - node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, volume_or_disk=volume_or_disk)) - disks = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201903' and active = 1".format(name)).splitlines() - assert disks == ["external"]*2 + node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, + volume_or_disk=volume_or_disk)) + disks = node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201903' and active = 1".format( + name)).splitlines() + assert disks == ["external"] * 2 assert node1.query("SELECT COUNT() FROM {}".format(name)) == "2\n" @@ -690,15 +738,19 @@ def test_alter_double_move_partition(start_cluster, volume_or_disk): assert all(d.startswith("jbod") for d in used_disks), "All writes should go to jbods" time.sleep(1) - node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, volume_or_disk=volume_or_disk)) - disks = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201903' and active = 1".format(name)).splitlines() - assert disks == ["external"]*2 + node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, + volume_or_disk=volume_or_disk)) + disks = node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201903' and active = 1".format( + name)).splitlines() + assert disks == ["external"] * 2 assert node1.query("SELECT COUNT() FROM {}".format(name)) == "2\n" time.sleep(1) with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, volume_or_disk=volume_or_disk)) + node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, + volume_or_disk=volume_or_disk)) finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) @@ -709,14 +761,14 @@ def produce_alter_move(node, name): if move_type == "PART": for _ in range(10): try: - parts = node1.query("SELECT name from system.parts where table = '{}' and active = 1".format(name)).strip().split('\n') + parts = node1.query( + "SELECT name from system.parts where table = '{}' and active = 1".format(name)).strip().split('\n') break except QueryRuntimeException: pass else: raise Exception("Cannot select from system.parts") - move_part = random.choice(["'" + part + "'" for part in parts]) else: move_part = random.choice([201903, 201904]) @@ -734,8 +786,9 @@ def produce_alter_move(node, name): @pytest.mark.parametrize("name,engine", [ - ("concurrently_altering_mt","MergeTree()"), - ("concurrently_altering_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_altering_replicated_mt', '1')",), + ("concurrently_altering_mt", "MergeTree()"), + ("concurrently_altering_replicated_mt", + "ReplicatedMergeTree('/clickhouse/concurrently_altering_replicated_mt', '1')",), ]) def test_concurrent_alter_move(start_cluster, name, engine): try: @@ -749,7 +802,7 @@ def test_concurrent_alter_move(start_cluster, name, engine): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - values = list({ random.randint(1, 1000000) for _ in range(0, 1000) }) + values = list({random.randint(1, 1000000) for _ in range(0, 1000)}) def insert(num): for i in range(num): @@ -786,9 +839,11 @@ def test_concurrent_alter_move(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + @pytest.mark.parametrize("name,engine", [ - ("concurrently_dropping_mt","MergeTree()"), - ("concurrently_dropping_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_dropping_replicated_mt', '1')",), + ("concurrently_dropping_mt", "MergeTree()"), + ("concurrently_dropping_replicated_mt", + "ReplicatedMergeTree('/clickhouse/concurrently_dropping_replicated_mt', '1')",), ]) def test_concurrent_alter_move_and_drop(start_cluster, name, engine): try: @@ -802,7 +857,7 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - values = list({ random.randint(1, 1000000) for _ in range(0, 1000) }) + values = list({random.randint(1, 1000000) for _ in range(0, 1000)}) def insert(num): for i in range(num): @@ -839,8 +894,8 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): @pytest.mark.parametrize("name,engine", [ - ("detach_attach_mt","MergeTree()"), - ("replicated_detach_attach_mt","ReplicatedMergeTree('/clickhouse/replicated_detach_attach_mt', '1')",), + ("detach_attach_mt", "MergeTree()"), + ("replicated_detach_attach_mt", "ReplicatedMergeTree('/clickhouse/replicated_detach_attach_mt', '1')",), ]) def test_detach_attach(start_cluster, name, engine): try: @@ -852,9 +907,9 @@ def test_detach_attach(start_cluster, name, engine): SETTINGS storage_policy='moving_jbod_with_external' """.format(name=name, engine=engine)) - data = [] # 5MB in total + data = [] # 5MB in total for i in range(5): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) node1.query("ALTER TABLE {} DETACH PARTITION tuple()".format(name)) @@ -870,8 +925,8 @@ def test_detach_attach(start_cluster, name, engine): @pytest.mark.parametrize("name,engine", [ - ("mutating_mt","MergeTree()"), - ("replicated_mutating_mt","ReplicatedMergeTree('/clickhouse/replicated_mutating_mt', '1')",), + ("mutating_mt", "MergeTree()"), + ("replicated_mutating_mt", "ReplicatedMergeTree('/clickhouse/replicated_mutating_mt', '1')",), ]) def test_mutate_to_another_disk(start_cluster, name, engine): try: @@ -884,9 +939,9 @@ def test_mutate_to_another_disk(start_cluster, name, engine): """.format(name=name, engine=engine)) for i in range(5): - data = [] # 5MB in total + data = [] # 5MB in total for i in range(5): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) node1.query("ALTER TABLE {} UPDATE s1 = concat(s1, 'x') WHERE 1".format(name)) @@ -898,7 +953,7 @@ def test_mutate_to_another_disk(start_cluster, name, engine): if node1.query("SELECT latest_fail_reason FROM system.mutations WHERE table = '{}'".format(name)) == "": assert node1.query("SELECT sum(endsWith(s1, 'x')) FROM {}".format(name)) == "25\n" - else: # mutation failed, let's try on another disk + else: # mutation failed, let's try on another disk print "Mutation failed" node1.query("OPTIMIZE TABLE {} FINAL".format(name)) node1.query("ALTER TABLE {} UPDATE s1 = concat(s1, 'x') WHERE 1".format(name)) @@ -914,9 +969,10 @@ def test_mutate_to_another_disk(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + @pytest.mark.parametrize("name,engine", [ - ("alter_modifying_mt","MergeTree()"), - ("replicated_alter_modifying_mt","ReplicatedMergeTree('/clickhouse/replicated_alter_modifying_mt', '1')",), + ("alter_modifying_mt", "MergeTree()"), + ("replicated_alter_modifying_mt", "ReplicatedMergeTree('/clickhouse/replicated_alter_modifying_mt', '1')",), ]) def test_concurrent_alter_modify(start_cluster, name, engine): try: @@ -930,7 +986,7 @@ def test_concurrent_alter_modify(start_cluster, name, engine): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - values = list({ random.randint(1, 1000000) for _ in range(0, 1000) }) + values = list({random.randint(1, 1000000) for _ in range(0, 1000)}) def insert(num): for i in range(num): @@ -971,6 +1027,7 @@ def test_concurrent_alter_modify(start_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + def test_simple_replication_and_moves(start_cluster): try: for i, node in enumerate([node1, node2]): @@ -983,17 +1040,18 @@ def test_simple_replication_and_moves(start_cluster): """.format(i + 1)) def insert(num): - for i in range(num): - node = random.choice([node1, node2]) - data = [] # 1MB in total - for i in range(2): - data.append(get_random_string(512 * 1024)) # 500KB value - node.query("INSERT INTO replicated_table_for_moves VALUES {}".format(','.join(["('" + x + "')" for x in data]))) + for i in range(num): + node = random.choice([node1, node2]) + data = [] # 1MB in total + for i in range(2): + data.append(get_random_string(512 * 1024)) # 500KB value + node.query("INSERT INTO replicated_table_for_moves VALUES {}".format( + ','.join(["('" + x + "')" for x in data]))) def optimize(num): - for i in range(num): - node = random.choice([node1, node2]) - node.query("OPTIMIZE TABLE replicated_table_for_moves FINAL") + for i in range(num): + node = random.choice([node1, node2]) + node.query("OPTIMIZE TABLE replicated_table_for_moves FINAL") p = Pool(60) tasks = [] @@ -1009,18 +1067,20 @@ def test_simple_replication_and_moves(start_cluster): node1.query("SELECT COUNT() FROM replicated_table_for_moves") == "40\n" node2.query("SELECT COUNT() FROM replicated_table_for_moves") == "40\n" - data = [] # 1MB in total + data = [] # 1MB in total for i in range(2): - data.append(get_random_string(512 * 1024)) # 500KB value + data.append(get_random_string(512 * 1024)) # 500KB value - time.sleep(3) # wait until old parts will be deleted + time.sleep(3) # wait until old parts will be deleted node1.query("SYSTEM STOP MERGES") node2.query("SYSTEM STOP MERGES") - node1.query("INSERT INTO replicated_table_for_moves VALUES {}".format(','.join(["('" + x + "')" for x in data]))) - node2.query("INSERT INTO replicated_table_for_moves VALUES {}".format(','.join(["('" + x + "')" for x in data]))) + node1.query( + "INSERT INTO replicated_table_for_moves VALUES {}".format(','.join(["('" + x + "')" for x in data]))) + node2.query( + "INSERT INTO replicated_table_for_moves VALUES {}".format(','.join(["('" + x + "')" for x in data]))) - time.sleep(3) # nothing was moved + time.sleep(3) # nothing was moved disks1 = get_used_disks_for_table(node1, "replicated_table_for_moves") disks2 = get_used_disks_for_table(node2, "replicated_table_for_moves") @@ -1034,6 +1094,7 @@ def test_simple_replication_and_moves(start_cluster): for node in [node1, node2]: node.query("DROP TABLE IF EXISTS replicated_table_for_moves") + def test_download_appropriate_disk(start_cluster): try: for i, node in enumerate([node1, node2]): @@ -1047,8 +1108,9 @@ def test_download_appropriate_disk(start_cluster): data = [] for i in range(50): - data.append(get_random_string(1024 * 1024)) # 1MB value - node1.query("INSERT INTO replicated_table_for_download VALUES {}".format(','.join(["('" + x + "')" for x in data]))) + data.append(get_random_string(1024 * 1024)) # 1MB value + node1.query( + "INSERT INTO replicated_table_for_download VALUES {}".format(','.join(["('" + x + "')" for x in data]))) for _ in range(10): try: @@ -1066,6 +1128,7 @@ def test_download_appropriate_disk(start_cluster): for node in [node1, node2]: node.query("DROP TABLE IF EXISTS replicated_table_for_download") + def test_rename(start_cluster): try: node1.query(""" @@ -1079,7 +1142,7 @@ def test_rename(start_cluster): for _ in range(5): data = [] for i in range(10): - data.append(get_random_string(1024 * 1024)) # 1MB value + data.append(get_random_string(1024 * 1024)) # 1MB value node1.query("INSERT INTO renaming_table VALUES {}".format(','.join(["('" + x + "')" for x in data]))) disks = get_used_disks_for_table(node1, "renaming_table") @@ -1121,9 +1184,10 @@ def test_freeze(start_cluster): data = [] dates = [] for i in range(10): - data.append(get_random_string(1024 * 1024)) # 1MB value + data.append(get_random_string(1024 * 1024)) # 1MB value dates.append("toDate('2019-03-05')") - node1.query("INSERT INTO freezing_table VALUES {}".format(','.join(["(" + d + ", '" + s + "')" for d, s in zip(dates, data)]))) + node1.query("INSERT INTO freezing_table VALUES {}".format( + ','.join(["(" + d + ", '" + s + "')" for d, s in zip(dates, data)]))) disks = get_used_disks_for_table(node1, "freezing_table") assert len(disks) > 1 @@ -1154,7 +1218,7 @@ def test_kill_while_insert(start_cluster): data = [] dates = [] for i in range(10): - data.append(get_random_string(1024 * 1024)) # 1MB value + data.append(get_random_string(1024 * 1024)) # 1MB value node1.query("INSERT INTO {name} VALUES {}".format(','.join(["('" + s + "')" for s in data]), name=name)) disks = get_used_disks_for_table(node1, name) @@ -1199,7 +1263,8 @@ def test_move_while_merge(start_cluster): node1.query("INSERT INTO {name} VALUES (1)".format(name=name)) node1.query("INSERT INTO {name} VALUES (2)".format(name=name)) - parts = node1.query("SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)).splitlines() + parts = node1.query( + "SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)).splitlines() assert len(parts) == 2 def optimize(): @@ -1219,7 +1284,8 @@ def test_move_while_merge(start_cluster): def alter(): while not exiting: try: - node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) + node1.query( + "ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) no_exception['missing'] = 'exception' break except QueryRuntimeException: diff --git a/tests/integration/test_mutations_hardlinks/test.py b/tests/integration/test_mutations_hardlinks/test.py index 56852f572ff..103cf7c2e36 100644 --- a/tests/integration/test_mutations_hardlinks/test.py +++ b/tests/integration/test_mutations_hardlinks/test.py @@ -1,16 +1,16 @@ -import pytest - import os import time -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry from multiprocessing.dummy import Pool +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1') + @pytest.fixture(scope="module") def started_cluster(): try: @@ -38,20 +38,23 @@ def check_exists(table, part_path, column_file): def test_update_mutation(started_cluster): - node1.query("CREATE TABLE table_for_update(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") + node1.query( + "CREATE TABLE table_for_update(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") node1.query("INSERT INTO table_for_update SELECT number, number, toString(number) from numbers(100)") assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(range(100)) - node1.query("ALTER TABLE table_for_update UPDATE value1 = value1 * value1 WHERE 1", settings={"mutations_sync" : "2"}) + node1.query("ALTER TABLE table_for_update UPDATE value1 = value1 * value1 WHERE 1", + settings={"mutations_sync": "2"}) assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(i * i for i in range(100)) check_hardlinks("table_for_update", "all_1_1_0_2", "key.bin", 2) check_hardlinks("table_for_update", "all_1_1_0_2", "value2.bin", 2) check_hardlinks("table_for_update", "all_1_1_0_2", "value1.bin", 1) - node1.query("ALTER TABLE table_for_update UPDATE key=key, value1=value1, value2=value2 WHERE 1", settings={"mutations_sync": "2"}) + node1.query("ALTER TABLE table_for_update UPDATE key=key, value1=value1, value2=value2 WHERE 1", + settings={"mutations_sync": "2"}) assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(i * i for i in range(100)) @@ -61,13 +64,14 @@ def test_update_mutation(started_cluster): def test_modify_mutation(started_cluster): - node1.query("CREATE TABLE table_for_modify(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") + node1.query( + "CREATE TABLE table_for_modify(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") node1.query("INSERT INTO table_for_modify SELECT number, number, toString(number) from numbers(100)") assert int(node1.query("SELECT sum(value1) FROM table_for_modify").strip()) == sum(range(100)) - node1.query("ALTER TABLE table_for_modify MODIFY COLUMN value2 UInt64", settings={"mutations_sync" : "2"}) + node1.query("ALTER TABLE table_for_modify MODIFY COLUMN value2 UInt64", settings={"mutations_sync": "2"}) assert int(node1.query("SELECT sum(value2) FROM table_for_modify").strip()) == sum(range(100)) @@ -77,7 +81,8 @@ def test_modify_mutation(started_cluster): def test_drop_mutation(started_cluster): - node1.query("CREATE TABLE table_for_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") + node1.query( + "CREATE TABLE table_for_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") node1.query("INSERT INTO table_for_drop SELECT number, number, toString(number) from numbers(100)") @@ -95,7 +100,8 @@ def test_drop_mutation(started_cluster): def test_delete_and_drop_mutation(started_cluster): - node1.query("CREATE TABLE table_for_delete_and_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") + node1.query( + "CREATE TABLE table_for_delete_and_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") node1.query("INSERT INTO table_for_delete_and_drop SELECT number, number, toString(number) from numbers(100)") @@ -110,7 +116,8 @@ def test_delete_and_drop_mutation(started_cluster): p.apply_async(mutate) for _ in range(1, 100): - result = node1.query("SELECT COUNT() FROM system.mutations WHERE table = 'table_for_delete_and_drop' and is_done=0") + result = node1.query( + "SELECT COUNT() FROM system.mutations WHERE table = 'table_for_delete_and_drop' and is_done=0") try: if int(result.strip()) == 2: break @@ -122,7 +129,8 @@ def test_delete_and_drop_mutation(started_cluster): node1.query("SYSTEM START MERGES") - assert_eq_with_retry(node1, "SELECT COUNT() FROM table_for_delete_and_drop", str(sum(1 for i in range(100) if i % 2 != 0))) + assert_eq_with_retry(node1, "SELECT COUNT() FROM table_for_delete_and_drop", + str(sum(1 for i in range(100) if i % 2 != 0))) check_hardlinks("table_for_delete_and_drop", "all_1_1_0_3", "key.bin", 1) check_hardlinks("table_for_delete_and_drop", "all_1_1_0_3", "value1.bin", 1) diff --git a/tests/integration/test_mutations_with_merge_tree/test.py b/tests/integration/test_mutations_with_merge_tree/test.py index c52e3343712..019f8c2ea40 100644 --- a/tests/integration/test_mutations_with_merge_tree/test.py +++ b/tests/integration/test_mutations_with_merge_tree/test.py @@ -1,21 +1,22 @@ -from contextlib import contextmanager - import time -import pytest +import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance_test_mutations = cluster.add_instance('test_mutations_with_merge_tree', main_configs=['configs/config.xml'], user_configs=['configs/users.xml']) +instance_test_mutations = cluster.add_instance('test_mutations_with_merge_tree', main_configs=['configs/config.xml'], + user_configs=['configs/users.xml']) @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - instance_test_mutations.query('''CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree(date, (a, date), 8192)''') - instance_test_mutations.query('''INSERT INTO test_mutations_with_ast_elements SELECT '2019-07-29' AS date, 1, toString(number) FROM numbers(1)''') + instance_test_mutations.query( + '''CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree(date, (a, date), 8192)''') + instance_test_mutations.query( + '''INSERT INTO test_mutations_with_ast_elements SELECT '2019-07-29' AS date, 1, toString(number) FROM numbers(1)''') yield cluster finally: cluster.shutdown() @@ -26,31 +27,36 @@ def test_mutations_with_merge_background_task(started_cluster): ## The number of asts per query is 15 for execution_times_for_mutation in range(100): - instance_test_mutations.query('''ALTER TABLE test_mutations_with_ast_elements DELETE WHERE 1 = 1 AND toUInt32(b) IN (1)''') + instance_test_mutations.query( + '''ALTER TABLE test_mutations_with_ast_elements DELETE WHERE 1 = 1 AND toUInt32(b) IN (1)''') all_done = False - for wait_times_for_mutation in range(100): # wait for replication 80 seconds max + for wait_times_for_mutation in range(100): # wait for replication 80 seconds max time.sleep(0.8) def get_done_mutations(instance): instance_test_mutations.query('''DETACH TABLE test_mutations_with_ast_elements''') instance_test_mutations.query('''ATTACH TABLE test_mutations_with_ast_elements''') - return int(instance.query("SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations_with_ast_elements'").rstrip()) + return int(instance.query( + "SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations_with_ast_elements'").rstrip()) if get_done_mutations(instance_test_mutations) == 100: all_done = True break - print instance_test_mutations.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' FORMAT TSVWithNames") + print instance_test_mutations.query( + "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' FORMAT TSVWithNames") assert all_done + def test_mutations_with_truncate_table(started_cluster): instance_test_mutations.query('''SYSTEM STOP MERGES test_mutations_with_ast_elements''') ## The number of asts per query is 15 for execute_number in range(100): - instance_test_mutations.query('''ALTER TABLE test_mutations_with_ast_elements DELETE WHERE 1 = 1 AND toUInt32(b) IN (1)''') + instance_test_mutations.query( + '''ALTER TABLE test_mutations_with_ast_elements DELETE WHERE 1 = 1 AND toUInt32(b) IN (1)''') instance_test_mutations.query("TRUNCATE TABLE test_mutations_with_ast_elements") - assert instance_test_mutations.query("SELECT COUNT() FROM system.mutations WHERE table = 'test_mutations_with_ast_elements'").rstrip() == '0' - + assert instance_test_mutations.query( + "SELECT COUNT() FROM system.mutations WHERE table = 'test_mutations_with_ast_elements'").rstrip() == '0' diff --git a/tests/integration/test_mysql_database_engine/test.py b/tests/integration/test_mysql_database_engine/test.py index 86cfa0364d3..399b9263123 100644 --- a/tests/integration/test_mysql_database_engine/test.py +++ b/tests/integration/test_mysql_database_engine/test.py @@ -1,13 +1,11 @@ -import time import contextlib +import time +from string import Template import pymysql.cursors import pytest - -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException - -from string import Template +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) clickhouse_node = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql=True) @@ -32,7 +30,8 @@ class MySQLNodeInstance: def query(self, execution_query): if self.mysql_connection is None: - self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, port=self.port) + self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, + port=self.port) with self.mysql_connection.cursor() as cursor: def execute(query): res = cursor.execute(query) @@ -59,19 +58,25 @@ def test_mysql_ddl_for_mysql_database(started_cluster): with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', 'test_database', 'root', 'clickhouse')") + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', 'test_database', 'root', 'clickhouse')") assert 'test_database' in clickhouse_node.query('SHOW DATABASES') - mysql_node.query('CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') + mysql_node.query( + 'CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') - time.sleep(3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained + time.sleep( + 3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained mysql_node.query('ALTER TABLE `test_database`.`test_table` ADD COLUMN `add_column` int(11)') - assert 'add_column' in clickhouse_node.query("SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") + assert 'add_column' in clickhouse_node.query( + "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") - time.sleep(3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained + time.sleep( + 3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained mysql_node.query('ALTER TABLE `test_database`.`test_table` DROP COLUMN `add_column`') - assert 'add_column' not in clickhouse_node.query("SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") + assert 'add_column' not in clickhouse_node.query( + "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") mysql_node.query('DROP TABLE `test_database`.`test_table`;') assert 'test_table' not in clickhouse_node.query('SHOW TABLES FROM test_database') @@ -85,9 +90,11 @@ def test_mysql_ddl_for_mysql_database(started_cluster): def test_clickhouse_ddl_for_mysql_database(started_cluster): with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") - mysql_node.query('CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') + mysql_node.query( + 'CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', 'test_database', 'root', 'clickhouse')") + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', 'test_database', 'root', 'clickhouse')") assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') clickhouse_node.query("DROP TABLE test_database.test_table") @@ -108,8 +115,10 @@ def test_clickhouse_ddl_for_mysql_database(started_cluster): def test_clickhouse_dml_for_mysql_database(started_cluster): with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") - mysql_node.query('CREATE TABLE `test_database`.`test_table` ( `i``d` int(11) NOT NULL, PRIMARY KEY (`i``d`)) ENGINE=InnoDB;') - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', test_database, 'root', 'clickhouse')") + mysql_node.query( + 'CREATE TABLE `test_database`.`test_table` ( `i``d` int(11) NOT NULL, PRIMARY KEY (`i``d`)) ENGINE=InnoDB;') + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MySQL('mysql1:3306', test_database, 'root', 'clickhouse')") assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '0' clickhouse_node.query("INSERT INTO `test_database`.`test_table`(`i``d`) select number from numbers(10000)") @@ -130,8 +139,10 @@ def test_clickhouse_join_for_mysql_database(started_cluster): "service VARCHAR(5) DEFAULT '' NOT NULL," "opco VARCHAR(5) DEFAULT ''" ")") - clickhouse_node.query("CREATE TABLE default.t1_remote_mysql AS mysql('mysql1:3306','test','t1_mysql_local','root','clickhouse')") - clickhouse_node.query("CREATE TABLE default.t2_remote_mysql AS mysql('mysql1:3306','test','t2_mysql_local','root','clickhouse')") + clickhouse_node.query( + "CREATE TABLE default.t1_remote_mysql AS mysql('mysql1:3306','test','t1_mysql_local','root','clickhouse')") + clickhouse_node.query( + "CREATE TABLE default.t2_remote_mysql AS mysql('mysql1:3306','test','t2_mysql_local','root','clickhouse')") assert clickhouse_node.query("SELECT s.pays " "FROM default.t1_remote_mysql AS s " "LEFT JOIN default.t1_remote_mysql AS s_ref " @@ -143,7 +154,8 @@ def test_bad_arguments_for_mysql_database_engine(started_cluster): with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: with pytest.raises(QueryRuntimeException) as exception: mysql_node.query("CREATE DATABASE IF NOT EXISTS test_bad_arguments DEFAULT CHARACTER SET 'utf8'") - clickhouse_node.query("CREATE DATABASE test_database_bad_arguments ENGINE = MySQL('mysql1:3306', test_bad_arguments, root, 'clickhouse')") + clickhouse_node.query( + "CREATE DATABASE test_database_bad_arguments ENGINE = MySQL('mysql1:3306', test_bad_arguments, root, 'clickhouse')") assert 'Database engine MySQL requested literal argument.' in str(exception.value) mysql_node.query("DROP DATABASE test_bad_arguments") @@ -152,46 +164,57 @@ decimal_values = [0.123, 0.4, 5.67, 8.91011, 123456789.123, -0.123, -0.4, -5.67, timestamp_values = ['2015-05-18 07:40:01.123', '2019-09-16 19:20:11.123'] timestamp_values_no_subsecond = ['2015-05-18 07:40:01', '2019-09-16 19:20:11'] + @pytest.mark.parametrize("case_name, mysql_type, expected_ch_type, mysql_values, setting_mysql_datatypes_support_level", -[ - ("decimal_default", "decimal NOT NULL", "Decimal(10, 0)", decimal_values, "decimal,datetime64"), - ("decimal_default_nullable", "decimal", "Nullable(Decimal(10, 0))", decimal_values, "decimal,datetime64"), - ("decimal_18_6", "decimal(18, 6) NOT NULL", "Decimal(18, 6)", decimal_values, "decimal,datetime64"), - ("decimal_38_6", "decimal(38, 6) NOT NULL", "Decimal(38, 6)", decimal_values, "decimal,datetime64"), + [ + ("decimal_default", "decimal NOT NULL", "Decimal(10, 0)", decimal_values, + "decimal,datetime64"), + ("decimal_default_nullable", "decimal", "Nullable(Decimal(10, 0))", decimal_values, + "decimal,datetime64"), + ("decimal_18_6", "decimal(18, 6) NOT NULL", "Decimal(18, 6)", decimal_values, + "decimal,datetime64"), + ("decimal_38_6", "decimal(38, 6) NOT NULL", "Decimal(38, 6)", decimal_values, + "decimal,datetime64"), - # Due to python DB driver roundtrip MySQL timestamp and datetime values - # are printed with 6 digits after decimal point, so to simplify tests a bit, - # we only validate precision of 0 and 6. - ("timestamp_default", "timestamp", "DateTime", timestamp_values, "decimal,datetime64"), - ("timestamp_6", "timestamp(6)", "DateTime64(6)", timestamp_values, "decimal,datetime64"), - ("datetime_default", "DATETIME NOT NULL", "DateTime64(0)", timestamp_values, "decimal,datetime64"), - ("datetime_6", "DATETIME(6) NOT NULL", "DateTime64(6)", timestamp_values, "decimal,datetime64"), + # Due to python DB driver roundtrip MySQL timestamp and datetime values + # are printed with 6 digits after decimal point, so to simplify tests a bit, + # we only validate precision of 0 and 6. + ("timestamp_default", "timestamp", "DateTime", timestamp_values, "decimal,datetime64"), + ("timestamp_6", "timestamp(6)", "DateTime64(6)", timestamp_values, "decimal,datetime64"), + ("datetime_default", "DATETIME NOT NULL", "DateTime64(0)", timestamp_values, + "decimal,datetime64"), + ("datetime_6", "DATETIME(6) NOT NULL", "DateTime64(6)", timestamp_values, + "decimal,datetime64"), - # right now precision bigger than 39 is not supported by ClickHouse's Decimal, hence fall back to String - ("decimal_40_6", "decimal(40, 6) NOT NULL", "String", decimal_values, "decimal,datetime64"), - ("decimal_18_6", "decimal(18, 6) NOT NULL", "String", decimal_values, "datetime64"), - ("decimal_18_6", "decimal(18, 6) NOT NULL", "String", decimal_values, ""), - ("datetime_6", "DATETIME(6) NOT NULL", "DateTime", timestamp_values_no_subsecond, "decimal"), - ("datetime_6", "DATETIME(6) NOT NULL", "DateTime", timestamp_values_no_subsecond, ""), -]) -def test_mysql_types(started_cluster, case_name, mysql_type, expected_ch_type, mysql_values, setting_mysql_datatypes_support_level): + # right now precision bigger than 39 is not supported by ClickHouse's Decimal, hence fall back to String + ( + "decimal_40_6", "decimal(40, 6) NOT NULL", "String", decimal_values, + "decimal,datetime64"), + ("decimal_18_6", "decimal(18, 6) NOT NULL", "String", decimal_values, "datetime64"), + ("decimal_18_6", "decimal(18, 6) NOT NULL", "String", decimal_values, ""), + ("datetime_6", "DATETIME(6) NOT NULL", "DateTime", timestamp_values_no_subsecond, + "decimal"), + ("datetime_6", "DATETIME(6) NOT NULL", "DateTime", timestamp_values_no_subsecond, ""), + ]) +def test_mysql_types(started_cluster, case_name, mysql_type, expected_ch_type, mysql_values, + setting_mysql_datatypes_support_level): """ Verify that values written to MySQL can be read on ClickHouse side via DB engine MySQL, or Table engine MySQL, or mysql() table function. Make sure that type is converted properly and values match exactly. """ substitutes = dict( - mysql_db = 'decimal_support', - table_name = case_name, - mysql_type = mysql_type, - mysql_values = ', '.join('({})'.format(repr(x)) for x in mysql_values), - ch_mysql_db = 'mysql_db', - ch_mysql_table = 'mysql_table_engine_' + case_name, - expected_ch_type = expected_ch_type, + mysql_db='decimal_support', + table_name=case_name, + mysql_type=mysql_type, + mysql_values=', '.join('({})'.format(repr(x)) for x in mysql_values), + ch_mysql_db='mysql_db', + ch_mysql_table='mysql_table_engine_' + case_name, + expected_ch_type=expected_ch_type, ) clickhouse_query_settings = dict( - mysql_datatypes_support_level = setting_mysql_datatypes_support_level + mysql_datatypes_support_level=setting_mysql_datatypes_support_level ) def execute_query(node, query, **kwargs): @@ -216,9 +239,8 @@ def test_mysql_types(started_cluster, case_name, mysql_type, expected_ch_type, m ]) assert execute_query(mysql_node, "SELECT COUNT(*) FROM ${mysql_db}.${table_name}") \ - == \ - "{}".format(len(mysql_values)) - + == \ + "{}".format(len(mysql_values)) # MySQL TABLE ENGINE execute_query(clickhouse_node, [ @@ -229,18 +251,17 @@ def test_mysql_types(started_cluster, case_name, mysql_type, expected_ch_type, m # Validate type assert \ execute_query(clickhouse_node, "SELECT toTypeName(value) FROM ${ch_mysql_table} LIMIT 1", - settings=clickhouse_query_settings) \ + settings=clickhouse_query_settings) \ == \ expected_ch_type # Validate values assert \ execute_query(clickhouse_node, "SELECT value FROM ${ch_mysql_table}", - settings=clickhouse_query_settings) \ + settings=clickhouse_query_settings) \ == \ execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}") - # MySQL DATABASE ENGINE execute_query(clickhouse_node, [ "DROP DATABASE IF EXISTS ${ch_mysql_db}", @@ -250,22 +271,23 @@ def test_mysql_types(started_cluster, case_name, mysql_type, expected_ch_type, m # Validate type assert \ execute_query(clickhouse_node, "SELECT toTypeName(value) FROM ${ch_mysql_db}.${table_name} LIMIT 1", - settings=clickhouse_query_settings) \ + settings=clickhouse_query_settings) \ == \ expected_ch_type # Validate values assert \ execute_query(clickhouse_node, "SELECT value FROM ${ch_mysql_db}.${table_name}", - settings=clickhouse_query_settings) \ + settings=clickhouse_query_settings) \ == \ execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}") # MySQL TABLE FUNCTION # Validate type assert \ - execute_query(clickhouse_node, "SELECT toTypeName(value) FROM mysql('mysql1:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse') LIMIT 1", - settings=clickhouse_query_settings) \ + execute_query(clickhouse_node, + "SELECT toTypeName(value) FROM mysql('mysql1:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse') LIMIT 1", + settings=clickhouse_query_settings) \ == \ expected_ch_type @@ -273,5 +295,6 @@ def test_mysql_types(started_cluster, case_name, mysql_type, expected_ch_type, m assert \ execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}") \ == \ - execute_query(clickhouse_node, "SELECT value FROM mysql('mysql1:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse')", - settings=clickhouse_query_settings) + execute_query(clickhouse_node, + "SELECT value FROM mysql('mysql1:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse')", + settings=clickhouse_query_settings) diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index c1d53fc2b34..b5fd312007a 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -1,19 +1,17 @@ # coding: utf-8 -import docker import datetime import math import os -import pytest import subprocess import time + +import docker import pymysql.connections - +import pytest from docker.models.containers import Container - from helpers.cluster import ClickHouseCluster, get_docker_compose_path - SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() @@ -37,7 +35,8 @@ def server_address(): @pytest.fixture(scope='module') def mysql_client(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_client.yml') - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) yield docker.from_env().containers.get(cluster.project_name + '_mysql1_1') @@ -63,28 +62,32 @@ def mysql_server(mysql_client): @pytest.fixture(scope='module') def golang_container(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_golang_client.yml') - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) yield docker.from_env().containers.get(cluster.project_name + '_golang1_1') @pytest.fixture(scope='module') def php_container(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_php_client.yml') - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) yield docker.from_env().containers.get(cluster.project_name + '_php1_1') @pytest.fixture(scope='module') def nodejs_container(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_js_client.yml') - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) yield docker.from_env().containers.get(cluster.project_name + '_mysqljs1_1') @pytest.fixture(scope='module') def java_container(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_java_client.yml') - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) yield docker.from_env().containers.get(cluster.project_name + '_java1_1') @@ -121,7 +124,7 @@ def test_mysql_client(mysql_client, server_address): assert stdout == 'count()\n1\n' assert stderr[0:182] == "mysql: [Warning] Using a password on the command line interface can be insecure.\n" \ - "ERROR 81 (00000) at line 1: Code: 81, e.displayText() = DB::Exception: Database system2 doesn't exist" + "ERROR 81 (00000) at line 1: Code: 81, e.displayText() = DB::Exception: Database system2 doesn't exist" code, (stdout, stderr) = mysql_client.exec_run(''' mysql --protocol tcp -h {host} -P {port} default -u default --password=123 @@ -139,15 +142,16 @@ def test_mysql_client(mysql_client, server_address): assert stdout == '\n'.join(['column', '0', '0', '1', '1', '5', '5', 'tmp_column', '0', '1', '']) + def test_mysql_client_exception(mysql_client, server_address): - # Poco exception. + # Poco exception. code, (stdout, stderr) = mysql_client.exec_run(''' mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "CREATE TABLE default.t1_remote_mysql AS mysql('127.0.0.1:10086','default','t1_local','default','');" '''.format(host=server_address, port=server_port), demux=True) assert stderr[0:266] == "mysql: [Warning] Using a password on the command line interface can be insecure.\n" \ - "ERROR 1000 (00000) at line 1: Poco::Exception. Code: 1000, e.code() = 2002, e.displayText() = mysqlxx::ConnectionFailed: Can't connect to MySQL server on '127.0.0.1' (115) ((nullptr):0)" + "ERROR 1000 (00000) at line 1: Poco::Exception. Code: 1000, e.code() = 2002, e.displayText() = mysqlxx::ConnectionFailed: Can't connect to MySQL server on '127.0.0.1' (115) ((nullptr):0)" def test_mysql_replacement_query(mysql_client, server_address): @@ -304,28 +308,34 @@ def test_mysql_set_variables(mysql_client, server_address): def test_python_client(server_address): - client = pymysql.connections.Connection(host=server_address, user='user_with_double_sha1', password='abacaba', database='default', port=server_port) + client = pymysql.connections.Connection(host=server_address, user='user_with_double_sha1', password='abacaba', + database='default', port=server_port) with pytest.raises(pymysql.InternalError) as exc_info: client.query('select name from tables') - assert exc_info.value.args[1][0:77] == "Code: 60, e.displayText() = DB::Exception: Table default.tables doesn't exist" + assert exc_info.value.args[1][ + 0:77] == "Code: 60, e.displayText() = DB::Exception: Table default.tables doesn't exist" cursor = client.cursor(pymysql.cursors.DictCursor) cursor.execute("select 1 as a, 'тест' as b") assert cursor.fetchall() == [{'a': 1, 'b': 'тест'}] with pytest.raises(pymysql.InternalError) as exc_info: - pymysql.connections.Connection(host=server_address, user='default', password='abacab', database='default', port=server_port) + pymysql.connections.Connection(host=server_address, user='default', password='abacab', database='default', + port=server_port) - assert exc_info.value.args == (516, 'default: Authentication failed: password is incorrect or there is no user with such name') + assert exc_info.value.args == ( + 516, 'default: Authentication failed: password is incorrect or there is no user with such name') - client = pymysql.connections.Connection(host=server_address, user='default', password='123', database='default', port=server_port) + client = pymysql.connections.Connection(host=server_address, user='default', password='123', database='default', + port=server_port) with pytest.raises(pymysql.InternalError) as exc_info: client.query('select name from tables') - assert exc_info.value.args[1][0:77] == "Code: 60, e.displayText() = DB::Exception: Table default.tables doesn't exist" + assert exc_info.value.args[1][ + 0:77] == "Code: 60, e.displayText() = DB::Exception: Table default.tables doesn't exist" cursor = client.cursor(pymysql.cursors.DictCursor) cursor.execute("select 1 as a, 'тест' as b") @@ -353,55 +363,71 @@ def test_golang_client(server_address, golang_container): with open(os.path.join(SCRIPT_DIR, 'golang.reference')) as fp: reference = fp.read() - code, (stdout, stderr) = golang_container.exec_run('./main --host {host} --port {port} --user default --password 123 --database ' - 'abc'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = golang_container.exec_run( + './main --host {host} --port {port} --user default --password 123 --database ' + 'abc'.format(host=server_address, port=server_port), demux=True) assert code == 1 assert stderr == "Error 81: Database abc doesn't exist\n" - code, (stdout, stderr) = golang_container.exec_run('./main --host {host} --port {port} --user default --password 123 --database ' - 'default'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = golang_container.exec_run( + './main --host {host} --port {port} --user default --password 123 --database ' + 'default'.format(host=server_address, port=server_port), demux=True) assert code == 0 assert stdout == reference - code, (stdout, stderr) = golang_container.exec_run('./main --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database ' - 'default'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = golang_container.exec_run( + './main --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database ' + 'default'.format(host=server_address, port=server_port), demux=True) assert code == 0 assert stdout == reference def test_php_client(server_address, php_container): # type: (str, Container) -> None - code, (stdout, stderr) = php_container.exec_run('php -f test.php {host} {port} default 123'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = php_container.exec_run( + 'php -f test.php {host} {port} default 123'.format(host=server_address, port=server_port), demux=True) assert code == 0 assert stdout == 'tables\n' - code, (stdout, stderr) = php_container.exec_run('php -f test_ssl.php {host} {port} default 123'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = php_container.exec_run( + 'php -f test_ssl.php {host} {port} default 123'.format(host=server_address, port=server_port), demux=True) assert code == 0 assert stdout == 'tables\n' - code, (stdout, stderr) = php_container.exec_run('php -f test.php {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = php_container.exec_run( + 'php -f test.php {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), + demux=True) assert code == 0 assert stdout == 'tables\n' - code, (stdout, stderr) = php_container.exec_run('php -f test_ssl.php {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = php_container.exec_run( + 'php -f test_ssl.php {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), + demux=True) assert code == 0 assert stdout == 'tables\n' def test_mysqljs_client(server_address, nodejs_container): - code, (_, stderr) = nodejs_container.exec_run('node test.js {host} {port} user_with_sha256 abacaba'.format(host=server_address, port=server_port), demux=True) + code, (_, stderr) = nodejs_container.exec_run( + 'node test.js {host} {port} user_with_sha256 abacaba'.format(host=server_address, port=server_port), demux=True) assert code == 1 assert 'MySQL is requesting the sha256_password authentication method, which is not supported.' in stderr - code, (_, stderr) = nodejs_container.exec_run('node test.js {host} {port} user_with_empty_password ""'.format(host=server_address, port=server_port), demux=True) + code, (_, stderr) = nodejs_container.exec_run( + 'node test.js {host} {port} user_with_empty_password ""'.format(host=server_address, port=server_port), + demux=True) assert code == 0 - code, (_, _) = nodejs_container.exec_run('node test.js {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), demux=True) + code, (_, _) = nodejs_container.exec_run( + 'node test.js {host} {port} user_with_double_sha1 abacaba'.format(host=server_address, port=server_port), + demux=True) assert code == 0 - code, (_, _) = nodejs_container.exec_run('node test.js {host} {port} user_with_empty_password 123'.format(host=server_address, port=server_port), demux=True) + code, (_, _) = nodejs_container.exec_run( + 'node test.js {host} {port} user_with_empty_password 123'.format(host=server_address, port=server_port), + demux=True) assert code == 1 @@ -411,31 +437,36 @@ def test_java_client(server_address, java_container): reference = fp.read() # database not exists exception. - code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database ' - 'abc'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = java_container.exec_run( + 'java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database ' + 'abc'.format(host=server_address, port=server_port), demux=True) assert code == 1 # empty password passed. - code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database ' - 'default'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = java_container.exec_run( + 'java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database ' + 'default'.format(host=server_address, port=server_port), demux=True) assert code == 0 assert stdout == reference # non-empty password passed. - code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database ' - 'default'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = java_container.exec_run( + 'java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database ' + 'default'.format(host=server_address, port=server_port), demux=True) assert code == 0 assert stdout == reference # double-sha1 password passed. - code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database ' - 'default'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = java_container.exec_run( + 'java JavaConnectorTest --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database ' + 'default'.format(host=server_address, port=server_port), demux=True) assert code == 0 assert stdout == reference def test_types(server_address): - client = pymysql.connections.Connection(host=server_address, user='default', password='123', database='default', port=server_port) + client = pymysql.connections.Connection(host=server_address, user='default', password='123', database='default', + port=server_port) cursor = client.cursor(pymysql.cursors.DictCursor) cursor.execute( diff --git a/tests/integration/test_no_local_metadata_node/test.py b/tests/integration/test_no_local_metadata_node/test.py index ef240cd710c..ae69f5e1384 100644 --- a/tests/integration/test_no_local_metadata_node/test.py +++ b/tests/integration/test_no_local_metadata_node/test.py @@ -1,4 +1,3 @@ -import time import pytest from helpers.cluster import ClickHouseCluster @@ -6,6 +5,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): try: diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index 2c099e84e89..4706d8efbdd 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -1,17 +1,23 @@ -import time -import pytest -import string import random +import string +import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/zstd_compression_by_default.xml'], user_configs=['configs/allow_suspicious_codecs.xml']) -node2 = cluster.add_instance('node2', main_configs=['configs/lz4hc_compression_by_default.xml'], user_configs=['configs/allow_suspicious_codecs.xml']) -node3 = cluster.add_instance('node3', main_configs=['configs/custom_compression_by_default.xml'], user_configs=['configs/allow_suspicious_codecs.xml']) -node4 = cluster.add_instance('node4', user_configs=['configs/enable_uncompressed_cache.xml', 'configs/allow_suspicious_codecs.xml']) -node5 = cluster.add_instance('node5', main_configs=['configs/zstd_compression_by_default.xml'], user_configs=['configs/enable_uncompressed_cache.xml', 'configs/allow_suspicious_codecs.xml']) +node1 = cluster.add_instance('node1', main_configs=['configs/zstd_compression_by_default.xml'], + user_configs=['configs/allow_suspicious_codecs.xml']) +node2 = cluster.add_instance('node2', main_configs=['configs/lz4hc_compression_by_default.xml'], + user_configs=['configs/allow_suspicious_codecs.xml']) +node3 = cluster.add_instance('node3', main_configs=['configs/custom_compression_by_default.xml'], + user_configs=['configs/allow_suspicious_codecs.xml']) +node4 = cluster.add_instance('node4', user_configs=['configs/enable_uncompressed_cache.xml', + 'configs/allow_suspicious_codecs.xml']) +node5 = cluster.add_instance('node5', main_configs=['configs/zstd_compression_by_default.xml'], + user_configs=['configs/enable_uncompressed_cache.xml', + 'configs/allow_suspicious_codecs.xml']) + @pytest.fixture(scope="module") def start_cluster(): @@ -33,17 +39,23 @@ def test_preconfigured_default_codec(start_cluster): somecolumn Float64 ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; """) - node.query("INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)") + node.query( + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)") assert node.query("SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0") == "2\n" - assert node.query("SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id") == "777.777\n88.88\n99.99\n" - assert node.query("SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100") == "hello\nworld\n" + assert node.query( + "SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id") == "777.777\n88.88\n99.99\n" + assert node.query( + "SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100") == "hello\nworld\n" - node.query("INSERT INTO compression_codec_multiple_with_key SELECT toDate('2018-10-12'), number, toString(number), 1.0 FROM system.numbers LIMIT 10000") + node.query( + "INSERT INTO compression_codec_multiple_with_key SELECT toDate('2018-10-12'), number, toString(number), 1.0 FROM system.numbers LIMIT 10000") assert node.query("SELECT COUNT(id) FROM compression_codec_multiple_with_key WHERE id % 10 == 0") == "1001\n" - assert node.query("SELECT SUM(somecolumn) FROM compression_codec_multiple_with_key") == str(777.777 + 88.88 + 99.99 + 1.0 * 10000) + "\n" + assert node.query("SELECT SUM(somecolumn) FROM compression_codec_multiple_with_key") == str( + 777.777 + 88.88 + 99.99 + 1.0 * 10000) + "\n" assert node.query("SELECT count(*) FROM compression_codec_multiple_with_key GROUP BY somedate") == "10003\n" + def test_preconfigured_custom_codec(start_cluster): node3.query(""" CREATE TABLE compression_codec_multiple_with_key ( @@ -54,23 +66,39 @@ def test_preconfigured_custom_codec(start_cluster): ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; """) - node3.query("INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)") + node3.query( + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)") assert node3.query("SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0") == "2\n" - assert node3.query("SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id") == "777.777\n88.88\n99.99\n" - assert node3.query("SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100") == "hello\nworld\n" + assert node3.query( + "SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id") == "777.777\n88.88\n99.99\n" + assert node3.query( + "SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100") == "hello\nworld\n" - node3.query("INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10000)))) + node3.query( + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format( + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10000)))) node3.query("OPTIMIZE TABLE compression_codec_multiple_with_key FINAL") - assert node3.query("SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + assert node3.query( + "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" for i in xrange(10): - node3.query("INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), {}, '{}', 88.88)".format(i, ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10000)))) + node3.query( + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), {}, '{}', 88.88)".format(i, + ''.join( + random.choice( + string.ascii_uppercase + string.digits) + for + _ + in + range( + 10000)))) node3.query("OPTIMIZE TABLE compression_codec_multiple_with_key FINAL") assert node3.query("SELECT COUNT(*) from compression_codec_multiple_with_key WHERE length(data) = 10000") == "11\n" + def test_uncompressed_cache_custom_codec(start_cluster): node4.query(""" CREATE TABLE compression_codec_multiple_with_key ( @@ -81,12 +109,17 @@ def test_uncompressed_cache_custom_codec(start_cluster): ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; """) - node4.query("INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10000)))) + node4.query( + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format( + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10000)))) # two equal requests one by one, to get into UncompressedCache for the first block - assert node4.query("SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + assert node4.query( + "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + + assert node4.query( + "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" - assert node4.query("SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" def test_uncompressed_cache_plus_zstd_codec(start_cluster): node5.query(""" @@ -98,6 +131,9 @@ def test_uncompressed_cache_plus_zstd_codec(start_cluster): ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; """) - node5.query("INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format('a' * 10000)) + node5.query( + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format( + 'a' * 10000)) - assert node5.query("SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + assert node5.query( + "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 0577917ded8..f527b4cc66e 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -1,17 +1,20 @@ import time -import pytest -import os -import pymysql.cursors import psycopg2 -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +import pymysql.cursors +import pytest from helpers.cluster import ClickHouseCluster - +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_odbc_drivers=True, with_mysql=True, main_configs=['configs/openssl.xml','configs/odbc_logging.xml','configs/enable_dictionaries.xml','configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml','configs/dictionaries/sqlite3_odbc_cached_dictionary.xml','configs/dictionaries/postgres_odbc_hashed_dictionary.xml'], stay_alive=True) +node1 = cluster.add_instance('node1', with_odbc_drivers=True, with_mysql=True, + main_configs=['configs/openssl.xml', 'configs/odbc_logging.xml', + 'configs/enable_dictionaries.xml', + 'configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml', + 'configs/dictionaries/sqlite3_odbc_cached_dictionary.xml', + 'configs/dictionaries/postgres_odbc_hashed_dictionary.xml'], stay_alive=True) -create_table_sql_template = """ +create_table_sql_template = """ CREATE TABLE `clickhouse`.`{}` ( `id` int(11) NOT NULL, `name` varchar(50) NOT NULL, @@ -20,19 +23,24 @@ create_table_sql_template = """ `column_x` int default NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB; """ + + def get_mysql_conn(): conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.1', port=3308) return conn + def create_mysql_db(conn, name): with conn.cursor() as cursor: cursor.execute( "CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(name)) + def create_mysql_table(conn, table_name): with conn.cursor() as cursor: cursor.execute(create_table_sql_template.format(table_name)) + def get_postgres_conn(): conn_string = "host='localhost' user='postgres' password='mysecretpassword'" conn = psycopg2.connect(conn_string) @@ -40,10 +48,12 @@ def get_postgres_conn(): conn.autocommit = True return conn + def create_postgres_db(conn, name): cursor = conn.cursor() cursor.execute("CREATE SCHEMA {}".format(name)) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -51,10 +61,18 @@ def started_cluster(): sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] print "sqlite data received" - node1.exec_in_container(["bash", "-c", "echo 'CREATE TABLE t1(x INTEGER PRIMARY KEY ASC, y, z);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') - node1.exec_in_container(["bash", "-c", "echo 'CREATE TABLE t2(X INTEGER PRIMARY KEY ASC, Y, Z);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') - node1.exec_in_container(["bash", "-c", "echo 'CREATE TABLE t3(X INTEGER PRIMARY KEY ASC, Y, Z);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') - node1.exec_in_container(["bash", "-c", "echo 'CREATE TABLE t4(X INTEGER PRIMARY KEY ASC, Y, Z);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + node1.exec_in_container( + ["bash", "-c", "echo 'CREATE TABLE t1(x INTEGER PRIMARY KEY ASC, y, z);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') + node1.exec_in_container( + ["bash", "-c", "echo 'CREATE TABLE t2(X INTEGER PRIMARY KEY ASC, Y, Z);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') + node1.exec_in_container( + ["bash", "-c", "echo 'CREATE TABLE t3(X INTEGER PRIMARY KEY ASC, Y, Z);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') + node1.exec_in_container( + ["bash", "-c", "echo 'CREATE TABLE t4(X INTEGER PRIMARY KEY ASC, Y, Z);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') print "sqlite tables created" mysql_conn = get_mysql_conn() print "mysql connection received" @@ -69,7 +87,8 @@ def started_cluster(): print "postgres db created" cursor = postgres_conn.cursor() - cursor.execute("create table if not exists clickhouse.test_table (column1 int primary key, column2 varchar(40) not null)") + cursor.execute( + "create table if not exists clickhouse.test_table (column1 int primary key, column2 varchar(40) not null)") yield cluster @@ -79,6 +98,7 @@ def started_cluster(): finally: cluster.shutdown() + def test_mysql_simple_select_works(started_cluster): mysql_setup = node1.odbc_drivers["MySQL"] @@ -88,17 +108,25 @@ def test_mysql_simple_select_works(started_cluster): # Check that NULL-values are handled correctly by the ODBC-bridge with conn.cursor() as cursor: - cursor.execute("INSERT INTO clickhouse.{} VALUES(50, 'null-guy', 127, 255, NULL), (100, 'non-null-guy', 127, 255, 511);".format(table_name)) + cursor.execute( + "INSERT INTO clickhouse.{} VALUES(50, 'null-guy', 127, 255, NULL), (100, 'non-null-guy', 127, 255, 511);".format( + table_name)) conn.commit() - assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name), settings={"external_table_functions_use_nulls": "1"}) == '\\N\n511\n' - assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name), settings={"external_table_functions_use_nulls": "0"}) == '0\n511\n' + assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name), + settings={"external_table_functions_use_nulls": "1"}) == '\\N\n511\n' + assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name), + settings={"external_table_functions_use_nulls": "0"}) == '0\n511\n' node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, column_x Nullable(UInt32)) ENGINE = MySQL('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse'); '''.format(table_name, table_name)) - node1.query("INSERT INTO {}(id, name, money, column_x) select number, concat('name_', toString(number)), 3, NULL from numbers(49) ".format(table_name)) - node1.query("INSERT INTO {}(id, name, money, column_x) select number, concat('name_', toString(number)), 3, 42 from numbers(51, 49) ".format(table_name)) + node1.query( + "INSERT INTO {}(id, name, money, column_x) select number, concat('name_', toString(number)), 3, NULL from numbers(49) ".format( + table_name)) + node1.query( + "INSERT INTO {}(id, name, money, column_x) select number, concat('name_', toString(number)), 3, 42 from numbers(51, 49) ".format( + table_name)) assert node1.query("SELECT COUNT () FROM {} WHERE column_x IS NOT NULL".format(table_name)) == '50\n' assert node1.query("SELECT COUNT () FROM {} WHERE column_x IS NULL".format(table_name)) == '50\n' @@ -110,6 +138,7 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, column_x Nulla conn.close() + def test_mysql_insert(started_cluster): mysql_setup = node1.odbc_drivers["MySQL"] table_name = 'test_insert' @@ -117,20 +146,26 @@ def test_mysql_insert(started_cluster): create_mysql_table(conn, table_name) odbc_args = "'DSN={}', '{}', '{}'".format(mysql_setup["DSN"], mysql_setup["Database"], table_name) - node1.query("create table mysql_insert (id Int64, name String, age UInt8, money Float, column_x Nullable(Int16)) Engine=ODBC({})".format(odbc_args)) + node1.query( + "create table mysql_insert (id Int64, name String, age UInt8, money Float, column_x Nullable(Int16)) Engine=ODBC({})".format( + odbc_args)) node1.query("insert into mysql_insert values (1, 'test', 11, 111, 1111), (2, 'odbc', 22, 222, NULL)") assert node1.query("select * from mysql_insert") == "1\ttest\t11\t111\t1111\n2\todbc\t22\t222\t\\N\n" node1.query("insert into table function odbc({}) values (3, 'insert', 33, 333, 3333)".format(odbc_args)) - node1.query("insert into table function odbc({}) (id, name, age, money) select id*4, upper(name), age*4, money*4 from odbc({}) where id=1".format(odbc_args, odbc_args)) - assert node1.query("select * from mysql_insert where id in (3, 4)") == "3\tinsert\t33\t333\t3333\n4\tTEST\t44\t444\t\\N\n" + node1.query( + "insert into table function odbc({}) (id, name, age, money) select id*4, upper(name), age*4, money*4 from odbc({}) where id=1".format( + odbc_args, odbc_args)) + assert node1.query( + "select * from mysql_insert where id in (3, 4)") == "3\tinsert\t33\t333\t3333\n4\tTEST\t44\t444\t\\N\n" def test_sqlite_simple_select_function_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t1 values(1, 2, 3);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t1 values(1, 2, 3);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') assert node1.query("select * from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\t2\t3\n" assert node1.query("select y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "2\n" @@ -138,14 +173,18 @@ def test_sqlite_simple_select_function_works(started_cluster): assert node1.query("select x from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\n" assert node1.query("select x, y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\t2\n" assert node1.query("select z, x, y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "3\t1\t2\n" - assert node1.query("select count(), sum(x) from odbc('DSN={}', '{}') group by x".format(sqlite_setup["DSN"], 't1')) == "1\t1\n" + assert node1.query( + "select count(), sum(x) from odbc('DSN={}', '{}') group by x".format(sqlite_setup["DSN"], 't1')) == "1\t1\n" + def test_sqlite_simple_select_storage_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t4 values(1, 2, 3);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') - node1.query("create table SqliteODBC (x Int32, y String, z String) engine = ODBC('DSN={}', '', 't4')".format(sqlite_setup["DSN"])) + node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t4 values(1, 2, 3);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') + node1.query("create table SqliteODBC (x Int32, y String, z String) engine = ODBC('DSN={}', '', 't4')".format( + sqlite_setup["DSN"])) assert node1.query("select * from SqliteODBC") == "1\t2\t3\n" assert node1.query("select y from SqliteODBC") == "2\n" @@ -155,32 +194,38 @@ def test_sqlite_simple_select_storage_works(started_cluster): assert node1.query("select z, x, y from SqliteODBC") == "3\t1\t2\n" assert node1.query("select count(), sum(x) from SqliteODBC group by x") == "1\t1\n" + def test_sqlite_odbc_hashed_dictionary(started_cluster): - sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] - node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t2 values(1, 2, 3);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] + node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t2 values(1, 2, 3);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))") == "3\n" - assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))") == "1\n" # default + assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))") == "1\n" # default - time.sleep(5) # first reload - node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t2 values(200, 2, 7);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + time.sleep(5) # first reload + node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t2 values(200, 2, 7);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') # No reload because of invalidate query time.sleep(5) assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))") == "3\n" - assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))") == "1\n" # still default + assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))") == "1\n" # still default - node1.exec_in_container(["bash", "-c", "echo 'REPLACE INTO t2 values(1, 2, 5);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + node1.exec_in_container(["bash", "-c", "echo 'REPLACE INTO t2 values(1, 2, 5);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') # waiting for reload time.sleep(5) assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))") == "5\n" - assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))") == "7\n" # new value + assert node1.query("select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))") == "7\n" # new value + def test_sqlite_odbc_cached_dictionary(started_cluster): - sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] - node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t3 values(1, 2, 3);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] + node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t3 values(1, 2, 3);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))") == "3\n" @@ -188,16 +233,19 @@ def test_sqlite_odbc_cached_dictionary(started_cluster): node1.exec_in_container(["bash", "-c", "chmod a+rw /tmp"], privileged=True, user='root') node1.exec_in_container(["bash", "-c", "chmod a+rw {}".format(sqlite_db)], privileged=True, user='root') - node1.query("insert into table function odbc('DSN={};', '', 't3') values (200, 2, 7)".format(node1.odbc_drivers["SQLite3"]["DSN"])) + node1.query("insert into table function odbc('DSN={};', '', 't3') values (200, 2, 7)".format( + node1.odbc_drivers["SQLite3"]["DSN"])) - assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(200))") == "7\n" # new value + assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(200))") == "7\n" # new value - node1.exec_in_container(["bash", "-c", "echo 'REPLACE INTO t3 values(1, 2, 12);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + node1.exec_in_container(["bash", "-c", "echo 'REPLACE INTO t3 values(1, 2, 12);' | sqlite3 {}".format(sqlite_db)], + privileged=True, user='root') time.sleep(5) assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))") == "12\n" + def test_postgres_odbc_hached_dictionary_with_schema(started_cluster): conn = get_postgres_conn() cursor = conn.cursor() @@ -206,6 +254,7 @@ def test_postgres_odbc_hached_dictionary_with_schema(started_cluster): assert node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))") == "hello\n" assert node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(2))") == "world\n" + def test_postgres_odbc_hached_dictionary_no_tty_pipe_overflow(started_cluster): conn = get_postgres_conn() cursor = conn.cursor() @@ -218,6 +267,7 @@ def test_postgres_odbc_hached_dictionary_no_tty_pipe_overflow(started_cluster): assert node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(3))") == "xxx\n" + def test_postgres_insert(started_cluster): conn = get_postgres_conn() conn.cursor().execute("truncate table clickhouse.test_table") @@ -226,13 +276,17 @@ def test_postgres_insert(started_cluster): # postgres .yml file). This is needed to check parsing, validation and # reconstruction of connection string. - node1.query("create table pg_insert (column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')") + node1.query( + "create table pg_insert (column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')") node1.query("insert into pg_insert values (1, 'hello'), (2, 'world')") assert node1.query("select * from pg_insert") == '1\thello\n2\tworld\n' node1.query("insert into table function odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table') format CSV 3,test") - node1.query("insert into table function odbc('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table') select number, 's' || toString(number) from numbers (4, 7)") + node1.query( + "insert into table function odbc('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table') select number, 's' || toString(number) from numbers (4, 7)") assert node1.query("select sum(column1), count(column1) from pg_insert") == "55\t10\n" - assert node1.query("select sum(n), count(n) from (select (*,).1 as n from (select * from odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table')))") == "55\t10\n" + assert node1.query( + "select sum(n), count(n) from (select (*,).1 as n from (select * from odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table')))") == "55\t10\n" + def test_bridge_dies_with_parent(started_cluster): node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))") @@ -251,13 +305,14 @@ def test_bridge_dies_with_parent(started_cluster): time.sleep(1) for i in range(5): - time.sleep(1) # just for sure, that odbc-bridge caught signal + time.sleep(1) # just for sure, that odbc-bridge caught signal bridge_pid = node1.get_process_pid("odbc-bridge") if bridge_pid is None: break if bridge_pid: - out = node1.exec_in_container(["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], privileged=True, user='root') + out = node1.exec_in_container(["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], + privileged=True, user='root') print("Bridge is running, gdb output:") print(out) diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index 0336a1196c4..1870ecf4c9d 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -1,28 +1,30 @@ - -import time -import os import pytest from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool -from helpers.client import QueryRuntimeException, QueryTimeoutExceedException from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) -node18_14 = cluster.add_instance('node18_14', image='yandex/clickhouse-server', tag='18.14.19', with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_1 = cluster.add_instance('node19_1', image='yandex/clickhouse-server', tag='19.1.16', with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_4 = cluster.add_instance('node19_4', image='yandex/clickhouse-server', tag='19.4.5.35', with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_8 = cluster.add_instance('node19_8', image='yandex/clickhouse-server', tag='19.8.3.8', with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_11 = cluster.add_instance('node19_11', image='yandex/clickhouse-server', tag='19.11.13.74', with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_13 = cluster.add_instance('node19_13', image='yandex/clickhouse-server', tag='19.13.7.57', with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_16 = cluster.add_instance('node19_16', image='yandex/clickhouse-server', tag='19.16.2.2', with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) +node18_14 = cluster.add_instance('node18_14', image='yandex/clickhouse-server', tag='18.14.19', + with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) +node19_1 = cluster.add_instance('node19_1', image='yandex/clickhouse-server', tag='19.1.16', with_installed_binary=True, + main_configs=["configs/config.d/test_cluster.xml"]) +node19_4 = cluster.add_instance('node19_4', image='yandex/clickhouse-server', tag='19.4.5.35', + with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) +node19_8 = cluster.add_instance('node19_8', image='yandex/clickhouse-server', tag='19.8.3.8', + with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) +node19_11 = cluster.add_instance('node19_11', image='yandex/clickhouse-server', tag='19.11.13.74', + with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) +node19_13 = cluster.add_instance('node19_13', image='yandex/clickhouse-server', tag='19.13.7.57', + with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) +node19_16 = cluster.add_instance('node19_16', image='yandex/clickhouse-server', tag='19.16.2.2', + with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) old_nodes = [node18_14, node19_1, node19_4, node19_8, node19_11, node19_13, node19_16] new_node = cluster.add_instance('node_new') def query_from_one_node_to_another(client_node, server_node, query): - client_node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.name, query)]) + client_node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.name, query)]) @pytest.fixture(scope="module") @@ -34,7 +36,8 @@ def setup_nodes(): n.query('''CREATE TABLE test_table (id UInt32, value UInt64) ENGINE = MergeTree() ORDER BY tuple()''') for n in old_nodes: - n.query('''CREATE TABLE dist_table AS test_table ENGINE = Distributed('test_cluster', 'default', 'test_table')''') + n.query( + '''CREATE TABLE dist_table AS test_table ENGINE = Distributed('test_cluster', 'default', 'test_table')''') yield cluster finally: @@ -70,5 +73,7 @@ def test_distributed_query_initiator_is_older_than_shard(setup_nodes): for i, initiator in enumerate(distributed_query_initiator_old_nodes): initiator.query("INSERT INTO dist_table VALUES (3, {})".format(i)) - assert_eq_with_retry(shard, "SELECT COUNT() FROM test_table WHERE id=3", str(len(distributed_query_initiator_old_nodes))) - assert_eq_with_retry(initiator, "SELECT COUNT() FROM dist_table WHERE id=3", str(len(distributed_query_initiator_old_nodes))) + assert_eq_with_retry(shard, "SELECT COUNT() FROM test_table WHERE id=3", + str(len(distributed_query_initiator_old_nodes))) + assert_eq_with_retry(initiator, "SELECT COUNT() FROM dist_table WHERE id=3", + str(len(distributed_query_initiator_old_nodes))) diff --git a/tests/integration/test_on_cluster_timeouts/test.py b/tests/integration/test_on_cluster_timeouts/test.py index 965bf8fae1b..544153d0d00 100644 --- a/tests/integration/test_on_cluster_timeouts/test.py +++ b/tests/integration/test_on_cluster_timeouts/test.py @@ -1,14 +1,17 @@ import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users_config.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users_config.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users_config.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users_config.xml'], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], + user_configs=['configs/users_config.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], + user_configs=['configs/users_config.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], + user_configs=['configs/users_config.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], + user_configs=['configs/users_config.xml'], with_zookeeper=True) @pytest.fixture(scope="module") @@ -22,19 +25,24 @@ def started_cluster(): def test_long_query(started_cluster): - node1.query("CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/1/cluster_table', '1') ORDER BY tuple()") - node2.query("CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/1/cluster_table', '2') ORDER BY tuple()") + node1.query( + "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/1/cluster_table', '1') ORDER BY tuple()") + node2.query( + "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/1/cluster_table', '2') ORDER BY tuple()") node1.query("INSERT INTO cluster_table SELECT number, toString(number) FROM numbers(20)") node2.query("SYSTEM SYNC REPLICA cluster_table") - node3.query("CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/2/cluster_table', '1') ORDER BY tuple()") + node3.query( + "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/2/cluster_table', '1') ORDER BY tuple()") - node4.query("CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/2/cluster_table', '2') ORDER BY tuple()") + node4.query( + "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/2/cluster_table', '2') ORDER BY tuple()") node3.query("INSERT INTO cluster_table SELECT number, toString(number) FROM numbers(20)") node4.query("SYSTEM SYNC REPLICA cluster_table") - node1.query("ALTER TABLE cluster_table ON CLUSTER 'test_cluster' UPDATE key = 1 WHERE sleepEachRow(1) == 0", settings={"mutations_sync": "2"}) + node1.query("ALTER TABLE cluster_table ON CLUSTER 'test_cluster' UPDATE key = 1 WHERE sleepEachRow(1) == 0", + settings={"mutations_sync": "2"}) assert node1.query("SELECT SUM(key) FROM cluster_table") == "20\n" assert node2.query("SELECT SUM(key) FROM cluster_table") == "20\n" diff --git a/tests/integration/test_part_log_table/test.py b/tests/integration/test_part_log_table/test.py index 455ca7eed54..63adde432b5 100644 --- a/tests/integration/test_part_log_table/test.py +++ b/tests/integration/test_part_log_table/test.py @@ -1,4 +1,3 @@ -import time import pytest from helpers.cluster import ClickHouseCluster @@ -8,6 +7,7 @@ node1 = cluster.add_instance("node1") node2 = cluster.add_instance("node2", main_configs=["configs/config_with_standard_part_log.xml"]) node3 = cluster.add_instance("node3", main_configs=["configs/config_with_non_standard_part_log.xml"]) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -16,6 +16,7 @@ def start_cluster(): finally: cluster.shutdown() + def test_config_without_part_log(start_cluster): assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") node1.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() ORDER BY value") @@ -24,6 +25,7 @@ def test_config_without_part_log(start_cluster): node1.query("SYSTEM FLUSH LOGS") assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") + # Note: if part_log is defined, we cannot say when the table will be created - because of metric_log, trace_log, text_log, query_log... def test_config_with_standard_part_log(start_cluster): @@ -32,6 +34,7 @@ def test_config_with_standard_part_log(start_cluster): node2.query("SYSTEM FLUSH LOGS") assert node2.query("SELECT * FROM system.part_log") != "" + def test_config_with_non_standard_part_log(start_cluster): node3.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value") node3.query("INSERT INTO test_table VALUES ('name', 1)") diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index 80fbe947316..58a37c405cb 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -3,7 +3,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance') q = instance.query @@ -109,7 +108,6 @@ def partition_table_complex(started_cluster): def test_partition_complex(partition_table_complex): - partition_complex_assert_columns_txt() q("ALTER TABLE test.partition FREEZE") @@ -131,7 +129,7 @@ def test_partition_complex(partition_table_complex): expected = TSV('31\t1\t2\n' '1\t2\t3') res = q("SELECT toUInt16(p), k, v1 FROM test.partition ORDER BY k") - assert(TSV(res) == expected) + assert (TSV(res) == expected) @pytest.fixture @@ -175,7 +173,7 @@ def test_attach_check_all_parts(attach_check_all_parts_table): path_to_detached = path_to_data + 'data/test/attach_partition/detached/' exec_bash('mkdir {}'.format(path_to_detached + '0_5_5_0')) exec_bash('cp -pr {} {}'.format(path_to_detached + '0_1_1_0', path_to_detached + 'attaching_0_6_6_0')) - exec_bash('cp -pr {} {}'.format(path_to_detached + '0_3_3_0', path_to_detached + 'deleting_0_7_7_0')) + exec_bash('cp -pr {} {}'.format(path_to_detached + '0_3_3_0', path_to_detached + 'deleting_0_7_7_0')) error = instance.client.query_and_get_error("ALTER TABLE test.attach_partition ATTACH PARTITION 0") assert 0 <= error.find('No columns in part 0_5_5_0') @@ -224,15 +222,18 @@ def test_drop_detached_parts(drop_detached_parts_table): exec_bash('mkdir {}'.format(path_to_detached + 'any_other_name')) exec_bash('mkdir {}'.format(path_to_detached + 'prefix_1_2_2_0_0')) - error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0'", settings=s) + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0'", + settings=s) assert 0 <= error.find('Invalid part name') q("ALTER TABLE test.drop_detached DROP DETACHED PART '0_1_1_0'", settings=s) - error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0'", settings=s) + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0'", + settings=s) assert 0 <= error.find('Cannot drop part') - error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0'", settings=s) + error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0'", + settings=s) assert 0 <= error.find('Cannot drop part') q("ALTER TABLE test.drop_detached DROP DETACHED PART 'any_other_name'", settings=s) diff --git a/tests/integration/test_parts_delete_zookeeper/test.py b/tests/integration/test_parts_delete_zookeeper/test.py index 7e4a8d36741..7489b2411f9 100644 --- a/tests/integration/test_parts_delete_zookeeper/test.py +++ b/tests/integration/test_parts_delete_zookeeper/test.py @@ -1,11 +1,10 @@ import time + import pytest - -from helpers.network import PartitionManager from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) @@ -55,7 +54,7 @@ def test_merge_doesnt_work_without_zookeeper(start_cluster): with PartitionManager() as pm: node1.query("OPTIMIZE TABLE test_table FINAL") pm.drop_instance_zk_connections(node1) - time.sleep(10) # > old_parts_lifetime + time.sleep(10) # > old_parts_lifetime assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "3\n" assert_eq_with_retry(node1, "SELECT count(*) from system.parts where table = 'test_table' and active = 1", "1") diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index ed89f768d4c..1729817cd53 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -1,75 +1,85 @@ -import time -import pytest +import os import random import string -import os import struct -from helpers.test_tools import TSV -from helpers.test_tools import assert_eq_with_retry +import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager -from multiprocessing.dummy import Pool +from helpers.test_tools import TSV +from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) + def get_random_array(): return [random.randint(0, 1000) % 1000 for _ in range(random.randint(0, 1000))] + def get_random_string(): length = random.randint(0, 1000) return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + def insert_random_data(table, node, size): data = [ - '(' + ','.join(( - "'2019-10-11'", - str(i), - "'" + get_random_string() + "'", - str(get_random_array()))) + - ')' for i in range(size) + '(' + ','.join(( + "'2019-10-11'", + str(i), + "'" + get_random_string() + "'", + str(get_random_array()))) + + ')' for i in range(size) ] node.query("INSERT INTO {} VALUES {}".format(table, ','.join(data))) + def create_tables(name, nodes, node_settings, shard): for i, (node, settings) in enumerate(zip(nodes, node_settings)): node.query( - ''' - CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/{name}', '{repl}') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes}, - min_rows_for_wide_part = {min_rows_for_wide_part}, min_rows_for_compact_part = {min_rows_for_compact_part}, - in_memory_parts_enable_wal = 1 - '''.format(name=name, shard=shard, repl=i, **settings)) + ''' + CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/{name}', '{repl}') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes}, + min_rows_for_wide_part = {min_rows_for_wide_part}, min_rows_for_compact_part = {min_rows_for_compact_part}, + in_memory_parts_enable_wal = 1 + '''.format(name=name, shard=shard, repl=i, **settings)) + def create_tables_old_format(name, nodes, shard): for i, node in enumerate(nodes): node.query( - ''' - CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/{name}', '{repl}', date, id, 64) - '''.format(name=name, shard=shard, repl=i)) + ''' + CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/{name}', '{repl}', date, id, 64) + '''.format(name=name, shard=shard, repl=i)) -node1 = cluster.add_instance('node1', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], with_zookeeper=True) -settings_default = {'index_granularity_bytes' : 10485760, 'min_rows_for_wide_part' : 512, 'min_rows_for_compact_part' : 0} -settings_compact_only = {'index_granularity_bytes' : 10485760, 'min_rows_for_wide_part' : 1000000, 'min_rows_for_compact_part' : 0} -settings_not_adaptive = {'index_granularity_bytes' : 0, 'min_rows_for_wide_part' : 512, 'min_rows_for_compact_part' : 0} +node1 = cluster.add_instance('node1', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], + with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], + with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], with_zookeeper=True) -node4 = cluster.add_instance('node4', user_configs=["configs/users.d/not_optimize_count.xml"], main_configs=['configs/no_leader.xml'], with_zookeeper=True) +settings_default = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 512, 'min_rows_for_compact_part': 0} +settings_compact_only = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 1000000, + 'min_rows_for_compact_part': 0} +settings_not_adaptive = {'index_granularity_bytes': 0, 'min_rows_for_wide_part': 512, 'min_rows_for_compact_part': 0} -settings_compact = {'index_granularity_bytes' : 10485760, 'min_rows_for_wide_part' : 512, 'min_rows_for_compact_part' : 0} -settings_wide = {'index_granularity_bytes' : 10485760, 'min_rows_for_wide_part' : 0, 'min_rows_for_compact_part' : 0} +node3 = cluster.add_instance('node3', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], + with_zookeeper=True) +node4 = cluster.add_instance('node4', user_configs=["configs/users.d/not_optimize_count.xml"], + main_configs=['configs/no_leader.xml'], with_zookeeper=True) + +settings_compact = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 512, 'min_rows_for_compact_part': 0} +settings_wide = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 0, 'min_rows_for_compact_part': 0} node5 = cluster.add_instance('node5', main_configs=['configs/compact_parts.xml'], with_zookeeper=True) node6 = cluster.add_instance('node6', main_configs=['configs/compact_parts.xml'], with_zookeeper=True) -settings_in_memory = {'index_granularity_bytes' : 10485760, 'min_rows_for_wide_part' : 512, 'min_rows_for_compact_part' : 256} +settings_in_memory = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 512, + 'min_rows_for_compact_part': 256} node9 = cluster.add_instance('node9', with_zookeeper=True, stay_alive=True) node10 = cluster.add_instance('node10', with_zookeeper=True) @@ -77,6 +87,7 @@ node10 = cluster.add_instance('node10', with_zookeeper=True) node11 = cluster.add_instance('node11', main_configs=['configs/do_not_merge.xml'], with_zookeeper=True, stay_alive=True) node12 = cluster.add_instance('node12', main_configs=['configs/do_not_merge.xml'], with_zookeeper=True, stay_alive=True) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -100,11 +111,12 @@ def start_cluster(): finally: cluster.shutdown() + @pytest.mark.parametrize( ('first_node', 'second_node'), [ - (node1, node2), # compact parts - (node5, node6), # compact parts, old-format + (node1, node2), # compact parts + (node5, node6), # compact parts, old-format ] ) def test_polymorphic_parts_basics(start_cluster, first_node, second_node): @@ -121,9 +133,11 @@ def test_polymorphic_parts_basics(start_cluster, first_node, second_node): expected = "Compact\t2\nWide\t1\n" assert TSV(first_node.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'polymorphic_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV(expected) + "WHERE table = 'polymorphic_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + expected) assert TSV(second_node.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'polymorphic_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV(expected) + "WHERE table = 'polymorphic_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + expected) first_node.query("SYSTEM START MERGES") second_node.query("SYSTEM START MERGES") @@ -144,8 +158,10 @@ def test_polymorphic_parts_basics(start_cluster, first_node, second_node): assert first_node.query("SELECT count() FROM polymorphic_table") == "2000\n" assert second_node.query("SELECT count() FROM polymorphic_table") == "2000\n" - assert first_node.query("SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' AND active") == "Wide\n" - assert second_node.query("SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' AND active") == "Wide\n" + assert first_node.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' AND active") == "Wide\n" + assert second_node.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' AND active") == "Wide\n" # Check alters and mutations also work first_node.query("ALTER TABLE polymorphic_table ADD COLUMN ss String") @@ -159,6 +175,7 @@ def test_polymorphic_parts_basics(start_cluster, first_node, second_node): second_node.query("SELECT count(ss) FROM polymorphic_table") == "2000\n" second_node.query("SELECT uniqExact(ss) FROM polymorphic_table") == "600\n" + # Checks mostly that merge from compact part to compact part works. def test_compact_parts_only(start_cluster): for i in range(20): @@ -171,8 +188,10 @@ def test_compact_parts_only(start_cluster): assert node1.query("SELECT count() FROM compact_parts_only") == "4000\n" assert node2.query("SELECT count() FROM compact_parts_only") == "4000\n" - assert node1.query("SELECT DISTINCT part_type FROM system.parts WHERE table = 'compact_parts_only' AND active") == "Compact\n" - assert node2.query("SELECT DISTINCT part_type FROM system.parts WHERE table = 'compact_parts_only' AND active") == "Compact\n" + assert node1.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'compact_parts_only' AND active") == "Compact\n" + assert node2.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'compact_parts_only' AND active") == "Compact\n" node1.query("OPTIMIZE TABLE compact_parts_only FINAL") node2.query("SYSTEM SYNC REPLICA compact_parts_only", timeout=20) @@ -180,9 +199,12 @@ def test_compact_parts_only(start_cluster): expected = "Compact\t1\n" assert TSV(node1.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'compact_parts_only' AND active GROUP BY part_type ORDER BY part_type")) == TSV(expected) + "WHERE table = 'compact_parts_only' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + expected) assert TSV(node2.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'compact_parts_only' AND active GROUP BY part_type ORDER BY part_type")) == TSV(expected) + "WHERE table = 'compact_parts_only' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + expected) + # Check that follower replicas create parts of the same type, which leader has chosen at merge. @pytest.mark.parametrize( @@ -208,16 +230,21 @@ def test_different_part_types_on_replicas(start_cluster, table, part_type): expected = "{}\t1\n".format(part_type) assert TSV(leader.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format(table))) == TSV(expected) + "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format( + table))) == TSV(expected) assert TSV(follower.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format(table))) == TSV(expected) + "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format( + table))) == TSV(expected) -node7 = cluster.add_instance('node7', user_configs=["configs_old/users.d/not_optimize_count.xml"], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) +node7 = cluster.add_instance('node7', user_configs=["configs_old/users.d/not_optimize_count.xml"], with_zookeeper=True, + image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, + with_installed_binary=True) node8 = cluster.add_instance('node8', user_configs=["configs/users.d/not_optimize_count.xml"], with_zookeeper=True) -settings7 = {'index_granularity_bytes' : 10485760} -settings8 = {'index_granularity_bytes' : 10485760, 'min_rows_for_wide_part' : 512, 'min_rows_for_compact_part' : 0} +settings7 = {'index_granularity_bytes': 10485760} +settings8 = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 512, 'min_rows_for_compact_part': 0} + @pytest.fixture(scope="module") def start_cluster_diff_versions(): @@ -225,24 +252,24 @@ def start_cluster_diff_versions(): for name in ['polymorphic_table', 'polymorphic_table_2']: cluster.start() node7.query( - ''' - CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard5/{name}', '1') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes} - '''.format(name=name, **settings7) + ''' + CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard5/{name}', '1') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes} + '''.format(name=name, **settings7) ) node8.query( - ''' - CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard5/{name}', '2') - PARTITION BY toYYYYMM(date) - ORDER BY id - SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes}, - min_rows_for_wide_part = {min_rows_for_wide_part}, min_bytes_for_wide_part = {min_bytes_for_wide_part} - '''.format(name=name, **settings8) + ''' + CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard5/{name}', '2') + PARTITION BY toYYYYMM(date) + ORDER BY id + SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes}, + min_rows_for_wide_part = {min_rows_for_wide_part}, min_bytes_for_wide_part = {min_bytes_for_wide_part} + '''.format(name=name, **settings8) ) yield cluster @@ -262,7 +289,8 @@ def test_polymorphic_parts_diff_versions(start_cluster_diff_versions): node8.query("SYSTEM SYNC REPLICA polymorphic_table", timeout=20) assert node8.query("SELECT count() FROM polymorphic_table") == "100\n" - assert node8.query("SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' and active") == "Wide\n" + assert node8.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' and active") == "Wide\n" @pytest.mark.skip(reason="compatability is temporary broken") @@ -286,7 +314,8 @@ def test_polymorphic_parts_diff_versions_2(start_cluster_diff_versions): # Works after update assert node_old.query("SELECT count() FROM polymorphic_table_2") == "100\n" - assert node_old.query("SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table_2' and active") == "Compact\n" + assert node_old.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table_2' and active") == "Compact\n" def test_polymorphic_parts_non_adaptive(start_cluster): @@ -300,11 +329,15 @@ def test_polymorphic_parts_non_adaptive(start_cluster): node1.query("SYSTEM SYNC REPLICA non_adaptive_table", timeout=20) assert TSV(node1.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'non_adaptive_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV("Wide\t2\n") + "WHERE table = 'non_adaptive_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + "Wide\t2\n") assert TSV(node2.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'non_adaptive_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV("Wide\t2\n") + "WHERE table = 'non_adaptive_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + "Wide\t2\n") + + assert node1.contains_in_log( + " default.non_adaptive_table: Table can't create parts with adaptive granularity") - assert node1.contains_in_log(" default.non_adaptive_table: Table can't create parts with adaptive granularity") def test_in_memory(start_cluster): node9.query("SYSTEM STOP MERGES") @@ -320,9 +353,11 @@ def test_in_memory(start_cluster): expected = "Compact\t1\nInMemory\t2\nWide\t1\n" assert TSV(node9.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV(expected) + "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + expected) assert TSV(node10.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV(expected) + "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + expected) node9.query("SYSTEM START MERGES") node10.query("SYSTEM START MERGES") @@ -334,9 +369,12 @@ def test_in_memory(start_cluster): assert node10.query("SELECT count() FROM in_memory_table") == "1300\n" assert TSV(node9.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV("Wide\t1\n") + "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + "Wide\t1\n") assert TSV(node10.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV("Wide\t1\n") + "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( + "Wide\t1\n") + def test_in_memory_wal(start_cluster): # Merges are disabled in config @@ -347,7 +385,9 @@ def test_in_memory_wal(start_cluster): def check(node, rows, parts): node.query("SELECT count() FROM wal_table") == "{}\n".format(rows) - node.query("SELECT count() FROM system.parts WHERE table = 'wal_table' AND part_type = 'InMemory'") == "{}\n".format(parts) + node.query( + "SELECT count() FROM system.parts WHERE table = 'wal_table' AND part_type = 'InMemory'") == "{}\n".format( + parts) check(node11, 250, 5) check(node12, 250, 5) @@ -383,7 +423,7 @@ def test_in_memory_wal(start_cluster): node11.query("SYSTEM SYNC REPLICA wal_table", timeout=20) check(node11, 300, 6) - #Check that new data is written to new wal, but old is still exists for restoring + # Check that new data is written to new wal, but old is still exists for restoring assert os.path.getsize(wal_file) > 0 assert os.path.exists(broken_wal_file) @@ -398,6 +438,7 @@ def test_in_memory_wal(start_cluster): node11.restart_clickhouse(kill=True) check(node11, 300, 6) + def test_in_memory_wal_rotate(start_cluster): # Write every part to single wal node11.query("ALTER TABLE restore_table MODIFY SETTING write_ahead_log_max_bytes = 10") @@ -409,7 +450,8 @@ def test_in_memory_wal_rotate(start_cluster): assert os.path.exists(wal_file) for node in [node11, node12]: - node.query("ALTER TABLE restore_table MODIFY SETTING number_of_free_entries_in_pool_to_lower_max_size_of_merge = 0") + node.query( + "ALTER TABLE restore_table MODIFY SETTING number_of_free_entries_in_pool_to_lower_max_size_of_merge = 0") node.query("ALTER TABLE restore_table MODIFY SETTING max_bytes_to_merge_at_max_space_in_pool = 10000000") assert_eq_with_retry(node11, "OPTIMIZE TABLE restore_table FINAL SETTINGS optimize_throw_if_noop = 1", "") @@ -425,6 +467,7 @@ def test_in_memory_wal_rotate(start_cluster): assert os.path.exists(wal_file) assert os.path.getsize(wal_file) == 0 + def test_in_memory_deduplication(start_cluster): for i in range(3): node9.query("INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')") @@ -436,13 +479,15 @@ def test_in_memory_deduplication(start_cluster): assert node9.query("SELECT date, id, s FROM deduplication_table") == "2020-03-03\t1\tfoo\n" assert node10.query("SELECT date, id, s FROM deduplication_table") == "2020-03-03\t1\tfoo\n" + # Checks that restoring from WAL works after table schema changed def test_in_memory_alters(start_cluster): def check_parts_type(parts_num): assert node9.query("SELECT part_type, count() FROM system.parts WHERE table = 'alters_table' \ AND active GROUP BY part_type") == "InMemory\t{}\n".format(parts_num) - node9.query("INSERT INTO alters_table (date, id, s) VALUES (toDate('2020-10-10'), 1, 'ab'), (toDate('2020-10-10'), 2, 'cd')") + node9.query( + "INSERT INTO alters_table (date, id, s) VALUES (toDate('2020-10-10'), 1, 'ab'), (toDate('2020-10-10'), 2, 'cd')") node9.query("ALTER TABLE alters_table ADD COLUMN col1 UInt32") node9.restart_clickhouse(kill=True) @@ -462,6 +507,7 @@ def test_in_memory_alters(start_cluster): expected = expected = "1\t0_foo\n2\t0_foo\n3\t100_foo\n" assert node9.query("SELECT id, col1 || '_foo' FROM alters_table") + def test_polymorphic_parts_index(start_cluster): node1.query(''' CREATE TABLE index_compact(a UInt32, s String) diff --git a/tests/integration/test_postgresql_protocol/test.py b/tests/integration/test_postgresql_protocol/test.py index 939e8231931..513bb75fcab 100644 --- a/tests/integration/test_postgresql_protocol/test.py +++ b/tests/integration/test_postgresql_protocol/test.py @@ -4,16 +4,16 @@ from __future__ import print_function import datetime import decimal +import os +import subprocess +import sys +import time +import uuid + import docker import psycopg2 as py_psql import psycopg2.extras import pytest -import os -import sys -import subprocess -import time -import uuid - from helpers.cluster import ClickHouseCluster, get_docker_compose_path psycopg2.extras.register_uuid() @@ -24,7 +24,8 @@ DOCKER_COMPOSE_PATH = get_docker_compose_path() cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=["configs/postresql.xml", "configs/log.xml", "configs/ssl_conf.xml", "configs/dhparam.pem", "configs/server.crt", "configs/server.key"], - user_configs=["configs/default_passwd.xml"], env_variables={'UBSAN_OPTIONS': 'print_stacktrace=1'}) + user_configs=["configs/default_passwd.xml"], + env_variables={'UBSAN_OPTIONS': 'print_stacktrace=1'}) server_port = 5433 @@ -41,7 +42,8 @@ def server_address(): @pytest.fixture(scope='module') def psql_client(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_postgesql.yml') - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build']) yield docker.from_env().containers.get(cluster.project_name + '_psql_1') @@ -64,7 +66,8 @@ def psql_server(psql_client): @pytest.fixture(scope='module') def java_container(): docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_postgesql_java_client.yml') - subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build']) + subprocess.check_call( + ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build']) yield docker.from_env().containers.get(cluster.project_name + '_java_1') @@ -73,7 +76,7 @@ def test_psql_is_ready(psql_server): def test_psql_client(psql_client, server_address): - cmd_prefix = 'psql "sslmode=require host={server_address} port={server_port} user=default dbname=default password=123" '\ + cmd_prefix = 'psql "sslmode=require host={server_address} port={server_port} user=default dbname=default password=123" ' \ .format(server_address=server_address, server_port=server_port) cmd_prefix += "--no-align --field-separator=' ' " @@ -116,7 +119,8 @@ def test_python_client(server_address): cur = ch.cursor() cur.execute('select name from tables;') - assert exc_info.value.args == ("Query execution failed.\nDB::Exception: Table default.tables doesn't exist.\nSSL connection has been closed unexpectedly\n",) + assert exc_info.value.args == ( + "Query execution failed.\nDB::Exception: Table default.tables doesn't exist.\nSSL connection has been closed unexpectedly\n",) ch = py_psql.connect(host=server_address, port=server_port, user='default', password='123', database='') cur = ch.cursor() @@ -127,10 +131,14 @@ def test_python_client(server_address): cur.execute('CREATE DATABASE x') cur.execute('USE x') - cur.execute('CREATE TEMPORARY TABLE tmp2 (ch Int8, i64 Int64, f64 Float64, str String, date Date, dec Decimal(19, 10), uuid UUID) ENGINE = Memory') - cur.execute("insert into tmp2 (ch, i64, f64, str, date, dec, uuid) values (44, 534324234, 0.32423423, 'hello', '2019-01-23', 0.333333, '61f0c404-5cb3-11e7-907b-a6006ad3dba0')") + cur.execute( + 'CREATE TEMPORARY TABLE tmp2 (ch Int8, i64 Int64, f64 Float64, str String, date Date, dec Decimal(19, 10), uuid UUID) ENGINE = Memory') + cur.execute( + "insert into tmp2 (ch, i64, f64, str, date, dec, uuid) values (44, 534324234, 0.32423423, 'hello', '2019-01-23', 0.333333, '61f0c404-5cb3-11e7-907b-a6006ad3dba0')") cur.execute('select * from tmp2') - assert cur.fetchall()[0] == ('44', 534324234, 0.32423423, 'hello', datetime.date(2019, 1, 23), decimal.Decimal('0.3333330000'), uuid.UUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')) + assert cur.fetchall()[0] == ( + '44', 534324234, 0.32423423, 'hello', datetime.date(2019, 1, 23), decimal.Decimal('0.3333330000'), + uuid.UUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')) def test_java_client(server_address, java_container): @@ -138,13 +146,15 @@ def test_java_client(server_address, java_container): reference = fp.read() # database not exists exception. - code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user default --database ' - 'abc'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = java_container.exec_run( + 'java JavaConnectorTest --host {host} --port {port} --user default --database ' + 'abc'.format(host=server_address, port=server_port), demux=True) assert code == 1 # non-empty password passed. - code, (stdout, stderr) = java_container.exec_run('java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database ' - 'default'.format(host=server_address, port=server_port), demux=True) + code, (stdout, stderr) = java_container.exec_run( + 'java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database ' + 'default'.format(host=server_address, port=server_port), demux=True) print(stdout, stderr, file=sys.stderr) assert code == 0 assert stdout == reference diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index e2cb10499e7..3d65a489610 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -1,11 +1,8 @@ import logging -import random -import string -import time import re -import requests import pytest +import requests from helpers.cluster import ClickHouseCluster logging.getLogger().setLevel(logging.INFO) @@ -17,7 +14,8 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", "configs/log.xml", "configs/query_log.xml", "configs/ssl_conf.xml"], with_minio=True) + cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", "configs/log.xml", + "configs/query_log.xml", "configs/ssl_conf.xml"], with_minio=True) logging.info("Starting cluster...") cluster.start() @@ -29,20 +27,21 @@ def cluster(): init_list = { - "S3ReadMicroseconds" : 0, - "S3ReadBytes" : 0, - "S3ReadRequestsCount" : 0, - "S3ReadRequestsErrorsTotal" : 0, - "S3ReadRequestsErrors503" : 0, - "S3ReadRequestsRedirects" : 0, - "S3WriteMicroseconds" : 0, - "S3WriteBytes" : 0, - "S3WriteRequestsCount" : 0, - "S3WriteRequestsErrorsTotal" : 0, - "S3WriteRequestsErrors503" : 0, - "S3WriteRequestsRedirects" : 0, + "S3ReadMicroseconds": 0, + "S3ReadBytes": 0, + "S3ReadRequestsCount": 0, + "S3ReadRequestsErrorsTotal": 0, + "S3ReadRequestsErrors503": 0, + "S3ReadRequestsRedirects": 0, + "S3WriteMicroseconds": 0, + "S3WriteBytes": 0, + "S3WriteRequestsCount": 0, + "S3WriteRequestsErrorsTotal": 0, + "S3WriteRequestsErrors503": 0, + "S3WriteRequestsRedirects": 0, } + def get_s3_events(instance): result = init_list.copy() events = instance.query("SELECT event,value FROM system.events WHERE event LIKE 'S3%'").split("\n") @@ -55,13 +54,14 @@ def get_s3_events(instance): def get_minio_stat(cluster): result = { - "get_requests" : 0, - "set_requests" : 0, - "errors" : 0, - "rx_bytes" : 0, - "tx_bytes" : 0, + "get_requests": 0, + "set_requests": 0, + "errors": 0, + "rx_bytes": 0, + "tx_bytes": 0, } - stat = requests.get(url="http://{}:{}/minio/prometheus/metrics".format("localhost", cluster.minio_port)).text.split("\n") + stat = requests.get(url="http://{}:{}/minio/prometheus/metrics".format("localhost", cluster.minio_port)).text.split( + "\n") for line in stat: x = re.search("s3_requests_total(\{.*\})?\s(\d+)(\s.*)?", line) if x != None: @@ -126,8 +126,10 @@ def test_profile_events(cluster): metrics1 = get_s3_events(instance) minio1 = get_minio_stat(cluster) - assert metrics1["S3ReadRequestsCount"] - metrics0["S3ReadRequestsCount"] == minio1["get_requests"] - minio0["get_requests"] - 1 # 1 from get_minio_size - assert metrics1["S3WriteRequestsCount"] - metrics0["S3WriteRequestsCount"] == minio1["set_requests"] - minio0["set_requests"] + assert metrics1["S3ReadRequestsCount"] - metrics0["S3ReadRequestsCount"] == minio1["get_requests"] - minio0[ + "get_requests"] - 1 # 1 from get_minio_size + assert metrics1["S3WriteRequestsCount"] - metrics0["S3WriteRequestsCount"] == minio1["set_requests"] - minio0[ + "set_requests"] stat1 = get_query_stat(instance, query1) for metric in stat1: assert stat1[metric] == metrics1[metric] - metrics0[metric] @@ -140,8 +142,10 @@ def test_profile_events(cluster): metrics2 = get_s3_events(instance) minio2 = get_minio_stat(cluster) - assert metrics2["S3ReadRequestsCount"] - metrics1["S3ReadRequestsCount"] == minio2["get_requests"] - minio1["get_requests"] - 1 # 1 from get_minio_size - assert metrics2["S3WriteRequestsCount"] - metrics1["S3WriteRequestsCount"] == minio2["set_requests"] - minio1["set_requests"] + assert metrics2["S3ReadRequestsCount"] - metrics1["S3ReadRequestsCount"] == minio2["get_requests"] - minio1[ + "get_requests"] - 1 # 1 from get_minio_size + assert metrics2["S3WriteRequestsCount"] - metrics1["S3WriteRequestsCount"] == minio2["set_requests"] - minio1[ + "set_requests"] stat2 = get_query_stat(instance, query2) for metric in stat2: assert stat2[metric] == metrics2[metric] - metrics1[metric] @@ -153,8 +157,10 @@ def test_profile_events(cluster): metrics3 = get_s3_events(instance) minio3 = get_minio_stat(cluster) - assert metrics3["S3ReadRequestsCount"] - metrics2["S3ReadRequestsCount"] == minio3["get_requests"] - minio2["get_requests"] - assert metrics3["S3WriteRequestsCount"] - metrics2["S3WriteRequestsCount"] == minio3["set_requests"] - minio2["set_requests"] + assert metrics3["S3ReadRequestsCount"] - metrics2["S3ReadRequestsCount"] == minio3["get_requests"] - minio2[ + "get_requests"] + assert metrics3["S3WriteRequestsCount"] - metrics2["S3WriteRequestsCount"] == minio3["set_requests"] - minio2[ + "set_requests"] stat3 = get_query_stat(instance, query3) for metric in stat3: assert stat3[metric] == metrics3[metric] - metrics2[metric] diff --git a/tests/integration/test_prometheus_endpoint/test.py b/tests/integration/test_prometheus_endpoint/test.py index 25d83cfb47c..909dbf139b9 100644 --- a/tests/integration/test_prometheus_endpoint/test.py +++ b/tests/integration/test_prometheus_endpoint/test.py @@ -1,15 +1,16 @@ from __future__ import print_function -import pytest import re -import requests import time +import pytest +import requests from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=['configs/prom_conf.xml']) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -66,7 +67,6 @@ def get_and_check_metrics(retries): def test_prometheus_endpoint(start_cluster): - metrics_dict = get_and_check_metrics(10) assert metrics_dict['ClickHouseProfileEvents_Query'] >= 0 prev_query_count = metrics_dict['ClickHouseProfileEvents_Query'] diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index e89611c0d99..0adee0afc64 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -1,9 +1,8 @@ import time import pytest - -from helpers.test_tools import TSV from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) @@ -19,10 +18,12 @@ second = cluster.add_instance("second", user_configs=["configs/users.d/settings. macros={"cluster": "anime", "shard": "0", "replica": "second"}, with_zookeeper=True) + def execute_on_all_cluster(query_): for node in [zero, first, second]: node.query(query_) + @pytest.fixture(scope="module") def started_cluster(): global cluster @@ -47,7 +48,7 @@ def test_simple_add_replica(started_cluster): first.query("SYSTEM STOP FETCHES test_simple") - zero.query("INSERT INTO test_simple VALUES (1, '2011-01-01')", settings={'insert_quorum' : 1}) + zero.query("INSERT INTO test_simple VALUES (1, '2011-01-01')", settings={'insert_quorum': 1}) assert '1\t2011-01-01\n' == zero.query("SELECT * from test_simple") assert '' == first.query("SELECT * from test_simple") @@ -69,7 +70,6 @@ def test_simple_add_replica(started_cluster): execute_on_all_cluster("DROP TABLE IF EXISTS test_simple") - def test_drop_replica_and_achieve_quorum(started_cluster): execute_on_all_cluster("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum") @@ -86,23 +86,24 @@ def test_drop_replica_and_achieve_quorum(started_cluster): first.query("SYSTEM STOP FETCHES test_drop_replica_and_achieve_quorum") print("Insert to other replica. This query will fail.") - quorum_timeout = zero.query_and_get_error("INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (1, '2011-01-01')", - settings={'insert_quorum_timeout' : 5000}) + quorum_timeout = zero.query_and_get_error( + "INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (1, '2011-01-01')", + settings={'insert_quorum_timeout': 5000}) assert "Timeout while waiting for quorum" in quorum_timeout, "Query must fail." assert TSV("1\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum", - settings={'select_sequential_consistency' : 0})) + settings={'select_sequential_consistency': 0})) assert TSV("") == TSV(zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum", - settings={'select_sequential_consistency' : 1})) + settings={'select_sequential_consistency': 1})) - #TODO:(Mikhaylov) begin; maybe delete this lines. I want clickhouse to fetch parts and update quorum. + # TODO:(Mikhaylov) begin; maybe delete this lines. I want clickhouse to fetch parts and update quorum. print("START FETCHES first replica") first.query("SYSTEM START FETCHES test_drop_replica_and_achieve_quorum") print("SYNC first replica") first.query("SYSTEM SYNC REPLICA test_drop_replica_and_achieve_quorum", timeout=20) - #TODO:(Mikhaylov) end + # TODO:(Mikhaylov) end print("Add second replica") second.query(create_query) @@ -112,14 +113,17 @@ def test_drop_replica_and_achieve_quorum(started_cluster): print("Quorum for previous insert achieved.") assert TSV("1\t2011-01-01\n") == TSV(second.query("SELECT * FROM test_drop_replica_and_achieve_quorum", - settings={'select_sequential_consistency' : 1})) + settings={'select_sequential_consistency': 1})) print("Now we can insert some other data.") zero.query("INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (2, '2012-02-02')") - assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV(zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) - assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV(first.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) - assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV(second.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV( + zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV( + first.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV( + second.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) execute_on_all_cluster("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum") @@ -131,7 +135,6 @@ def test_drop_replica_and_achieve_quorum(started_cluster): True ] ) - def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): execute_on_all_cluster("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition") @@ -186,21 +189,20 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): True ] ) - def test_insert_quorum_with_move_partition(started_cluster, add_new_data): execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source") execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination") create_source = "CREATE TABLE test_insert_quorum_with_move_partition_source " \ - "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ - "PARTITION BY d ORDER BY a " - - create_destination = "CREATE TABLE test_insert_quorum_with_move_partition_destination " \ "(a Int8, d Date) " \ "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ "PARTITION BY d ORDER BY a " + create_destination = "CREATE TABLE test_insert_quorum_with_move_partition_destination " \ + "(a Int8, d Date) " \ + "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "PARTITION BY d ORDER BY a " + print("Create source Replicated table with three replicas") zero.query(create_source) first.query(create_source) @@ -218,7 +220,8 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data): zero.query("INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(1, '2011-01-01')") print("Drop partition.") - zero.query("ALTER TABLE test_insert_quorum_with_move_partition_source MOVE PARTITION '2011-01-01' TO TABLE test_insert_quorum_with_move_partition_destination") + zero.query( + "ALTER TABLE test_insert_quorum_with_move_partition_source MOVE PARTITION '2011-01-01' TO TABLE test_insert_quorum_with_move_partition_destination") if (add_new_data): print("Insert to deleted partition") @@ -237,7 +240,8 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data): print("Select from updated partition.") if (add_new_data): assert TSV("2\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) - assert TSV("2\t2011-01-01\n") == TSV(second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) + assert TSV("2\t2011-01-01\n") == TSV( + second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) else: assert TSV("") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) assert TSV("") == TSV(second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) @@ -265,12 +269,13 @@ def test_insert_quorum_with_ttl(started_cluster): print("Insert should fail since it can not reach the quorum.") quorum_timeout = zero.query_and_get_error("INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", - settings={'insert_quorum_timeout' : 5000}) + settings={'insert_quorum_timeout': 5000}) assert "Timeout while waiting for quorum" in quorum_timeout, "Query must fail." print("Wait 10 seconds and TTL merge have to be executed. But it won't delete data.") time.sleep(10) - assert TSV("1\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 0})) + assert TSV("1\t2011-01-01\n") == TSV( + zero.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency': 0})) print("Resume fetches for test_insert_quorum_with_ttl at first replica.") first.query("SYSTEM START FETCHES test_insert_quorum_with_ttl") @@ -279,7 +284,7 @@ def test_insert_quorum_with_ttl(started_cluster): first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") zero.query("INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", - settings={'insert_quorum_timeout' : 5000}) + settings={'insert_quorum_timeout': 5000}) print("Inserts should resume.") zero.query("INSERT INTO test_insert_quorum_with_ttl(a, d) VALUES(2, '2012-02-02')") @@ -288,7 +293,9 @@ def test_insert_quorum_with_ttl(started_cluster): first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") zero.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") - assert TSV("2\t2012-02-02\n") == TSV(first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 0})) - assert TSV("2\t2012-02-02\n") == TSV(first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency' : 1})) + assert TSV("2\t2012-02-02\n") == TSV( + first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency': 0})) + assert TSV("2\t2012-02-02\n") == TSV( + first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency': 1})) execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_ttl") diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index 4c97d127ad0..5d2a4acffe6 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -1,12 +1,15 @@ -import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV import os import re import time +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, TSV + cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', user_configs=["configs/users.d/assign_myquota.xml", "configs/users.d/drop_default_quota.xml", "configs/users.d/quota.xml"]) +instance = cluster.add_instance('instance', user_configs=["configs/users.d/assign_myquota.xml", + "configs/users.d/drop_default_quota.xml", + "configs/users.d/quota.xml"]) def check_system_quotas(canonical): @@ -15,35 +18,40 @@ def check_system_quotas(canonical): print("system_quotas: {},\ncanonical: {}".format(r, TSV(canonical_tsv))) assert r == canonical_tsv + def system_quota_limits(canonical): canonical_tsv = TSV(canonical) r = TSV(instance.query("SELECT * FROM system.quota_limits ORDER BY quota_name, duration")) print("system_quota_limits: {},\ncanonical: {}".format(r, TSV(canonical_tsv))) assert r == canonical_tsv + def system_quota_usage(canonical): canonical_tsv = TSV(canonical) - query = "SELECT quota_name, quota_key, duration, queries, max_queries, errors, max_errors, result_rows, max_result_rows,"\ - "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time "\ + query = "SELECT quota_name, quota_key, duration, queries, max_queries, errors, max_errors, result_rows, max_result_rows," \ + "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time " \ "FROM system.quota_usage ORDER BY duration" r = TSV(instance.query(query)) print("system_quota_usage: {},\ncanonical: {}".format(r, TSV(canonical_tsv))) assert r == canonical_tsv + def system_quotas_usage(canonical): canonical_tsv = TSV(canonical) - query = "SELECT quota_name, quota_key, is_current, duration, queries, max_queries, errors, max_errors, result_rows, max_result_rows, "\ - "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time "\ + query = "SELECT quota_name, quota_key, is_current, duration, queries, max_queries, errors, max_errors, result_rows, max_result_rows, " \ + "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time " \ "FROM system.quotas_usage ORDER BY quota_name, quota_key, duration" r = TSV(instance.query(query)) print("system_quotas_usage: {},\ncanonical: {}".format(r, TSV(canonical_tsv))) assert r == canonical_tsv -def copy_quota_xml(local_file_name, reload_immediately = True): + +def copy_quota_xml(local_file_name, reload_immediately=True): script_dir = os.path.dirname(os.path.realpath(__file__)) - instance.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/quota.xml') + instance.copy_file_to_container(os.path.join(script_dir, local_file_name), + '/etc/clickhouse-server/users.d/quota.xml') if reload_immediately: - instance.query("SYSTEM RELOAD CONFIG") + instance.query("SYSTEM RELOAD CONFIG") @pytest.fixture(scope="module", autouse=True) @@ -66,52 +74,63 @@ def reset_quotas_and_usage_info(): yield finally: instance.query("DROP QUOTA IF EXISTS qA, qB") - copy_quota_xml('simpliest.xml') # To reset usage info. + copy_quota_xml('simpliest.xml') # To reset usage info. copy_quota_xml('normal_limits.xml') def test_quota_from_users_xml(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) - system_quotas_usage([["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) + system_quotas_usage( + [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) instance.query("SELECT * from test_table") - system_quota_usage([["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]]) instance.query("SELECT COUNT() from test_table") - system_quota_usage([["myQuota", "default", 31556952, 2, 1000, 0, "\N", 51, "\N", 208, "\N", 50, 1000, 200, "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", 31556952, 2, 1000, 0, "\N", 51, "\N", 208, "\N", 50, 1000, 200, "\N", "\N"]]) def test_simpliest_quota(): # Simpliest quota doesn't even track usage. copy_quota_xml('simpliest.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[]", 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[]", 0, + "['default']", "[]"]]) system_quota_limits("") - system_quota_usage([["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) instance.query("SELECT * from test_table") - system_quota_usage([["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) def test_tracking_quota(): # Now we're tracking usage. copy_quota_xml('tracking.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) system_quota_usage([["myQuota", "default", 31556952, 0, "\N", 0, "\N", 0, "\N", 0, "\N", 0, "\N", 0, "\N", "\N"]]) instance.query("SELECT * from test_table") - system_quota_usage([["myQuota", "default", 31556952, 1, "\N", 0, "\N", 50, "\N", 200, "\N", 50, "\N", 200, "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", 31556952, 1, "\N", 0, "\N", 50, "\N", 200, "\N", 50, "\N", 200, "\N", "\N"]]) instance.query("SELECT COUNT() from test_table") - system_quota_usage([["myQuota", "default", 31556952, 2, "\N", 0, "\N", 51, "\N", 208, "\N", 50, "\N", 200, "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", 31556952, 2, "\N", 0, "\N", 51, "\N", 208, "\N", 50, "\N", 200, "\N", "\N"]]) def test_exceed_quota(): # Change quota, now the limits are tiny so we will exceed the quota. copy_quota_xml('tiny_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]]) system_quota_usage([["myQuota", "default", 31556952, 0, 1, 0, 1, 0, 1, 0, "\N", 0, 1, 0, "\N", "\N"]]) @@ -120,75 +139,94 @@ def test_exceed_quota(): # Change quota, now the limits are enough to execute queries. copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) system_quota_usage([["myQuota", "default", 31556952, 1, 1000, 1, "\N", 0, "\N", 0, "\N", 50, 1000, 0, "\N", "\N"]]) instance.query("SELECT * from test_table") - system_quota_usage([["myQuota", "default", 31556952, 2, 1000, 1, "\N", 50, "\N", 200, "\N", 100, 1000, 200, "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", 31556952, 2, 1000, 1, "\N", 50, "\N", 200, "\N", 100, 1000, 200, "\N", "\N"]]) def test_add_remove_interval(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) # Add interval. copy_quota_xml('two_intervals.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952,63113904]", 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], - ["myQuota", 63113904, 1, "\N", "\N", "\N", 30000, "\N", 20000, 120]]) - system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"], - ["myQuota", "default", 63113904, 0, "\N", 0, "\N", 0, "\N", 0, 30000, 0, "\N", 0, 20000, 120]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", + "[31556952,63113904]", 0, "['default']", "[]"]]) + system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], + ["myQuota", 63113904, 1, "\N", "\N", "\N", 30000, "\N", 20000, 120]]) + system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"], + ["myQuota", "default", 63113904, 0, "\N", 0, "\N", 0, "\N", 0, 30000, 0, "\N", 0, 20000, 120]]) instance.query("SELECT * from test_table") - system_quota_usage([["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"], - ["myQuota", "default", 63113904, 1, "\N", 0, "\N", 50, "\N", 200, 30000, 50, "\N", 200, 20000, 120]]) + system_quota_usage( + [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"], + ["myQuota", "default", 63113904, 1, "\N", 0, "\N", 50, "\N", 200, 30000, 50, "\N", 200, 20000, 120]]) # Remove interval. copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) - system_quota_usage([["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]]) instance.query("SELECT * from test_table") - system_quota_usage([["myQuota", "default", 31556952, 2, 1000, 0, "\N", 100, "\N", 400, "\N", 100, 1000, 400, "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", 31556952, 2, 1000, 0, "\N", 100, "\N", 400, "\N", 100, 1000, 400, "\N", "\N"]]) # Remove all intervals. copy_quota_xml('simpliest.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[]", 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[]", 0, + "['default']", "[]"]]) system_quota_limits("") - system_quota_usage([["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) instance.query("SELECT * from test_table") - system_quota_usage([["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) + system_quota_usage( + [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]]) # Add one interval back. copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) def test_add_remove_quota(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) - system_quotas_usage([["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) + system_quotas_usage( + [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) # Add quota. copy_quota_xml('two_quotas.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"], - ["myQuota2", "4590510c-4d13-bf21-ec8a-c2187b092e73", "users.xml", "['client_key','user_name']", "[3600,2629746]", 0, "[]", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], - ["myQuota2", 3600, 1, "\N", "\N", 4000, 400000, 4000, 400000, 60], - ["myQuota2", 2629746, 0, "\N", "\N", "\N", "\N", "\N", "\N", 1800]]) - system_quotas_usage([["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", + 0, "['default']", "[]"], + ["myQuota2", "4590510c-4d13-bf21-ec8a-c2187b092e73", "users.xml", "['client_key','user_name']", + "[3600,2629746]", 0, "[]", "[]"]]) + system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], + ["myQuota2", 3600, 1, "\N", "\N", 4000, 400000, 4000, 400000, 60], + ["myQuota2", 2629746, 0, "\N", "\N", "\N", "\N", "\N", "\N", 1800]]) + system_quotas_usage( + [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) # Drop quota. copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) - system_quotas_usage([["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) + system_quotas_usage( + [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) # Drop all quotas. copy_quota_xml('no_quotas.xml') @@ -198,32 +236,42 @@ def test_add_remove_quota(): # Add one quota back. copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) - system_quotas_usage([["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) + system_quotas_usage( + [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]]) def test_reload_users_xml_by_timer(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", 0, "['default']", "[]"]]) + check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", + 0, "['default']", "[]"]]) system_quota_limits([["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]]) - time.sleep(1) # The modification time of the 'quota.xml' file should be different, - # because config files are reload by timer only when the modification time is changed. + time.sleep(1) # The modification time of the 'quota.xml' file should be different, + # because config files are reload by timer only when the modification time is changed. copy_quota_xml('tiny_limits.xml', reload_immediately=False) - assert_eq_with_retry(instance, "SELECT * FROM system.quotas", [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", ['user_name'], "[31556952]", 0, "['default']", "[]"]]) - assert_eq_with_retry(instance, "SELECT * FROM system.quota_limits", [["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]]) + assert_eq_with_retry(instance, "SELECT * FROM system.quotas", [ + ["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", ['user_name'], "[31556952]", 0, "['default']", + "[]"]]) + assert_eq_with_retry(instance, "SELECT * FROM system.quota_limits", + [["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]]) def test_dcl_introspection(): assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" - assert instance.query("SHOW CREATE QUOTAS") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" - assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t0\\t1000\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t1000\\t0\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert instance.query( + "SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" + assert instance.query( + "SHOW CREATE QUOTAS") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" + assert re.match( + "myQuota\\tdefault\\t.*\\t31556952\\t0\\t1000\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t1000\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert re.match( + "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) expected_access = "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" assert expected_access in instance.query("SHOW ACCESS") @@ -231,20 +279,26 @@ def test_dcl_introspection(): # Add interval. copy_quota_xml('two_intervals.xml') assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000, FOR RANDOMIZED INTERVAL 2 year MAX result_bytes = 30000, read_bytes = 20000, execution_time = 120 TO default\n" - assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n" - "myQuota\\tdefault\\t.*\\t63113904\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t30000\\t0\\t\\\\N\\t0\\t20000\\t.*\\t120", - instance.query("SHOW QUOTA")) + assert instance.query( + "SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000, FOR RANDOMIZED INTERVAL 2 year MAX result_bytes = 30000, read_bytes = 20000, execution_time = 120 TO default\n" + assert re.match( + "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n" + "myQuota\\tdefault\\t.*\\t63113904\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t30000\\t0\\t\\\\N\\t0\\t20000\\t.*\\t120", + instance.query("SHOW QUOTA")) # Drop interval, add quota. copy_quota_xml('two_quotas.xml') assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" - assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" - assert instance.query("SHOW CREATE QUOTAS") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n"\ - "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" - assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert instance.query( + "SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" + assert instance.query( + "SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" + assert instance.query( + "SHOW CREATE QUOTAS") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" \ + "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" + assert re.match( + "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) # Drop all quotas. copy_quota_xml('no_quotas.xml') @@ -258,41 +312,54 @@ def test_dcl_management(): assert instance.query("SHOW QUOTA") == "" instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER") - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA FOR INTERVAL 5 quarter MAX queries = 123 TO default\n" - assert re.match("qA\\t\\t.*\\t39446190\\t0\\t123\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert instance.query( + "SHOW CREATE QUOTA qA") == "CREATE QUOTA qA FOR INTERVAL 5 quarter MAX queries = 123 TO default\n" + assert re.match( + "qA\\t\\t.*\\t39446190\\t0\\t123\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - assert re.match("qA\\t\\t.*\\t39446190\\t1\\t123\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert re.match( + "qA\\t\\t.*\\t39446190\\t1\\t123\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) - instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 321, MAX ERRORS 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME 0.5") - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default\n" - assert re.match("qA\\t\\t.*\\t1800\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t0.5\n" - "qA\\t\\t.*\\t39446190\\t1\\t321\\t0\\t10\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query( + "ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 321, MAX ERRORS 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME 0.5") + assert instance.query( + "SHOW CREATE QUOTA qA") == "CREATE QUOTA qA FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default\n" + assert re.match( + "qA\\t\\t.*\\t1800\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t0.5\n" + "qA\\t\\t.*\\t39446190\\t1\\t321\\t0\\t10\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - assert re.match("qA\\t\\t.*\\t1800\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t0.5\n" - "qA\\t\\t.*\\t39446190\\t2\\t321\\t0\\t10\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert re.match( + "qA\\t\\t.*\\t1800\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t0.5\n" + "qA\\t\\t.*\\t39446190\\t2\\t321\\t0\\t10\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) - instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS") - assert re.match("qA\\t\\t.*\\t42075936\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query( + "ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS") + assert re.match( + "qA\\t\\t.*\\t42075936\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - assert re.match("qA\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert re.match( + "qA\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("ALTER QUOTA qA RENAME TO qB") - assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB FOR RANDOMIZED INTERVAL 16 month TRACKING ONLY TO default\n" - assert re.match("qB\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert instance.query( + "SHOW CREATE QUOTA qB") == "CREATE QUOTA qB FOR RANDOMIZED INTERVAL 16 month TRACKING ONLY TO default\n" + assert re.match( + "qB\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - assert re.match("qB\\t\\t.*\\t42075936\\t2\\t\\\\N\\t0\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + assert re.match( + "qB\\t\\t.*\\t42075936\\t2\\t\\\\N\\t0\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("DROP QUOTA qB") assert instance.query("SHOW QUOTA") == "" diff --git a/tests/integration/test_random_inserts/test.py b/tests/integration/test_random_inserts/test.py index 4e3d8db7e53..f43581b6482 100644 --- a/tests/integration/test_random_inserts/test.py +++ b/tests/integration/test_random_inserts/test.py @@ -1,23 +1,25 @@ -import time import os -import threading import random -from contextlib import contextmanager +import threading +import time import pytest - +from helpers.client import CommandRequest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV -from helpers.client import CommandRequest - cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml" ], with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1}) -node2 = cluster.add_instance('node2', main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml" ], with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2}) +node1 = cluster.add_instance('node1', + main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], + with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1}) +node2 = cluster.add_instance('node2', + main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], + with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2}) nodes = [node1, node2] + @pytest.fixture(scope="module") def started_cluster(): try: @@ -31,7 +33,7 @@ def started_cluster(): def test_random_inserts(started_cluster): # Duration of the test, reduce it if don't want to wait - DURATION_SECONDS = 10# * 60 + DURATION_SECONDS = 10 # * 60 node1.query(""" CREATE TABLE simple ON CLUSTER test_cluster (date Date, i UInt32, s String) @@ -39,9 +41,9 @@ def test_random_inserts(started_cluster): with PartitionManager() as pm_random_drops: for sacrifice in nodes: - pass # This test doesn't work with partition problems still - #pm_random_drops._add_rule({'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) - #pm_random_drops._add_rule({'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) + pass # This test doesn't work with partition problems still + # pm_random_drops._add_rule({'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) + # pm_random_drops._add_rule({'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) min_timestamp = int(time.time()) max_timestamp = min_timestamp + DURATION_SECONDS @@ -50,18 +52,21 @@ def test_random_inserts(started_cluster): bash_script = os.path.join(os.path.dirname(__file__), "test.sh") inserters = [] for node in nodes: - cmd = ['/bin/bash', bash_script, node.ip_address, str(min_timestamp), str(max_timestamp), str(cluster.get_client_cmd())] + cmd = ['/bin/bash', bash_script, node.ip_address, str(min_timestamp), str(max_timestamp), + str(cluster.get_client_cmd())] inserters.append(CommandRequest(cmd, timeout=DURATION_SECONDS * 2, stdin='')) print node.name, node.ip_address for inserter in inserters: inserter.get_answer() - answer="{}\t{}\t{}\t{}\n".format(num_timestamps, num_timestamps, min_timestamp, max_timestamp) + answer = "{}\t{}\t{}\t{}\n".format(num_timestamps, num_timestamps, min_timestamp, max_timestamp) for node in nodes: - res = node.query_with_retry("SELECT count(), uniqExact(i), min(i), max(i) FROM simple", check_callback=lambda res: TSV(res) == TSV(answer)) - assert TSV(res) == TSV(answer), node.name + " : " + node.query("SELECT groupArray(_part), i, count() AS c FROM simple GROUP BY i ORDER BY c DESC LIMIT 1") + res = node.query_with_retry("SELECT count(), uniqExact(i), min(i), max(i) FROM simple", + check_callback=lambda res: TSV(res) == TSV(answer)) + assert TSV(res) == TSV(answer), node.name + " : " + node.query( + "SELECT groupArray(_part), i, count() AS c FROM simple GROUP BY i ORDER BY c DESC LIMIT 1") node1.query("""DROP TABLE simple ON CLUSTER test_cluster""") @@ -114,13 +119,14 @@ def test_insert_multithreaded(started_cluster): node.query("DROP TABLE IF EXISTS repl_test") for node in nodes: - node.query("CREATE TABLE repl_test(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/repl_test', '{replica}') ORDER BY x PARTITION BY toYYYYMM(d)") + node.query( + "CREATE TABLE repl_test(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/repl_test', '{replica}') ORDER BY x PARTITION BY toYYYYMM(d)") runner = Runner() threads = [] for thread_num in range(5): - threads.append(threading.Thread(target=runner.do_insert, args=(thread_num, ))) + threads.append(threading.Thread(target=runner.do_insert, args=(thread_num,))) for t in threads: t.start() @@ -135,7 +141,7 @@ def test_insert_multithreaded(started_cluster): assert runner.total_inserted > 0 all_replicated = False - for i in range(100): # wait for replication 50 seconds max + for i in range(100): # wait for replication 50 seconds max time.sleep(0.5) def get_delay(node): diff --git a/tests/integration/test_range_hashed_dictionary_types/test.py b/tests/integration/test_range_hashed_dictionary_types/test.py index 24d4d5d4094..198e2e27db8 100644 --- a/tests/integration/test_range_hashed_dictionary_types/test.py +++ b/tests/integration/test_range_hashed_dictionary_types/test.py @@ -1,7 +1,7 @@ import pytest - from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1') @@ -39,4 +39,5 @@ def test_range_hashed_dict(started_cluster): """) node1.query("SYSTEM RELOAD DICTIONARY default.rates") - assert node1.query("SELECT dictGetString('default.rates', 'currency', toUInt64(4990954156238030839), toDateTime('2019-10-01 00:00:00'))") == "RU\n" + assert node1.query( + "SELECT dictGetString('default.rates', 'currency', toUInt64(4990954156238030839), toDateTime('2019-10-01 00:00:00'))") == "RU\n" diff --git a/tests/integration/test_read_temporary_tables_on_failure/test.py b/tests/integration/test_read_temporary_tables_on_failure/test.py index ad1a41b8979..f7df52f67e9 100644 --- a/tests/integration/test_read_temporary_tables_on_failure/test.py +++ b/tests/integration/test_read_temporary_tables_on_failure/test.py @@ -1,13 +1,12 @@ import pytest -import time - -from helpers.cluster import ClickHouseCluster from helpers.client import QueryTimeoutExceedException, QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node') + @pytest.fixture(scope="module") def start_cluster(): try: @@ -17,6 +16,7 @@ def start_cluster(): finally: cluster.shutdown() + def test_different_versions(start_cluster): with pytest.raises(QueryTimeoutExceedException): node.query("SELECT sleep(3)", timeout=1) diff --git a/tests/integration/test_recompression_ttl/test.py b/tests/integration/test_recompression_ttl/test.py index 4707a5c41ad..9a96151d04a 100644 --- a/tests/integration/test_recompression_ttl/test.py +++ b/tests/integration/test_recompression_ttl/test.py @@ -1,15 +1,13 @@ import time + import pytest - -import helpers.client as client from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/background_pool_config.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/background_pool_config.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -47,12 +45,15 @@ def optimize_final_table_until_success(node, table_name, retries=40): def wait_part_and_get_compression_codec(node, table, part_name, retries=40): if wait_part_in_parts(node, table, part_name, retries): - return node.query("SELECT default_compression_codec FROM system.parts where name = '{}' and table = '{}'".format(part_name, table)).strip() + return node.query( + "SELECT default_compression_codec FROM system.parts where name = '{}' and table = '{}'".format(part_name, + table)).strip() return None def test_recompression_simple(started_cluster): - node1.query("CREATE TABLE table_for_recompression (d DateTime, key UInt64, data String) ENGINE MergeTree() ORDER BY tuple() TTL d + INTERVAL 10 SECOND RECOMPRESS CODEC(ZSTD(10)) SETTINGS merge_with_recompression_ttl_timeout = 0") + node1.query( + "CREATE TABLE table_for_recompression (d DateTime, key UInt64, data String) ENGINE MergeTree() ORDER BY tuple() TTL d + INTERVAL 10 SECOND RECOMPRESS CODEC(ZSTD(10)) SETTINGS merge_with_recompression_ttl_timeout = 0") node1.query("INSERT INTO table_for_recompression VALUES (now(), 1, '1')") assert node1.query("SELECT default_compression_codec FROM system.parts where name = 'all_1_1_0'") == "LZ4\n" @@ -106,20 +107,24 @@ def test_recompression_multiple_ttls(started_cluster): assert node2.query("SELECT default_compression_codec FROM system.parts where name = 'all_1_1_4'") == "ZSTD(12)\n" - assert node2.query("SELECT recompression_ttl_info.expression FROM system.parts where name = 'all_1_1_4'") == "['plus(d, toIntervalSecond(10))','plus(d, toIntervalSecond(15))','plus(d, toIntervalSecond(5))']\n" + assert node2.query( + "SELECT recompression_ttl_info.expression FROM system.parts where name = 'all_1_1_4'") == "['plus(d, toIntervalSecond(10))','plus(d, toIntervalSecond(15))','plus(d, toIntervalSecond(5))']\n" def test_recompression_replicated(started_cluster): for i, node in enumerate([node1, node2]): node.query("CREATE TABLE recompression_replicated (d DateTime, key UInt64, data String) \ ENGINE ReplicatedMergeTree('/test/rr', '{}') ORDER BY tuple() \ - TTL d + INTERVAL 10 SECOND RECOMPRESS CODEC(ZSTD(13)) SETTINGS merge_with_recompression_ttl_timeout = 0".format(i + 1)) + TTL d + INTERVAL 10 SECOND RECOMPRESS CODEC(ZSTD(13)) SETTINGS merge_with_recompression_ttl_timeout = 0".format( + i + 1)) node1.query("INSERT INTO recompression_replicated VALUES (now(), 1, '1')") node2.query("SYSTEM SYNC REPLICA recompression_replicated", timeout=5) - assert node1.query("SELECT default_compression_codec FROM system.parts where name = 'all_0_0_0' and table = 'recompression_replicated'") == "LZ4\n" - assert node2.query("SELECT default_compression_codec FROM system.parts where name = 'all_0_0_0' and table = 'recompression_replicated'") == "LZ4\n" + assert node1.query( + "SELECT default_compression_codec FROM system.parts where name = 'all_0_0_0' and table = 'recompression_replicated'") == "LZ4\n" + assert node2.query( + "SELECT default_compression_codec FROM system.parts where name = 'all_0_0_0' and table = 'recompression_replicated'") == "LZ4\n" codec1 = wait_part_and_get_compression_codec(node1, "recompression_replicated", "all_0_0_1") if not codec1: diff --git a/tests/integration/test_recovery_replica/test.py b/tests/integration/test_recovery_replica/test.py index d2a53144d98..74e773cfb83 100644 --- a/tests/integration/test_recovery_replica/test.py +++ b/tests/integration/test_recovery_replica/test.py @@ -1,23 +1,26 @@ import time -import pytest +import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry + def fill_nodes(nodes, shard): for node in nodes: node.query( - ''' - CREATE DATABASE test; + ''' + CREATE DATABASE test; + + CREATE TABLE test_table(date Date, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; + '''.format(shard=shard, replica=node.name)) - CREATE TABLE test_table(date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -33,6 +36,7 @@ def start_cluster(): finally: cluster.shutdown() + def test_recovery(start_cluster): node1.query("INSERT INTO test_table VALUES (1, 1)") time.sleep(1) @@ -41,6 +45,7 @@ def test_recovery(start_cluster): for i in range(100): node1.query("INSERT INTO test_table VALUES (1, {})".format(i)) - node2.query_with_retry("ATTACH TABLE test_table", check_callback=lambda x: len(node2.query("select * from test_table")) > 0) + node2.query_with_retry("ATTACH TABLE test_table", + check_callback=lambda x: len(node2.query("select * from test_table")) > 0) assert_eq_with_retry(node2, "SELECT count(*) FROM test_table", node1.query("SELECT count(*) FROM test_table")) diff --git a/tests/integration/test_redirect_url_storage/test.py b/tests/integration/test_redirect_url_storage/test.py index cf64e84b96b..f93548af0db 100644 --- a/tests/integration/test_redirect_url_storage/test.py +++ b/tests/integration/test_redirect_url_storage/test.py @@ -1,11 +1,11 @@ import pytest -from helpers.hdfs_api import HDFSApi - from helpers.cluster import ClickHouseCluster +from helpers.hdfs_api import HDFSApi cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=False, with_hdfs=True) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -15,25 +15,30 @@ def started_cluster(): finally: cluster.shutdown() + def test_url_without_redirect(started_cluster): hdfs_api = HDFSApi("root") hdfs_api.write_data("/simple_storage", "1\tMark\t72.53\n") assert hdfs_api.read_data("/simple_storage") == "1\tMark\t72.53\n" # access datanode port directly - node1.query("create table WebHDFSStorage (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") + node1.query( + "create table WebHDFSStorage (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") assert node1.query("select * from WebHDFSStorage") == "1\tMark\t72.53\n" + def test_url_with_redirect_not_allowed(started_cluster): hdfs_api = HDFSApi("root") hdfs_api.write_data("/simple_storage", "1\tMark\t72.53\n") assert hdfs_api.read_data("/simple_storage") == "1\tMark\t72.53\n" # access proxy port without allowing redirects - node1.query("create table WebHDFSStorageWithoutRedirect (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") + node1.query( + "create table WebHDFSStorageWithoutRedirect (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") with pytest.raises(Exception): assert node1.query("select * from WebHDFSStorageWithoutRedirect") == "1\tMark\t72.53\n" + def test_url_with_redirect_allowed(started_cluster): hdfs_api = HDFSApi("root") hdfs_api.write_data("/simple_storage", "1\tMark\t72.53\n") @@ -41,5 +46,6 @@ def test_url_with_redirect_allowed(started_cluster): # access proxy port with allowing redirects # http://localhost:50070/webhdfs/v1/b?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0 - node1.query("create table WebHDFSStorageWithRedirect (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") + node1.query( + "create table WebHDFSStorageWithRedirect (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") assert node1.query("SET max_http_get_redirects=1; select * from WebHDFSStorageWithRedirect") == "1\tMark\t72.53\n" diff --git a/tests/integration/test_relative_filepath/test.py b/tests/integration/test_relative_filepath/test.py index a8e2341a3cd..45c969b86f5 100644 --- a/tests/integration/test_relative_filepath/test.py +++ b/tests/integration/test_relative_filepath/test.py @@ -6,6 +6,7 @@ cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=['configs/config.xml']) path_to_userfiles_from_defaut_config = "user_files" + @pytest.fixture(scope="module") def start_cluster(): try: @@ -14,6 +15,7 @@ def start_cluster(): finally: cluster.shutdown() + def test_filepath(start_cluster): # 2 rows data some_data = "Test\t111.222\nData\t333.444" diff --git a/tests/integration/test_reload_max_table_size_to_drop/test.py b/tests/integration/test_reload_max_table_size_to_drop/test.py index 9d0bc244521..d6bdcc83945 100644 --- a/tests/integration/test_reload_max_table_size_to_drop/test.py +++ b/tests/integration/test_reload_max_table_size_to_drop/test.py @@ -1,11 +1,9 @@ - -import time -import pytest import os +import time +import pytest from helpers.cluster import ClickHouseCluster - cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=["configs/max_table_size_to_drop.xml"]) diff --git a/tests/integration/test_reloading_storage_configuration/test.py b/tests/integration/test_reloading_storage_configuration/test.py index a30d4029d7c..0c3139c7fdd 100644 --- a/tests/integration/test_reloading_storage_configuration/test.py +++ b/tests/integration/test_reloading_storage_configuration/test.py @@ -5,27 +5,27 @@ import shutil import time import xml.etree.ElementTree as ET -import pytest - import helpers.client import helpers.cluster - +import pytest cluster = helpers.cluster.ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml'], - with_zookeeper=True, - stay_alive=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/jbod3:size=40M', '/jbod4:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 1} ) + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + stay_alive=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/jbod3:size=40M', '/jbod4:size=40M', + '/external:size=200M'], + macros={"shard": 0, "replica": 1}) node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml'], - with_zookeeper=True, - stay_alive=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/jbod3:size=40M', '/jbod4:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 2} ) + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + stay_alive=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/jbod3:size=40M', '/jbod4:size=40M', + '/external:size=200M'], + macros={"shard": 0, "replica": 2}) def get_log(node): @@ -43,7 +43,8 @@ def started_cluster(): def start_over(): - shutil.copy(os.path.join(os.path.dirname(__file__), "configs/config.d/storage_configuration.xml"), os.path.join(node1.config_d_dir, "storage_configuration.xml")) + shutil.copy(os.path.join(os.path.dirname(__file__), "configs/config.d/storage_configuration.xml"), + os.path.join(node1.config_d_dir, "storage_configuration.xml")) for node in (node1, node2): separate_configuration_path = os.path.join(node.config_d_dir, "separate_configuration.xml") @@ -62,7 +63,8 @@ def add_disk(node, name, path, separate_file=False): else: tree = ET.parse(os.path.join(node.config_d_dir, "storage_configuration.xml")) except: - tree = ET.ElementTree(ET.fromstring('')) + tree = ET.ElementTree( + ET.fromstring('')) root = tree.getroot() new_disk = ET.Element(name) new_path = ET.Element("path") @@ -178,8 +180,10 @@ def test_add_policy(started_cluster): disks = set(node1.query("SELECT name FROM system.disks").splitlines()) assert "cool_policy" in set(node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) - assert {"volume1"} == set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) - assert {"['jbod3','jbod4']"} == set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + assert {"volume1"} == set(node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + assert {"['jbod3','jbod4']"} == set( + node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) finally: try: @@ -218,7 +222,8 @@ def test_new_policy_works(started_cluster): start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") - add_policy(node1, "cool_policy", collections.OrderedDict([("volume1", ["jbod3"]), ("main", ["jbod1", "jbod2"]), ("external", ["external"])])) + add_policy(node1, "cool_policy", collections.OrderedDict( + [("volume1", ["jbod3"]), ("main", ["jbod1", "jbod2"]), ("external", ["external"])])) node1.query("SYSTEM RELOAD CONFIG") node1.query(""" @@ -228,7 +233,8 @@ def test_new_policy_works(started_cluster): node1.query(""" INSERT INTO TABLE {name} VALUES (1) """.format(name=name)) - assert {"jbod3"} == set(node1.query("SELECT disk_name FROM system.parts WHERE active = 1 AND table = '{name}'".format(name=name)).splitlines()) + assert {"jbod3"} == set(node1.query( + "SELECT disk_name FROM system.parts WHERE active = 1 AND table = '{name}'".format(name=name)).splitlines()) finally: try: @@ -263,8 +269,10 @@ def test_add_volume_to_policy(started_cluster): add_policy(node1, "cool_policy", collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])])) node1.query("SYSTEM RELOAD CONFIG") - volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) - disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + volumes = set(node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + disks_sets = set( + node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) assert {"volume1", "volume2"} == volumes assert {"['jbod3']", "['jbod4']"} == disks_sets @@ -298,11 +306,13 @@ def test_add_disk_to_policy(started_cluster): start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") - add_policy(node1, "cool_policy", {"volume1": ["jbod3","jbod4"]}) + add_policy(node1, "cool_policy", {"volume1": ["jbod3", "jbod4"]}) node1.query("SYSTEM RELOAD CONFIG") - volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) - disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + volumes = set(node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + disks_sets = set( + node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) assert {"volume1"} == volumes assert {"['jbod3','jbod4']"} == disks_sets @@ -365,14 +375,16 @@ def test_remove_policy(started_cluster): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - assert "remove_policy_cool_policy" in set(node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) + assert "remove_policy_cool_policy" in set( + node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") node1.query("SYSTEM RELOAD CONFIG") - assert "remove_policy_cool_policy" in set(node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) + assert "remove_policy_cool_policy" in set( + node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) assert re.search("Error.*remove_policy_cool_policy", get_log(node1)) finally: @@ -390,7 +402,8 @@ def test_remove_volume_from_policy(started_cluster): start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") - add_policy(node1, "test_remove_volume_from_policy_cool_policy", collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])])) + add_policy(node1, "test_remove_volume_from_policy_cool_policy", + collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])])) node1.restart_clickhouse(kill=True) time.sleep(2) @@ -402,8 +415,10 @@ def test_remove_volume_from_policy(started_cluster): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) - disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + volumes = set(node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + disks_sets = set(node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) assert {"volume1", "volume2"} == volumes assert {"['jbod3']", "['jbod4']"} == disks_sets @@ -413,8 +428,10 @@ def test_remove_volume_from_policy(started_cluster): add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) node1.query("SYSTEM RELOAD CONFIG") - volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) - disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + volumes = set(node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + disks_sets = set(node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) assert {"volume1", "volume2"} == volumes assert {"['jbod3']", "['jbod4']"} == disks_sets assert re.search("Error.*test_remove_volume_from_policy_cool_policy", get_log(node1)) @@ -434,7 +451,7 @@ def test_remove_disk_from_policy(started_cluster): start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") - add_policy(node1, "test_remove_disk_from_policy_cool_policy", {"volume1": ["jbod3","jbod4"]}) + add_policy(node1, "test_remove_disk_from_policy_cool_policy", {"volume1": ["jbod3", "jbod4"]}) node1.restart_clickhouse(kill=True) time.sleep(2) @@ -446,8 +463,10 @@ def test_remove_disk_from_policy(started_cluster): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) - disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + volumes = set(node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + disks_sets = set(node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) assert {"volume1"} == volumes assert {"['jbod3','jbod4']"} == disks_sets @@ -457,8 +476,10 @@ def test_remove_disk_from_policy(started_cluster): add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) node1.query("SYSTEM RELOAD CONFIG") - volumes = set(node1.query("SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) - disks_sets = set(node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + volumes = set(node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + disks_sets = set(node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) assert {"volume1"} == volumes assert {"['jbod3','jbod4']"} == disks_sets assert re.search("Error.*test_remove_disk_from_policy_cool_policy", get_log(node1)) diff --git a/tests/integration/test_remote_prewhere/test.py b/tests/integration/test_remote_prewhere/test.py index 5cf3836213e..07d05797223 100644 --- a/tests/integration/test_remote_prewhere/test.py +++ b/tests/integration/test_remote_prewhere/test.py @@ -1,14 +1,12 @@ -import time import pytest from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException, QueryTimeoutExceedException - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/log_conf.xml']) node2 = cluster.add_instance('node2', main_configs=['configs/log_conf.xml']) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -32,4 +30,5 @@ def start_cluster(): def test_remote(start_cluster): - assert node1.query("SELECT 1 FROM remote('node{1,2}', default.test_table) WHERE (APIKey = 137715) AND (CustomAttributeId IN (45, 66)) AND (ProfileIDHash != 0) LIMIT 1") == "" + assert node1.query( + "SELECT 1 FROM remote('node{1,2}', default.test_table) WHERE (APIKey = 137715) AND (CustomAttributeId IN (45, 66)) AND (ProfileIDHash != 0) LIMIT 1") == "" diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 9a108583347..51921c3385c 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -1,14 +1,12 @@ from __future__ import print_function -import time import random -import pytest - +import time from multiprocessing.dummy import Pool -from helpers.cluster import ClickHouseCluster + +import pytest from helpers.client import QueryRuntimeException -from helpers.network import PartitionManager -from helpers.test_tools import TSV +from helpers.cluster import ClickHouseCluster node_options = dict( with_zookeeper=True, @@ -83,7 +81,7 @@ def create_table(nodes, table_name, with_storage_policy=False, with_time_column= def create_distributed_table(node, table_name): - sql = """ + sql = """ CREATE TABLE %(table_name)s_replicated ON CLUSTER test_cluster ( num UInt32, @@ -92,12 +90,12 @@ def create_distributed_table(node, table_name): ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/%(table_name)s_replicated', '{replica}') ORDER BY num PARTITION BY num %% 100; """ % dict(table_name=table_name) - node.query(sql) - sql = """ + node.query(sql) + sql = """ CREATE TABLE %(table_name)s ON CLUSTER test_cluster AS %(table_name)s_replicated ENGINE = Distributed(test_cluster, default, %(table_name)s_replicated, rand()) """ % dict(table_name=table_name) - node.query(sql) + node.query(sql) def drop_distributed_table(node, table_name): @@ -107,7 +105,7 @@ def drop_distributed_table(node, table_name): def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_exception=False, - slow=False, with_many_parts=False, offset=0, with_time_column=False): + slow=False, with_many_parts=False, offset=0, with_time_column=False): if col_names is None: col_names = ['num', 'num2'] for i in range(iterations): @@ -118,28 +116,32 @@ def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_ex if with_time_column: query.append( "INSERT INTO {table_name} ({col0}, {col1}, time) SELECT number AS {col0}, number + 1 AS {col1}, now() + 10 AS time FROM numbers_mt({chunk})" - .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) elif slow: query.append( "INSERT INTO {table_name} ({col0}, {col1}) SELECT number + sleepEachRow(0.001) AS {col0}, number + 1 AS {col1} FROM numbers_mt({chunk})" - .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) else: query.append( "INSERT INTO {table_name} ({col0},{col1}) SELECT number + {offset} AS {col0}, number + 1 + {offset} AS {col1} FROM numbers_mt({chunk})" - .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1], offset=str(offset))) + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1], + offset=str(offset))) node.query(";\n".join(query)) except QueryRuntimeException as ex: if not ignore_exception: raise -def select(node, table_name, col_name="num", expected_result=None, iterations=1, ignore_exception=False, slow=False, poll=None): +def select(node, table_name, col_name="num", expected_result=None, iterations=1, ignore_exception=False, slow=False, + poll=None): for i in range(iterations): start_time = time.time() while True: try: if slow: - r = node.query("SELECT count() FROM (SELECT num2, sleepEachRow(0.5) FROM {} WHERE {} % 1000 > 0)".format(table_name, col_name)) + r = node.query( + "SELECT count() FROM (SELECT num2, sleepEachRow(0.5) FROM {} WHERE {} % 1000 > 0)".format( + table_name, col_name)) else: r = node.query("SELECT count() FROM {} WHERE {} % 1000 > 0".format(table_name, col_name)) if expected_result: @@ -180,7 +182,7 @@ def alter_move(node, table_name, iterations=1, ignore_exception=False): move_volume = 'external' try: node.query("ALTER TABLE {table_name} MOVE PARTITION '{move_part}' TO VOLUME '{move_volume}'" - .format(table_name=table_name, move_part=move_part, move_volume=move_volume)) + .format(table_name=table_name, move_part=move_part, move_volume=move_volume)) except QueryRuntimeException as ex: if not ignore_exception: raise @@ -306,7 +308,7 @@ def test_rename_with_parallel_merges(started_cluster): try: create_table(nodes, table_name) for i in range(20): - insert(node1, table_name, 100, ["num","num2"], 1, False, False, True, offset=i*100) + insert(node1, table_name, 100, ["num", "num2"], 1, False, False, True, offset=i * 100) def merge_parts(node, table_name, iterations=1): for i in range(iterations): @@ -347,7 +349,7 @@ def test_rename_with_parallel_slow_insert(started_cluster): p = Pool(15) tasks = [] tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, True))) - tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, True, True))) # deduplicated + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, True, True))) # deduplicated time.sleep(0.5) tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2"))) @@ -451,7 +453,9 @@ def test_rename_with_parallel_ttl_move(started_cluster): task.get(timeout=240) # check some parts got moved - assert "external" in set(node1.query("SELECT disk_name FROM system.parts WHERE table == '{}' AND active=1 ORDER BY modification_time".format(table_name)).strip().splitlines()) + assert "external" in set(node1.query( + "SELECT disk_name FROM system.parts WHERE table == '{}' AND active=1 ORDER BY modification_time".format( + table_name)).strip().splitlines()) # rename column back to original rename_column(node1, table_name, "foo2", "num2", 1, True) @@ -507,7 +511,7 @@ def test_rename_distributed(started_cluster): rename_column_on_cluster(node1, table_name, 'num2', 'foo2') rename_column_on_cluster(node1, '%s_replicated' % table_name, 'num2', 'foo2') - insert(node1, table_name, 1000, col_names=['num','foo2']) + insert(node1, table_name, 1000, col_names=['num', 'foo2']) select(node1, table_name, "foo2", '1998\n', poll=30) finally: @@ -524,11 +528,14 @@ def test_rename_distributed_parallel_insert_and_select(started_cluster): tasks = [] for i in range(1): tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'num2', 'foo2', 3, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'num2', 'foo2', 3, True))) + tasks.append( + p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'num2', 'foo2', 3, True))) tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo2', 'foo3', 3, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo2', 'foo3', 3, True))) + tasks.append( + p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo2', 'foo3', 3, True))) tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo3', 'num2', 3, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo3', 'num2', 3, True))) + tasks.append( + p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo3', 'num2', 3, True))) tasks.append(p.apply_async(insert, (node1, table_name, 10, ["num", "foo3"], 5, True))) tasks.append(p.apply_async(insert, (node2, table_name, 10, ["num", "num2"], 5, True))) tasks.append(p.apply_async(insert, (node3, table_name, 10, ["num", "foo2"], 5, True))) @@ -543,7 +550,7 @@ def test_rename_distributed_parallel_insert_and_select(started_cluster): rename_column_on_cluster(node1, table_name, 'foo3', 'num2', 1, True) rename_column_on_cluster(node1, '%s_replicated' % table_name, 'foo3', 'num2', 1, True) - insert(node1, table_name, 1000, col_names=['num','num2']) + insert(node1, table_name, 1000, col_names=['num', 'num2']) select(node1, table_name, "num2") select(node2, table_name, "num2") select(node3, table_name, "num2") diff --git a/tests/integration/test_replace_partition/test.py b/tests/integration/test_replace_partition/test.py index c771ed4c874..06e7f4be82b 100644 --- a/tests/integration/test_replace_partition/test.py +++ b/tests/integration/test_replace_partition/test.py @@ -1,6 +1,4 @@ import pytest -import time -import sys from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager @@ -9,26 +7,28 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) + def _fill_nodes(nodes, shard): for node in nodes: node.query( - ''' - CREATE DATABASE test; - - CREATE TABLE real_table(date Date, id UInt32, dummy UInt32) - ENGINE = MergeTree(date, id, 8192); - - CREATE TABLE other_table(date Date, id UInt32, dummy UInt32) - ENGINE = MergeTree(date, id, 8192); - - CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); - '''.format(shard=shard, replica=node.name)) + ''' + CREATE DATABASE test; + + CREATE TABLE real_table(date Date, id UInt32, dummy UInt32) + ENGINE = MergeTree(date, id, 8192); + + CREATE TABLE other_table(date Date, id UInt32, dummy UInt32) + ENGINE = MergeTree(date, id, 8192); + + CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + '''.format(shard=shard, replica=node.name)) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def normal_work(): try: @@ -41,6 +41,7 @@ def normal_work(): finally: cluster.shutdown() + def test_normal_work(normal_work): node1.query("insert into test_table values ('2017-06-16', 111, 0)") node1.query("insert into real_table values ('2017-06-16', 222, 0)") @@ -54,9 +55,11 @@ def test_normal_work(normal_work): assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", '222') assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", '222') + node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def drop_failover(): try: @@ -69,6 +72,7 @@ def drop_failover(): finally: cluster.shutdown() + def test_drop_failover(drop_failover): node3.query("insert into test_table values ('2017-06-16', 111, 0)") node3.query("insert into real_table values ('2017-06-16', 222, 0)") @@ -77,7 +81,6 @@ def test_drop_failover(drop_failover): assert_eq_with_retry(node3, "SELECT id FROM real_table order by id", '222') assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111') - with PartitionManager() as pm: # Hinder replication between replicas pm.partition_instances(node3, node4, port=9009) @@ -91,18 +94,22 @@ def test_drop_failover(drop_failover): # Network interrupted -- replace is not ok, but it's ok assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111') - #Drop partition on source node + # Drop partition on source node node3.query("ALTER TABLE test_table DROP PARTITION 201706") # connection restored - node4.query_with_retry("select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", check_callback=lambda x: 'Not found part' not in x, sleep_time=1) - assert 'Not found part' not in node4.query("select last_exception from system.replication_queue where type = 'REPLACE_RANGE'") + node4.query_with_retry("select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", + check_callback=lambda x: 'Not found part' not in x, sleep_time=1) + assert 'Not found part' not in node4.query( + "select last_exception from system.replication_queue where type = 'REPLACE_RANGE'") assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '') + node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def replace_after_replace_failover(): try: @@ -115,6 +122,7 @@ def replace_after_replace_failover(): finally: cluster.shutdown() + def test_replace_after_replace_failover(replace_after_replace_failover): node5.query("insert into test_table values ('2017-06-16', 111, 0)") node5.query("insert into real_table values ('2017-06-16', 222, 0)") @@ -125,7 +133,6 @@ def test_replace_after_replace_failover(replace_after_replace_failover): assert_eq_with_retry(node5, "SELECT id FROM other_table order by id", '333') assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '111') - with PartitionManager() as pm: # Hinder replication between replicas pm.partition_instances(node5, node6, port=9009) @@ -139,11 +146,13 @@ def test_replace_after_replace_failover(replace_after_replace_failover): # Network interrupted -- replace is not ok, but it's ok assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '111') - #Replace partition on source node + # Replace partition on source node node5.query("ALTER TABLE test_table REPLACE PARTITION 201706 FROM other_table") assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", '333') - node6.query_with_retry("select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", check_callback=lambda x: 'Not found part' not in x, sleep_time=1) - assert 'Not found part' not in node6.query("select last_exception from system.replication_queue where type = 'REPLACE_RANGE'") + node6.query_with_retry("select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", + check_callback=lambda x: 'Not found part' not in x, sleep_time=1) + assert 'Not found part' not in node6.query( + "select last_exception from system.replication_queue where type = 'REPLACE_RANGE'") assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '333') diff --git a/tests/integration/test_replica_can_become_leader/test.py b/tests/integration/test_replica_can_become_leader/test.py index 2ef518d5570..fae4fa28226 100644 --- a/tests/integration/test_replica_can_become_leader/test.py +++ b/tests/integration/test_replica_can_become_leader/test.py @@ -1,13 +1,13 @@ import pytest - -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/notleader.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/notleaderignorecase.xml'], with_zookeeper=True) node3 = cluster.add_instance('node3', with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): try: diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index a77a69b842b..de6f5e9f868 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -1,7 +1,6 @@ import logging import random import string -import time import pytest from helpers.cluster import ClickHouseCluster @@ -15,9 +14,12 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, with_minio=True, with_zookeeper=True) - cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, with_zookeeper=True) - cluster.add_instance("node3", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, with_zookeeper=True) + cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, + with_minio=True, with_zookeeper=True) + cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, + with_zookeeper=True) + cluster.add_instance("node3", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, + with_zookeeper=True) logging.info("Starting cluster...") cluster.start() @@ -39,7 +41,7 @@ def random_string(length): def generate_values(date_str, count, sign=1): - data = [[date_str, sign*(i + 1), random_string(10)] for i in range(count)] + data = [[date_str, sign * (i + 1), random_string(10)] for i in range(count)] data.sort(key=lambda tup: tup[1]) return ",".join(["('{}',{},'{}')".format(x, y, z) for x, y, z in data]) @@ -87,7 +89,9 @@ def test_insert_select_replicated(cluster): for node_idx in range(1, 4): node = cluster.instances["node" + str(node_idx)] - assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values", settings={"select_sequential_consistency": 1}) == all_values + assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values", + settings={"select_sequential_consistency": 1}) == all_values minio = cluster.minio_client - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 3 * (FILES_OVERHEAD + FILES_OVERHEAD_PER_PART * 3) + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 3 * ( + FILES_OVERHEAD + FILES_OVERHEAD_PER_PART * 3) diff --git a/tests/integration/test_replicated_mutations/test.py b/tests/integration/test_replicated_mutations/test.py index 0347ba4782c..cf3cef3a9e6 100644 --- a/tests/integration/test_replicated_mutations/test.py +++ b/tests/integration/test_replicated_mutations/test.py @@ -1,26 +1,28 @@ -import time -import threading import random +import threading +import time from collections import Counter import pytest - from helpers.cluster import ClickHouseCluster - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', macros={'cluster': 'test1'}, with_zookeeper=True) # Check, that limits on max part size for merges doesn`t affect mutations -node2 = cluster.add_instance('node2', macros={'cluster': 'test1'}, main_configs=["configs/merge_tree.xml"], with_zookeeper=True) +node2 = cluster.add_instance('node2', macros={'cluster': 'test1'}, main_configs=["configs/merge_tree.xml"], + with_zookeeper=True) -node3 = cluster.add_instance('node3', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], with_zookeeper=True) -node4 = cluster.add_instance('node4', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], with_zookeeper=True) +node3 = cluster.add_instance('node3', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], + with_zookeeper=True) +node4 = cluster.add_instance('node4', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], + with_zookeeper=True) node5 = cluster.add_instance('node5', macros={'cluster': 'test3'}, main_configs=["configs/merge_tree_max_parts.xml"]) all_nodes = [node1, node2, node3, node4, node5] + @pytest.fixture(scope="module") def started_cluster(): try: @@ -30,9 +32,11 @@ def started_cluster(): node.query("DROP TABLE IF EXISTS test_mutations") for node in [node1, node2, node3, node4]: - node.query("CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/test/test_mutations', '{instance}') ORDER BY x PARTITION BY toYYYYMM(d)") + node.query( + "CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/test/test_mutations', '{instance}') ORDER BY x PARTITION BY toYYYYMM(d)") - node5.query("CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE MergeTree() ORDER BY x PARTITION BY toYYYYMM(d)") + node5.query( + "CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE MergeTree() ORDER BY x PARTITION BY toYYYYMM(d)") yield cluster @@ -184,7 +188,8 @@ def test_mutations(started_cluster): print "Total mutations: ", runner.total_mutations for node in nodes: - print node.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") + print node.query( + "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") assert all_done expected_sum = runner.total_inserted_xs - runner.total_deleted_xs @@ -195,10 +200,10 @@ def test_mutations(started_cluster): @pytest.mark.parametrize( - ('nodes', ), + ('nodes',), [ - ([node5, ], ), # MergeTree - ([node3, node4], ), # ReplicatedMergeTree + ([node5, ],), # MergeTree + ([node3, node4],), # ReplicatedMergeTree ] ) def test_mutations_dont_prevent_merges(started_cluster, nodes): @@ -228,8 +233,10 @@ def test_mutations_dont_prevent_merges(started_cluster, nodes): t.join() for node in nodes: - print node.query("SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") - print node.query("SELECT partition, count(name), sum(active), sum(active*rows) FROM system.parts WHERE table ='test_mutations' GROUP BY partition FORMAT TSVWithNames") + print node.query( + "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames") + print node.query( + "SELECT partition, count(name), sum(active), sum(active*rows) FROM system.parts WHERE table ='test_mutations' GROUP BY partition FORMAT TSVWithNames") assert all_done assert all([str(e).find("Too many parts") < 0 for e in runner.exceptions]) diff --git a/tests/integration/test_replicated_parse_zk_metadata/test.py b/tests/integration/test_replicated_parse_zk_metadata/test.py index dc8eb0129cc..4bdd77393b3 100644 --- a/tests/integration/test_replicated_parse_zk_metadata/test.py +++ b/tests/integration/test_replicated_parse_zk_metadata/test.py @@ -5,6 +5,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', with_zookeeper=True) + @pytest.fixture(scope='module', autouse=True) def started_cluster(): try: @@ -13,17 +14,18 @@ def started_cluster(): finally: cluster.shutdown() + def test_replicated_engine_parse_metadata_on_attach(): node.query( - ''' - CREATE TABLE data ( - key Int, - INDEX key_idx0 key+0 TYPE minmax GRANULARITY 1, - INDEX key_idx1 key+1 TYPE minmax GRANULARITY 1 - ) - ENGINE = ReplicatedMergeTree('/ch/tables/default/data', 'node') - ORDER BY key; - ''') + ''' + CREATE TABLE data ( + key Int, + INDEX key_idx0 key+0 TYPE minmax GRANULARITY 1, + INDEX key_idx1 key+1 TYPE minmax GRANULARITY 1 + ) + ENGINE = ReplicatedMergeTree('/ch/tables/default/data', 'node') + ORDER BY key; + ''') node.query('DETACH TABLE data') zk = cluster.get_kazoo_client('zoo1') diff --git a/tests/integration/test_replicating_constants/test.py b/tests/integration/test_replicating_constants/test.py index b72b9089f65..13a605f2650 100644 --- a/tests/integration/test_replicating_constants/test.py +++ b/tests/integration/test_replicating_constants/test.py @@ -5,7 +5,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', + with_installed_binary=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -16,6 +18,6 @@ def start_cluster(): finally: cluster.shutdown() -def test_different_versions(start_cluster): +def test_different_versions(start_cluster): assert node1.query("SELECT uniqExact(x) FROM (SELECT version() as x from remote('node{1,2}', system.one))") == "2\n" diff --git a/tests/integration/test_replication_credentials/test.py b/tests/integration/test_replication_credentials/test.py index ad5f05e04d9..4f07d6966a6 100644 --- a/tests/integration/test_replication_credentials/test.py +++ b/tests/integration/test_replication_credentials/test.py @@ -1,22 +1,25 @@ import time -import pytest +import pytest from helpers.cluster import ClickHouseCluster def _fill_nodes(nodes, shard): for node in nodes: node.query( - ''' - CREATE DATABASE test; + ''' + CREATE DATABASE test; + + CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + '''.format(shard=shard, replica=node.name)) - CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); - '''.format(shard=shard, replica=node.name)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], + with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], + with_zookeeper=True) @pytest.fixture(scope="module") @@ -31,6 +34,7 @@ def same_credentials_cluster(): finally: cluster.shutdown() + def test_same_credentials(same_credentials_cluster): node1.query("insert into test_table values ('2017-06-16', 111, 0)") time.sleep(1) @@ -45,8 +49,11 @@ def test_same_credentials(same_credentials_cluster): assert node2.query("SELECT id FROM test_table order by id") == '111\n222\n' -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], + with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], + with_zookeeper=True) + @pytest.fixture(scope="module") def no_credentials_cluster(): @@ -74,8 +81,12 @@ def test_no_credentials(no_credentials_cluster): assert node3.query("SELECT id FROM test_table order by id") == '111\n222\n' assert node4.query("SELECT id FROM test_table order by id") == '111\n222\n' -node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], with_zookeeper=True) -node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/credentials2.xml'], with_zookeeper=True) + +node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], + with_zookeeper=True) +node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/credentials2.xml'], + with_zookeeper=True) + @pytest.fixture(scope="module") def different_credentials_cluster(): @@ -89,6 +100,7 @@ def different_credentials_cluster(): finally: cluster.shutdown() + def test_different_credentials(different_credentials_cluster): node5.query("insert into test_table values ('2017-06-20', 111, 0)") time.sleep(1) @@ -102,8 +114,12 @@ def test_different_credentials(different_credentials_cluster): assert node5.query("SELECT id FROM test_table order by id") == '111\n' assert node6.query("SELECT id FROM test_table order by id") == '222\n' -node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], with_zookeeper=True) -node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], with_zookeeper=True) + +node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], + with_zookeeper=True) +node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], + with_zookeeper=True) + @pytest.fixture(scope="module") def credentials_and_no_credentials_cluster(): @@ -117,6 +133,7 @@ def credentials_and_no_credentials_cluster(): finally: cluster.shutdown() + def test_credentials_and_no_credentials(credentials_and_no_credentials_cluster): node7.query("insert into test_table values ('2017-06-21', 111, 0)") time.sleep(1) @@ -129,4 +146,3 @@ def test_credentials_and_no_credentials(credentials_and_no_credentials_cluster): assert node7.query("SELECT id FROM test_table order by id") == '111\n' assert node8.query("SELECT id FROM test_table order by id") == '222\n' - diff --git a/tests/integration/test_replication_without_zookeeper/test.py b/tests/integration/test_replication_without_zookeeper/test.py index 45c7f10acd6..90f060d991a 100644 --- a/tests/integration/test_replication_without_zookeeper/test.py +++ b/tests/integration/test_replication_without_zookeeper/test.py @@ -1,8 +1,7 @@ import time -import pytest +import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, stay_alive=True) @@ -50,4 +49,3 @@ def test_startup_without_zookeeper(start_cluster): assert node1.query("SELECT COUNT(*) from test_table") == "3\n" assert node1.query("SELECT is_readonly from system.replicas where table='test_table'") == "1\n" - diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index 667347be017..31fddd3df8c 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -1,7 +1,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -import re cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance') @@ -11,7 +10,7 @@ instance = cluster.add_instance('instance') def started_cluster(): try: cluster.start() - + instance.query("CREATE TABLE test_table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()") instance.query("INSERT INTO test_table VALUES (1,5), (2,10)") @@ -35,7 +34,7 @@ def test_create_role(): instance.query('CREATE ROLE R1') assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') - + instance.query('GRANT SELECT ON test_table TO R1') assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') @@ -52,13 +51,13 @@ def test_grant_role_to_role(): instance.query('CREATE ROLE R2') assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') - + instance.query('GRANT R1 TO A') assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') - + instance.query('GRANT R2 TO R1') assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') - + instance.query('GRANT SELECT ON test_table TO R2') assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" @@ -69,12 +68,12 @@ def test_combine_privileges(): instance.query('CREATE ROLE R2') assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') - + instance.query('GRANT R1 TO A') instance.query('GRANT SELECT(x) ON test_table TO R1') assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') assert instance.query("SELECT x FROM test_table", user='A') == "1\n2\n" - + instance.query('GRANT SELECT(y) ON test_table TO R2') instance.query('GRANT R2 TO A') assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" @@ -100,7 +99,7 @@ def test_admin_option(): def test_revoke_requires_admin_option(): instance.query("CREATE USER A, B") instance.query("CREATE ROLE R1, R2") - + instance.query("GRANT R1 TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT R1 TO B\n" @@ -150,25 +149,28 @@ def test_introspection(): instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') instance.query('REVOKE SELECT(x) ON test.table FROM R2') - assert instance.query("SHOW ROLES") == TSV([ "R1", "R2" ]) - assert instance.query("SHOW CREATE ROLE R1") == TSV([ "CREATE ROLE R1" ]) - assert instance.query("SHOW CREATE ROLE R2") == TSV([ "CREATE ROLE R2" ]) - assert instance.query("SHOW CREATE ROLES R1, R2") == TSV([ "CREATE ROLE R1", "CREATE ROLE R2" ]) - assert instance.query("SHOW CREATE ROLES") == TSV([ "CREATE ROLE R1", "CREATE ROLE R2" ]) + assert instance.query("SHOW ROLES") == TSV(["R1", "R2"]) + assert instance.query("SHOW CREATE ROLE R1") == TSV(["CREATE ROLE R1"]) + assert instance.query("SHOW CREATE ROLE R2") == TSV(["CREATE ROLE R2"]) + assert instance.query("SHOW CREATE ROLES R1, R2") == TSV(["CREATE ROLE R1", "CREATE ROLE R2"]) + assert instance.query("SHOW CREATE ROLES") == TSV(["CREATE ROLE R1", "CREATE ROLE R2"]) - assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) - assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) + assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A", "GRANT R1 TO A"]) + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION"]) assert instance.query("SHOW GRANTS FOR R1") == "" - assert instance.query("SHOW GRANTS FOR R2") == TSV([ "GRANT SELECT ON test.table TO R2", "REVOKE SELECT(x) ON test.table FROM R2" ]) + assert instance.query("SHOW GRANTS FOR R2") == TSV( + ["GRANT SELECT ON test.table TO R2", "REVOKE SELECT(x) ON test.table FROM R2"]) - assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) - assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) - assert instance.query("SHOW CURRENT ROLES", user='A') == TSV([[ "R1", 0, 1 ]]) - assert instance.query("SHOW CURRENT ROLES", user='B') == TSV([[ "R2", 1, 1 ]]) - assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]]) - assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]]) + assert instance.query("SHOW GRANTS", user='A') == TSV(["GRANT SELECT ON test.table TO A", "GRANT R1 TO A"]) + assert instance.query("SHOW GRANTS", user='B') == TSV( + ["GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION"]) + assert instance.query("SHOW CURRENT ROLES", user='A') == TSV([["R1", 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user='B') == TSV([["R2", 1, 1]]) + assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([["R1", 0, 1, 1]]) + assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([["R2", 1, 1, 1]]) - expected_access1 = "CREATE ROLE R1\n"\ + expected_access1 = "CREATE ROLE R1\n" \ "CREATE ROLE R2\n" expected_access2 = "GRANT R1 TO A\n" expected_access3 = "GRANT R2 TO B WITH ADMIN OPTION" @@ -176,21 +178,23 @@ def test_introspection(): assert expected_access2 in instance.query("SHOW ACCESS") assert expected_access3 in instance.query("SHOW ACCESS") - assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\ - TSV([[ "R1", "local directory" ], - [ "R2", "local directory" ]]) + assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") == \ + TSV([["R1", "local directory"], + ["R2", "local directory"]]) - assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, grant_option") ==\ - TSV([[ "A", "\N", "SELECT", "test", "table", "\N", 0, 0 ], - [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 1 ], - [ "\N", "R2", "SELECT", "test", "table", "\N", 0, 0 ], - [ "\N", "R2", "SELECT", "test", "table", "x", 1, 0 ]]) + assert instance.query( + "SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, grant_option") == \ + TSV([["A", "\N", "SELECT", "test", "table", "\N", 0, 0], + ["B", "\N", "CREATE", "\N", "\N", "\N", 0, 1], + ["\N", "R2", "SELECT", "test", "table", "\N", 0, 0], + ["\N", "R2", "SELECT", "test", "table", "x", 1, 0]]) - assert instance.query("SELECT * from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name") ==\ - TSV([[ "A", "\N", "R1", 1, 0 ], - [ "B", "\N", "R2", 1, 1 ]]) + assert instance.query( + "SELECT * from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name") == \ + TSV([["A", "\N", "R1", 1, 0], + ["B", "\N", "R2", 1, 1]]) - assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='A') == TSV([[ "R1", 0, 1 ]]) - assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='B') == TSV([[ "R2", 1, 1 ]]) - assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='A') == TSV([[ "R1", 0, 1, 1 ]]) - assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='B') == TSV([[ "R2", 1, 1, 1 ]]) + assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='A') == TSV([["R1", 0, 1]]) + assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='B') == TSV([["R2", 1, 1]]) + assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='A') == TSV([["R1", 0, 1, 1]]) + assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='B') == TSV([["R2", 1, 1, 1]]) diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index dd0495df237..a407f0b2c7a 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -1,20 +1,28 @@ -import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV import os import re import time +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, TSV + cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/config.d/remote_servers.xml"], user_configs=["configs/users.d/row_policy.xml", "configs/users.d/another_user.xml", "configs/users.d/any_join_distinct_right_table_keys.xml"], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=["configs/config.d/remote_servers.xml"], user_configs=["configs/users.d/row_policy.xml", "configs/users.d/another_user.xml", "configs/users.d/any_join_distinct_right_table_keys.xml"], with_zookeeper=True) +node = cluster.add_instance('node', main_configs=["configs/config.d/remote_servers.xml"], + user_configs=["configs/users.d/row_policy.xml", "configs/users.d/another_user.xml", + "configs/users.d/any_join_distinct_right_table_keys.xml"], + with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=["configs/config.d/remote_servers.xml"], + user_configs=["configs/users.d/row_policy.xml", "configs/users.d/another_user.xml", + "configs/users.d/any_join_distinct_right_table_keys.xml"], + with_zookeeper=True) nodes = [node, node2] -def copy_policy_xml(local_file_name, reload_immediately = True): +def copy_policy_xml(local_file_name, reload_immediately=True): script_dir = os.path.dirname(os.path.realpath(__file__)) for current_node in nodes: - current_node.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/row_policy.xml') + current_node.copy_file_to_container(os.path.join(script_dir, local_file_name), + '/etc/clickhouse-server/users.d/row_policy.xml') if reload_immediately: current_node.query("SYSTEM RELOAD CONFIG") @@ -66,7 +74,7 @@ def reset_policies(): def test_smoke(): - assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1,0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) @@ -86,8 +94,12 @@ def test_smoke(): def test_join(): - assert node.query("SELECT * FROM mydb.filtered_table1 as t1 ANY LEFT JOIN mydb.filtered_table1 as t2 ON t1.a = t2.b") == TSV([[1, 0, 1, 1], [1, 1, 1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table1 as t2 ANY RIGHT JOIN mydb.filtered_table1 as t1 ON t2.b = t1.a") == TSV([[1, 1, 1, 0]]) + assert node.query( + "SELECT * FROM mydb.filtered_table1 as t1 ANY LEFT JOIN mydb.filtered_table1 as t2 ON t1.a = t2.b") == TSV( + [[1, 0, 1, 1], [1, 1, 1, 1]]) + assert node.query( + "SELECT * FROM mydb.filtered_table1 as t2 ANY RIGHT JOIN mydb.filtered_table1 as t1 ON t2.b = t1.a") == TSV( + [[1, 1, 1, 0]]) def test_cannot_trick_row_policy_with_keyword_with(): @@ -104,17 +116,19 @@ def test_prewhere_not_supported(): assert expected_error in node.query_and_get_error("SELECT * FROM mydb.filtered_table3 PREWHERE 1") # However PREWHERE should still work for user without filtering. - assert node.query("SELECT * FROM mydb.filtered_table1 PREWHERE 1", user="another") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table1 PREWHERE 1", user="another") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]]) def test_policy_from_users_xml_affects_only_user_assigned(): - assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1,0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) assert node.query("SELECT * FROM mydb.filtered_table1", user="another") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) - assert node.query("SELECT * FROM mydb.filtered_table2", user="another") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2", user="another") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) - assert node.query("SELECT * FROM mydb.local") == TSV([[1,0], [1, 1], [2, 0], [2, 1]]) + assert node.query("SELECT * FROM mydb.local") == TSV([[1, 0], [1, 1], [2, 0], [2, 1]]) assert node.query("SELECT * FROM mydb.local", user="another") == TSV([[1, 0], [1, 1]]) @@ -126,7 +140,8 @@ def test_change_of_users_xml_changes_row_policies(): copy_policy_xml('all_rows.xml') assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) copy_policy_xml('no_rows.xml') @@ -141,7 +156,8 @@ def test_change_of_users_xml_changes_row_policies(): copy_policy_xml('no_filters.xml') assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) copy_policy_xml('normal_filters.xml') @@ -156,13 +172,14 @@ def test_reload_users_xml_by_timer(): assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. + time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. copy_policy_xml('all_rows.xml', False) assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table1", [[0, 0], [0, 1], [1, 0], [1, 1]]) - assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", + [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table3", [[0, 0], [0, 1], [1, 0], [1, 1]]) - time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. + time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. copy_policy_xml('normal_filters.xml', False) assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table1", [[1, 0], [1, 1]]) assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", [[0, 0, 0, 0], [0, 0, 6, 0]]) @@ -171,57 +188,109 @@ def test_reload_users_xml_by_timer(): def test_introspection(): policies = [ - ["another ON mydb.filtered_table1", "another", "mydb", "filtered_table1", "6068883a-0e9d-f802-7e22-0144f8e66d3c", "users.xml", "1", 0, 0, "['another']", "[]"], - ["another ON mydb.filtered_table2", "another", "mydb", "filtered_table2", "c019e957-c60b-d54e-cc52-7c90dac5fb01", "users.xml", "1", 0, 0, "['another']", "[]"], - ["another ON mydb.filtered_table3", "another", "mydb", "filtered_table3", "4cb080d0-44e8-dbef-6026-346655143628", "users.xml", "1", 0, 0, "['another']", "[]"], - ["another ON mydb.local", "another", "mydb", "local", "5b23c389-7e18-06bf-a6bc-dd1afbbc0a97", "users.xml", "a = 1", 0, 0, "['another']", "[]"], - ["default ON mydb.filtered_table1", "default", "mydb", "filtered_table1", "9e8a8f62-4965-2b5e-8599-57c7b99b3549", "users.xml", "a = 1", 0, 0, "['default']", "[]"], - ["default ON mydb.filtered_table2", "default", "mydb", "filtered_table2", "cffae79d-b9bf-a2ef-b798-019c18470b25", "users.xml", "a + b < 1 or c - d > 5", 0, 0, "['default']", "[]"], - ["default ON mydb.filtered_table3", "default", "mydb", "filtered_table3", "12fc5cef-e3da-3940-ec79-d8be3911f42b", "users.xml", "c = 1", 0, 0, "['default']", "[]"], - ["default ON mydb.local", "default", "mydb", "local", "cdacaeb5-1d97-f99d-2bb0-4574f290629c", "users.xml", "1", 0, 0, "['default']", "[]"] + ["another ON mydb.filtered_table1", "another", "mydb", "filtered_table1", + "6068883a-0e9d-f802-7e22-0144f8e66d3c", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.filtered_table2", "another", "mydb", "filtered_table2", + "c019e957-c60b-d54e-cc52-7c90dac5fb01", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.filtered_table3", "another", "mydb", "filtered_table3", + "4cb080d0-44e8-dbef-6026-346655143628", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.local", "another", "mydb", "local", "5b23c389-7e18-06bf-a6bc-dd1afbbc0a97", "users.xml", + "a = 1", 0, 0, "['another']", "[]"], + ["default ON mydb.filtered_table1", "default", "mydb", "filtered_table1", + "9e8a8f62-4965-2b5e-8599-57c7b99b3549", "users.xml", "a = 1", 0, 0, "['default']", "[]"], + ["default ON mydb.filtered_table2", "default", "mydb", "filtered_table2", + "cffae79d-b9bf-a2ef-b798-019c18470b25", "users.xml", "a + b < 1 or c - d > 5", 0, 0, "['default']", "[]"], + ["default ON mydb.filtered_table3", "default", "mydb", "filtered_table3", + "12fc5cef-e3da-3940-ec79-d8be3911f42b", "users.xml", "c = 1", 0, 0, "['default']", "[]"], + ["default ON mydb.local", "default", "mydb", "local", "cdacaeb5-1d97-f99d-2bb0-4574f290629c", "users.xml", "1", + 0, 0, "['default']", "[]"] ] assert node.query("SELECT * from system.row_policies ORDER BY short_name, database, table") == TSV(policies) def test_dcl_introspection(): - assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", "default ON mydb.local"]) + assert node.query("SHOW POLICIES") == TSV( + ["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", + "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", + "default ON mydb.filtered_table3", "default ON mydb.local"]) - assert node.query("SHOW POLICIES ON mydb.filtered_table1") == TSV([ "another", "default" ]) - assert node.query("SHOW POLICIES ON mydb.local") == TSV([ "another", "default" ]) - assert node.query("SHOW POLICIES ON mydb.*") == TSV([ "another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", "default ON mydb.local" ]) - assert node.query("SHOW POLICIES default") == TSV([ "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", "default ON mydb.local" ]) + assert node.query("SHOW POLICIES ON mydb.filtered_table1") == TSV(["another", "default"]) + assert node.query("SHOW POLICIES ON mydb.local") == TSV(["another", "default"]) + assert node.query("SHOW POLICIES ON mydb.*") == TSV( + ["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", + "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", + "default ON mydb.filtered_table3", "default ON mydb.local"]) + assert node.query("SHOW POLICIES default") == TSV( + ["default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", + "default ON mydb.local"]) - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" - assert node.query("SHOW CREATE POLICY default ON mydb.local") == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" + assert node.query( + "SHOW CREATE POLICY default ON mydb.local") == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" - assert node.query("SHOW CREATE POLICY default") == TSV([ "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default" ]) - assert node.query("SHOW CREATE POLICIES ON mydb.filtered_table1") == TSV([ "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default" ]) - assert node.query("SHOW CREATE POLICIES ON mydb.*") == TSV([ "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default" ]) - assert node.query("SHOW CREATE POLICIES") == TSV([ "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default" ]) + assert node.query("SHOW CREATE POLICY default") == TSV( + ["CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", + "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default"]) + assert node.query("SHOW CREATE POLICIES ON mydb.filtered_table1") == TSV( + ["CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default"]) + assert node.query("SHOW CREATE POLICIES ON mydb.*") == TSV( + ["CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", + "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default"]) + assert node.query("SHOW CREATE POLICIES") == TSV( + ["CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", + "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default"]) - expected_access = "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another\n"\ - "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another\n"\ - "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another\n"\ - "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another\n"\ - "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n"\ - "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n"\ - "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n"\ + expected_access = "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another\n" \ + "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another\n" \ + "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another\n" \ + "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another\n" \ + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" \ + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" \ + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" \ "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" assert expected_access in node.query("SHOW ACCESS") copy_policy_xml('all_rows.xml') - assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING 1 TO default\n" - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING 1 TO default\n" + assert node.query("SHOW POLICIES") == TSV( + ["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", + "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING 1 TO default\n" + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING 1 TO default\n" copy_policy_xml('no_rows.xml') - assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING NULL TO default\n" - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING NULL TO default\n" - assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING NULL TO default\n" + assert node.query("SHOW POLICIES") == TSV( + ["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", + "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING NULL TO default\n" + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING NULL TO default\n" + assert node.query( + "SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING NULL TO default\n" copy_policy_xml('no_filters.xml') assert node.query("SHOW POLICIES") == "" @@ -245,7 +314,8 @@ def test_dcl_management(): node.query("ALTER POLICY pA ON mydb.filtered_table1 RENAME TO pB") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0]]) assert node.query("SHOW POLICIES ON mydb.filtered_table1") == "pB\n" - assert node.query("SHOW CREATE POLICY pB ON mydb.filtered_table1") == "CREATE ROW POLICY pB ON mydb.filtered_table1 FOR SELECT USING a > b TO default\n" + assert node.query( + "SHOW CREATE POLICY pB ON mydb.filtered_table1") == "CREATE ROW POLICY pB ON mydb.filtered_table1 FOR SELECT USING a > b TO default\n" node.query("DROP POLICY pB ON mydb.filtered_table1") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) @@ -258,16 +328,17 @@ def test_users_xml_is_readonly(): def test_tags_with_db_and_table_names(): copy_policy_xml('tags_with_db_and_table_names.xml') - + assert node.query("SELECT * FROM mydb.table") == TSV([[0, 0], [0, 1]]) assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 6, 0]]) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0]]) assert node.query("SELECT * FROM mydb.`.filtered_table4`") == TSV([[1, 1]]) - assert node.query("SHOW CREATE POLICIES default") == TSV(["CREATE ROW POLICY default ON mydb.`.filtered_table4` FOR SELECT USING c = 2 TO default", - "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING c > (d + 5) TO default", - "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 0 TO default", - "CREATE ROW POLICY default ON mydb.table FOR SELECT USING a = 0 TO default"]) + assert node.query("SHOW CREATE POLICIES default") == TSV( + ["CREATE ROW POLICY default ON mydb.`.filtered_table4` FOR SELECT USING c = 2 TO default", + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING c > (d + 5) TO default", + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 0 TO default", + "CREATE ROW POLICY default ON mydb.table FOR SELECT USING a = 0 TO default"]) def test_miscellaneous_engines(): @@ -275,7 +346,8 @@ def test_miscellaneous_engines(): # ReplicatedMergeTree node.query("DROP TABLE mydb.filtered_table1") - node.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1') ORDER BY a") + node.query( + "CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1') ORDER BY a") node.query("INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1)") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) @@ -287,12 +359,15 @@ def test_miscellaneous_engines(): # ReplicatedCollapsingMergeTree node.query("DROP TABLE mydb.filtered_table1") - node.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1', b) ORDER BY a") + node.query( + "CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1', b) ORDER BY a") node.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 1], [1, 1]]) # DistributedMergeTree node.query("DROP TABLE IF EXISTS mydb.not_filtered_table") - node.query("CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") + node.query( + "CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") assert node.query("SELECT * FROM mydb.not_filtered_table", user="another") == TSV([[1, 0], [1, 1], [1, 0], [1, 1]]) - assert node.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == TSV([[2, 0], [2, 1]]) + assert node.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == TSV( + [[2, 0], [2, 1]]) diff --git a/tests/integration/test_s3_with_https/test.py b/tests/integration/test_s3_with_https/test.py index 2b40e02e701..17f24bba794 100644 --- a/tests/integration/test_s3_with_https/test.py +++ b/tests/integration/test_s3_with_https/test.py @@ -18,7 +18,9 @@ def check_proxy_logs(cluster, proxy_instance): def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", "configs/config.d/log_conf.xml", "configs/config.d/ssl.xml"], with_minio=True, minio_certs_dir='minio_certs') + cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", "configs/config.d/log_conf.xml", + "configs/config.d/ssl.xml"], with_minio=True, + minio_certs_dir='minio_certs') logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -43,7 +45,7 @@ def test_s3_with_https(cluster, policy): ORDER BY id SETTINGS storage_policy='{}' """ - .format(policy) + .format(policy) ) node.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") diff --git a/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py b/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py index 520c018cbad..87fe4ce30f6 100644 --- a/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py +++ b/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py @@ -1,6 +1,7 @@ -import bottle import random +import bottle + @bottle.route('/hostname') def index(): diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index daf53c2e27b..9df209826f9 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -21,7 +21,9 @@ def run_resolver(cluster): def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/config.d/log_conf.xml", "configs/config.d/storage_conf.xml"], with_minio=True) + cluster.add_instance("node", + main_configs=["configs/config.d/log_conf.xml", "configs/config.d/storage_conf.xml"], + with_minio=True) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -56,7 +58,7 @@ def test_s3_with_proxy_list(cluster, policy): ORDER BY id SETTINGS storage_policy='{}' """ - .format(policy) + .format(policy) ) node.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") diff --git a/tests/integration/test_send_crash_reports/fake_sentry_server.py b/tests/integration/test_send_crash_reports/fake_sentry_server.py index 74f0592504f..49463bdb133 100644 --- a/tests/integration/test_send_crash_reports/fake_sentry_server.py +++ b/tests/integration/test_send_crash_reports/fake_sentry_server.py @@ -2,6 +2,7 @@ import BaseHTTPServer RESULT_PATH = '/result.txt' + class SentryHandler(BaseHTTPServer.BaseHTTPRequestHandler): def do_POST(self): post_data = self.__read_and_decode_post_data() diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py index ff4b55da99b..4c832d9d67c 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test.py @@ -1,12 +1,11 @@ import os import time -import pytest - import helpers.cluster import helpers.test_tools -import fake_sentry_server +import pytest +import fake_sentry_server SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -24,7 +23,7 @@ def started_node(): cluster.shutdown() -def test_send_segfault(started_node,): +def test_send_segfault(started_node, ): started_node.copy_file_to_container(os.path.join(SCRIPT_DIR, "fake_sentry_server.py"), "/fake_sentry_server.py") started_node.exec_in_container(["bash", "-c", "python2 /fake_sentry_server.py"], detach=True, user="root") time.sleep(0.5) diff --git a/tests/integration/test_send_request_to_leader_replica/test.py b/tests/integration/test_send_request_to_leader_replica/test.py index 913a5e6ff7a..8c58d02d104 100644 --- a/tests/integration/test_send_request_to_leader_replica/test.py +++ b/tests/integration/test_send_request_to_leader_replica/test.py @@ -1,5 +1,3 @@ -import time - import pytest from helpers.cluster import ClickHouseCluster @@ -7,17 +5,21 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], user_configs=['configs/user_good_restricted.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], user_configs=['configs/user_good_restricted.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], user_configs=['configs/user_good_allowed.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], user_configs=['configs/user_good_allowed.xml'], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], + user_configs=['configs/user_good_restricted.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], + user_configs=['configs/user_good_restricted.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], + user_configs=['configs/user_good_allowed.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], + user_configs=['configs/user_good_allowed.xml'], with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - for node in [node1, node2]: node.query(''' CREATE TABLE sometable(date Date, id UInt32, value Int32) @@ -30,18 +32,18 @@ def started_cluster(): ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/someothertable', '{replica}', date, id, 8192); '''.format(replica=node.name), user='good') - yield cluster finally: cluster.shutdown() + @pytest.mark.parametrize("table,query,expected,n1,n2", [ - ("sometable","ALTER TABLE sometable DROP PARTITION 201706", '1', node1, node2), - ("sometable","TRUNCATE TABLE sometable", '0', node1, node2), + ("sometable", "ALTER TABLE sometable DROP PARTITION 201706", '1', node1, node2), + ("sometable", "TRUNCATE TABLE sometable", '0', node1, node2), ("sometable", "OPTIMIZE TABLE sometable", '4', node1, node2), - ("someothertable","ALTER TABLE someothertable DROP PARTITION 201706", '1', node3, node4), - ("someothertable","TRUNCATE TABLE someothertable", '0', node3, node4), + ("someothertable", "ALTER TABLE someothertable DROP PARTITION 201706", '1', node3, node4), + ("someothertable", "TRUNCATE TABLE someothertable", '0', node3, node4), ("someothertable", "OPTIMIZE TABLE someothertable", '4', node3, node4), ]) def test_alter_table_drop_partition(started_cluster, table, query, expected, n1, n2): diff --git a/tests/integration/test_server_initialization/test.py b/tests/integration/test_server_initialization/test.py index ebf50e7dc51..08032436982 100644 --- a/tests/integration/test_server_initialization/test.py +++ b/tests/integration/test_server_initialization/test.py @@ -1,8 +1,8 @@ -import time import pytest from helpers.cluster import ClickHouseCluster + @pytest.fixture(scope="module") def started_cluster(): try: @@ -14,7 +14,7 @@ def started_cluster(): instance_fail = cluster_fail.add_instance('dummy_fail', clickhouse_path_dir='clickhouse_path_fail') with pytest.raises(Exception): cluster_fail.start() - cluster_fail.shutdown() # cleanup + cluster_fail.shutdown() # cleanup yield cluster @@ -30,9 +30,10 @@ def test_sophisticated_default(started_cluster): def test_partially_dropped_tables(started_cluster): instance = started_cluster.instances['dummy'] - assert instance.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/*/default -name *.sql* | sort'], privileged=True, user='root') \ - == "/var/lib/clickhouse/metadata/default/should_be_restored.sql\n" \ - "/var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" + assert instance.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/*/default -name *.sql* | sort'], + privileged=True, user='root') \ + == "/var/lib/clickhouse/metadata/default/should_be_restored.sql\n" \ + "/var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" assert instance.query("SELECT n FROM should_be_restored") == "1\n2\n3\n" assert instance.query("SELECT count() FROM system.tables WHERE name='should_be_dropped'") == "0\n" @@ -42,5 +43,6 @@ def test_live_view_dependency(started_cluster): instance.query("CREATE DATABASE a_load_first") instance.query("CREATE DATABASE b_load_second") instance.query("CREATE TABLE b_load_second.mt (a Int32) Engine=MergeTree order by tuple()") - instance.query("CREATE LIVE VIEW a_load_first.lv AS SELECT sum(a) FROM b_load_second.mt", settings={'allow_experimental_live_view': 1}) + instance.query("CREATE LIVE VIEW a_load_first.lv AS SELECT sum(a) FROM b_load_second.mt", + settings={'allow_experimental_live_view': 1}) instance.restart_clickhouse() diff --git a/tests/integration/test_settings_constraints/test.py b/tests/integration/test_settings_constraints/test.py index b2dcd80448f..90e639685f0 100644 --- a/tests/integration/test_settings_constraints/test.py +++ b/tests/integration/test_settings_constraints/test.py @@ -5,8 +5,6 @@ cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', user_configs=["configs/users.xml"]) - - @pytest.fixture(scope="module") def started_cluster(): try: @@ -18,13 +16,15 @@ def started_cluster(): def test_system_settings(started_cluster): - assert instance.query("SELECT name, value, min, max, readonly from system.settings WHERE name = 'force_index_by_date'") ==\ + assert instance.query( + "SELECT name, value, min, max, readonly from system.settings WHERE name = 'force_index_by_date'") == \ "force_index_by_date\t0\t\\N\t\\N\t1\n" - assert instance.query("SELECT name, value, min, max, readonly from system.settings WHERE name = 'max_memory_usage'") ==\ + assert instance.query( + "SELECT name, value, min, max, readonly from system.settings WHERE name = 'max_memory_usage'") == \ "max_memory_usage\t10000000000\t5000000000\t20000000000\t0\n" - assert instance.query("SELECT name, value, min, max, readonly from system.settings WHERE name = 'readonly'") ==\ + assert instance.query("SELECT name, value, min, max, readonly from system.settings WHERE name = 'readonly'") == \ "readonly\t0\t\\N\t\\N\t0\n" diff --git a/tests/integration/test_settings_constraints_distributed/test.py b/tests/integration/test_settings_constraints_distributed/test.py index 94afa0d6d2d..acb73f0ef49 100644 --- a/tests/integration/test_settings_constraints_distributed/test.py +++ b/tests/integration/test_settings_constraints_distributed/test.py @@ -1,16 +1,16 @@ -import time - import pytest -from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/config.d/remote_servers.xml"], user_configs=["configs/users.d/allow_introspection_functions.xml"]) -node2 = cluster.add_instance('node2', main_configs=["configs/config.d/remote_servers.xml"], user_configs=["configs/users.d/allow_introspection_functions.xml"]) -distributed = cluster.add_instance('distributed', main_configs=["configs/config.d/remote_servers.xml"], user_configs=["configs/users.d/allow_introspection_functions.xml"], stay_alive=True) +node1 = cluster.add_instance('node1', main_configs=["configs/config.d/remote_servers.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml"]) +node2 = cluster.add_instance('node2', main_configs=["configs/config.d/remote_servers.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml"]) +distributed = cluster.add_instance('distributed', main_configs=["configs/config.d/remote_servers.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml"], stay_alive=True) @pytest.fixture(scope="module", autouse=True) @@ -24,8 +24,10 @@ def started_cluster(): node.query("CREATE USER shard") node.query("GRANT ALL ON *.* TO shard") - distributed.query("CREATE TABLE proxy (date Date, id UInt32, value Int32) ENGINE = Distributed(test_cluster, default, sometable, toUInt64(date));") - distributed.query("CREATE TABLE shard_settings (name String, value String) ENGINE = Distributed(test_cluster, system, settings);") + distributed.query( + "CREATE TABLE proxy (date Date, id UInt32, value Int32) ENGINE = Distributed(test_cluster, default, sometable, toUInt64(date));") + distributed.query( + "CREATE TABLE shard_settings (name String, value String) ENGINE = Distributed(test_cluster, system, settings);") distributed.query("CREATE ROLE admin") distributed.query("GRANT ALL ON *.* TO admin") @@ -53,49 +55,52 @@ def test_select_clamps_settings(): # Check that shards doesn't throw exceptions on constraints violation query = "SELECT COUNT() FROM proxy" assert distributed.query(query) == '2\n' - assert distributed.query(query, user = 'normal') == '2\n' - assert distributed.query(query, user = 'wasteful') == '2\n' - assert distributed.query(query, user = 'readonly') == '2\n' + assert distributed.query(query, user='normal') == '2\n' + assert distributed.query(query, user='wasteful') == '2\n' + assert distributed.query(query, user='readonly') == '2\n' assert distributed.query(query, settings={"max_memory_usage": 40000000, "readonly": 2}) == '2\n' assert distributed.query(query, settings={"max_memory_usage": 3000000000, "readonly": 2}) == '2\n' query = "SELECT COUNT() FROM remote('node{1,2}', 'default', 'sometable')" assert distributed.query(query) == '2\n' - assert distributed.query(query, user = 'normal') == '2\n' - assert distributed.query(query, user = 'wasteful') == '2\n' + assert distributed.query(query, user='normal') == '2\n' + assert distributed.query(query, user='wasteful') == '2\n' # Check that shards clamp passed settings. query = "SELECT hostName() as host, name, value FROM shard_settings WHERE name = 'max_memory_usage' OR name = 'readonly' ORDER BY host, name, value" - assert distributed.query(query) == 'node1\tmax_memory_usage\t99999999\n'\ - 'node1\treadonly\t0\n'\ - 'node2\tmax_memory_usage\t10000000000\n'\ + assert distributed.query(query) == 'node1\tmax_memory_usage\t99999999\n' \ + 'node1\treadonly\t0\n' \ + 'node2\tmax_memory_usage\t10000000000\n' \ 'node2\treadonly\t1\n' - assert distributed.query(query, user = 'normal') == 'node1\tmax_memory_usage\t80000000\n'\ - 'node1\treadonly\t0\n'\ - 'node2\tmax_memory_usage\t10000000000\n'\ + assert distributed.query(query, user='normal') == 'node1\tmax_memory_usage\t80000000\n' \ + 'node1\treadonly\t0\n' \ + 'node2\tmax_memory_usage\t10000000000\n' \ + 'node2\treadonly\t1\n' + assert distributed.query(query, user='wasteful') == 'node1\tmax_memory_usage\t99999999\n' \ + 'node1\treadonly\t0\n' \ + 'node2\tmax_memory_usage\t10000000000\n' \ + 'node2\treadonly\t1\n' + assert distributed.query(query, user='readonly') == 'node1\tmax_memory_usage\t99999999\n' \ + 'node1\treadonly\t1\n' \ + 'node2\tmax_memory_usage\t10000000000\n' \ 'node2\treadonly\t1\n' - assert distributed.query(query, user = 'wasteful') == 'node1\tmax_memory_usage\t99999999\n'\ - 'node1\treadonly\t0\n'\ - 'node2\tmax_memory_usage\t10000000000\n'\ - 'node2\treadonly\t1\n' - assert distributed.query(query, user = 'readonly') == 'node1\tmax_memory_usage\t99999999\n'\ - 'node1\treadonly\t1\n'\ - 'node2\tmax_memory_usage\t10000000000\n'\ - 'node2\treadonly\t1\n' - assert distributed.query(query, settings={"max_memory_usage": 1}) == 'node1\tmax_memory_usage\t11111111\n'\ - 'node1\treadonly\t0\n'\ - 'node2\tmax_memory_usage\t10000000000\n'\ + assert distributed.query(query, settings={"max_memory_usage": 1}) == 'node1\tmax_memory_usage\t11111111\n' \ + 'node1\treadonly\t0\n' \ + 'node2\tmax_memory_usage\t10000000000\n' \ 'node2\treadonly\t1\n' - assert distributed.query(query, settings={"max_memory_usage": 40000000, "readonly": 2}) == 'node1\tmax_memory_usage\t40000000\n'\ - 'node1\treadonly\t2\n'\ - 'node2\tmax_memory_usage\t10000000000\n'\ - 'node2\treadonly\t1\n' - assert distributed.query(query, settings={"max_memory_usage": 3000000000, "readonly": 2}) == 'node1\tmax_memory_usage\t99999999\n'\ - 'node1\treadonly\t2\n'\ - 'node2\tmax_memory_usage\t10000000000\n'\ - 'node2\treadonly\t1\n' + assert distributed.query(query, settings={"max_memory_usage": 40000000, + "readonly": 2}) == 'node1\tmax_memory_usage\t40000000\n' \ + 'node1\treadonly\t2\n' \ + 'node2\tmax_memory_usage\t10000000000\n' \ + 'node2\treadonly\t1\n' + assert distributed.query(query, settings={"max_memory_usage": 3000000000, + "readonly": 2}) == 'node1\tmax_memory_usage\t99999999\n' \ + 'node1\treadonly\t2\n' \ + 'node2\tmax_memory_usage\t10000000000\n' \ + 'node2\treadonly\t1\n' + def test_insert_clamps_settings(): node1.query("ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999") diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 71a1d9aca54..f7901dc1fe6 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -7,7 +7,9 @@ instance = cluster.add_instance('instance') def system_settings_profile(profile_name): - return TSV(instance.query("SELECT name, storage, num_elements, apply_to_all, apply_to_list, apply_to_except FROM system.settings_profiles WHERE name='" + profile_name + "'")) + return TSV(instance.query( + "SELECT name, storage, num_elements, apply_to_all, apply_to_list, apply_to_except FROM system.settings_profiles WHERE name='" + profile_name + "'")) + def system_settings_profile_elements(profile_name=None, user_name=None, role_name=None): where = "" @@ -45,33 +47,46 @@ def reset_after_test(): def test_smoke(): # Set settings and constraints via CREATE SETTINGS PROFILE ... TO user - instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [[ "xyz", "local directory", 1, 0, "['robin']", "[]" ]] - assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000001, 90000000, 110000000, "\N", "\N" ]] + instance.query( + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin") + assert instance.query( + "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "100000001\n" + assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error( + "SET max_memory_usage = 80000000", user="robin") + assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [["xyz", "local directory", 1, 0, "['robin']", "[]"]] + assert system_settings_profile_elements(profile_name="xyz") == [ + ["xyz", "\N", "\N", 0, "max_memory_usage", 100000001, 90000000, 110000000, "\N", "\N"]] instance.query("ALTER SETTINGS PROFILE xyz TO NONE") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" + assert instance.query( + "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [[ "xyz", "local directory", 1, 0, "[]", "[]" ]] + assert system_settings_profile("xyz") == [["xyz", "local directory", 1, 0, "[]", "[]"]] assert system_settings_profile_elements(user_name="robin") == [] # Set settings and constraints via CREATE USER ... SETTINGS PROFILE instance.query("ALTER USER robin SETTINGS PROFILE xyz") assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin SETTINGS PROFILE xyz\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile_elements(user_name="robin") == [[ "\N", "robin", "\N", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "100000001\n" + assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error( + "SET max_memory_usage = 80000000", user="robin") + assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile_elements(user_name="robin") == [ + ["\N", "robin", "\N", 0, "\N", "\N", "\N", "\N", "\N", "xyz"]] instance.query("ALTER USER robin SETTINGS NONE") assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") assert system_settings_profile_elements(user_name="robin") == [] @@ -79,70 +94,95 @@ def test_smoke(): def test_settings_from_granted_role(): # Set settings and constraints via granted role - instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000") + instance.query( + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000") instance.query("CREATE ROLE worker SETTINGS PROFILE xyz") instance.query("GRANT worker TO robin") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" + assert instance.query( + "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker SETTINGS PROFILE xyz\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "100000001\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_ast_depth'", user="robin") == "2000\n" - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [[ "xyz", "local directory", 2, 0, "[]", "[]" ]] - assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000001, "\N", 110000000, "\N", "\N" ], - [ "xyz", "\N", "\N", 1, "max_ast_depth", 2000, "\N", "\N", "\N", "\N" ]] - assert system_settings_profile_elements(role_name="worker") == [[ "\N", "\N", "worker", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] + assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [["xyz", "local directory", 2, 0, "[]", "[]"]] + assert system_settings_profile_elements(profile_name="xyz") == [ + ["xyz", "\N", "\N", 0, "max_memory_usage", 100000001, "\N", 110000000, "\N", "\N"], + ["xyz", "\N", "\N", 1, "max_ast_depth", 2000, "\N", "\N", "\N", "\N"]] + assert system_settings_profile_elements(role_name="worker") == [ + ["\N", "\N", "worker", 0, "\N", "\N", "\N", "\N", "\N", "xyz"]] instance.query("REVOKE worker FROM robin") - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 120000000", user="robin") instance.query("ALTER ROLE worker SETTINGS NONE") instance.query("GRANT worker TO robin") assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 120000000", user="robin") assert system_settings_profile_elements(role_name="worker") == [] # Set settings and constraints via CREATE SETTINGS PROFILE ... TO granted role instance.query("ALTER SETTINGS PROFILE xyz TO worker") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000 TO worker\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [[ "xyz", "local directory", 2, 0, "['worker']", "[]" ]] + assert instance.query( + "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000 TO worker\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "100000001\n" + assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [["xyz", "local directory", 2, 0, "['worker']", "[]"]] instance.query("ALTER SETTINGS PROFILE xyz TO NONE") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" + assert instance.query( + "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [[ "xyz", "local directory", 2, 0, "[]", "[]" ]] + assert system_settings_profile("xyz") == [["xyz", "local directory", 2, 0, "[]", "[]"]] def test_inheritance(): instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY") instance.query("CREATE SETTINGS PROFILE alpha SETTINGS PROFILE xyz TO robin") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" - assert instance.query("SHOW CREATE SETTINGS PROFILE alpha") == "CREATE SETTINGS PROFILE alpha SETTINGS INHERIT xyz TO robin\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000002\n" - assert "Setting max_memory_usage should not be changed" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + assert instance.query( + "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" + assert instance.query( + "SHOW CREATE SETTINGS PROFILE alpha") == "CREATE SETTINGS PROFILE alpha SETTINGS INHERIT xyz TO robin\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "100000002\n" + assert "Setting max_memory_usage should not be changed" in instance.query_and_get_error( + "SET max_memory_usage = 80000000", user="robin") - assert system_settings_profile("xyz") == [[ "xyz", "local directory", 1, 0, "[]", "[]" ]] - assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000002, "\N", "\N", 1, "\N" ]] - assert system_settings_profile("alpha") == [[ "alpha", "local directory", 1, 0, "['robin']", "[]" ]] - assert system_settings_profile_elements(profile_name="alpha") == [[ "alpha", "\N", "\N", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] + assert system_settings_profile("xyz") == [["xyz", "local directory", 1, 0, "[]", "[]"]] + assert system_settings_profile_elements(profile_name="xyz") == [ + ["xyz", "\N", "\N", 0, "max_memory_usage", 100000002, "\N", "\N", 1, "\N"]] + assert system_settings_profile("alpha") == [["alpha", "local directory", 1, 0, "['robin']", "[]"]] + assert system_settings_profile_elements(profile_name="alpha") == [ + ["alpha", "\N", "\N", 0, "\N", "\N", "\N", "\N", "\N", "xyz"]] assert system_settings_profile_elements(user_name="robin") == [] def test_alter_and_drop(): - instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000003 MIN 90000000 MAX 110000000 TO robin") - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000003\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + instance.query( + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000003 MIN 90000000 MAX 110000000 TO robin") + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "100000003\n" + assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error( + "SET max_memory_usage = 80000000", user="robin") + assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin") instance.query("ALTER SETTINGS PROFILE xyz SETTINGS readonly=1") - assert "Cannot modify 'max_memory_usage' setting in readonly mode" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + assert "Cannot modify 'max_memory_usage' setting in readonly mode" in instance.query_and_get_error( + "SET max_memory_usage = 80000000", user="robin") instance.query("DROP SETTINGS PROFILE xyz") - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") @@ -151,25 +191,29 @@ def test_show_profiles(): instance.query("CREATE SETTINGS PROFILE xyz") assert instance.query("SHOW SETTINGS PROFILES") == "default\nreadonly\nxyz\n" assert instance.query("SHOW PROFILES") == "default\nreadonly\nxyz\n" - + assert instance.query("SHOW CREATE PROFILE xyz") == "CREATE SETTINGS PROFILE xyz\n" - assert instance.query("SHOW CREATE SETTINGS PROFILE default") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n" - assert instance.query("SHOW CREATE PROFILES") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n"\ - "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n"\ - "CREATE SETTINGS PROFILE xyz\n" - - expected_access = "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n"\ - "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n"\ + assert instance.query( + "SHOW CREATE SETTINGS PROFILE default") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n" + assert instance.query( + "SHOW CREATE PROFILES") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n" \ + "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n" \ + "CREATE SETTINGS PROFILE xyz\n" + + expected_access = "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, use_uncompressed_cache = 0, load_balancing = \\'random\\'\n" \ + "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n" \ "CREATE SETTINGS PROFILE xyz\n" assert expected_access in instance.query("SHOW ACCESS") def test_allow_ddl(): assert "Not enough privileges" in instance.query_and_get_error("CREATE TABLE tbl(a Int32) ENGINE=Log", user="robin") - assert "DDL queries are prohibited" in instance.query_and_get_error("CREATE TABLE tbl(a Int32) ENGINE=Log", settings={"allow_ddl":0}) + assert "DDL queries are prohibited" in instance.query_and_get_error("CREATE TABLE tbl(a Int32) ENGINE=Log", + settings={"allow_ddl": 0}) assert "Not enough privileges" in instance.query_and_get_error("GRANT CREATE ON tbl TO robin", user="robin") - assert "DDL queries are prohibited" in instance.query_and_get_error("GRANT CREATE ON tbl TO robin", settings={"allow_ddl":0}) + assert "DDL queries are prohibited" in instance.query_and_get_error("GRANT CREATE ON tbl TO robin", + settings={"allow_ddl": 0}) instance.query("GRANT CREATE ON tbl TO robin") instance.query("CREATE TABLE tbl(a Int32) ENGINE=Log", user="robin") @@ -179,14 +223,16 @@ def test_allow_ddl(): def test_allow_introspection(): assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')") assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin") - assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin", settings={"allow_introspection_functions":1}) + assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin", + settings={"allow_introspection_functions": 1}) assert "Introspection functions are disabled" in instance.query_and_get_error("GRANT demangle ON *.* TO robin") assert "Not enough privileges" in instance.query_and_get_error("GRANT demangle ON *.* TO robin", user="robin") - assert "Not enough privileges" in instance.query_and_get_error("GRANT demangle ON *.* TO robin", user="robin", settings={"allow_introspection_functions":1}) + assert "Not enough privileges" in instance.query_and_get_error("GRANT demangle ON *.* TO robin", user="robin", + settings={"allow_introspection_functions": 1}) - assert instance.query("SELECT demangle('a')", settings={"allow_introspection_functions":1}) == "signed char\n" - instance.query("GRANT demangle ON *.* TO robin", settings={"allow_introspection_functions":1}) + assert instance.query("SELECT demangle('a')", settings={"allow_introspection_functions": 1}) == "signed char\n" + instance.query("GRANT demangle ON *.* TO robin", settings={"allow_introspection_functions": 1}) assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')", user="robin") instance.query("ALTER USER robin SETTINGS allow_introspection_functions=1") @@ -201,5 +247,5 @@ def test_allow_introspection(): instance.query("DROP SETTINGS PROFILE xyz") assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')", user="robin") - instance.query("REVOKE demangle ON *.* FROM robin", settings={"allow_introspection_functions":1}) + instance.query("REVOKE demangle ON *.* FROM robin", settings={"allow_introspection_functions": 1}) assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin") diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 20613bde1bc..ed2a4e0140d 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -1,19 +1,14 @@ -import time -import pytest -import requests -from tempfile import NamedTemporaryFile -from helpers.hdfs_api import HDFSApi - import os +import pytest from helpers.cluster import ClickHouseCluster -import subprocess - +from helpers.hdfs_api import HDFSApi SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_hdfs=True, user_configs=[], main_configs=['configs/log_conf.xml']) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -27,21 +22,28 @@ def started_cluster(): finally: cluster.shutdown() + def test_read_write_storage(started_cluster): hdfs_api = HDFSApi("root") - node1.query("create table SimpleHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/simple_storage', 'TSV')") + node1.query( + "create table SimpleHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/simple_storage', 'TSV')") node1.query("insert into SimpleHDFSStorage values (1, 'Mark', 72.53)") assert hdfs_api.read_data("/simple_storage") == "1\tMark\t72.53\n" assert node1.query("select * from SimpleHDFSStorage") == "1\tMark\t72.53\n" + def test_read_write_storage_with_globs(started_cluster): hdfs_api = HDFSApi("root") - node1.query("create table HDFSStorageWithRange (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage{1..5}', 'TSV')") - node1.query("create table HDFSStorageWithEnum (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage{1,2,3,4,5}', 'TSV')") - node1.query("create table HDFSStorageWithQuestionMark (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage?', 'TSV')") - node1.query("create table HDFSStorageWithAsterisk (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage*', 'TSV')") + node1.query( + "create table HDFSStorageWithRange (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage{1..5}', 'TSV')") + node1.query( + "create table HDFSStorageWithEnum (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage{1,2,3,4,5}', 'TSV')") + node1.query( + "create table HDFSStorageWithQuestionMark (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage?', 'TSV')") + node1.query( + "create table HDFSStorageWithAsterisk (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage*', 'TSV')") for i in ["1", "2", "3"]: hdfs_api.write_data("/storage" + i, i + "\tMark\t72.53\n") @@ -73,6 +75,7 @@ def test_read_write_storage_with_globs(started_cluster): print ex assert "in readonly mode" in str(ex) + def test_read_write_table(started_cluster): hdfs_api = HDFSApi("root") data = "1\tSerialize\t555.222\n2\tData\t777.333\n" @@ -80,42 +83,50 @@ def test_read_write_table(started_cluster): assert hdfs_api.read_data("/simple_table_function") == data - assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')") == data + assert node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')") == data def test_write_table(started_cluster): hdfs_api = HDFSApi("root") - node1.query("create table OtherHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/other_storage', 'TSV')") + node1.query( + "create table OtherHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/other_storage', 'TSV')") node1.query("insert into OtherHDFSStorage values (10, 'tomas', 55.55), (11, 'jack', 32.54)") result = "10\ttomas\t55.55\n11\tjack\t32.54\n" assert hdfs_api.read_data("/other_storage") == result assert node1.query("select * from OtherHDFSStorage order by id") == result + def test_bad_hdfs_uri(started_cluster): try: - node1.query("create table BadStorage1 (id UInt32, name String, weight Float64) ENGINE = HDFS('hads:hgsdfs100500:9000/other_storage', 'TSV')") + node1.query( + "create table BadStorage1 (id UInt32, name String, weight Float64) ENGINE = HDFS('hads:hgsdfs100500:9000/other_storage', 'TSV')") except Exception as ex: print ex assert "Illegal HDFS URI" in str(ex) try: - node1.query("create table BadStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs100500:9000/other_storage', 'TSV')") + node1.query( + "create table BadStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs100500:9000/other_storage', 'TSV')") except Exception as ex: print ex assert "Unable to create builder to connect to HDFS" in str(ex) try: - node1.query("create table BadStorage3 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/<>', 'TSV')") + node1.query( + "create table BadStorage3 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/<>', 'TSV')") except Exception as ex: print ex assert "Unable to open HDFS file" in str(ex) + def test_globs_in_read_table(started_cluster): hdfs_api = HDFSApi("root") some_data = "1\tSerialize\t555.222\n2\tData\t777.333\n" globs_dir = "/dir_for_test_with_globs/" - files = ["dir1/dir_dir/file1", "dir2/file2", "simple_table_function", "dir/file", "some_dir/dir1/file", "some_dir/dir2/file", "some_dir/file", "table1_function", "table2_function", "table3_function"] + files = ["dir1/dir_dir/file1", "dir2/file2", "simple_table_function", "dir/file", "some_dir/dir1/file", + "some_dir/dir2/file", "some_dir/file", "table1_function", "table2_function", "table3_function"] for filename in files: hdfs_api.write_data(globs_dir + filename, some_data) @@ -135,8 +146,11 @@ def test_globs_in_read_table(started_cluster): for pattern, paths_amount, files_amount in test_requests: inside_table_func = "'hdfs://hdfs1:9000" + globs_dir + pattern + "', 'TSV', 'id UInt64, text String, number Float64'" assert node1.query("select * from hdfs(" + inside_table_func + ")") == paths_amount * some_data - assert node1.query("select count(distinct _path) from hdfs(" + inside_table_func + ")").rstrip() == str(paths_amount) - assert node1.query("select count(distinct _file) from hdfs(" + inside_table_func + ")").rstrip() == str(files_amount) + assert node1.query("select count(distinct _path) from hdfs(" + inside_table_func + ")").rstrip() == str( + paths_amount) + assert node1.query("select count(distinct _file) from hdfs(" + inside_table_func + ")").rstrip() == str( + files_amount) + def test_read_write_gzip_table(started_cluster): hdfs_api = HDFSApi("root") @@ -145,7 +159,9 @@ def test_read_write_gzip_table(started_cluster): assert hdfs_api.read_gzip_data("/simple_table_function.gz") == data - assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64')") == data + assert node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64')") == data + def test_read_write_gzip_table_with_parameter_gzip(started_cluster): hdfs_api = HDFSApi("root") @@ -154,7 +170,9 @@ def test_read_write_gzip_table_with_parameter_gzip(started_cluster): assert hdfs_api.read_gzip_data("/simple_table_function") == data - assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64', 'gzip')") == data + assert node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64', 'gzip')") == data + def test_read_write_table_with_parameter_none(started_cluster): hdfs_api = HDFSApi("root") @@ -163,7 +181,9 @@ def test_read_write_table_with_parameter_none(started_cluster): assert hdfs_api.read_data("/simple_table_function.gz") == data - assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'none')") == data + assert node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'none')") == data + def test_read_write_gzip_table_with_parameter_auto_gz(started_cluster): hdfs_api = HDFSApi("root") @@ -172,20 +192,25 @@ def test_read_write_gzip_table_with_parameter_auto_gz(started_cluster): assert hdfs_api.read_gzip_data("/simple_table_function.gz") == data - assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'auto')") == data + assert node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'auto')") == data + def test_write_gz_storage(started_cluster): hdfs_api = HDFSApi("root") - node1.query("create table GZHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage.gz', 'TSV')") + node1.query( + "create table GZHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage.gz', 'TSV')") node1.query("insert into GZHDFSStorage values (1, 'Mark', 72.53)") assert hdfs_api.read_gzip_data("/storage.gz") == "1\tMark\t72.53\n" assert node1.query("select * from GZHDFSStorage") == "1\tMark\t72.53\n" + def test_write_gzip_storage(started_cluster): hdfs_api = HDFSApi("root") - node1.query("create table GZIPHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/gzip_storage', 'TSV', 'gzip')") + node1.query( + "create table GZIPHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/gzip_storage', 'TSV', 'gzip')") node1.query("insert into GZIPHDFSStorage values (1, 'Mark', 72.53)") assert hdfs_api.read_gzip_data("/gzip_storage") == "1\tMark\t72.53\n" assert node1.query("select * from GZIPHDFSStorage") == "1\tMark\t72.53\n" diff --git a/tests/integration/test_storage_kafka/kafka_pb2.py b/tests/integration/test_storage_kafka/kafka_pb2.py index 79890682125..a9dcab1a85a 100644 --- a/tests/integration/test_storage_kafka/kafka_pb2.py +++ b/tests/integration/test_storage_kafka/kafka_pb2.py @@ -2,75 +2,70 @@ # source: clickhouse_path/format_schemas/kafka.proto import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - - DESCRIPTOR = _descriptor.FileDescriptor( - name='clickhouse_path/format_schemas/kafka.proto', - package='', - syntax='proto3', - serialized_pb=_b('\n*clickhouse_path/format_schemas/kafka.proto\"*\n\x0cKeyValuePair\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3') + name='clickhouse_path/format_schemas/kafka.proto', + package='', + syntax='proto3', + serialized_pb=_b( + '\n*clickhouse_path/format_schemas/kafka.proto\"*\n\x0cKeyValuePair\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - _KEYVALUEPAIR = _descriptor.Descriptor( - name='KeyValuePair', - full_name='KeyValuePair', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='KeyValuePair.key', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='KeyValuePair.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=46, - serialized_end=88, + name='KeyValuePair', + full_name='KeyValuePair', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='KeyValuePair.key', index=0, + number=1, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='KeyValuePair.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=46, + serialized_end=88, ) DESCRIPTOR.message_types_by_name['KeyValuePair'] = _KEYVALUEPAIR KeyValuePair = _reflection.GeneratedProtocolMessageType('KeyValuePair', (_message.Message,), dict( - DESCRIPTOR = _KEYVALUEPAIR, - __module__ = 'clickhouse_path.format_schemas.kafka_pb2' - # @@protoc_insertion_point(class_scope:KeyValuePair) - )) + DESCRIPTOR=_KEYVALUEPAIR, + __module__='clickhouse_path.format_schemas.kafka_pb2' + # @@protoc_insertion_point(class_scope:KeyValuePair) +)) _sym_db.RegisterMessage(KeyValuePair) - # @@protoc_insertion_point(module_scope) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 8f605e1bbd4..90422bf98e9 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1,29 +1,25 @@ +import json import os.path as p import random +import socket +import subprocess import threading import time -import pytest -import io - -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.client import QueryRuntimeException -from helpers.network import PartitionManager - -import json -import subprocess -import kafka.errors -from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection -from kafka.admin import NewTopic -from kafka.protocol.admin import DescribeGroupsResponse_v1, DescribeGroupsRequest_v1 -from kafka.protocol.group import MemberAssignment import avro.schema +import kafka.errors +import pytest from confluent.schemaregistry.client import CachedSchemaRegistryClient from confluent.schemaregistry.serializers.MessageSerializer import MessageSerializer - -import socket from google.protobuf.internal.encoder import _VarintBytes +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import TSV +from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection +from kafka.admin import NewTopic +from kafka.protocol.admin import DescribeGroupsRequest_v1 +from kafka.protocol.group import MemberAssignment """ protoc --version @@ -34,13 +30,12 @@ protoc --python_out=. kafka.proto """ import kafka_pb2 - # TODO: add test for run-time offset update in CH, if we manually update it on Kafka side. # TODO: add test for SELECT LIMIT is working. cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', - main_configs=['configs/kafka.xml', 'configs/log_conf.xml', 'configs/kafka_macros.xml' ], + main_configs=['configs/kafka.xml', 'configs/log_conf.xml', 'configs/kafka_macros.xml'], with_kafka=True, with_zookeeper=True, clickhouse_path_dir='clickhouse_path') @@ -80,6 +75,8 @@ def kafka_produce(topic, messages, timestamp=None): for message in messages: producer.send(topic=topic, value=message, timestamp_ms=timestamp) producer.flush() + + # print ("Produced {} messages for topic {}".format(len(messages), topic)) @@ -107,6 +104,7 @@ def kafka_produce_protobuf_messages(topic, start_index, num_messages): producer.flush() print("Produced {} messages for topic {}".format(num_messages, topic)) + def avro_confluent_message(schema_registry_client, value): # type: (CachedSchemaRegistryClient, dict) -> str @@ -124,9 +122,11 @@ def avro_confluent_message(schema_registry_client, value): }) return serializer.encode_record_with_schema('test_subject', schema, value) + @pytest.mark.timeout(180) def test_kafka_json_as_string(kafka_cluster): - kafka_produce('kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) + kafka_produce('kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', + '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) instance.query(''' CREATE TABLE test.kafka (field String) @@ -145,7 +145,9 @@ def test_kafka_json_as_string(kafka_cluster): {"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} ''' assert TSV(result) == TSV(expected) - assert instance.contains_in_log("Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows") + assert instance.contains_in_log( + "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows") + @pytest.mark.timeout(300) def test_kafka_formats(kafka_cluster): @@ -155,8 +157,8 @@ def test_kafka_formats(kafka_cluster): all_formats = { ## Text formats ## # dumped with clickhouse-client ... | perl -pe 's/\n/\\n/; s/\t/\\t/g;' - 'JSONEachRow' : { - 'data_sample' : [ + 'JSONEachRow': { + 'data_sample': [ '{"id":"0","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', '{"id":"1","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"2","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"3","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"4","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"5","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"6","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"7","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"8","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"9","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"10","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"11","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"12","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"13","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"14","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"15","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', '{"id":"0","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', @@ -164,16 +166,16 @@ def test_kafka_formats(kafka_cluster): 'supports_empty_value': True, }, # JSONAsString doesn't fit to that test, and tested separately - 'JSONCompactEachRow' : { - 'data_sample' : [ + 'JSONCompactEachRow': { + 'data_sample': [ '["0", 0, "AM", 0.5, 1]\n', '["1", 0, "AM", 0.5, 1]\n["2", 0, "AM", 0.5, 1]\n["3", 0, "AM", 0.5, 1]\n["4", 0, "AM", 0.5, 1]\n["5", 0, "AM", 0.5, 1]\n["6", 0, "AM", 0.5, 1]\n["7", 0, "AM", 0.5, 1]\n["8", 0, "AM", 0.5, 1]\n["9", 0, "AM", 0.5, 1]\n["10", 0, "AM", 0.5, 1]\n["11", 0, "AM", 0.5, 1]\n["12", 0, "AM", 0.5, 1]\n["13", 0, "AM", 0.5, 1]\n["14", 0, "AM", 0.5, 1]\n["15", 0, "AM", 0.5, 1]\n', '["0", 0, "AM", 0.5, 1]\n', ], 'supports_empty_value': True, }, - 'JSONCompactEachRowWithNamesAndTypes' : { - 'data_sample' : [ + 'JSONCompactEachRowWithNamesAndTypes': { + 'data_sample': [ '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["0", 0, "AM", 0.5, 1]\n', '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["1", 0, "AM", 0.5, 1]\n["2", 0, "AM", 0.5, 1]\n["3", 0, "AM", 0.5, 1]\n["4", 0, "AM", 0.5, 1]\n["5", 0, "AM", 0.5, 1]\n["6", 0, "AM", 0.5, 1]\n["7", 0, "AM", 0.5, 1]\n["8", 0, "AM", 0.5, 1]\n["9", 0, "AM", 0.5, 1]\n["10", 0, "AM", 0.5, 1]\n["11", 0, "AM", 0.5, 1]\n["12", 0, "AM", 0.5, 1]\n["13", 0, "AM", 0.5, 1]\n["14", 0, "AM", 0.5, 1]\n["15", 0, "AM", 0.5, 1]\n', '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["0", 0, "AM", 0.5, 1]\n', @@ -184,8 +186,8 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/IRowInputFormat.cpp:0: DB::IRowInputFormat::generate() @ 0x1de72710 in /usr/bin/clickhouse ], }, - 'TSKV' : { - 'data_sample' : [ + 'TSKV': { + 'data_sample': [ 'id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', 'id=1\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=2\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=3\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=4\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=5\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=6\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=7\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=8\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=9\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=10\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=11\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=12\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=13\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=14\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=15\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', 'id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', @@ -196,24 +198,24 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/IRowInputFormat.cpp:64: DB::IRowInputFormat::generate() @ 0x1de727cf in /usr/bin/clickhouse ], }, - 'CSV' : { - 'data_sample' : [ + 'CSV': { + 'data_sample': [ '0,0,"AM",0.5,1\n', '1,0,"AM",0.5,1\n2,0,"AM",0.5,1\n3,0,"AM",0.5,1\n4,0,"AM",0.5,1\n5,0,"AM",0.5,1\n6,0,"AM",0.5,1\n7,0,"AM",0.5,1\n8,0,"AM",0.5,1\n9,0,"AM",0.5,1\n10,0,"AM",0.5,1\n11,0,"AM",0.5,1\n12,0,"AM",0.5,1\n13,0,"AM",0.5,1\n14,0,"AM",0.5,1\n15,0,"AM",0.5,1\n', '0,0,"AM",0.5,1\n', ], 'supports_empty_value': True, }, - 'TSV' : { - 'data_sample' : [ + 'TSV': { + 'data_sample': [ '0\t0\tAM\t0.5\t1\n', '1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', '0\t0\tAM\t0.5\t1\n', ], 'supports_empty_value': True, }, - 'CSVWithNames' : { - 'data_sample' : [ + 'CSVWithNames': { + 'data_sample': [ '"id","blockNo","val1","val2","val3"\n0,0,"AM",0.5,1\n', '"id","blockNo","val1","val2","val3"\n1,0,"AM",0.5,1\n2,0,"AM",0.5,1\n3,0,"AM",0.5,1\n4,0,"AM",0.5,1\n5,0,"AM",0.5,1\n6,0,"AM",0.5,1\n7,0,"AM",0.5,1\n8,0,"AM",0.5,1\n9,0,"AM",0.5,1\n10,0,"AM",0.5,1\n11,0,"AM",0.5,1\n12,0,"AM",0.5,1\n13,0,"AM",0.5,1\n14,0,"AM",0.5,1\n15,0,"AM",0.5,1\n', '"id","blockNo","val1","val2","val3"\n0,0,"AM",0.5,1\n', @@ -227,24 +229,24 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/ISource.cpp:48: DB::ISource::work() @ 0x1dd79737 in /usr/bin/clickhouse ], }, - 'Values' : { - 'data_sample' : [ + 'Values': { + 'data_sample': [ "(0,0,'AM',0.5,1)", "(1,0,'AM',0.5,1),(2,0,'AM',0.5,1),(3,0,'AM',0.5,1),(4,0,'AM',0.5,1),(5,0,'AM',0.5,1),(6,0,'AM',0.5,1),(7,0,'AM',0.5,1),(8,0,'AM',0.5,1),(9,0,'AM',0.5,1),(10,0,'AM',0.5,1),(11,0,'AM',0.5,1),(12,0,'AM',0.5,1),(13,0,'AM',0.5,1),(14,0,'AM',0.5,1),(15,0,'AM',0.5,1)", "(0,0,'AM',0.5,1)", ], 'supports_empty_value': True, }, - 'TSVWithNames' : { - 'data_sample' : [ + 'TSVWithNames': { + 'data_sample': [ 'id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n', 'id\tblockNo\tval1\tval2\tval3\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', 'id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n', ], 'supports_empty_value': True, }, - 'TSVWithNamesAndTypes' : { - 'data_sample' : [ + 'TSVWithNamesAndTypes': { + 'data_sample': [ 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n', 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n', @@ -266,8 +268,8 @@ def test_kafka_formats(kafka_cluster): # ], # 'extra_settings': ", format_template_row='template_row.format'" # }, - 'Regexp' : { - 'data_sample' : [ + 'Regexp': { + 'data_sample': [ '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', '(id = 1, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 2, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 3, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 4, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 5, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 6, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 7, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 8, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 9, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 10, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 11, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 12, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 13, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 14, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 15, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', @@ -281,7 +283,7 @@ def test_kafka_formats(kafka_cluster): ## BINARY FORMATS # dumped with # clickhouse-client ... | xxd -ps -c 200 | tr -d '\n' | sed 's/\(..\)/\\x\1/g' - 'Native' : { + 'Native': { 'data_sample': [ '\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01', '\x05\x0f\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01', @@ -297,8 +299,8 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/ISource.cpp:48: DB::ISource::work() @ 0x1dd79737 in /usr/bin/clickhouse ], }, - 'MsgPack' : { - 'data_sample' : [ + 'MsgPack': { + 'data_sample': [ '\x00\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01', '\x01\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x02\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x03\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x04\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x05\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x06\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x07\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x08\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x09\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0a\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0b\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0c\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0d\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0e\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0f\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01', '\x00\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01', @@ -307,8 +309,8 @@ def test_kafka_formats(kafka_cluster): # coming from Processors/Formats/Impl/MsgPackRowInputFormat.cpp:170 ], }, - 'RowBinary' : { - 'data_sample' : [ + 'RowBinary': { + 'data_sample': [ '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', '\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', @@ -321,12 +323,12 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/Impl/BinaryRowInputFormat.cpp:22: DB::BinaryRowInputFormat::readRow(std::__1::vector::mutable_ptr, std::__1::allocator::mutable_ptr > >&, DB::RowReadExtension&) @ 0x1dea2c0b in /usr/bin/clickhouse ], }, - 'RowBinaryWithNamesAndTypes' : { - 'data_sample' : [ + 'RowBinaryWithNamesAndTypes': { + 'data_sample': [ '\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', '\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', '\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - #'' + # '' # !!! On empty message segfault: Address not mapped to object # /contrib/FastMemcpy/FastMemcpy.h:666: memcpy_fast @ 0x21742d65 in /usr/bin/clickhouse # /contrib/FastMemcpy/memcpy_wrapper.c:5: memcpy @ 0x21738235 in /usr/bin/clickhouse @@ -336,8 +338,8 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/Impl/BinaryRowInputFormat.cpp:22: DB::BinaryRowInputFormat::readRow(std::__1::vector::mutable_ptr, std::__1::allocator::mutable_ptr > >&, DB::RowReadExtension&) @ 0x1dea2c0b in /usr/bin/clickhouse ], }, - 'Protobuf' : { - 'data_sample' : [ + 'Protobuf': { + 'data_sample': [ '\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01', '\x0d\x08\x01\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x02\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x03\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x04\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x05\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x06\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x07\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x08\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x09\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0a\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0c\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0d\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0e\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0f\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01', '\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01', @@ -351,19 +353,19 @@ def test_kafka_formats(kafka_cluster): ], 'extra_settings': ", kafka_schema='test:TestMessage'" }, - 'ORC' : { - 'data_sample' : [ + 'ORC': { + 'data_sample': [ '\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', '\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x0f\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x7e\x25\x0e\x2e\x46\x43\x21\x46\x4b\x09\xad\x00\x06\x00\x33\x00\x00\x0a\x17\x0a\x03\x00\x00\x00\x12\x10\x08\x0f\x22\x0a\x0a\x02\x41\x4d\x12\x02\x41\x4d\x18\x3c\x50\x00\x3a\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x7e\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x66\x73\x3d\xd3\x00\x06\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x02\x10\x02\x18\x1e\x50\x00\x05\x00\x00\x0c\x00\x2b\x00\x00\x31\x32\x33\x34\x35\x36\x37\x38\x39\x31\x30\x31\x31\x31\x32\x31\x33\x31\x34\x31\x35\x09\x00\x00\x06\x01\x03\x02\x09\x00\x00\xc0\x0e\x00\x00\x07\x00\x00\x42\x00\x80\x05\x00\x00\x41\x4d\x0a\x00\x00\xe3\xe2\x42\x01\x00\x09\x00\x00\xc0\x0e\x02\x00\x05\x00\x00\x0c\x01\x94\x00\x00\x2d\xca\xc1\x0e\x80\x30\x08\x03\xd0\xc1\x60\x2e\xf3\x62\x76\x6a\xe2\x0e\xfe\xff\x57\x5a\x3b\x0f\xe4\x51\xe8\x68\xbd\x5d\x05\xe7\xf8\x34\x40\x3a\x6e\x59\xb1\x64\xe0\x91\xa9\xbf\xb1\x97\xd2\x95\x9d\x1e\xca\x55\x3a\x6d\xb4\xd2\xdd\x0b\x74\x9a\x74\xf7\x12\x39\xbd\x97\x7f\x7c\x06\xbb\xa6\x8d\x97\x17\xb4\x00\x00\xe3\x4a\xe6\x62\xe1\xe0\x0f\x60\xe0\xe2\xe3\xe0\x17\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\xe0\x57\xe2\xe0\x62\x34\x14\x62\xb4\x94\xd0\x02\x8a\xc8\x73\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\xc2\x06\x28\x26\xc4\x25\xca\xc1\x6f\xc4\xcb\xc5\x68\x20\xc4\x6c\xa0\x67\x2a\xc5\x6c\xae\x67\x0a\x14\xe6\x87\x1a\xc6\x24\xc0\x24\x21\x07\x32\x0c\x00\x4a\x01\x00\xe3\x60\x16\x58\xc3\x24\xc5\xcd\xc1\x2c\x30\x89\x51\xc2\x4b\xc1\x57\x83\x5f\x49\x83\x83\x47\x88\x95\x91\x89\x99\x85\x55\x8a\x3d\x29\x27\x3f\x39\xdb\x2f\x5f\x8a\x29\x33\x45\x8a\xa5\x2c\x31\xc7\x10\x4c\x1a\x81\x49\x63\x25\x26\x0e\x46\x20\x66\x07\x63\x36\x0e\x3e\x0d\x26\x03\x10\x9f\xd1\x80\xdf\x8a\x85\x83\x3f\x80\xc1\x8a\x8f\x83\x5f\x88\x8d\x83\x41\x80\x41\x82\x21\x80\x21\x82\xd5\x4a\x80\x83\x5f\x89\x83\x8b\xd1\x50\x88\xd1\x52\x42\x0b\x28\x22\x6f\x25\x04\x14\xe1\xe2\x62\x72\xf4\x15\x02\x62\x09\x1b\xa0\x98\x90\x95\x28\x07\xbf\x11\x2f\x17\xa3\x81\x10\xb3\x81\x9e\xa9\x14\xb3\xb9\x9e\x29\x50\x98\x1f\x6a\x18\x93\x00\x93\x84\x1c\xc8\x30\x87\x09\x7e\x1e\x0c\x00\x08\xa8\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x5d\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', '\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', - #'' + # '' # On empty message exception: IOError: File size too small, Stack trace (when copying this message, always include the lines below): # /src/Processors/Formats/Impl/ORCBlockInputFormat.cpp:36: DB::ORCBlockInputFormat::generate() @ 0x1df282a6 in /usr/bin/clickhouse # /src/Processors/ISource.cpp:48: DB::ISource::work() @ 0x1dd79737 in /usr/bin/clickhouse ], }, - 'CapnProto' : { - 'data_sample' : [ + 'CapnProto': { + 'data_sample': [ '\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00', '\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00', '\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00', @@ -376,7 +378,6 @@ def test_kafka_formats(kafka_cluster): 'extra_settings': ", kafka_schema='test:TestRecordStruct'" }, - # 'Parquet' : { # not working at all with Kafka: DB::Exception: IOError: Invalid Parquet file size is 0 bytes # /contrib/libcxx/include/exception:129: std::exception::capture() @ 0x15c33fe8 in /usr/bin/clickhouse @@ -416,13 +417,17 @@ def test_kafka_formats(kafka_cluster): # '\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x82\x03\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x72\x6f\x77\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x69\x64\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x6c\x6f\x6e\x67\x22\x7d\x2c\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x62\x6c\x6f\x63\x6b\x4e\x6f\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x69\x6e\x74\x22\x7d\x2c\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x76\x61\x6c\x31\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67\x22\x7d\x2c\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x76\x61\x6c\x32\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x66\x6c\x6f\x61\x74\x22\x7d\x2c\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x76\x61\x6c\x33\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x69\x6e\x74\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x73\x65\x4f\x7c\xd9\x33\xe1\x18\xdd\x30\xe8\x22\x2a\x58\x20\x6f\x02\x14\x00\x00\x04\x41\x4d\x00\x00\x00\x3f\x02\x73\x65\x4f\x7c\xd9\x33\xe1\x18\xdd\x30\xe8\x22\x2a\x58\x20\x6f', # ], # }, - 'AvroConfluent' : { + 'AvroConfluent': { 'data_sample': [ - avro_confluent_message(cluster.schema_registry_client, {'id':0L,'blockNo':0,'val1':unicode('AM'),'val2':0.5,"val3":1}), + avro_confluent_message(cluster.schema_registry_client, + {'id': 0L, 'blockNo': 0, 'val1': unicode('AM'), 'val2': 0.5, "val3": 1}), - ''.join(map(lambda id: avro_confluent_message(cluster.schema_registry_client, {'id':id,'blockNo':0,'val1':unicode('AM'),'val2':0.5,"val3":1}), range(1,16))), + ''.join(map(lambda id: avro_confluent_message(cluster.schema_registry_client, + {'id': id, 'blockNo': 0, 'val1': unicode('AM'), + 'val2': 0.5, "val3": 1}), range(1, 16))), - avro_confluent_message(cluster.schema_registry_client, {'id':0L,'blockNo':0,'val1':unicode('AM'),'val2':0.5,"val3":1}), + avro_confluent_message(cluster.schema_registry_client, + {'id': 0L, 'blockNo': 0, 'val1': unicode('AM'), 'val2': 0.5, "val3": 1}), ], 'extra_settings': ", format_avro_schema_registry_url='http://{}:{}'".format( cluster.schema_registry_host, @@ -476,7 +481,7 @@ def test_kafka_formats(kafka_cluster): for format_name, format_opts in all_formats.items(): print('Set up {}'.format(format_name)) - topic_name='format_tests_{}'.format(format_name) + topic_name = 'format_tests_{}'.format(format_name) data_sample = format_opts['data_sample'] data_prefix = [] # prepend empty value when supported @@ -503,15 +508,16 @@ def test_kafka_formats(kafka_cluster): CREATE MATERIALIZED VIEW test.kafka_{format_name}_mv Engine=Log AS SELECT *, _topic, _partition, _offset FROM test.kafka_{format_name}; - '''.format(topic_name=topic_name, format_name=format_name, extra_settings=format_opts.get('extra_settings') or '')) + '''.format(topic_name=topic_name, format_name=format_name, + extra_settings=format_opts.get('extra_settings') or '')) time.sleep(12) for format_name, format_opts in all_formats.items(): print('Checking {}'.format(format_name)) - topic_name='format_tests_{}'.format(format_name) + topic_name = 'format_tests_{}'.format(format_name) # shift offsets by 1 if format supports empty value - offsets = [1,2,3] if format_opts.get('supports_empty_value', False) else [0,1,2] + offsets = [1, 2, 3] if format_opts.get('supports_empty_value', False) else [0, 1, 2] result = instance.query('SELECT * FROM test.kafka_{format_name}_mv;'.format(format_name=format_name)) expected = '''\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} @@ -531,13 +537,13 @@ def test_kafka_formats(kafka_cluster): 14 0 AM 0.5 1 {topic_name} 0 {offset_1} 15 0 AM 0.5 1 {topic_name} 0 {offset_1} 0 0 AM 0.5 1 {topic_name} 0 {offset_2} -'''.format(topic_name=topic_name, offset_0 = offsets[0], offset_1 = offsets[1], offset_2 = offsets[2]) +'''.format(topic_name=topic_name, offset_0=offsets[0], offset_1=offsets[1], offset_2=offsets[2]) assert TSV(result) == TSV(expected), 'Proper result for format: {}'.format(format_name) # Since everything is async and shaky when receiving messages from Kafka, # we may want to try and check results multiple times in a loop. -def kafka_check_result(result, check=False, ref_file='test_kafka_json.reference'): +def kafka_check_result(result, check=False, ref_file='test_kafka_json.reference'): fpath = p.join(p.dirname(__file__), ref_file) with open(fpath) as reference: if check: @@ -545,6 +551,7 @@ def kafka_check_result(result, check=False, ref_file='test_kafka_json.reference else: return TSV(result) == TSV(reference) + # https://stackoverflow.com/a/57692111/1555175 def describe_consumer_group(name): client = BrokerConnection('localhost', 9092, socket.AF_INET) @@ -567,11 +574,12 @@ def describe_consumer_group(name): member_info['client_host'] = client_host member_topics_assignment = [] for (topic, partitions) in MemberAssignment.decode(member_assignment).assignment: - member_topics_assignment.append({'topic':topic, 'partitions':partitions}) + member_topics_assignment.append({'topic': topic, 'partitions': partitions}) member_info['assignment'] = member_topics_assignment res.append(member_info) return res + # Fixtures @pytest.fixture(scope="module") @@ -594,11 +602,13 @@ def kafka_setup_teardown(): # print("kafka is available - running test") yield # run test + # Tests @pytest.mark.timeout(180) def test_kafka_settings_old_syntax(kafka_cluster): - assert TSV(instance.query("SELECT * FROM system.macros WHERE macro like 'kafka%' ORDER BY macro", ignore_error=True)) == TSV('''kafka_broker kafka1 + assert TSV(instance.query("SELECT * FROM system.macros WHERE macro like 'kafka%' ORDER BY macro", + ignore_error=True)) == TSV('''kafka_broker kafka1 kafka_client_id instance kafka_format_json_each_row JSONEachRow kafka_group_name_new new @@ -631,6 +641,7 @@ kafka_topic_old old assert members[0]['client_id'] == u'ClickHouse-instance-test-kafka' # text_desc = kafka_cluster.exec_in_container(kafka_cluster.get_container_id('kafka1'),"kafka-consumer-groups --bootstrap-server localhost:9092 --describe --members --group old --verbose")) + @pytest.mark.timeout(180) def test_kafka_settings_new_syntax(kafka_cluster): instance.query(''' @@ -674,7 +685,8 @@ def test_kafka_settings_new_syntax(kafka_cluster): @pytest.mark.timeout(180) def test_kafka_issue11308(kafka_cluster): # Check that matview does respect Kafka SETTINGS - kafka_produce('issue11308', ['{"t": 123, "e": {"x": "woof"} }', '{"t": 123, "e": {"x": "woof"} }', '{"t": 124, "e": {"x": "test"} }']) + kafka_produce('issue11308', ['{"t": 123, "e": {"x": "woof"} }', '{"t": 123, "e": {"x": "woof"} }', + '{"t": 124, "e": {"x": "test"} }']) instance.query(''' CREATE TABLE test.persistent_kafka ( @@ -722,7 +734,7 @@ def test_kafka_issue11308(kafka_cluster): @pytest.mark.timeout(180) def test_kafka_issue4116(kafka_cluster): # Check that format_csv_delimiter parameter works now - as part of all available format settings. - kafka_produce('issue4116', ['1|foo', '2|bar', '42|answer','100|multi\n101|row\n103|message']) + kafka_produce('issue4116', ['1|foo', '2|bar', '42|answer', '100|multi\n101|row\n103|message']) instance.query(''' CREATE TABLE test.kafka (a UInt64, b String) @@ -750,7 +762,6 @@ def test_kafka_issue4116(kafka_cluster): @pytest.mark.timeout(180) def test_kafka_consumer_hang(kafka_cluster): - instance.query(''' DROP TABLE IF EXISTS test.kafka; DROP TABLE IF EXISTS test.view; @@ -781,7 +792,7 @@ def test_kafka_consumer_hang(kafka_cluster): # print("Attempt to drop") instance.query('DROP TABLE test.kafka') - #kafka_cluster.open_bash_shell('instance') + # kafka_cluster.open_bash_shell('instance') instance.query(''' DROP TABLE test.consumer; @@ -796,9 +807,9 @@ def test_kafka_consumer_hang(kafka_cluster): # 'dr'||'op' to avoid self matching assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 + @pytest.mark.timeout(180) def test_kafka_consumer_hang2(kafka_cluster): - instance.query(''' DROP TABLE IF EXISTS test.kafka; @@ -824,8 +835,8 @@ def test_kafka_consumer_hang2(kafka_cluster): # consumer, try to poll some data instance.query('SELECT * FROM test.kafka2') -#echo 'SELECT * FROM test.kafka; SELECT * FROM test.kafka2; DROP TABLE test.kafka;' | clickhouse client -mn & -# kafka_cluster.open_bash_shell('instance') + # echo 'SELECT * FROM test.kafka; SELECT * FROM test.kafka2; DROP TABLE test.kafka;' | clickhouse client -mn & + # kafka_cluster.open_bash_shell('instance') # first consumer has pending rebalance callback unprocessed (no poll after select) # one of those queries was failing because of @@ -834,7 +845,6 @@ def test_kafka_consumer_hang2(kafka_cluster): instance.query('DROP TABLE test.kafka') instance.query('DROP TABLE test.kafka2') - # from a user perspective: we expect no hanging 'drop' queries # 'dr'||'op' to avoid self matching assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 @@ -1123,7 +1133,7 @@ def test_kafka_flush_on_big_message(kafka_cluster): while True: result = instance.query('SELECT count() FROM test.view') - if int(result) == kafka_messages*batch_messages: + if int(result) == kafka_messages * batch_messages: break instance.query(''' @@ -1131,7 +1141,7 @@ def test_kafka_flush_on_big_message(kafka_cluster): DROP TABLE test.view; ''') - assert int(result) == kafka_messages*batch_messages, 'ClickHouse lost some messages: {}'.format(result) + assert int(result) == kafka_messages * batch_messages, 'ClickHouse lost some messages: {}'.format(result) @pytest.mark.timeout(180) @@ -1157,7 +1167,9 @@ def test_kafka_virtual_columns(kafka_cluster): result = '' while True: - result += instance.query('''SELECT _key, key, _topic, value, _offset, _partition, _timestamp = 0 ? '0000-00-00 00:00:00' : toString(_timestamp) AS _timestamp FROM test.kafka''', ignore_error=True) + result += instance.query( + '''SELECT _key, key, _topic, value, _offset, _partition, _timestamp = 0 ? '0000-00-00 00:00:00' : toString(_timestamp) AS _timestamp FROM test.kafka''', + ignore_error=True) if kafka_check_result(result, False, 'test_kafka_virtual1.reference'): break @@ -1258,6 +1270,7 @@ def test_kafka_produce_consume(kafka_cluster): ''') messages_num = 10000 + def insert(): values = [] for i in range(messages_num): @@ -1322,6 +1335,7 @@ def test_kafka_commit_on_block_write(kafka_cluster): cancel = threading.Event() i = [0] + def produce(): while not cancel.is_set(): messages = [] @@ -1373,7 +1387,6 @@ def test_kafka_commit_on_block_write(kafka_cluster): @pytest.mark.timeout(180) def test_kafka_virtual_columns2(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") topic_list = [] topic_list.append(NewTopic(name="virt2_0", num_partitions=2, replication_factor=1)) @@ -1396,13 +1409,17 @@ def test_kafka_virtual_columns2(kafka_cluster): producer = KafkaProducer(bootstrap_servers="localhost:9092") - producer.send(topic='virt2_0', value=json.dumps({'value': 1}), partition=0, key='k1', timestamp_ms=1577836801001, headers=[('content-encoding', b'base64')]) - producer.send(topic='virt2_0', value=json.dumps({'value': 2}), partition=0, key='k2', timestamp_ms=1577836802002, headers=[('empty_value', ''),('', 'empty name'), ('',''), ('repetition', '1'), ('repetition', '2')]) + producer.send(topic='virt2_0', value=json.dumps({'value': 1}), partition=0, key='k1', timestamp_ms=1577836801001, + headers=[('content-encoding', b'base64')]) + producer.send(topic='virt2_0', value=json.dumps({'value': 2}), partition=0, key='k2', timestamp_ms=1577836802002, + headers=[('empty_value', ''), ('', 'empty name'), ('', ''), ('repetition', '1'), ('repetition', '2')]) producer.flush() time.sleep(1) - producer.send(topic='virt2_0', value=json.dumps({'value': 3}), partition=1, key='k3', timestamp_ms=1577836803003, headers=[('b', 'b'),('a', 'a')]) - producer.send(topic='virt2_0', value=json.dumps({'value': 4}), partition=1, key='k4', timestamp_ms=1577836804004, headers=[('a', 'a'),('b', 'b')]) + producer.send(topic='virt2_0', value=json.dumps({'value': 3}), partition=1, key='k3', timestamp_ms=1577836803003, + headers=[('b', 'b'), ('a', 'a')]) + producer.send(topic='virt2_0', value=json.dumps({'value': 4}), partition=1, key='k4', timestamp_ms=1577836804004, + headers=[('a', 'a'), ('b', 'b')]) producer.flush() time.sleep(1) @@ -1418,7 +1435,7 @@ def test_kafka_virtual_columns2(kafka_cluster): time.sleep(10) members = describe_consumer_group('virt2') - #pprint.pprint(members) + # pprint.pprint(members) members[0]['client_id'] = u'ClickHouse-instance-test-kafka-0' members[1]['client_id'] = u'ClickHouse-instance-test-kafka-1' @@ -1438,7 +1455,6 @@ def test_kafka_virtual_columns2(kafka_cluster): assert TSV(result) == TSV(expected) - @pytest.mark.timeout(240) def test_kafka_produce_key_timestamp(kafka_cluster): instance.query(''' @@ -1464,10 +1480,16 @@ def test_kafka_produce_key_timestamp(kafka_cluster): SELECT key, value, inserted_key, toUnixTimestamp(inserted_timestamp), _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp) FROM test.kafka; ''') - instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(1,1,'k1',1577836801)) - instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(2,2,'k2',1577836802)) - instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({})),({},{},'{}',toDateTime({}))".format(3,3,'k3',1577836803,4,4,'k4',1577836804)) - instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(5,5,'k5',1577836805)) + instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(1, 1, 'k1', 1577836801)) + instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(2, 2, 'k2', 1577836802)) + instance.query( + "INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({})),({},{},'{}',toDateTime({}))".format(3, 3, + 'k3', + 1577836803, + 4, 4, + 'k4', + 1577836804)) + instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(5, 5, 'k5', 1577836805)) while int(instance.query("SELECT count() FROM test.view")) < 5: time.sleep(1) @@ -1487,7 +1509,6 @@ def test_kafka_produce_key_timestamp(kafka_cluster): assert TSV(result) == TSV(expected) - @pytest.mark.timeout(600) def test_kafka_flush_by_time(kafka_cluster): instance.query(''' @@ -1581,7 +1602,8 @@ def test_kafka_flush_by_block_size(kafka_cluster): ''') # Wait for Kafka engine to consume this data - while 1 != int(instance.query("SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'")): + while 1 != int(instance.query( + "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'")): time.sleep(0.5) cancel.set() @@ -1596,10 +1618,10 @@ def test_kafka_flush_by_block_size(kafka_cluster): DROP TABLE test.view; ''') - # 100 = first poll should return 100 messages (and rows) # not waiting for stream_flush_interval_ms - assert int(result) == 100, 'Messages from kafka should be flushed when block of size kafka_max_block_size is formed!' + assert int( + result) == 100, 'Messages from kafka should be flushed when block of size kafka_max_block_size is formed!' @pytest.mark.timeout(600) @@ -1631,7 +1653,7 @@ def test_kafka_lot_of_partitions_partial_commit_of_bulk(kafka_cluster): count = 0 for dummy_msg in range(1000): rows = [] - for dummy_row in range(random.randrange(3,10)): + for dummy_row in range(random.randrange(3, 10)): count = count + 1 rows.append(json.dumps({'key': count, 'value': count})) messages.append("\n".join(rows)) @@ -1641,17 +1663,17 @@ def test_kafka_lot_of_partitions_partial_commit_of_bulk(kafka_cluster): result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.view') print(result) - assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(count) ) + assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(count)) instance.query(''' DROP TABLE test.consumer; DROP TABLE test.view; ''') + @pytest.mark.timeout(1200) def test_kafka_rebalance(kafka_cluster): - - NUMBER_OF_CONSURRENT_CONSUMERS=11 + NUMBER_OF_CONSURRENT_CONSUMERS = 11 instance.query(''' DROP TABLE IF EXISTS test.destination; @@ -1669,9 +1691,9 @@ def test_kafka_rebalance(kafka_cluster): ORDER BY key; ''') - # kafka_cluster.open_bash_shell('instance') + # kafka_cluster.open_bash_shell('instance') - #time.sleep(2) + # time.sleep(2) admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") topic_list = [] @@ -1681,6 +1703,7 @@ def test_kafka_rebalance(kafka_cluster): cancel = threading.Event() msg_index = [0] + def produce(): while not cancel.is_set(): messages = [] @@ -1719,17 +1742,20 @@ def test_kafka_rebalance(kafka_cluster): FROM test.{0}; '''.format(table_name)) # kafka_cluster.open_bash_shell('instance') - while int(instance.query("SELECT count() FROM test.destination WHERE _consumed_by='{}'".format(table_name))) == 0: + while int( + instance.query("SELECT count() FROM test.destination WHERE _consumed_by='{}'".format(table_name))) == 0: print("Waiting for test.kafka_consumer{} to start consume".format(consumer_index)) time.sleep(1) cancel.set() # I leave last one working by intent (to finish consuming after all rebalances) - for consumer_index in range(NUMBER_OF_CONSURRENT_CONSUMERS-1): + for consumer_index in range(NUMBER_OF_CONSURRENT_CONSUMERS - 1): print("Dropping test.kafka_consumer{}".format(consumer_index)) instance.query('DROP TABLE IF EXISTS test.kafka_consumer{}'.format(consumer_index)) - while int(instance.query("SELECT count() FROM system.tables WHERE database='test' AND name='kafka_consumer{}'".format(consumer_index))) == 1: + while int(instance.query( + "SELECT count() FROM system.tables WHERE database='test' AND name='kafka_consumer{}'".format( + consumer_index))) == 1: time.sleep(1) # print(instance.query('SELECT count(), uniqExact(key), max(key) + 1 FROM test.destination')) @@ -1740,7 +1766,7 @@ def test_kafka_rebalance(kafka_cluster): if messages_consumed >= msg_index[0]: break time.sleep(1) - print("Waiting for finishing consuming (have {}, should be {})".format(messages_consumed,msg_index[0])) + print("Waiting for finishing consuming (have {}, should be {})".format(messages_consumed, msg_index[0])) print(instance.query('SELECT count(), uniqExact(key), max(key) + 1 FROM test.destination')) @@ -1782,9 +1808,10 @@ def test_kafka_rebalance(kafka_cluster): assert result == 1, 'Messages from kafka get duplicated!' + @pytest.mark.timeout(1200) def test_kafka_no_holes_when_write_suffix_failed(kafka_cluster): - messages = [json.dumps({'key': j+1, 'value': 'x' * 300}) for j in range(22)] + messages = [json.dumps({'key': j + 1, 'value': 'x' * 300}) for j in range(22)] kafka_produce('no_holes_when_write_suffix_failed', messages) instance.query(''' @@ -1864,13 +1891,13 @@ def test_exception_from_destructor(kafka_cluster): DROP TABLE test.kafka; ''') - #kafka_cluster.open_bash_shell('instance') + # kafka_cluster.open_bash_shell('instance') assert TSV(instance.query('SELECT 1')) == TSV('1') @pytest.mark.timeout(120) def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): - messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(1)] + messages = [json.dumps({'key': j + 1, 'value': j + 1}) for j in range(1)] kafka_produce('commits_of_unprocessed_messages_on_drop', messages) instance.query(''' @@ -1915,6 +1942,7 @@ def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): cancel = threading.Event() i = [2] + def produce(): while not cancel.is_set(): messages = [] @@ -1945,7 +1973,7 @@ def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): cancel.set() time.sleep(15) - #kafka_cluster.open_bash_shell('instance') + # kafka_cluster.open_bash_shell('instance') # SELECT key, _timestamp, _offset FROM test.destination where runningDifference(key) <> 1 ORDER BY key; result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.destination') @@ -1957,13 +1985,12 @@ def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): ''') kafka_thread.join() - assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(i[0]-1)), 'Missing data!' - + assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(i[0] - 1)), 'Missing data!' @pytest.mark.timeout(120) def test_bad_reschedule(kafka_cluster): - messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(20000)] + messages = [json.dumps({'key': j + 1, 'value': j + 1}) for j in range(20000)] kafka_produce('test_bad_reschedule', messages) instance.query(''' @@ -1997,7 +2024,7 @@ def test_bad_reschedule(kafka_cluster): @pytest.mark.timeout(1200) def test_kafka_duplicates_when_commit_failed(kafka_cluster): - messages = [json.dumps({'key': j+1, 'value': 'x' * 300}) for j in range(22)] + messages = [json.dumps({'key': j + 1, 'value': 'x' * 300}) for j in range(22)] kafka_produce('duplicates_when_commit_failed', messages) instance.query(''' @@ -2021,20 +2048,21 @@ def test_kafka_duplicates_when_commit_failed(kafka_cluster): WHERE NOT sleepEachRow(0.5); ''') - #print time.strftime("%m/%d/%Y %H:%M:%S") - time.sleep(12) # 5-6 sec to connect to kafka, do subscription, and fetch 20 rows, another 10 sec for MV, after that commit should happen + # print time.strftime("%m/%d/%Y %H:%M:%S") + time.sleep( + 12) # 5-6 sec to connect to kafka, do subscription, and fetch 20 rows, another 10 sec for MV, after that commit should happen - #print time.strftime("%m/%d/%Y %H:%M:%S") + # print time.strftime("%m/%d/%Y %H:%M:%S") kafka_cluster.pause_container('kafka1') # that timeout it VERY important, and picked after lot of experiments # when too low (<30sec) librdkafka will not report any timeout (alternative is to decrease the default session timeouts for librdkafka) # when too high (>50sec) broker will decide to remove us from the consumer group, and will start answering "Broker: Unknown member" time.sleep(40) - #print time.strftime("%m/%d/%Y %H:%M:%S") + # print time.strftime("%m/%d/%Y %H:%M:%S") kafka_cluster.unpause_container('kafka1') - #kafka_cluster.open_bash_shell('instance') + # kafka_cluster.open_bash_shell('instance') # connection restored and it will take a while until next block will be flushed # it takes years on CI :\ @@ -2057,6 +2085,7 @@ def test_kafka_duplicates_when_commit_failed(kafka_cluster): # impossible. So we have a duplicate in that scenario, but we report that situation properly. assert TSV(result) == TSV('42\t22\t22') + # if we came to partition end we will repeat polling until reaching kafka_max_block_size or flush_interval # that behavior is a bit quesionable - we can just take a bigger pauses between polls instead - # to do more job in a single pass, and give more rest for a thread. @@ -2087,7 +2116,7 @@ def test_premature_flush_on_eof(kafka_cluster): ORDER BY key; ''') - messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(1)] + messages = [json.dumps({'key': j + 1, 'value': j + 1}) for j in range(1)] kafka_produce('premature_flush_on_eof', messages) instance.query(''' @@ -2103,7 +2132,6 @@ def test_premature_flush_on_eof(kafka_cluster): FROM test.kafka; ''') - # all subscriptions/assignments done during select, so it start sending data to test.destination # immediately after creation of MV time.sleep(2) @@ -2125,7 +2153,7 @@ def test_premature_flush_on_eof(kafka_cluster): @pytest.mark.timeout(180) def test_kafka_unavailable(kafka_cluster): - messages = [json.dumps({'key': j+1, 'value': j+1}) for j in range(20000)] + messages = [json.dumps({'key': j + 1, 'value': j + 1}) for j in range(20000)] kafka_produce('test_bad_reschedule', messages) kafka_cluster.pause_container('kafka1') @@ -2163,6 +2191,7 @@ def test_kafka_unavailable(kafka_cluster): print("Waiting for consume") time.sleep(1) + @pytest.mark.timeout(180) def test_kafka_issue14202(kafka_cluster): instance.query(''' @@ -2184,7 +2213,8 @@ def test_kafka_issue14202(kafka_cluster): time.sleep(3) - instance.query('INSERT INTO test.kafka_q SELECT t, some_string FROM ( SELECT dt AS t, some_string FROM test.empty_table )') + instance.query( + 'INSERT INTO test.kafka_q SELECT t, some_string FROM ( SELECT dt AS t, some_string FROM test.empty_table )') # check instance is alive assert TSV(instance.query('SELECT 1')) == TSV('1') instance.query(''' @@ -2192,6 +2222,7 @@ def test_kafka_issue14202(kafka_cluster): DROP TABLE test.kafka_q; ''') + @pytest.mark.timeout(180) def test_kafka_csv_with_thread_per_consumer(kafka_cluster): instance.query(''' @@ -2219,6 +2250,7 @@ def test_kafka_csv_with_thread_per_consumer(kafka_cluster): kafka_check_result(result, True) + if __name__ == '__main__': cluster.start() raw_input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py index 90534949b0b..f75a9aac237 100644 --- a/tests/integration/test_storage_mongodb/test.py +++ b/tests/integration/test_storage_mongodb/test.py @@ -3,7 +3,6 @@ import pymongo import pytest from helpers.client import QueryRuntimeException - from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) @@ -36,7 +35,8 @@ def test_simple_select(started_cluster): data.append({'key': i, 'data': hex(i * i)}) simple_mongo_table.insert_many(data) - node.query("CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')") + node.query( + "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')") assert node.query("SELECT COUNT() FROM simple_mongo_table") == '100\n' assert node.query("SELECT sum(key) FROM simple_mongo_table") == str(sum(range(0, 100))) + '\n' @@ -51,10 +51,11 @@ def test_complex_data_type(started_cluster): incomplete_mongo_table = db['complex_table'] data = [] for i in range(0, 100): - data.append({'key': i, 'data': hex(i * i), 'dict': {'a' : i, 'b': str(i)}}) + data.append({'key': i, 'data': hex(i * i), 'dict': {'a': i, 'b': str(i)}}) incomplete_mongo_table.insert_many(data) - node.query("CREATE TABLE incomplete_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse')") + node.query( + "CREATE TABLE incomplete_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse')") assert node.query("SELECT COUNT() FROM incomplete_mongo_table") == '100\n' assert node.query("SELECT sum(key) FROM incomplete_mongo_table") == str(sum(range(0, 100))) + '\n' @@ -72,7 +73,8 @@ def test_incorrect_data_type(started_cluster): data.append({'key': i, 'data': hex(i * i), 'aaaa': 'Hello'}) strange_mongo_table.insert_many(data) - node.query("CREATE TABLE strange_mongo_table(key String, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')") + node.query( + "CREATE TABLE strange_mongo_table(key String, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')") with pytest.raises(QueryRuntimeException): node.query("SELECT COUNT() FROM strange_mongo_table") @@ -80,7 +82,8 @@ def test_incorrect_data_type(started_cluster): with pytest.raises(QueryRuntimeException): node.query("SELECT uniq(key) FROM strange_mongo_table") - node.query("CREATE TABLE strange_mongo_table2(key UInt64, data String, bbbb String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')") + node.query( + "CREATE TABLE strange_mongo_table2(key UInt64, data String, bbbb String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')") with pytest.raises(QueryRuntimeException): node.query("SELECT bbbb FROM strange_mongo_table2") diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index a2a5fe9263b..0b73866eaee 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -1,10 +1,8 @@ from contextlib import contextmanager -import pytest - ## sudo -H pip install PyMySQL import pymysql.cursors - +import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) @@ -43,7 +41,9 @@ def test_insert_select(started_cluster): node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse'); '''.format(table_name, table_name)) - node1.query("INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format(table_name)) + node1.query( + "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( + table_name)) assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == '10000' assert node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '30000' conn.close() @@ -57,8 +57,12 @@ def test_replace_select(started_cluster): node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse', 1); '''.format(table_name, table_name)) - node1.query("INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format(table_name)) - node1.query("INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format(table_name)) + node1.query( + "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( + table_name)) + node1.query( + "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( + table_name)) assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == '10000' assert node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '30000' conn.close() @@ -72,8 +76,12 @@ def test_insert_on_duplicate_select(started_cluster): node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse', 0, 'update money = money + values(money)'); '''.format(table_name, table_name)) - node1.query("INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format(table_name)) - node1.query("INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format(table_name)) + node1.query( + "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( + table_name)) + node1.query( + "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( + table_name)) assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == '10000' assert node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '60000' conn.close() @@ -86,13 +94,17 @@ def test_where(started_cluster): node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse'); '''.format(table_name, table_name)) - node1.query("INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format(table_name)) + node1.query( + "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( + table_name)) assert node1.query("SELECT count() FROM {} WHERE name LIKE '%name_%'".format(table_name)).rstrip() == '10000' assert node1.query("SELECT count() FROM {} WHERE name NOT LIKE '%tmp_%'".format(table_name)).rstrip() == '10000' assert node1.query("SELECT count() FROM {} WHERE money IN (1, 2, 3)".format(table_name)).rstrip() == '10000' assert node1.query("SELECT count() FROM {} WHERE money IN (1, 2, 4, 5, 6)".format(table_name)).rstrip() == '0' - assert node1.query("SELECT count() FROM {} WHERE money NOT IN (1, 2, 4, 5, 6)".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT count() FROM {} WHERE name LIKE concat('name_', toString(1))".format(table_name)).rstrip() == '1' + assert node1.query( + "SELECT count() FROM {} WHERE money NOT IN (1, 2, 4, 5, 6)".format(table_name)).rstrip() == '10000' + assert node1.query( + "SELECT count() FROM {} WHERE name LIKE concat('name_', toString(1))".format(table_name)).rstrip() == '1' conn.close() @@ -101,16 +113,19 @@ def test_table_function(started_cluster): create_mysql_table(conn, 'table_function') table_function = "mysql('mysql1:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('table_function') assert node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == '0' - node1.query("INSERT INTO {} (id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000)".format( - 'TABLE FUNCTION ' + table_function)) + node1.query( + "INSERT INTO {} (id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000)".format( + 'TABLE FUNCTION ' + table_function)) assert node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == '10000' assert node1.query("SELECT sum(c) FROM (" "SELECT count() as c FROM {} WHERE id % 3 == 0" " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 1" - " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 2)".format(table_function, table_function, + " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 2)".format(table_function, + table_function, table_function)).rstrip() == '10000' assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '30000' - node1.query("INSERT INTO {} (id, name, age, money) SELECT id + 100000, name, age, money FROM {}".format('TABLE FUNCTION ' + table_function, table_function)) + node1.query("INSERT INTO {} (id, name, age, money) SELECT id + 100000, name, age, money FROM {}".format( + 'TABLE FUNCTION ' + table_function, table_function)) assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '60000' conn.close() @@ -127,7 +142,6 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, source Enum8(' conn.close() - def get_mysql_conn(): conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.1', port=3308) return conn diff --git a/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py b/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py index bd03d3c21d6..6abc087dc75 100644 --- a/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py +++ b/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py @@ -6,72 +6,66 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - - DESCRIPTOR = _descriptor.FileDescriptor( - name='clickhouse_path/format_schemas/rabbitmq.proto', - package='', - syntax='proto3', - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n-clickhouse_path/format_schemas/rabbitmq.proto\"+\n\rKeyValueProto\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3' + name='clickhouse_path/format_schemas/rabbitmq.proto', + package='', + syntax='proto3', + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n-clickhouse_path/format_schemas/rabbitmq.proto\"+\n\rKeyValueProto\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3' ) - - - _KEYVALUEPROTO = _descriptor.Descriptor( - name='KeyValueProto', - full_name='KeyValueProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='KeyValueProto.key', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='value', full_name='KeyValueProto.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=49, - serialized_end=92, + name='KeyValueProto', + full_name='KeyValueProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='KeyValueProto.key', index=0, + number=1, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='KeyValueProto.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=49, + serialized_end=92, ) DESCRIPTOR.message_types_by_name['KeyValueProto'] = _KEYVALUEPROTO _sym_db.RegisterFileDescriptor(DESCRIPTOR) KeyValueProto = _reflection.GeneratedProtocolMessageType('KeyValueProto', (_message.Message,), { - 'DESCRIPTOR' : _KEYVALUEPROTO, - '__module__' : 'clickhouse_path.format_schemas.rabbitmq_pb2' - # @@protoc_insertion_point(class_scope:KeyValueProto) - }) + 'DESCRIPTOR': _KEYVALUEPROTO, + '__module__': 'clickhouse_path.format_schemas.rabbitmq_pb2' + # @@protoc_insertion_point(class_scope:KeyValueProto) +}) _sym_db.RegisterMessage(KeyValueProto) - # @@protoc_insertion_point(module_scope) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 370515956ea..4d892eaa72c 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1,29 +1,23 @@ +import json import os.path as p import random +import subprocess import threading import time -import pytest - from random import randrange -import pika -from sys import getdefaultencoding +import pika +import pytest +from google.protobuf.internal.encoder import _VarintBytes +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -from helpers.client import QueryRuntimeException -from helpers.network import PartitionManager - -import json -import subprocess - -from confluent.schemaregistry.serializers.MessageSerializer import MessageSerializer -from google.protobuf.internal.encoder import _VarintBytes import rabbitmq_pb2 cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', - main_configs=['configs/rabbitmq.xml','configs/log_conf.xml'], + main_configs=['configs/rabbitmq.xml', 'configs/log_conf.xml'], with_rabbitmq=True) # clickhouse_path_dir='clickhouse_path') rabbitmq_id = '' @@ -542,7 +536,6 @@ def test_rabbitmq_big_message(rabbitmq_cluster): @pytest.mark.timeout(420) def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): - NUM_CONSUMERS = 10 NUM_QUEUES = 2 @@ -570,6 +563,7 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): credentials = pika.PlainCredentials('root', 'clickhouse') parameters = pika.ConnectionParameters('localhost', 5672, '/', credentials) + def produce(): connection = pika.BlockingConnection(parameters) channel = connection.channel() @@ -582,7 +576,8 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): for message in messages: current += 1 mes_id = str(current) - channel.basic_publish(exchange='test_sharding', routing_key='', properties=pika.BasicProperties(message_id=mes_id), body=message) + channel.basic_publish(exchange='test_sharding', routing_key='', + properties=pika.BasicProperties(message_id=mes_id), body=message) connection.close() threads = [] @@ -612,7 +607,6 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): @pytest.mark.timeout(420) def test_rabbitmq_mv_combo(rabbitmq_cluster): - NUM_MV = 5 NUM_CONSUMERS = 4 @@ -646,6 +640,7 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): credentials = pika.PlainCredentials('root', 'clickhouse') parameters = pika.ConnectionParameters('localhost', 5672, '/', credentials) + def produce(): connection = pika.BlockingConnection(parameters) channel = connection.channel() @@ -656,7 +651,7 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): i[0] += 1 for msg_id in range(messages_num): channel.basic_publish(exchange='combo', routing_key='', - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) connection.close() threads = [] @@ -685,7 +680,6 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): DROP TABLE test.combo_{0}; '''.format(mv_id)) - assert int(result) == messages_num * threads_num * NUM_MV, 'ClickHouse lost some messages: {}'.format(result) @@ -727,6 +721,7 @@ def test_rabbitmq_insert(rabbitmq_cluster): raise insert_messages = [] + def onReceived(channel, method, properties, body): i = 0 insert_messages.append(body.decode()) @@ -762,7 +757,7 @@ def test_rabbitmq_insert_headers_exchange(rabbitmq_cluster): result = consumer.queue_declare(queue='') queue_name = result.method.queue consumer.queue_bind(exchange='insert_headers', queue=queue_name, routing_key="", - arguments={'x-match':'all', 'test':'insert', 'topic':'headers'}) + arguments={'x-match': 'all', 'test': 'insert', 'topic': 'headers'}) values = [] for i in range(50): @@ -780,6 +775,7 @@ def test_rabbitmq_insert_headers_exchange(rabbitmq_cluster): raise insert_messages = [] + def onReceived(channel, method, properties, body): i = 0 insert_messages.append(body.decode()) @@ -826,6 +822,7 @@ def test_rabbitmq_many_inserts(rabbitmq_cluster): ''') messages_num = 1000 + def insert(): values = [] for i in range(messages_num): @@ -904,6 +901,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): ''') messages_num = 100000 + def insert(): values = [] for i in range(messages_num): @@ -997,8 +995,8 @@ def test_rabbitmq_direct_exchange(rabbitmq_cluster): for message in messages: mes_id = str(randrange(10)) channel.basic_publish( - exchange='direct_exchange_testing', routing_key=key, - properties=pika.BasicProperties(message_id=mes_id), body=message) + exchange='direct_exchange_testing', routing_key=key, + properties=pika.BasicProperties(message_id=mes_id), body=message) connection.close() @@ -1065,7 +1063,7 @@ def test_rabbitmq_fanout_exchange(rabbitmq_cluster): for msg_id in range(messages_num): channel.basic_publish(exchange='fanout_exchange_testing', routing_key='', - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) connection.close() @@ -1160,7 +1158,7 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): current = 0 for msg_id in range(messages_num): channel.basic_publish(exchange='topic_exchange_testing', routing_key=key, - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) connection.close() @@ -1180,7 +1178,9 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): DROP TABLE test.destination; ''') - assert int(result) == messages_num * num_tables + messages_num * num_tables, 'ClickHouse lost some messages: {}'.format(result) + assert int( + result) == messages_num * num_tables + messages_num * num_tables, 'ClickHouse lost some messages: {}'.format( + result) @pytest.mark.timeout(420) @@ -1228,7 +1228,7 @@ def test_rabbitmq_hash_exchange(rabbitmq_cluster): i[0] += 1 for msg_id in range(messages_num): channel.basic_publish(exchange='hash_exchange_testing', routing_key=str(msg_id), - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) connection.close() threads = [] @@ -1267,7 +1267,6 @@ def test_rabbitmq_hash_exchange(rabbitmq_cluster): assert int(result2) == 4 * num_tables - @pytest.mark.timeout(420) def test_rabbitmq_multiple_bindings(rabbitmq_cluster): instance.query(''' @@ -1402,14 +1401,15 @@ def test_rabbitmq_headers_exchange(rabbitmq_cluster): messages.append(json.dumps({'key': i[0], 'value': i[0]})) i[0] += 1 - fields={} - fields['format']='logs' - fields['type']='report' - fields['year']='2020' + fields = {} + fields['format'] = 'logs' + fields['type'] = 'report' + fields['year'] = '2020' for msg_id in range(messages_num): channel.basic_publish(exchange='headers_exchange_testing', routing_key='', - properties=pika.BasicProperties(headers=fields, message_id=str(msg_id)), body=messages[msg_id]) + properties=pika.BasicProperties(headers=fields, message_id=str(msg_id)), + body=messages[msg_id]) connection.close() @@ -1535,7 +1535,8 @@ def test_rabbitmq_virtual_columns_with_materialized_view(rabbitmq_cluster): connection.close() - result = instance.query("SELECT key, value, exchange_name, SUBSTRING(channel_id, 1, 3), delivery_tag, redelivered FROM test.view ORDER BY delivery_tag") + result = instance.query( + "SELECT key, value, exchange_name, SUBSTRING(channel_id, 1, 3), delivery_tag, redelivered FROM test.view ORDER BY delivery_tag") expected = '''\ 0 0 virtuals_mv 1_0 1 0 1 1 virtuals_mv 1_0 2 0 @@ -1591,6 +1592,7 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): credentials = pika.PlainCredentials('root', 'clickhouse') parameters = pika.ConnectionParameters('localhost', 5672, '/', credentials) + def produce(): connection = pika.BlockingConnection(parameters) channel = connection.channel() @@ -1601,7 +1603,7 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): i[0] += 1 for msg_id in range(messages_num): channel.basic_publish(exchange='many_consumers', routing_key='', - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) connection.close() threads = [] @@ -1739,7 +1741,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): i += 1 for msg_id in range(messages_num): channel.basic_publish(exchange='consumer_reconnect', routing_key='', body=messages[msg_id], - properties=pika.BasicProperties(delivery_mode = 2, message_id=str(msg_id))) + properties=pika.BasicProperties(delivery_mode=2, message_id=str(msg_id))) connection.close() instance.query(''' @@ -1759,12 +1761,12 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): time.sleep(8) revive_rabbitmq() - #while int(instance.query('SELECT count() FROM test.view')) == 0: + # while int(instance.query('SELECT count() FROM test.view')) == 0: # time.sleep(0.1) - #kill_rabbitmq() - #time.sleep(2) - #revive_rabbitmq() + # kill_rabbitmq() + # time.sleep(2) + # revive_rabbitmq() while True: result = instance.query('SELECT count(DISTINCT key) FROM test.view') @@ -1808,6 +1810,7 @@ def test_rabbitmq_commit_on_block_write(rabbitmq_cluster): cancel = threading.Event() i = [0] + def produce(): while not cancel.is_set(): messages = [] diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index af530808e27..e39296525d0 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -1,14 +1,12 @@ import json import logging +import os import random import threading -import os - -import pytest - -from helpers.cluster import ClickHouseCluster, ClickHouseInstance import helpers.client +import pytest +from helpers.cluster import ClickHouseCluster, ClickHouseInstance logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) @@ -292,7 +290,7 @@ def test_s3_glob_scheherazade(cluster): cluster.minio_host, cluster.minio_port, bucket, path, table_format, values) run_query(instance, query) - jobs.append(threading.Thread(target=add_tales, args=(night, min(night+nights_per_job, 1001)))) + jobs.append(threading.Thread(target=add_tales, args=(night, min(night + nights_per_job, 1001)))) jobs[-1].start() for job in jobs: @@ -313,9 +311,10 @@ def run_s3_mock(cluster): def test_custom_auth_headers(cluster): - ping_response = cluster.exec_in_container(cluster.get_container_id('resolver'), ["curl", "-s", "http://resolver:8080"]) + ping_response = cluster.exec_in_container(cluster.get_container_id('resolver'), + ["curl", "-s", "http://resolver:8080"]) assert ping_response == 'OK', 'Expected "OK", but got "{}"'.format(ping_response) - + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" filename = "test.csv" get_query = "select * from s3('http://resolver:8080/{bucket}/{file}', 'CSV', '{table_format}')".format( @@ -345,4 +344,3 @@ def test_infinite_redirect(cluster): exception_raised = True finally: assert exception_raised - diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py index 15e5b1c0835..07e6f7331d9 100644 --- a/tests/integration/test_system_merges/test.py +++ b/tests/integration/test_system_merges/test.py @@ -1,19 +1,20 @@ -import pytest import threading import time + +import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml'], - with_zookeeper=True, - macros={"shard": 0, "replica": 1} ) + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + macros={"shard": 0, "replica": 1}) node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml'], - with_zookeeper=True, - macros={"shard": 0, "replica": 2} ) + main_configs=['configs/logs_config.xml'], + with_zookeeper=True, + macros={"shard": 0, "replica": 2}) @pytest.fixture(scope="module") @@ -29,7 +30,7 @@ def started_cluster(): def split_tsv(data): - return [ x.split("\t") for x in data.splitlines() ] + return [x.split("\t") for x in data.splitlines()] @pytest.mark.parametrize("replicated", [ @@ -62,8 +63,8 @@ def test_merge_simple(started_cluster, replicated): node1.query("INSERT INTO {name} VALUES (2)".format(name=name)) node1.query("INSERT INTO {name} VALUES (3)".format(name=name)) - parts = ["all_{}_{}_0".format(x, x) for x in range(starting_block, starting_block+3)] - result_part = "all_{}_{}_1".format(starting_block, starting_block+2) + parts = ["all_{}_{}_0".format(x, x) for x in range(starting_block, starting_block + 3)] + result_part = "all_{}_{}_1".format(starting_block, starting_block + 2) def optimize(): node1.query("OPTIMIZE TABLE {name}".format(name=name)) @@ -84,7 +85,8 @@ def test_merge_simple(started_cluster, replicated): table_name, "3", "['{}','{}','{}']".format(*parts), - "['{clickhouse}/{table_path}/{}/','{clickhouse}/{table_path}/{}/','{clickhouse}/{table_path}/{}/']".format(*parts, clickhouse=clickhouse_path, table_path=table_path), + "['{clickhouse}/{table_path}/{}/','{clickhouse}/{table_path}/{}/','{clickhouse}/{table_path}/{}/']".format( + *parts, clickhouse=clickhouse_path, table_path=table_path), result_part, "{clickhouse}/{table_path}/{}/".format(result_part, clickhouse=clickhouse_path, table_path=table_path), "all", @@ -129,7 +131,7 @@ def test_mutation_simple(started_cluster, replicated): node1.query("INSERT INTO {name} VALUES (1)".format(name=name)) part = "all_{}_{}_0".format(starting_block, starting_block) - result_part = "all_{}_{}_0_{}".format(starting_block, starting_block, starting_block+1) + result_part = "all_{}_{}_0_{}".format(starting_block, starting_block, starting_block + 1) def alter(): node1.query("ALTER TABLE {name} UPDATE a = 42 WHERE sleep(2) OR 1".format(name=name)) diff --git a/tests/integration/test_system_queries/test.py b/tests/integration/test_system_queries/test.py index db9cf5ccf3c..18a164da805 100644 --- a/tests/integration/test_system_queries/test.py +++ b/tests/integration/test_system_queries/test.py @@ -1,11 +1,10 @@ import os -import os.path as p import sys import time -import datetime -import pytest from contextlib import contextmanager +import pytest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV @@ -18,8 +17,10 @@ def started_cluster(): global instance try: cluster = ClickHouseCluster(__file__) - cluster.add_instance('ch1', main_configs=["configs/config.d/clusters_config.xml", "configs/config.d/query_log.xml"], - dictionaries=["configs/dictionaries/dictionary_clickhouse_cache.xml", "configs/dictionaries/dictionary_clickhouse_flat.xml"]) + cluster.add_instance('ch1', + main_configs=["configs/config.d/clusters_config.xml", "configs/config.d/query_log.xml"], + dictionaries=["configs/dictionaries/dictionary_clickhouse_cache.xml", + "configs/dictionaries/dictionary_clickhouse_flat.xml"]) cluster.start() instance = cluster.instances['ch1'] @@ -39,20 +40,32 @@ def test_SYSTEM_RELOAD_DICTIONARY(started_cluster): instance = cluster.instances['ch1'] instance.query("SYSTEM RELOAD DICTIONARIES") - assert TSV(instance.query("SELECT dictHas('clickhouse_flat', toUInt64(0)), dictHas('clickhouse_flat', toUInt64(1))")) == TSV("0\t0\n") + assert TSV(instance.query( + "SELECT dictHas('clickhouse_flat', toUInt64(0)), dictHas('clickhouse_flat', toUInt64(1))")) == TSV("0\t0\n") instance.query("INSERT INTO dictionary_source VALUES (0, 0)") - assert TSV(instance.query("SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictHas('clickhouse_cache', toUInt64(1))")) == TSV("0\t0\n") + assert TSV(instance.query( + "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictHas('clickhouse_cache', toUInt64(1))")) == TSV( + "0\t0\n") instance.query("INSERT INTO dictionary_source VALUES (1, 1)") - assert TSV(instance.query("SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictHas('clickhouse_cache', toUInt64(1))")) == TSV("0\t0\n") + assert TSV(instance.query( + "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictHas('clickhouse_cache', toUInt64(1))")) == TSV( + "0\t0\n") instance.query("SYSTEM RELOAD DICTIONARY clickhouse_cache") - assert TSV(instance.query("SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictGetUInt8('clickhouse_cache', 'value', toUInt64(1))")) == TSV("0\t1\n") - assert TSV(instance.query("SELECT dictHas('clickhouse_flat', toUInt64(0)), dictHas('clickhouse_flat', toUInt64(1))")) == TSV("0\t0\n") + assert TSV(instance.query( + "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictGetUInt8('clickhouse_cache', 'value', toUInt64(1))")) == TSV( + "0\t1\n") + assert TSV(instance.query( + "SELECT dictHas('clickhouse_flat', toUInt64(0)), dictHas('clickhouse_flat', toUInt64(1))")) == TSV("0\t0\n") instance.query("SYSTEM RELOAD DICTIONARIES") - assert TSV(instance.query("SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictGetUInt8('clickhouse_cache', 'value', toUInt64(1))")) == TSV("0\t1\n") - assert TSV(instance.query("SELECT dictGetUInt8('clickhouse_flat', 'value', toUInt64(0)), dictGetUInt8('clickhouse_flat', 'value', toUInt64(1))")) == TSV("0\t1\n") + assert TSV(instance.query( + "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictGetUInt8('clickhouse_cache', 'value', toUInt64(1))")) == TSV( + "0\t1\n") + assert TSV(instance.query( + "SELECT dictGetUInt8('clickhouse_flat', 'value', toUInt64(0)), dictGetUInt8('clickhouse_flat', 'value', toUInt64(1))")) == TSV( + "0\t1\n") def test_DROP_DNS_CACHE(started_cluster): @@ -61,13 +74,15 @@ def test_DROP_DNS_CACHE(started_cluster): instance.exec_in_container(['bash', '-c', 'echo 127.0.0.1 localhost > /etc/hosts'], privileged=True, user='root') instance.exec_in_container(['bash', '-c', 'echo ::1 localhost >> /etc/hosts'], privileged=True, user='root') - instance.exec_in_container(['bash', '-c', 'echo 127.255.255.255 lost_host >> /etc/hosts'], privileged=True, user='root') + instance.exec_in_container(['bash', '-c', 'echo 127.255.255.255 lost_host >> /etc/hosts'], privileged=True, + user='root') instance.query("SYSTEM DROP DNS CACHE") with pytest.raises(QueryRuntimeException): instance.query("SELECT * FROM remote('lost_host', 'system', 'one')") - instance.query("CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')") + instance.query( + "CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')") with pytest.raises(QueryRuntimeException): instance.query("SELECT * FROM distributed_lost_host") @@ -79,11 +94,12 @@ def test_DROP_DNS_CACHE(started_cluster): instance.query("SELECT * FROM remote('lost_host', 'system', 'one')") instance.query("SELECT * FROM distributed_lost_host") - assert TSV(instance.query("SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV("lost_host\t127.0.0.1\n") + assert TSV(instance.query( + "SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV( + "lost_host\t127.0.0.1\n") def test_RELOAD_CONFIG_AND_MACROS(started_cluster): - macros = "ro" create_macros = 'echo "{}" > /etc/clickhouse-server/config.d/macros.xml'.format(macros) @@ -120,6 +136,6 @@ def test_SYSTEM_FLUSH_LOGS(started_cluster): if __name__ == '__main__': with contextmanager(started_cluster)() as cluster: - for name, instance in cluster.instances.items(): - print name, instance.ip_address - raw_input("Cluster created, press any key to destroy...") + for name, instance in cluster.instances.items(): + print name, instance.ip_address + raw_input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_text_log_level/test.py b/tests/integration/test_text_log_level/test.py index 799ae9021cb..44679481266 100644 --- a/tests/integration/test_text_log_level/test.py +++ b/tests/integration/test_text_log_level/test.py @@ -2,14 +2,14 @@ # pylint: disable=redefined-outer-name import pytest - -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"]) + @pytest.fixture(scope='module') def start_cluster(): try: @@ -19,6 +19,7 @@ def start_cluster(): finally: cluster.shutdown() + def test_basic(start_cluster): with pytest.raises(QueryRuntimeException): # generates log with "Error" level diff --git a/tests/integration/test_timezone_config/test.py b/tests/integration/test_timezone_config/test.py index 22e11daa72e..ac12eddc709 100644 --- a/tests/integration/test_timezone_config/test.py +++ b/tests/integration/test_timezone_config/test.py @@ -5,6 +5,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=['configs/config.xml']) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -13,5 +14,6 @@ def start_cluster(): finally: cluster.shutdown() + def test_check_timezone_config(start_cluster): assert node.query("SELECT toDateTime(1111111111)") == "2005-03-17 17:58:31\n" diff --git a/tests/integration/test_tmp_policy/test.py b/tests/integration/test_tmp_policy/test.py index 728c62d82fb..f7174c3b695 100644 --- a/tests/integration/test_tmp_policy/test.py +++ b/tests/integration/test_tmp_policy/test.py @@ -8,8 +8,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', - main_configs=["configs/config.d/storage_configuration.xml"], - tmpfs=['/disk1:size=100M', '/disk2:size=100M']) + main_configs=["configs/config.d/storage_configuration.xml"], + tmpfs=['/disk1:size=100M', '/disk2:size=100M']) + @pytest.fixture(scope='module') def start_cluster(): @@ -19,11 +20,12 @@ def start_cluster(): finally: cluster.shutdown() + def test_different_versions(start_cluster): query = 'SELECT count(ignore(*)) FROM (SELECT * FROM system.numbers LIMIT 1e7) GROUP BY number' settings = { - 'max_bytes_before_external_group_by': 1<<20, - 'max_bytes_before_external_sort': 1<<20, + 'max_bytes_before_external_group_by': 1 << 20, + 'max_bytes_before_external_sort': 1 << 20, } assert node.contains_in_log('Setting up /disk1/ to store temporary data in it') diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index d0db52287ca..ad822bc6545 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1,29 +1,30 @@ -import json -import pytest import random -import re import string import threading import time from multiprocessing.dummy import Pool + +import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml', "configs/config.d/instant_moves.xml", "configs/config.d/storage_configuration.xml", "configs/config.d/cluster.xml",], - with_zookeeper=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 1} ) + main_configs=['configs/logs_config.xml', "configs/config.d/instant_moves.xml", + "configs/config.d/storage_configuration.xml", + "configs/config.d/cluster.xml", ], + with_zookeeper=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], + macros={"shard": 0, "replica": 1}) node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml', "configs/config.d/instant_moves.xml", "configs/config.d/storage_configuration.xml", "configs/config.d/cluster.xml",], - with_zookeeper=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 2} ) + main_configs=['configs/logs_config.xml', "configs/config.d/instant_moves.xml", + "configs/config.d/storage_configuration.xml", + "configs/config.d/cluster.xml", ], + with_zookeeper=True, + tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], + macros={"shard": 0, "replica": 2}) @pytest.fixture(scope="module") @@ -38,7 +39,7 @@ def started_cluster(): def get_random_string(length): symbols = bytes(string.ascii_uppercase + string.digits) - result_list = bytearray([0])*length + result_list = bytearray([0]) * length for i in range(length): result_list[i] = random.choice(symbols) return str(result_list) @@ -56,6 +57,7 @@ def get_used_disks_for_table(node, table_name, partition=None): ORDER BY modification_time """.format(name=table_name, suffix=suffix)).strip().split('\n') + def check_used_disks_with_retry(node, table_name, expected_disks, retries): for _ in range(retries): used_disks = get_used_disks_for_table(node, table_name) @@ -64,11 +66,14 @@ def check_used_disks_with_retry(node, table_name, expected_disks, retries): time.sleep(0.5) return False + @pytest.mark.parametrize("name,engine,alter", [ - ("mt_test_rule_with_invalid_destination","MergeTree()",0), - ("replicated_mt_test_rule_with_invalid_destination","ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')",0), - ("mt_test_rule_with_invalid_destination","MergeTree()",1), - ("replicated_mt_test_rule_with_invalid_destination","ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')",1), + ("mt_test_rule_with_invalid_destination", "MergeTree()", 0), + ("replicated_mt_test_rule_with_invalid_destination", + "ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')", 0), + ("mt_test_rule_with_invalid_destination", "MergeTree()", 1), + ("replicated_mt_test_rule_with_invalid_destination", + "ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')", 1), ]) def test_rule_with_invalid_destination(started_cluster, name, engine, alter): try: @@ -124,10 +129,12 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): @pytest.mark.parametrize("name,engine,positive", [ - ("mt_test_inserts_to_disk_do_not_work","MergeTree()",0), - ("replicated_mt_test_inserts_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_do_not_work', '1')",0), - ("mt_test_inserts_to_disk_work","MergeTree()",1), - ("replicated_mt_test_inserts_to_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_work', '1')",1), + ("mt_test_inserts_to_disk_do_not_work", "MergeTree()", 0), + ("replicated_mt_test_inserts_to_disk_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_do_not_work', '1')", 0), + ("mt_test_inserts_to_disk_work", "MergeTree()", 1), + ("replicated_mt_test_inserts_to_disk_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_work', '1')", 1), ]) def test_inserts_to_disk_work(started_cluster, name, engine, positive): try: @@ -141,9 +148,10 @@ def test_inserts_to_disk_work(started_cluster, name, engine, positive): SETTINGS storage_policy='small_jbod_with_external' """.format(name=name, engine=engine)) - data = [] # 10MB in total + data = [] # 10MB in total for i in range(10): - data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time()-1 if i > 0 or positive else time.time()+300))) # 1MB row + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format( + time.time() - 1 if i > 0 or positive else time.time() + 300))) # 1MB row node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) @@ -159,8 +167,9 @@ def test_inserts_to_disk_work(started_cluster, name, engine, positive): @pytest.mark.parametrize("name,engine", [ - ("mt_test_moves_work_after_storage_policy_change","MergeTree()"), - ("replicated_mt_test_moves_work_after_storage_policy_change","ReplicatedMergeTree('/clickhouse/test_moves_work_after_storage_policy_change', '1')"), + ("mt_test_moves_work_after_storage_policy_change", "MergeTree()"), + ("replicated_mt_test_moves_work_after_storage_policy_change", + "ReplicatedMergeTree('/clickhouse/test_moves_work_after_storage_policy_change', '1')"), ]) def test_moves_work_after_storage_policy_change(started_cluster, name, engine): try: @@ -172,10 +181,13 @@ def test_moves_work_after_storage_policy_change(started_cluster, name, engine): ORDER BY tuple() """.format(name=name, engine=engine)) - node1.query("""ALTER TABLE {name} MODIFY SETTING storage_policy='default_with_small_jbod_with_external'""".format(name=name)) + node1.query( + """ALTER TABLE {name} MODIFY SETTING storage_policy='default_with_small_jbod_with_external'""".format( + name=name)) # Second expression is preferred because d1 > now()-3600. - node1.query("""ALTER TABLE {name} MODIFY TTL now()-3600 TO DISK 'jbod1', d1 TO DISK 'external'""".format(name=name)) + node1.query( + """ALTER TABLE {name} MODIFY TTL now()-3600 TO DISK 'jbod1', d1 TO DISK 'external'""".format(name=name)) wait_expire_1 = 12 wait_expire_2 = 4 @@ -185,16 +197,16 @@ def test_moves_work_after_storage_policy_change(started_cluster, name, engine): wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) wait_expire_1_thread.start() - data = [] # 10MB in total + data = [] # 10MB in total for i in range(10): - data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1))) # 1MB row + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1))) # 1MB row node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} wait_expire_1_thread.join() - time.sleep(wait_expire_2/2) + time.sleep(wait_expire_2 / 2) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external"} @@ -206,10 +218,12 @@ def test_moves_work_after_storage_policy_change(started_cluster, name, engine): @pytest.mark.parametrize("name,engine,positive", [ - ("mt_test_moves_to_disk_do_not_work","MergeTree()",0), - ("replicated_mt_test_moves_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_do_not_work', '1')",0), - ("mt_test_moves_to_disk_work","MergeTree()",1), - ("replicated_mt_test_moves_to_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_work', '1')",1), + ("mt_test_moves_to_disk_do_not_work", "MergeTree()", 0), + ("replicated_mt_test_moves_to_disk_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_do_not_work', '1')", 0), + ("mt_test_moves_to_disk_work", "MergeTree()", 1), + ("replicated_mt_test_moves_to_disk_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_work', '1')", 1), ]) def test_moves_to_disk_work(started_cluster, name, engine, positive): try: @@ -231,16 +245,17 @@ def test_moves_to_disk_work(started_cluster, name, engine, positive): wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) wait_expire_1_thread.start() - data = [] # 10MB in total + data = [] # 10MB in total for i in range(10): - data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row + data.append(("'{}'".format(get_random_string(1024 * 1024)), + "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} wait_expire_1_thread.join() - time.sleep(wait_expire_2/2) + time.sleep(wait_expire_2 / 2) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} @@ -252,8 +267,9 @@ def test_moves_to_disk_work(started_cluster, name, engine, positive): @pytest.mark.parametrize("name,engine", [ - ("mt_test_moves_to_volume_work","MergeTree()"), - ("replicated_mt_test_moves_to_volume_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_volume_work', '1')"), + ("mt_test_moves_to_volume_work", "MergeTree()"), + ("replicated_mt_test_moves_to_volume_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_volume_work', '1')"), ]) def test_moves_to_volume_work(started_cluster, name, engine): try: @@ -276,11 +292,13 @@ def test_moves_to_volume_work(started_cluster, name, engine): wait_expire_1_thread.start() for p in range(2): - data = [] # 10MB in total + data = [] # 10MB in total for i in range(5): - data.append((str(p), "'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1))) # 1MB row + data.append( + (str(p), "'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1))) # 1MB row - node1.query("INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {'jbod1', 'jbod2'} @@ -298,10 +316,12 @@ def test_moves_to_volume_work(started_cluster, name, engine): @pytest.mark.parametrize("name,engine,positive", [ - ("mt_test_inserts_to_volume_do_not_work","MergeTree()",0), - ("replicated_mt_test_inserts_to_volume_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_do_not_work', '1')",0), - ("mt_test_inserts_to_volume_work","MergeTree()",1), - ("replicated_mt_test_inserts_to_volume_work","ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_work', '1')",1), + ("mt_test_inserts_to_volume_do_not_work", "MergeTree()", 0), + ("replicated_mt_test_inserts_to_volume_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_do_not_work', '1')", 0), + ("mt_test_inserts_to_volume_work", "MergeTree()", 1), + ("replicated_mt_test_inserts_to_volume_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_work', '1')", 1), ]) def test_inserts_to_volume_work(started_cluster, name, engine, positive): try: @@ -320,11 +340,13 @@ def test_inserts_to_volume_work(started_cluster, name, engine, positive): node1.query("SYSTEM STOP MOVES {name}".format(name=name)) for p in range(2): - data = [] # 20MB in total + data = [] # 20MB in total for i in range(10): - data.append((str(p), "'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time()-1 if i > 0 or positive else time.time()+300))) # 1MB row + data.append((str(p), "'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format( + time.time() - 1 if i > 0 or positive else time.time() + 300))) # 1MB row - node1.query("INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} @@ -336,8 +358,9 @@ def test_inserts_to_volume_work(started_cluster, name, engine, positive): @pytest.mark.parametrize("name,engine", [ - ("mt_test_moves_to_disk_eventually_work","MergeTree()"), - ("replicated_mt_test_moves_to_disk_eventually_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_eventually_work', '1')"), + ("mt_test_moves_to_disk_eventually_work", "MergeTree()"), + ("replicated_mt_test_moves_to_disk_eventually_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_eventually_work', '1')"), ]) def test_moves_to_disk_eventually_work(started_cluster, name, engine): try: @@ -351,9 +374,9 @@ def test_moves_to_disk_eventually_work(started_cluster, name, engine): SETTINGS storage_policy='only_jbod2' """.format(name=name_temp)) - data = [] # 35MB in total + data = [] # 35MB in total for i in range(35): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name_temp, ",".join(["('" + x + "')" for x in data]))) used_disks = get_used_disks_for_table(node1, name_temp) @@ -369,9 +392,10 @@ def test_moves_to_disk_eventually_work(started_cluster, name, engine): SETTINGS storage_policy='jbod1_with_jbod2' """.format(name=name, engine=engine)) - data = [] # 10MB in total + data = [] # 10MB in total for i in range(10): - data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time()-1))) # 1MB row + data.append( + ("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time() - 1))) # 1MB row node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) @@ -407,7 +431,8 @@ def test_replicated_download_ttl_info(started_cluster): node1.query("SYSTEM STOP MOVES {}".format(name)) - node2.query("INSERT INTO {} (s1, d1) VALUES ('{}', toDateTime({}))".format(name, get_random_string(1024 * 1024), time.time()-100)) + node2.query("INSERT INTO {} (s1, d1) VALUES ('{}', toDateTime({}))".format(name, get_random_string(1024 * 1024), + time.time() - 100)) assert set(get_used_disks_for_table(node2, name)) == {"external"} time.sleep(1) @@ -424,10 +449,12 @@ def test_replicated_download_ttl_info(started_cluster): @pytest.mark.parametrize("name,engine,positive", [ - ("mt_test_merges_to_disk_do_not_work","MergeTree()",0), - ("replicated_mt_test_merges_to_disk_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_do_not_work', '1')",0), - ("mt_test_merges_to_disk_work","MergeTree()",1), - ("replicated_mt_test_merges_to_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_work', '1')",1), + ("mt_test_merges_to_disk_do_not_work", "MergeTree()", 0), + ("replicated_mt_test_merges_to_disk_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_do_not_work', '1')", 0), + ("mt_test_merges_to_disk_work", "MergeTree()", 1), + ("replicated_mt_test_merges_to_disk_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_work', '1')", 1), ]) def test_merges_to_disk_work(started_cluster, name, engine, positive): try: @@ -453,18 +480,21 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): wait_expire_1_thread.start() for _ in range(2): - data = [] # 16MB in total + data = [] # 16MB in total for i in range(8): - data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row + data.append(("'{}'".format(get_random_string(1024 * 1024)), + "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - assert "2" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert "2" == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() wait_expire_1_thread.join() - time.sleep(wait_expire_2/2) + time.sleep(wait_expire_2 / 2) node1.query("SYSTEM START MERGES {}".format(name)) node1.query("OPTIMIZE TABLE {}".format(name)) @@ -472,7 +502,8 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): time.sleep(1) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} - assert "1" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert "1" == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "16" @@ -481,8 +512,9 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): @pytest.mark.parametrize("name,engine", [ - ("mt_test_merges_with_full_disk_work","MergeTree()"), - ("replicated_mt_test_merges_with_full_disk_work","ReplicatedMergeTree('/clickhouse/replicated_test_merges_with_full_disk_work', '1')"), + ("mt_test_merges_with_full_disk_work", "MergeTree()"), + ("replicated_mt_test_merges_with_full_disk_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_merges_with_full_disk_work', '1')"), ]) def test_merges_with_full_disk_work(started_cluster, name, engine): try: @@ -496,9 +528,9 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): SETTINGS storage_policy='only_jbod2' """.format(name=name_temp)) - data = [] # 35MB in total + data = [] # 35MB in total for i in range(35): - data.append(get_random_string(1024 * 1024)) # 1MB row + data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name_temp, ",".join(["('" + x + "')" for x in data]))) used_disks = get_used_disks_for_table(node1, name_temp) @@ -521,14 +553,16 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): wait_expire_1_thread.start() for _ in range(2): - data = [] # 12MB in total + data = [] # 12MB in total for i in range(6): - data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1))) # 1MB row - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1))) # 1MB row + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - assert "2" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert "2" == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() wait_expire_1_thread.join() @@ -536,8 +570,9 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): time.sleep(1) used_disks = get_used_disks_for_table(node1, name) - assert set(used_disks) == {"jbod1"} # Merged to the same disk against the rule. - assert "1" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert set(used_disks) == {"jbod1"} # Merged to the same disk against the rule. + assert "1" == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "12" @@ -547,10 +582,12 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): @pytest.mark.parametrize("name,engine,positive", [ - ("mt_test_moves_after_merges_do_not_work","MergeTree()",0), - ("replicated_mt_test_moves_after_merges_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_do_not_work', '1')",0), - ("mt_test_moves_after_merges_work","MergeTree()",1), - ("replicated_mt_test_moves_after_merges_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_work', '1')",1), + ("mt_test_moves_after_merges_do_not_work", "MergeTree()", 0), + ("replicated_mt_test_moves_after_merges_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_do_not_work', '1')", 0), + ("mt_test_moves_after_merges_work", "MergeTree()", 1), + ("replicated_mt_test_moves_after_merges_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_work', '1')", 1), ]) def test_moves_after_merges_work(started_cluster, name, engine, positive): try: @@ -573,21 +610,24 @@ def test_moves_after_merges_work(started_cluster, name, engine, positive): wait_expire_1_thread.start() for _ in range(2): - data = [] # 14MB in total + data = [] # 14MB in total for i in range(7): - data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row + data.append(("'{}'".format(get_random_string(1024 * 1024)), + "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) node1.query("OPTIMIZE TABLE {}".format(name)) time.sleep(1) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - assert "1" == node1.query("SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert "1" == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() wait_expire_1_thread.join() - time.sleep(wait_expire_2/2) + time.sleep(wait_expire_2 / 2) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} @@ -599,14 +639,18 @@ def test_moves_after_merges_work(started_cluster, name, engine, positive): @pytest.mark.parametrize("name,engine,positive,bar", [ - ("mt_test_moves_after_alter_do_not_work","MergeTree()",0,"DELETE"), - ("replicated_mt_test_moves_after_alter_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')",0,"DELETE"), - ("mt_test_moves_after_alter_work","MergeTree()",1,"DELETE"), - ("replicated_mt_test_moves_after_alter_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')",1,"DELETE"), - ("mt_test_moves_after_alter_do_not_work","MergeTree()",0,"TO DISK 'external'"), - ("replicated_mt_test_moves_after_alter_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')",0,"TO DISK 'external'"), - ("mt_test_moves_after_alter_work","MergeTree()",1,"TO DISK 'external'"), - ("replicated_mt_test_moves_after_alter_work","ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')",1,"TO DISK 'external'"), + ("mt_test_moves_after_alter_do_not_work", "MergeTree()", 0, "DELETE"), + ("replicated_mt_test_moves_after_alter_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')", 0, "DELETE"), + ("mt_test_moves_after_alter_work", "MergeTree()", 1, "DELETE"), + ("replicated_mt_test_moves_after_alter_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')", 1, "DELETE"), + ("mt_test_moves_after_alter_do_not_work", "MergeTree()", 0, "TO DISK 'external'"), + ("replicated_mt_test_moves_after_alter_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')", 0, "TO DISK 'external'"), + ("mt_test_moves_after_alter_work", "MergeTree()", 1, "TO DISK 'external'"), + ("replicated_mt_test_moves_after_alter_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')", 1, "TO DISK 'external'"), ]) def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive, bar): try: @@ -625,11 +669,12 @@ def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive, b ALTER TABLE {name} MODIFY TTL d1 + INTERVAL 15 MINUTE {bar} - """.format(name=name, bar=bar)) # That shall disable TTL. + """.format(name=name, bar=bar)) # That shall disable TTL. - data = [] # 10MB in total + data = [] # 10MB in total for i in range(10): - data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time()-1))) # 1MB row + data.append( + ("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time() - 1))) # 1MB row node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) @@ -642,8 +687,9 @@ def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive, b @pytest.mark.parametrize("name,engine", [ - ("mt_test_materialize_ttl_in_partition","MergeTree()"), - ("replicated_mt_test_materialize_ttl_in_partition","ReplicatedMergeTree('/clickhouse/test_materialize_ttl_in_partition', '1')"), + ("mt_test_materialize_ttl_in_partition", "MergeTree()"), + ("replicated_mt_test_materialize_ttl_in_partition", + "ReplicatedMergeTree('/clickhouse/test_materialize_ttl_in_partition', '1')"), ]) def test_materialize_ttl_in_partition(started_cluster, name, engine): try: @@ -658,10 +704,12 @@ def test_materialize_ttl_in_partition(started_cluster, name, engine): SETTINGS storage_policy='small_jbod_with_external' """.format(name=name, engine=engine)) - data = [] # 5MB in total + data = [] # 5MB in total for i in range(5): - data.append((str(i), "'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format(time.time()-1))) # 1MB row - node1.query("INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + data.append((str(i), "'{}'".format(get_random_string(1024 * 1024)), + "toDateTime({})".format(time.time() - 1))) # 1MB row + node1.query( + "INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) time.sleep(0.5) @@ -705,9 +753,11 @@ def test_materialize_ttl_in_partition(started_cluster, name, engine): @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_alter_multiple_ttls_positive", "MergeTree()", True), - ("mt_replicated_test_alter_multiple_ttls_positive", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", True), + ("mt_replicated_test_alter_multiple_ttls_positive", + "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", True), ("mt_test_alter_multiple_ttls_negative", "MergeTree()", False), - ("mt_replicated_test_alter_multiple_ttls_negative", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_negative', '1')", False), + ("mt_replicated_test_alter_multiple_ttls_negative", + "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_negative', '1')", False), ]) def test_alter_multiple_ttls(started_cluster, name, engine, positive): """Copyright 2019, Altinity LTD @@ -754,11 +804,11 @@ limitations under the License.""" """.format(name=name)) for p in range(3): - data = [] # 6MB in total + data = [] # 6MB in total now = time.time() for i in range(2): p1 = p - s1 = get_random_string(1024 * 1024) # 1MB + s1 = get_random_string(1024 * 1024) # 1MB d1 = now - 1 if i > 0 or positive else now + 300 data.append("({}, '{}', toDateTime({}))".format(p1, s1, d1)) node1.query("INSERT INTO {name} (p1, s1, d1) VALUES {values}".format(name=name, values=",".join(data))) @@ -790,7 +840,6 @@ limitations under the License.""" node1.query("OPTIMIZE TABLE {name} FINAL".format(name=name)) time.sleep(0.5) - if positive: assert rows_count == 0 else: @@ -801,8 +850,9 @@ limitations under the License.""" @pytest.mark.parametrize("name,engine", [ - ("concurrently_altering_ttl_mt","MergeTree()"), - ("concurrently_altering_ttl_replicated_mt","ReplicatedMergeTree('/clickhouse/concurrently_altering_ttl_replicated_mt', '1')",), + ("concurrently_altering_ttl_mt", "MergeTree()"), + ("concurrently_altering_ttl_replicated_mt", + "ReplicatedMergeTree('/clickhouse/concurrently_altering_ttl_replicated_mt', '1')",), ]) def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): try: @@ -816,7 +866,7 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): SETTINGS storage_policy='jbods_with_external' """.format(name=name, engine=engine)) - values = list({ random.randint(1, 1000000) for _ in range(0, 1000) }) + values = list({random.randint(1, 1000000) for _ in range(0, 1000)}) def insert(num): for i in range(num): @@ -831,7 +881,9 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): if move_type == "PART": for _ in range(10): try: - parts = node1.query("SELECT name from system.parts where table = '{}' and active = 1".format(name)).strip().split('\n') + parts = node1.query( + "SELECT name from system.parts where table = '{}' and active = 1".format( + name)).strip().split('\n') break except QueryRuntimeException: pass @@ -864,7 +916,9 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): for i in range(num): ttls = [] for j in range(random.randint(1, 10)): - what = random.choice(["TO VOLUME 'main'", "TO VOLUME 'external'", "TO DISK 'jbod1'", "TO DISK 'jbod2'", "TO DISK 'external'"]) + what = random.choice( + ["TO VOLUME 'main'", "TO VOLUME 'external'", "TO DISK 'jbod1'", "TO DISK 'jbod2'", + "TO DISK 'external'"]) when = "now()+{}".format(random.randint(-1, 5)) ttls.append("{} {}".format(when, what)) try: @@ -874,7 +928,7 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): def optimize_table(num): for i in range(num): - try: # optimize may throw after concurrent alter + try: # optimize may throw after concurrent alter node1.query("OPTIMIZE TABLE {} FINAL".format(name), settings={'optimize_throw_if_noop': '1'}) break except: @@ -897,6 +951,7 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): finally: node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) + @pytest.mark.skip(reason="Flacky test") @pytest.mark.parametrize("name,positive", [ ("test_double_move_while_select_negative", 0), @@ -914,9 +969,11 @@ def test_double_move_while_select(started_cluster, name, positive): SETTINGS storage_policy='small_jbod_with_external' """.format(name=name)) - node1.query("INSERT INTO {name} VALUES (1, '{string}')".format(name=name, string=get_random_string(10 * 1024 * 1024))) + node1.query( + "INSERT INTO {name} VALUES (1, '{string}')".format(name=name, string=get_random_string(10 * 1024 * 1024))) - parts = node1.query("SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)).splitlines() + parts = node1.query( + "SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)).splitlines() assert len(parts) == 1 node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) @@ -933,14 +990,18 @@ def test_double_move_while_select(started_cluster, name, positive): node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'jbod1'".format(name=name, part=parts[0])) # Fill jbod1 to force ClickHouse to make move of partition 1 to external. - node1.query("INSERT INTO {name} VALUES (2, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) - node1.query("INSERT INTO {name} VALUES (3, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) - node1.query("INSERT INTO {name} VALUES (4, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) + node1.query( + "INSERT INTO {name} VALUES (2, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) + node1.query( + "INSERT INTO {name} VALUES (3, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) + node1.query( + "INSERT INTO {name} VALUES (4, '{string}')".format(name=name, string=get_random_string(9 * 1024 * 1024))) time.sleep(1) # If SELECT locked old part on external, move shall fail. - assert node1.query("SELECT disk_name FROM system.parts WHERE table = '{name}' AND active = 1 AND name = '{part}'" + assert node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{name}' AND active = 1 AND name = '{part}'" .format(name=name, part=parts[0])).splitlines() == ["jbod1" if positive else "external"] thread.join() @@ -952,10 +1013,12 @@ def test_double_move_while_select(started_cluster, name, positive): @pytest.mark.parametrize("name,engine,positive", [ - ("mt_test_alter_with_merge_do_not_work","MergeTree()",0), - ("replicated_mt_test_alter_with_merge_do_not_work","ReplicatedMergeTree('/clickhouse/replicated_test_alter_with_merge_do_not_work', '1')",0), - ("mt_test_alter_with_merge_work","MergeTree()",1), - ("replicated_mt_test_alter_with_merge_work","ReplicatedMergeTree('/clickhouse/replicated_test_alter_with_merge_work', '1')",1), + ("mt_test_alter_with_merge_do_not_work", "MergeTree()", 0), + ("replicated_mt_test_alter_with_merge_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_alter_with_merge_do_not_work', '1')", 0), + ("mt_test_alter_with_merge_work", "MergeTree()", 1), + ("replicated_mt_test_alter_with_merge_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_alter_with_merge_work', '1')", 1), ]) def test_alter_with_merge_work(started_cluster, name, engine, positive): """Copyright 2019, Altinity LTD @@ -984,20 +1047,19 @@ limitations under the License.""" SETTINGS storage_policy='jbods_with_external', merge_with_ttl_timeout=0 """.format(name=name, engine=engine)) - def optimize_table(num): for i in range(num): - try: # optimize may throw after concurrent alter + try: # optimize may throw after concurrent alter node1.query("OPTIMIZE TABLE {} FINAL".format(name), settings={'optimize_throw_if_noop': '1'}) break except: pass for p in range(3): - data = [] # 6MB in total + data = [] # 6MB in total now = time.time() for i in range(2): - s1 = get_random_string(1024 * 1024) # 1MB + s1 = get_random_string(1024 * 1024) # 1MB d1 = now - 1 if positive else now + 300 data.append("('{}', toDateTime({}))".format(s1, d1)) values = ",".join(data) @@ -1017,7 +1079,8 @@ limitations under the License.""" optimize_table(20) - assert node1.query("SELECT count() FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)) == "1\n" + assert node1.query( + "SELECT count() FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)) == "1\n" time.sleep(5) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 0f201f569b3..6e988023951 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -1,11 +1,10 @@ import time -import pytest import helpers.client as client +import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) @@ -35,15 +34,15 @@ def test_ttl_columns(started_cluster): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( - ''' - CREATE TABLE test_ttl(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') - ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0; - '''.format(replica=node.name)) + ''' + CREATE TABLE test_ttl(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0; + '''.format(replica=node.name)) node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1, 1, 3)") node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2, 2, 4)") - time.sleep(1) # sleep to allow use ttl merge selector for second time + time.sleep(1) # sleep to allow use ttl merge selector for second time node1.query("OPTIMIZE TABLE test_ttl FINAL") expected = "1\t0\t0\n2\t0\t0\n" @@ -56,17 +55,18 @@ def test_merge_with_ttl_timeout(started_cluster): drop_table([node1, node2], table) for node in [node1, node2]: node.query( - ''' - CREATE TABLE {table}(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') - ORDER BY id PARTITION BY toDayOfMonth(date); - '''.format(replica=node.name, table=table)) + ''' + CREATE TABLE {table}(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') + ORDER BY id PARTITION BY toDayOfMonth(date); + '''.format(replica=node.name, table=table)) node1.query("SYSTEM STOP TTL MERGES {table}".format(table=table)) node2.query("SYSTEM STOP TTL MERGES {table}".format(table=table)) for i in range(1, 4): - node1.query("INSERT INTO {table} VALUES (toDateTime('2000-10-{day:02d} 10:00:00'), 1, 2, 3)".format(day=i, table=table)) + node1.query( + "INSERT INTO {table} VALUES (toDateTime('2000-10-{day:02d} 10:00:00'), 1, 2, 3)".format(day=i, table=table)) assert node1.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "0\n" assert node2.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "0\n" @@ -74,12 +74,13 @@ def test_merge_with_ttl_timeout(started_cluster): node1.query("SYSTEM START TTL MERGES {table}".format(table=table)) node2.query("SYSTEM START TTL MERGES {table}".format(table=table)) - time.sleep(15) # TTL merges shall happen. + time.sleep(15) # TTL merges shall happen. for i in range(1, 4): - node1.query("INSERT INTO {table} VALUES (toDateTime('2000-10-{day:02d} 10:00:00'), 1, 2, 3)".format(day=i, table=table)) + node1.query( + "INSERT INTO {table} VALUES (toDateTime('2000-10-{day:02d} 10:00:00'), 1, 2, 3)".format(day=i, table=table)) - time.sleep(15) # TTL merges shall not happen. + time.sleep(15) # TTL merges shall not happen. assert node1.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "3\n" assert node2.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "3\n" @@ -89,15 +90,15 @@ def test_ttl_many_columns(started_cluster): drop_table([node1, node2], "test_ttl_2") for node in [node1, node2]: node.query( - ''' - CREATE TABLE test_ttl_2(date DateTime, id UInt32, - a Int32 TTL date, - _idx Int32 TTL date, - _offset Int32 TTL date, - _partition Int32 TTL date) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_2', '{replica}') - ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0; - '''.format(replica=node.name)) + ''' + CREATE TABLE test_ttl_2(date DateTime, id UInt32, + a Int32 TTL date, + _idx Int32 TTL date, + _offset Int32 TTL date, + _partition Int32 TTL date) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_2', '{replica}') + ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0; + '''.format(replica=node.name)) node1.query("SYSTEM STOP TTL MERGES test_ttl_2") node2.query("SYSTEM STOP TTL MERGES test_ttl_2") @@ -114,7 +115,7 @@ def test_ttl_many_columns(started_cluster): node1.query("SYSTEM START TTL MERGES test_ttl_2") node2.query("SYSTEM START TTL MERGES test_ttl_2") - time.sleep(1) # sleep to allow use ttl merge selector for second time + time.sleep(1) # sleep to allow use ttl merge selector for second time node1.query("OPTIMIZE TABLE test_ttl_2 FINAL", timeout=5) node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) @@ -132,32 +133,34 @@ def test_ttl_table(started_cluster, delete_suffix): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( - ''' - CREATE TABLE test_ttl(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') - ORDER BY id PARTITION BY toDayOfMonth(date) - TTL date + INTERVAL 1 DAY {delete_suffix} SETTINGS merge_with_ttl_timeout=0; - '''.format(replica=node.name, delete_suffix=delete_suffix)) + ''' + CREATE TABLE test_ttl(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + ORDER BY id PARTITION BY toDayOfMonth(date) + TTL date + INTERVAL 1 DAY {delete_suffix} SETTINGS merge_with_ttl_timeout=0; + '''.format(replica=node.name, delete_suffix=delete_suffix)) node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1)") node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2)") - time.sleep(1) # sleep to allow use ttl merge selector for second time + time.sleep(1) # sleep to allow use ttl merge selector for second time node1.query("OPTIMIZE TABLE test_ttl FINAL") assert TSV(node1.query("SELECT * FROM test_ttl")) == TSV("") assert TSV(node2.query("SELECT * FROM test_ttl")) == TSV("") + def test_modify_ttl(started_cluster): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( - ''' - CREATE TABLE test_ttl(d DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') - ORDER BY id - '''.format(replica=node.name)) + ''' + CREATE TABLE test_ttl(d DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + ORDER BY id + '''.format(replica=node.name)) - node1.query("INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)") + node1.query( + "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)") node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2") @@ -169,17 +172,19 @@ def test_modify_ttl(started_cluster): node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2") assert node2.query("SELECT id FROM test_ttl") == "" + def test_modify_column_ttl(started_cluster): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( - ''' - CREATE TABLE test_ttl(d DateTime, id UInt32 DEFAULT 42) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') - ORDER BY d - '''.format(replica=node.name)) + ''' + CREATE TABLE test_ttl(d DateTime, id UInt32 DEFAULT 42) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + ORDER BY d + '''.format(replica=node.name)) - node1.query("INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)") + node1.query( + "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)") node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2") @@ -191,6 +196,7 @@ def test_modify_column_ttl(started_cluster): node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2") assert node2.query("SELECT id FROM test_ttl") == "42\n42\n42\n" + def test_ttl_double_delete_rule_returns_error(started_cluster): drop_table([node1, node2], "test_ttl") try: @@ -206,6 +212,7 @@ def test_ttl_double_delete_rule_returns_error(started_cluster): except: assert False + @pytest.mark.parametrize("name,engine", [ ("test_ttl_alter_delete", "MergeTree()"), ("test_replicated_ttl_alter_delete", "ReplicatedMergeTree('/clickhouse/test_replicated_ttl_alter_delete', '1')"), @@ -238,21 +245,24 @@ limitations under the License.""" break except: time.sleep(0.5) + node1.query( - """ - CREATE TABLE {name} ( - s1 String, - d1 DateTime - ) ENGINE = {engine} - ORDER BY tuple() - TTL d1 + INTERVAL 1 DAY DELETE - """.format(name=name, engine=engine)) + """ + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 + INTERVAL 1 DAY DELETE + """.format(name=name, engine=engine)) node1.query("""ALTER TABLE {name} MODIFY COLUMN s1 String TTL d1 + INTERVAL 1 SECOND""".format(name=name)) node1.query("""ALTER TABLE {name} ADD COLUMN b1 Int32""".format(name=name)) - node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello1', 1, toDateTime({time}))""".format(name=name, time=time.time())) - node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello2', 2, toDateTime({time}))""".format(name=name, time=time.time() + 360)) + node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello1', 1, toDateTime({time}))""".format(name=name, + time=time.time())) + node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello2', 2, toDateTime({time}))""".format(name=name, + time=time.time() + 360)) time.sleep(1) @@ -261,7 +271,8 @@ limitations under the License.""" assert r == ["\t1", "hello2\t2"] node1.query("""ALTER TABLE {name} MODIFY COLUMN b1 Int32 TTL d1""".format(name=name)) - node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello3', 3, toDateTime({time}))""".format(name=name, time=time.time())) + node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello3', 3, toDateTime({time}))""".format(name=name, + time=time.time())) time.sleep(1) diff --git a/tests/integration/test_union_header/test.py b/tests/integration/test_union_header/test.py index 7b671f03386..edbf4dddecf 100644 --- a/tests/integration/test_union_header/test.py +++ b/tests/integration/test_union_header/test.py @@ -14,7 +14,6 @@ def started_cluster(): cluster.start() for node in (node1, node2): - node.query(''' CREATE TABLE default.t1_local ( diff --git a/tests/integration/test_user_directories/test.py b/tests/integration/test_user_directories/test.py index 0a6e037904e..84547293f0d 100644 --- a/tests/integration/test_user_directories/test.py +++ b/tests/integration/test_user_directories/test.py @@ -1,5 +1,6 @@ -import pytest import os + +import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV @@ -7,6 +8,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', stay_alive=True) + @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: @@ -22,42 +24,56 @@ def started_cluster(): def test_old_style(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/old_style.xml"), '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/old_style.xml"), + '/etc/clickhouse-server/config.d/z.xml') node.restart_clickhouse() - assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users2.xml"}', 1], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access2\\\\/"}', 2]]) + assert node.query("SELECT * FROM system.user_directories") == TSV( + [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users2.xml"}', 1], + ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access2\\\\/"}', 2]]) def test_local_directories(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/local_directories.xml"), '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/local_directories.xml"), + '/etc/clickhouse-server/config.d/z.xml') node.restart_clickhouse() - assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users3.xml"}', 1], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3\\\\/"}', 2], - ["local directory (ro)", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3-ro\\\\/","readonly":true}', 3]]) + assert node.query("SELECT * FROM system.user_directories") == TSV( + [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users3.xml"}', 1], + ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3\\\\/"}', 2], + ["local directory (ro)", "local directory", + '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3-ro\\\\/","readonly":true}', 3]]) def test_relative_path(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/relative_path.xml"), '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/relative_path.xml"), + '/etc/clickhouse-server/config.d/z.xml') node.restart_clickhouse() - assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users4.xml"}', 1]]) + assert node.query("SELECT * FROM system.user_directories") == TSV( + [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users4.xml"}', 1]]) def test_memory(): node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/memory.xml"), '/etc/clickhouse-server/config.d/z.xml') node.restart_clickhouse() - assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users5.xml"}', 1], - ["memory", "memory", '{}', 2]]) + assert node.query("SELECT * FROM system.user_directories") == TSV( + [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users5.xml"}', 1], + ["memory", "memory", '{}', 2]]) + def test_mixed_style(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/mixed_style.xml"), '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/mixed_style.xml"), + '/etc/clickhouse-server/config.d/z.xml') node.restart_clickhouse() - assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users6.xml"}', 1], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6\\\\/"}', 2], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6a\\\\/"}', 3], - ["memory", "memory", '{}', 4]]) + assert node.query("SELECT * FROM system.user_directories") == TSV( + [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users6.xml"}', 1], + ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6\\\\/"}', 2], + ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6a\\\\/"}', 3], + ["memory", "memory", '{}', 4]]) + def test_duplicates(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/duplicates.xml"), '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/duplicates.xml"), + '/etc/clickhouse-server/config.d/z.xml') node.restart_clickhouse() - assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users7.xml"}', 1], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access7\\\\/"}', 2]]) + assert node.query("SELECT * FROM system.user_directories") == TSV( + [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users7.xml"}', 1], + ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access7\\\\/"}', 2]]) diff --git a/tests/integration/test_user_ip_restrictions/test.py b/tests/integration/test_user_ip_restrictions/test.py index aee0819fe95..1f28fbde069 100644 --- a/tests/integration/test_user_ip_restrictions/test.py +++ b/tests/integration/test_user_ip_restrictions/test.py @@ -1,22 +1,26 @@ -import time import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) -node_ipv4 = cluster.add_instance('node_ipv4', main_configs=[], user_configs=['configs/users_ipv4.xml'], ipv4_address='10.5.172.77') +node_ipv4 = cluster.add_instance('node_ipv4', main_configs=[], user_configs=['configs/users_ipv4.xml'], + ipv4_address='10.5.172.77') client_ipv4_ok = cluster.add_instance('client_ipv4_ok', main_configs=[], user_configs=[], ipv4_address='10.5.172.10') -client_ipv4_ok_direct = cluster.add_instance('client_ipv4_ok_direct', main_configs=[], user_configs=[], ipv4_address='10.5.173.1') -client_ipv4_ok_full_mask = cluster.add_instance('client_ipv4_ok_full_mask', main_configs=[], user_configs=[], ipv4_address='10.5.175.77') +client_ipv4_ok_direct = cluster.add_instance('client_ipv4_ok_direct', main_configs=[], user_configs=[], + ipv4_address='10.5.173.1') +client_ipv4_ok_full_mask = cluster.add_instance('client_ipv4_ok_full_mask', main_configs=[], user_configs=[], + ipv4_address='10.5.175.77') client_ipv4_bad = cluster.add_instance('client_ipv4_bad', main_configs=[], user_configs=[], ipv4_address='10.5.173.10') -node_ipv6 = cluster.add_instance('node_ipv6', main_configs=["configs/config_ipv6.xml"], user_configs=['configs/users_ipv6.xml'], ipv6_address='2001:3984:3989::1:1000') -client_ipv6_ok = cluster.add_instance('client_ipv6_ok', main_configs=[], user_configs=[], ipv6_address='2001:3984:3989::5555') -client_ipv6_ok_direct = cluster.add_instance('client_ipv6_ok_direct', main_configs=[], user_configs=[], ipv6_address='2001:3984:3989::1:1111') -client_ipv6_bad = cluster.add_instance('client_ipv6_bad', main_configs=[], user_configs=[], ipv6_address='2001:3984:3989::1:1112') +node_ipv6 = cluster.add_instance('node_ipv6', main_configs=["configs/config_ipv6.xml"], + user_configs=['configs/users_ipv6.xml'], ipv6_address='2001:3984:3989::1:1000') +client_ipv6_ok = cluster.add_instance('client_ipv6_ok', main_configs=[], user_configs=[], + ipv6_address='2001:3984:3989::5555') +client_ipv6_ok_direct = cluster.add_instance('client_ipv6_ok_direct', main_configs=[], user_configs=[], + ipv6_address='2001:3984:3989::1:1111') +client_ipv6_bad = cluster.add_instance('client_ipv6_bad', main_configs=[], user_configs=[], + ipv6_address='2001:3984:3989::1:1112') @pytest.fixture(scope="module") @@ -31,42 +35,57 @@ def setup_cluster(): def test_ipv4(setup_cluster): try: - client_ipv4_ok.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, user='root') + client_ipv4_ok.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, + user='root') except Exception as ex: assert False, "allowed client with 10.5.172.10 cannot connect to server with allowed mask '10.5.172.0/24'" try: - client_ipv4_ok_direct.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, user='root') + client_ipv4_ok_direct.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, + user='root') except Exception as ex: assert False, "allowed client with 10.5.173.1 cannot connect to server with allowed ip '10.5.173.1'" try: - client_ipv4_ok_full_mask.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, user='root') + client_ipv4_ok_full_mask.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, + user='root') except Exception as ex: assert False, "allowed client with 10.5.175.77 cannot connect to server with allowed ip '10.5.175.0/255.255.255.0'" try: - client_ipv4_bad.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, user='root') + client_ipv4_bad.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, + user='root') assert False, "restricted client with 10.5.173.10 can connect to server with allowed mask '10.5.172.0/24'" except AssertionError: raise except Exception as ex: print ex + def test_ipv6(setup_cluster): try: - client_ipv6_ok.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989::1:1000 --query 'select 1'"], privileged=True, user='root') + client_ipv6_ok.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989::1:1000 --query 'select 1'"], + privileged=True, user='root') except Exception as ex: print ex assert False, "allowed client with 2001:3984:3989:0:0:0:1:1111 cannot connect to server with allowed mask '2001:3984:3989:0:0:0:0:0/112'" try: - client_ipv6_ok_direct.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'"], privileged=True, user='root') + client_ipv6_ok_direct.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'"], + privileged=True, user='root') except Exception as ex: assert False, "allowed client with 2001:3984:3989:0:0:0:1:1111 cannot connect to server with allowed ip '2001:3984:3989:0:0:0:1:1111'" try: - client_ipv6_bad.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'"], privileged=True, user='root') + client_ipv6_bad.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'"], + privileged=True, user='root') assert False, "restricted client with 2001:3984:3989:0:0:0:1:1112 can connect to server with allowed mask '2001:3984:3989:0:0:0:0:0/112'" except AssertionError: raise diff --git a/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py b/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py index 3af5c18544a..c5ea7ed60a0 100644 --- a/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py +++ b/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py @@ -1,9 +1,7 @@ -import time import pytest from helpers.cluster import ClickHouseCluster - cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', user_configs=["configs/users.xml"]) @@ -20,7 +18,8 @@ def start_cluster(): def test_user_zero_database_access(start_cluster): try: - node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'no_access' --query 'DROP DATABASE test'"], user='root') + node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --user 'no_access' --query 'DROP DATABASE test'"], user='root') assert False, "user with no access rights dropped database test" except AssertionError: raise @@ -28,17 +27,22 @@ def test_user_zero_database_access(start_cluster): print ex try: - node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'DROP DATABASE test'"], user='root') + node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'DROP DATABASE test'"], user='root') except Exception as ex: assert False, "user with access rights can't drop database test" try: - node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test'"], user='root') + node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test'"], + user='root') except Exception as ex: assert False, "user with access rights can't create database test" try: - node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'no_access' --query 'CREATE DATABASE test2'"], user='root') + node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --user 'no_access' --query 'CREATE DATABASE test2'"], + user='root') assert False, "user with no access rights created database test2" except AssertionError: raise @@ -46,7 +50,9 @@ def test_user_zero_database_access(start_cluster): print ex try: - node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test2'"], user='root') + node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test2'"], + user='root') assert False, "user with limited access rights created database test2 which is outside of his scope of rights" except AssertionError: raise @@ -54,11 +60,13 @@ def test_user_zero_database_access(start_cluster): print ex try: - node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'default' --query 'CREATE DATABASE test2'"], user='root') + node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --user 'default' --query 'CREATE DATABASE test2'"], user='root') except Exception as ex: assert False, "user with full access rights can't create database test2" try: - node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --user 'default' --query 'DROP DATABASE test2'"], user='root') + node.exec_in_container( + ["bash", "-c", "/usr/bin/clickhouse client --user 'default' --query 'DROP DATABASE test2'"], user='root') except Exception as ex: - assert False, "user with full access rights can't drop database test2" \ No newline at end of file + assert False, "user with full access rights can't drop database test2" diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index f78dbf18c0d..68c2c65cbf2 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -5,9 +5,13 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', with_installed_binary=True, stay_alive=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', with_installed_binary=True, stay_alive=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', with_installed_binary=True, stay_alive=True) +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', + with_installed_binary=True, stay_alive=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', + with_installed_binary=True, stay_alive=True) +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', + with_installed_binary=True, stay_alive=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -21,7 +25,9 @@ def start_cluster(): def test_mutate_and_upgrade(start_cluster): for node in [node1, node2]: - node.query("CREATE TABLE mt (EventDate Date, id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{}') ORDER BY tuple()".format(node.name)) + node.query( + "CREATE TABLE mt (EventDate Date, id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{}') ORDER BY tuple()".format( + node.name)) node1.query("INSERT INTO mt VALUES ('2020-02-13', 1), ('2020-02-13', 2);") @@ -52,7 +58,8 @@ def test_mutate_and_upgrade(start_cluster): assert node1.query("SELECT COUNT() FROM mt") == "2\n" assert node2.query("SELECT COUNT() FROM mt") == "2\n" - node1.query("ALTER TABLE mt MODIFY COLUMN id String DEFAULT '0'", settings={"replication_alter_partitions_sync": "2"}) + node1.query("ALTER TABLE mt MODIFY COLUMN id String DEFAULT '0'", + settings={"replication_alter_partitions_sync": "2"}) node2.query("OPTIMIZE TABLE mt FINAL") @@ -61,7 +68,8 @@ def test_mutate_and_upgrade(start_cluster): def test_upgrade_while_mutation(start_cluster): - node3.query("CREATE TABLE mt1 (EventDate Date, id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t1', 'node3') ORDER BY tuple()") + node3.query( + "CREATE TABLE mt1 (EventDate Date, id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t1', 'node3') ORDER BY tuple()") node3.query("INSERT INTO mt1 select '2020-02-13', number from numbers(100000)") diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 086b9ac0c73..9bc206d8da4 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -1,24 +1,30 @@ from __future__ import print_function -from helpers.cluster import ClickHouseCluster + +import time +from os import path as p, unlink +from tempfile import NamedTemporaryFile + import helpers import pytest -import time -from tempfile import NamedTemporaryFile -from os import path as p, unlink +from helpers.cluster import ClickHouseCluster def test_chroot_with_same_root(): - cluster_1 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_a.xml') cluster_2 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_a.xml') - node1 = cluster_1.add_instance('node1', main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"], with_zookeeper=True, zookeeper_use_tmpfs=False) - node2 = cluster_2.add_instance('node2', main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"], with_zookeeper=True, zookeeper_use_tmpfs=False) + node1 = cluster_1.add_instance('node1', + main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"], + with_zookeeper=True, zookeeper_use_tmpfs=False) + node2 = cluster_2.add_instance('node2', + main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"], + with_zookeeper=True, zookeeper_use_tmpfs=False) nodes = [node1, node2] def create_zk_root(zk): zk.ensure_path('/root_a') print(zk.get_children('/')) + cluster_1.add_zookeeper_startup_command(create_zk_root) try: @@ -31,7 +37,7 @@ def test_chroot_with_same_root(): CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); '''.format(replica=node.name)) - for j in range(2): # Second insert to test deduplication + for j in range(2): # Second insert to test deduplication node.query("INSERT INTO simple VALUES ({0}, {0})".format(i)) time.sleep(1) @@ -47,18 +53,22 @@ def test_chroot_with_same_root(): def test_chroot_with_different_root(): - cluster_1 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_a.xml') cluster_2 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_b.xml') - node1 = cluster_1.add_instance('node1', main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"], with_zookeeper=True, zookeeper_use_tmpfs=False) - node2 = cluster_2.add_instance('node2', main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_b.xml"], with_zookeeper=True, zookeeper_use_tmpfs=False) + node1 = cluster_1.add_instance('node1', + main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"], + with_zookeeper=True, zookeeper_use_tmpfs=False) + node2 = cluster_2.add_instance('node2', + main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_b.xml"], + with_zookeeper=True, zookeeper_use_tmpfs=False) nodes = [node1, node2] def create_zk_roots(zk): zk.ensure_path('/root_a') zk.ensure_path('/root_b') print(zk.get_children('/')) + cluster_1.add_zookeeper_startup_command(create_zk_roots) try: @@ -72,7 +82,7 @@ def test_chroot_with_different_root(): CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); '''.format(replica=node.name)) - for j in range(2): # Second insert to test deduplication + for j in range(2): # Second insert to test deduplication node.query("INSERT INTO simple VALUES ({0}, {0})".format(i)) assert node1.query('select count() from simple').strip() == '1' @@ -86,12 +96,14 @@ def test_chroot_with_different_root(): def test_identity(): - cluster_1 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_with_password.xml') cluster_2 = ClickHouseCluster(__file__) - node1 = cluster_1.add_instance('node1', main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_with_password.xml"], with_zookeeper=True, zookeeper_use_tmpfs=False) - node2 = cluster_2.add_instance('node2', main_configs=["configs/remote_servers.xml"], with_zookeeper=True, zookeeper_use_tmpfs=False) + node1 = cluster_1.add_instance('node1', main_configs=["configs/remote_servers.xml", + "configs/zookeeper_config_with_password.xml"], + with_zookeeper=True, zookeeper_use_tmpfs=False) + node2 = cluster_2.add_instance('node2', main_configs=["configs/remote_servers.xml"], with_zookeeper=True, + zookeeper_use_tmpfs=False) try: cluster_1.start() @@ -146,11 +158,15 @@ def test_secure_connection(): docker_compose.close() node1 = cluster.add_instance('node1', main_configs=["configs_secure/client.crt", "configs_secure/client.key", - "configs_secure/conf.d/remote_servers.xml", "configs_secure/conf.d/ssl_conf.xml"], - with_zookeeper=True, zookeeper_docker_compose_path=docker_compose.name, zookeeper_use_tmpfs=False) + "configs_secure/conf.d/remote_servers.xml", + "configs_secure/conf.d/ssl_conf.xml"], + with_zookeeper=True, zookeeper_docker_compose_path=docker_compose.name, + zookeeper_use_tmpfs=False) node2 = cluster.add_instance('node2', main_configs=["configs_secure/client.crt", "configs_secure/client.key", - "configs_secure/conf.d/remote_servers.xml", "configs_secure/conf.d/ssl_conf.xml"], - with_zookeeper=True, zookeeper_docker_compose_path=docker_compose.name, zookeeper_use_tmpfs=False) + "configs_secure/conf.d/remote_servers.xml", + "configs_secure/conf.d/ssl_conf.xml"], + with_zookeeper=True, zookeeper_docker_compose_path=docker_compose.name, + zookeeper_use_tmpfs=False) try: cluster.start() diff --git a/tests/queries/0_stateless/01084_defaults_on_aliases.reference b/tests/queries/0_stateless/01084_defaults_on_aliases.reference index 9b39b07db94..6c75649efd7 100644 --- a/tests/queries/0_stateless/01084_defaults_on_aliases.reference +++ b/tests/queries/0_stateless/01084_defaults_on_aliases.reference @@ -1,5 +1,6 @@ 1 1 1 1 1 +1 2 2 4 2 2 2 4 3 3 9 diff --git a/tests/queries/0_stateless/01084_defaults_on_aliases.sql b/tests/queries/0_stateless/01084_defaults_on_aliases.sql index 2e4be37cc73..2f9d8227338 100644 --- a/tests/queries/0_stateless/01084_defaults_on_aliases.sql +++ b/tests/queries/0_stateless/01084_defaults_on_aliases.sql @@ -2,12 +2,16 @@ DROP TABLE IF EXISTS table_with_defaults_on_aliases; CREATE TABLE table_with_defaults_on_aliases (col1 UInt32, col2 ALIAS col1, col3 DEFAULT col2) Engine = MergeTree() ORDER BY tuple(); +SYSTEM STOP MERGES table_with_defaults_on_aliases; + INSERT INTO table_with_defaults_on_aliases (col1) VALUES (1); SELECT * FROM table_with_defaults_on_aliases WHERE col1 = 1; SELECT col1, col2, col3 FROM table_with_defaults_on_aliases WHERE col1 = 1; +SELECT col3 FROM table_with_defaults_on_aliases; -- important to check without WHERE + ALTER TABLE table_with_defaults_on_aliases ADD COLUMN col4 UInt64 DEFAULT col2 * col3; INSERT INTO table_with_defaults_on_aliases (col1) VALUES (2); @@ -24,7 +28,6 @@ SELECT * FROM table_with_defaults_on_aliases WHERE col1 = 3; SELECT col1, col2, col3, col4, col5 FROM table_with_defaults_on_aliases WHERE col1 = 3; - ALTER TABLE table_with_defaults_on_aliases ADD COLUMN col6 UInt64 MATERIALIZED col2 * col4; DROP TABLE IF EXISTS table_with_defaults_on_aliases; diff --git a/tests/queries/0_stateless/01461_query_start_time_microseconds.sql b/tests/queries/0_stateless/01461_query_start_time_microseconds.sql index 56af94bfd73..bf56fefd841 100644 --- a/tests/queries/0_stateless/01461_query_start_time_microseconds.sql +++ b/tests/queries/0_stateless/01461_query_start_time_microseconds.sql @@ -1,9 +1,36 @@ -set log_queries = 1; +SET log_queries = 1; +SELECT '01461_query_log_query_start_time_milliseconds_test'; +SYSTEM FLUSH LOGS; +-- assumes that the query_start_time field is already accurate. +WITH ( + ( + SELECT query_start_time_microseconds + FROM system.query_log + ORDER BY query_start_time DESC + LIMIT 1 + ) AS time_with_microseconds, + ( + SELECT query_start_time + FROM system.query_log + ORDER BY query_start_time DESC + LIMIT 1 + ) AS t) +SELECT if(dateDiff('second', toDateTime(time_with_microseconds), toDateTime(t)) = 0, 'ok', 'fail'); -- -select '01461_query_log_query_start_time_milliseconds_test'; -system flush logs; -SELECT If((select count(query_start_time_microseconds) from system.query_log WHERE query LIKE '%01461_query_log_query_start_time_milliseconds_test%' AND query NOT LIKE '%system.query_log%') > 0, 'ok', 'fail'); - -select '01461_query_thread_log_query_start_time_milliseconds_test'; -system flush logs; -SELECT If((select count(query_start_time_microseconds) from system.query_log WHERE query LIKE '%01461_query_thread_log_query_start_time_milliseconds_test%' AND query NOT LIKE '%system.query_log%') > 0, 'ok', 'fail'); +SELECT '01461_query_thread_log_query_start_time_milliseconds_test'; +SYSTEM FLUSH LOGS; +-- assumes that the query_start_time field is already accurate. +WITH ( + ( + SELECT query_start_time_microseconds + FROM system.query_thread_log + ORDER BY query_start_time DESC + LIMIT 1 + ) AS time_with_microseconds, + ( + SELECT query_start_time + FROM system.query_thread_log + ORDER BY query_start_time DESC + LIMIT 1 + ) AS t) +SELECT if(dateDiff('second', toDateTime(time_with_microseconds), toDateTime(t)) = 0, 'ok', 'fail'); -- \ No newline at end of file diff --git a/tests/queries/0_stateless/01470_columns_transformers.reference b/tests/queries/0_stateless/01470_columns_transformers.reference index c0f02e51ccf..ba23352c420 100644 --- a/tests/queries/0_stateless/01470_columns_transformers.reference +++ b/tests/queries/0_stateless/01470_columns_transformers.reference @@ -61,3 +61,9 @@ SELECT j, k FROM columns_transformers +220 18 347 +SELECT + sum(i), + sum(j), + sum(k) +FROM columns_transformers diff --git a/tests/queries/0_stateless/01470_columns_transformers.sql b/tests/queries/0_stateless/01470_columns_transformers.sql index de6a1a89d81..55335110c97 100644 --- a/tests/queries/0_stateless/01470_columns_transformers.sql +++ b/tests/queries/0_stateless/01470_columns_transformers.sql @@ -33,4 +33,8 @@ EXPLAIN SYNTAX SELECT a.* APPLY(toDate) REPLACE(i + 1 AS i) APPLY(any) from colu -- Multiple REPLACE in a row EXPLAIN SYNTAX SELECT * REPLACE(i + 1 AS i) REPLACE(i + 1 AS i) from columns_transformers; +-- Explicit column list +SELECT COLUMNS(i, j, k) APPLY(sum) from columns_transformers; +EXPLAIN SYNTAX SELECT COLUMNS(i, j, k) APPLY(sum) from columns_transformers; + DROP TABLE columns_transformers; diff --git a/tests/queries/0_stateless/01497_alias_on_default_array.reference b/tests/queries/0_stateless/01497_alias_on_default_array.reference new file mode 100644 index 00000000000..8a4406e57f3 --- /dev/null +++ b/tests/queries/0_stateless/01497_alias_on_default_array.reference @@ -0,0 +1,6 @@ +a1 b1 +a2 b2 +a3 b3 +c1 +c2 +c3 diff --git a/tests/queries/0_stateless/01497_alias_on_default_array.sql b/tests/queries/0_stateless/01497_alias_on_default_array.sql new file mode 100644 index 00000000000..c0c26b05eb8 --- /dev/null +++ b/tests/queries/0_stateless/01497_alias_on_default_array.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS test_new_col; + +CREATE TABLE test_new_col +( + `_csv` String, + `csv_as_array` Array(String) ALIAS splitByChar(';',_csv), + `csv_col1` String DEFAULT csv_as_array[1], + `csv_col2` String DEFAULT csv_as_array[2] +) +ENGINE = MergeTree +ORDER BY tuple(); + +INSERT INTO test_new_col (_csv) VALUES ('a1;b1;c1;d1'), ('a2;b2;c2;d2'), ('a3;b3;c3;d3'); + +SELECT csv_col1, csv_col2 FROM test_new_col ORDER BY csv_col1; + +ALTER TABLE test_new_col ADD COLUMN `csv_col3` String DEFAULT csv_as_array[3]; + +SELECT csv_col3 FROM test_new_col ORDER BY csv_col3; + +DROP TABLE IF EXISTS test_new_col; diff --git a/tests/queries/0_stateless/01500_StorageFile_write_to_fd.reference b/tests/queries/0_stateless/01500_StorageFile_write_to_fd.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01500_StorageFile_write_to_fd.sh b/tests/queries/0_stateless/01500_StorageFile_write_to_fd.sh new file mode 100755 index 00000000000..589a578eb0b --- /dev/null +++ b/tests/queries/0_stateless/01500_StorageFile_write_to_fd.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +# The following command will execute: +# CREATE TABLE table (key UInt32) ENGINE = File(TSV, stdin); +# INSERT INTO `table` SELECT key FROM input('key UInt32') FORMAT TSV +${CLICKHOUSE_LOCAL} -S 'key UInt32' -q "INSERT INTO \`table\` SELECT key FROM input('key UInt32') FORMAT TSV" < /dev/null diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index aa8a9f48ce9..123b4c3d144 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -142,3 +142,4 @@ 01474_executable_dictionary 01474_bad_global_join 01473_event_time_microseconds +01461_query_start_time_microseconds