From 814d78af298b74cae0f830a33ab53dc66f4794d6 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Tue, 23 Jun 2020 16:42:52 +0300 Subject: [PATCH 01/12] Better namings --- src/Functions/FunctionsLogical.cpp | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index ade2fe960b0..07351f6f2c9 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -42,7 +42,7 @@ using UInt8Container = ColumnUInt8::Container; using UInt8ColumnPtrs = std::vector; -MutableColumnPtr convertFromTernaryData(const UInt8Container & ternary_data, const bool make_nullable) +MutableColumnPtr buildColumnFromTernaryData(const UInt8Container & ternary_data, const bool make_nullable) { const size_t rows_count = ternary_data.size(); @@ -63,7 +63,7 @@ MutableColumnPtr convertFromTernaryData(const UInt8Container & ternary_data, con } template -bool tryConvertColumnToUInt8(const IColumn * column, UInt8Container & res) +bool tryConvertColumnToBool(const IColumn * column, UInt8Container & res) { const auto col = checkAndGetColumn>(column); if (!col) @@ -76,17 +76,17 @@ bool tryConvertColumnToUInt8(const IColumn * column, UInt8Container & res) return true; } -void convertColumnToUInt8(const IColumn * column, UInt8Container & res) +void convertAnyColumnToBool(const IColumn * column, UInt8Container & res) { - if (!tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res) && - !tryConvertColumnToUInt8(column, res)) + if (!tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res) && + !tryConvertColumnToBool(column, res)) throw Exception("Unexpected type of column: " + column->getName(), ErrorCodes::ILLEGAL_COLUMN); } @@ -119,7 +119,7 @@ static bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res, Func && func) } template -inline bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res) +inline bool extractConstColumnsAsBool(ColumnRawPtrs & in, UInt8 & res) { return extractConstColumns( in, res, @@ -131,7 +131,7 @@ inline bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res) } template -inline bool extractConstColumnsTernary(ColumnRawPtrs & in, UInt8 & res_3v) +inline bool extractConstColumnsAsTernary(ColumnRawPtrs & in, UInt8 & res_3v) { return extractConstColumns( in, res_3v, @@ -188,7 +188,7 @@ private: /// A helper class used by AssociativeGenericApplierImpl /// Allows for on-the-fly conversion of any data type into intermediate ternary representation -using ValueGetter = std::function; +using TernaryValueGetter = std::function; template struct ValueGetterBuilderImpl; @@ -196,7 +196,7 @@ struct ValueGetterBuilderImpl; template struct ValueGetterBuilderImpl { - static ValueGetter build(const IColumn * x) + static TernaryValueGetter build(const IColumn * x) { if (const auto * nullable_column = typeid_cast(x)) { @@ -218,7 +218,7 @@ struct ValueGetterBuilderImpl template <> struct ValueGetterBuilderImpl<> { - static ValueGetter build(const IColumn * x) + static TernaryValueGetter build(const IColumn * x) { throw Exception( std::string("Unknown numeric column of type: ") + demangle(typeid(x).name()), @@ -253,7 +253,7 @@ public: } private: - const ValueGetter val_getter; + const TernaryValueGetter val_getter; const AssociativeGenericApplierImpl next; }; @@ -271,7 +271,7 @@ public: inline ResultValueType apply(const size_t i) const { return val_getter(i); } private: - const ValueGetter val_getter; + const TernaryValueGetter val_getter; }; @@ -338,7 +338,7 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn if (has_consts && (arguments.empty() || Op::isSaturatedValue(const_3v_value))) { result_info.column = ColumnConst::create( - convertFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()), + buildColumnFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()), input_rows_count ); return; @@ -349,7 +349,7 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn OperationApplier::apply(arguments, result_column->getData(), has_consts); - result_info.column = convertFromTernaryData(result_column->getData(), result_info.type->isNullable()); + result_info.column = buildColumnFromTernaryData(result_column->getData(), result_info.type->isNullable()); } @@ -447,7 +447,7 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re else { auto converted_column = ColumnUInt8::create(input_rows_count); - convertColumnToUInt8(column, converted_column->getData()); + convertAnyColumnToBool(column, converted_column->getData()); uint8_args.push_back(converted_column.get()); converted_columns_holder.emplace_back(std::move(converted_column)); } From 22aabf8da3ff5ef58f5db4a67aa56d7a489fc539 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 2 Jul 2020 17:58:13 +0200 Subject: [PATCH 02/12] Adding a simple example of using TestFlows. --- tests/testflows/README.md | 55 +++ tests/testflows/example/__init__.py | 0 .../example/configs/clickhouse/common.xml | 6 + .../configs/clickhouse/config.d/logs.xml | 17 + .../configs/clickhouse/config.d/ports.xml | 5 + .../configs/clickhouse/config.d/remote.xml | 107 +++++ .../configs/clickhouse/config.d/ssl.xml | 17 + .../configs/clickhouse/config.d/storage.xml | 20 + .../configs/clickhouse/config.d/zookeeper.xml | 10 + .../example/configs/clickhouse/config.xml | 436 ++++++++++++++++++ .../configs/clickhouse/ssl/dhparam.pem | 8 + .../example/configs/clickhouse/ssl/server.crt | 19 + .../example/configs/clickhouse/ssl/server.key | 28 ++ .../example/configs/clickhouse/users.xml | 133 ++++++ .../configs/clickhouse1/config.d/macros.xml | 8 + .../docker-compose/clickhouse-service.yml | 28 ++ .../example/docker-compose/docker-compose.yml | 31 ++ .../docker-compose/zookeeper-service.yml | 18 + tests/testflows/example/regression.py | 26 ++ .../example/requirements/__init__.py | 1 + .../example/requirements/requirements.py | 9 + tests/testflows/example/tests/example.py | 20 + tests/testflows/helpers/argparser.py | 13 + tests/testflows/helpers/cluster.py | 272 +++++++++++ tests/testflows/regression.py | 19 + 25 files changed, 1306 insertions(+) create mode 100644 tests/testflows/README.md create mode 100644 tests/testflows/example/__init__.py create mode 100644 tests/testflows/example/configs/clickhouse/common.xml create mode 100644 tests/testflows/example/configs/clickhouse/config.d/logs.xml create mode 100644 tests/testflows/example/configs/clickhouse/config.d/ports.xml create mode 100644 tests/testflows/example/configs/clickhouse/config.d/remote.xml create mode 100644 tests/testflows/example/configs/clickhouse/config.d/ssl.xml create mode 100644 tests/testflows/example/configs/clickhouse/config.d/storage.xml create mode 100644 tests/testflows/example/configs/clickhouse/config.d/zookeeper.xml create mode 100644 tests/testflows/example/configs/clickhouse/config.xml create mode 100644 tests/testflows/example/configs/clickhouse/ssl/dhparam.pem create mode 100644 tests/testflows/example/configs/clickhouse/ssl/server.crt create mode 100644 tests/testflows/example/configs/clickhouse/ssl/server.key create mode 100644 tests/testflows/example/configs/clickhouse/users.xml create mode 100644 tests/testflows/example/configs/clickhouse1/config.d/macros.xml create mode 100644 tests/testflows/example/docker-compose/clickhouse-service.yml create mode 100644 tests/testflows/example/docker-compose/docker-compose.yml create mode 100644 tests/testflows/example/docker-compose/zookeeper-service.yml create mode 100755 tests/testflows/example/regression.py create mode 100644 tests/testflows/example/requirements/__init__.py create mode 100644 tests/testflows/example/requirements/requirements.py create mode 100644 tests/testflows/example/tests/example.py create mode 100644 tests/testflows/helpers/argparser.py create mode 100644 tests/testflows/helpers/cluster.py create mode 100755 tests/testflows/regression.py diff --git a/tests/testflows/README.md b/tests/testflows/README.md new file mode 100644 index 00000000000..645ad541795 --- /dev/null +++ b/tests/testflows/README.md @@ -0,0 +1,55 @@ +## ClickHouse Tests in [TestFlows] + +This directory contains integration tests written using [TestFlows] +that involves several ClickHouse instances, custom configs, ZooKeeper, etc. + +## Supported environment + +* [Ubuntu] 18.04 +* [Python 3] >= 3.6 + +## Prerequisites + +* [Docker] [install](https://docs.docker.com/compose/install/) +* [Docker Compose] [install](https://docs.docker.com/engine/install/) +* [TestFlows] [install](https://testflows.com/handbook/#Installation) + +## Running tests locally + +You can run tests locally by passing `--local` and `--clickhouse-binary-path` to the `regression.py`. + +* `--local` specifies that regression will be run locally +* `--clickhouse-binary-path` specifies the path to the ClickHouse binary that will be used during the regression run + +> Note: you can pass `-h` or `--help` argument to the `regression.py` to see a help message. +> +> ```bash +> python3 regression.py -h +> ``` + +> Note: make sure that the ClickHouse binary has correct permissions. +> If you are using `/usr/bin/clickhouse` its owner and group is set to `root:root` by default +> and it needs to be changed to `clickhouse:clickhouse`. You can change the owner and the group +> using the following command. +> +> ```bash +> sudo chown clickhouse:clickhouse /usr/bin/clickhouse +> ``` + +Using the default ClickHouse installation and its server binary at `/usr/bin/clickhouse`, you can run +regressions locally using the following command. + +```bash +python3 regression.py --local --clickhouse-binary-path "/usr/bin/clickhouse" +``` + +## Output Verbosity + +You can control verbosity of the output by specifying the output format with `-o` or `--output` option. +See `--help` for more details. + +[Python 3]: https://www.python.org/ +[Ubuntu]: https://ubuntu.com/ +[TestFlows]: https://testflows.com +[Docker]: https://www.docker.com/ +[Docker Compose]: https://docs.docker.com/compose/ diff --git a/tests/testflows/example/__init__.py b/tests/testflows/example/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/example/configs/clickhouse/common.xml b/tests/testflows/example/configs/clickhouse/common.xml new file mode 100644 index 00000000000..df952b28c82 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + 0.0.0.0 + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/tests/testflows/example/configs/clickhouse/config.d/logs.xml b/tests/testflows/example/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/tests/testflows/example/configs/clickhouse/config.d/ports.xml b/tests/testflows/example/configs/clickhouse/config.d/ports.xml new file mode 100644 index 00000000000..fbc6cea74c0 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,5 @@ + + + 8443 + 9440 + \ No newline at end of file diff --git a/tests/testflows/example/configs/clickhouse/config.d/remote.xml b/tests/testflows/example/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..51be2a6e8e3 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,107 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + true + + clickhouse1 + 9440 + 1 + + + clickhouse2 + 9440 + 1 + + + clickhouse3 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9440 + 1 + + + + + clickhouse2 + 9440 + 1 + + + + + clickhouse3 + 9440 + 1 + + + + + diff --git a/tests/testflows/example/configs/clickhouse/config.d/ssl.xml b/tests/testflows/example/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 00000000000..ca65ffd5e04 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,17 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/tests/testflows/example/configs/clickhouse/config.d/storage.xml b/tests/testflows/example/configs/clickhouse/config.d/storage.xml new file mode 100644 index 00000000000..618fd6b6d24 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,20 @@ + + + + + + 1024 + + + + + + + default + + + + + + + diff --git a/tests/testflows/example/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/example/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/example/configs/clickhouse/config.xml b/tests/testflows/example/configs/clickhouse/config.xml new file mode 100644 index 00000000000..d34d2c35253 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/config.xml @@ -0,0 +1,436 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + /var/lib/clickhouse/access/ + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 + + + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + diff --git a/tests/testflows/example/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/example/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 00000000000..2e6cee0798d --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/example/configs/clickhouse/ssl/server.crt b/tests/testflows/example/configs/clickhouse/ssl/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/testflows/example/configs/clickhouse/ssl/server.key b/tests/testflows/example/configs/clickhouse/ssl/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/testflows/example/configs/clickhouse/users.xml b/tests/testflows/example/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/example/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/example/configs/clickhouse1/config.d/macros.xml b/tests/testflows/example/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/example/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/example/docker-compose/clickhouse-service.yml b/tests/testflows/example/docker-compose/clickhouse-service.yml new file mode 100644 index 00000000000..ed345bd7b04 --- /dev/null +++ b/tests/testflows/example/docker-compose/clickhouse-service.yml @@ -0,0 +1,28 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 3s + timeout: 5s + retries: 40 + start_period: 2s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/example/docker-compose/docker-compose.yml b/tests/testflows/example/docker-compose/docker-compose.yml new file mode 100644 index 00000000000..e7e57386dc4 --- /dev/null +++ b/tests/testflows/example/docker-compose/docker-compose.yml @@ -0,0 +1,31 @@ +version: '2.3' + +services: + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + zookeeper: + condition: service_healthy diff --git a/tests/testflows/example/docker-compose/zookeeper-service.yml b/tests/testflows/example/docker-compose/zookeeper-service.yml new file mode 100644 index 00000000000..f3df33358be --- /dev/null +++ b/tests/testflows/example/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable diff --git a/tests/testflows/example/regression.py b/tests/testflows/example/regression.py new file mode 100755 index 00000000000..2c0a778d39b --- /dev/null +++ b/tests/testflows/example/regression.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +import sys +from testflows.core import * + +append_path(sys.path, "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser + +@TestFeature +@Name("example") +@ArgumentParser(argparser) +def regression(self, local, clickhouse_binary_path): + """Simple example of how you can use TestFlows to test ClickHouse. + """ + nodes = { + "clickhouse": ("clickhouse1",), + } + + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + + Scenario(run=load("example.tests.example", "scenario")) + +if main(): + regression() diff --git a/tests/testflows/example/requirements/__init__.py b/tests/testflows/example/requirements/__init__.py new file mode 100644 index 00000000000..02f7d430154 --- /dev/null +++ b/tests/testflows/example/requirements/__init__.py @@ -0,0 +1 @@ +from .requirements import * diff --git a/tests/testflows/example/requirements/requirements.py b/tests/testflows/example/requirements/requirements.py new file mode 100644 index 00000000000..9a0b4922f19 --- /dev/null +++ b/tests/testflows/example/requirements/requirements.py @@ -0,0 +1,9 @@ +from testflows.core import Requirement + +RQ_ClickHouse_Select_1 = Requirement( + name='RQ.ClickHouse.Select.1', + version='1.0', + description=( + '[ClickHouse] SHALL return `1` when user sends a simple `SELECT 1` query.\n' + ) + ) diff --git a/tests/testflows/example/tests/example.py b/tests/testflows/example/tests/example.py new file mode 100644 index 00000000000..8559f290cc4 --- /dev/null +++ b/tests/testflows/example/tests/example.py @@ -0,0 +1,20 @@ +from testflows.core import * +from testflows.asserts import error + +from example.requirements import * + +@TestScenario +@Name("select 1") +@Requirements( + RQ_ClickHouse_Select_1("1.0") +) +def scenario(self, node="clickhouse1"): + """Check that ClickHouse returns 1 when user executes `SELECT 1` query. + """ + node = self.context.cluster.node(node) + + with When("I execute query select 1"): + r = node.query("SELECT 1").output.strip() + + with Then("the result should be 1"): + assert r == "1", error() diff --git a/tests/testflows/helpers/argparser.py b/tests/testflows/helpers/argparser.py new file mode 100644 index 00000000000..34b91453876 --- /dev/null +++ b/tests/testflows/helpers/argparser.py @@ -0,0 +1,13 @@ +import os + +def argparser(parser): + """Default argument parser for regressions. + """ + parser.add_argument("--local", + action="store_true", + help="run regression in local mode", default=False) + + parser.add_argument("--clickhouse-binary-path", + type=str, dest="clickhouse_binary_path", + help="path to ClickHouse binary, default: /usr/bin/clickhouse", metavar="path", + default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse")) diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py new file mode 100644 index 00000000000..e087b3b5b9d --- /dev/null +++ b/tests/testflows/helpers/cluster.py @@ -0,0 +1,272 @@ +import os +import time +import inspect +import threading +import tempfile + +from testflows.core import * +from testflows.asserts import error +from testflows.connect import Shell + +class QueryRuntimeException(Exception): + """Exception during query execution on the server. + """ + pass + +class Node(object): + """Generic cluster node. + """ + config_d_dir = "/etc/clickhouse-server/config.d/" + + def __init__(self, cluster, name): + self.cluster = cluster + self.name = name + + def repr(self): + return f"Node(name='{self.name}')" + + def restart(self, timeout=120, safe=True): + """Restart node. + """ + with self.cluster.lock: + for key in list(self.cluster._bash.keys()): + if key.endswith(f"-{self.name}"): + shell = self.cluster._bash.pop(key) + shell.__exit__(None, None, None) + + self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout) + + def command(self, *args, **kwargs): + return self.cluster.command(self.name, *args, **kwargs) + +class ClickHouseNode(Node): + """Node with ClickHouse server. + """ + def wait_healthy(self, timeout=120): + with By(f"waiting until container {self.name} is healthy"): + start_time = time.time() + while True: + if self.query("select 1", no_checks=1, timeout=120, steps=False).exitcode == 0: + break + if time.time() - start_time < timeout: + time.sleep(2) + continue + assert False, "container is not healthy" + + def restart(self, timeout=120, safe=True): + """Restart node. + """ + if safe: + self.query("SYSTEM STOP MOVES") + self.query("SYSTEM STOP MERGES") + self.query("SYSTEM FLUSH LOGS") + with By("waiting for 5 sec for moves and merges to stop"): + time.sleep(5) + with And("forcing to sync everything to disk"): + self.command("sync", timeout=30) + + with self.cluster.lock: + for key in list(self.cluster._bash.keys()): + if key.endswith(f"-{self.name}"): + shell = self.cluster._bash.pop(key) + shell.__exit__(None, None, None) + + self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout) + + self.wait_healthy(timeout) + + def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False, + raise_on_exception=False, step=By, settings=None, *args, **kwargs): + """Execute and check query. + + :param sql: sql query + :param message: expected message that should be in the output, default: None + :param exitcode: expected exitcode, default: None + """ + if len(sql) > 1024: + with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query: + query.write(sql) + query.flush() + command = f"cat \"{query.name}\" | {self.cluster.docker_compose} exec -T {self.name} clickhouse client -n" + for setting in settings or []: + name, value = setting + command += f" --{name} \"{value}\"" + description = f""" + echo -e \"{sql[:100]}...\" > {query.name} + {command} + """ + with step("executing command", description=description) if steps else NullStep(): + r = self.cluster.bash(None)(command, *args, **kwargs) + else: + command = f"echo -e \"{sql}\" | clickhouse client -n" + for setting in settings or []: + name, value = setting + command += f" --{name} \"{value}\"" + with step("executing command", description=command) if steps else NullStep(): + r = self.cluster.bash(self.name)(command, *args, **kwargs) + + if no_checks: + return r + + if exitcode is not None: + with Then(f"exitcode should be {exitcode}") if steps else NullStep(): + assert r.exitcode == exitcode, error(r.output) + + if message is not None: + with Then(f"output should contain message", description=message) if steps else NullStep(): + assert message in r.output, error(r.output) + + if message is None or "Exception:" not in message: + with Then("check if output has exception") if steps else NullStep(): + if "Exception:" in r.output: + if raise_on_exception: + raise QueryRuntimeException(r.output) + assert False, error(r.output) + + return r + +class Cluster(object): + """Simple object around docker-compose cluster. + """ + def __init__(self, local=False, + clickhouse_binary_path=None, configs_dir=None, + nodes=None, + docker_compose="docker-compose", docker_compose_project_dir=None, + docker_compose_file="docker-compose.yml"): + + self._bash = {} + self.clickhouse_binary_path = clickhouse_binary_path + self.configs_dir = configs_dir + self.local = local + self.nodes = nodes or {} + self.docker_compose = docker_compose + + frame = inspect.currentframe().f_back + caller_dir = os.path.dirname(os.path.abspath(frame.f_globals["__file__"])) + + # auto set configs directory + if self.configs_dir is None: + caller_configs_dir = caller_dir + if os.path.exists(caller_configs_dir): + self.configs_dir = caller_configs_dir + + if not os.path.exists(self.configs_dir): + raise TypeError("configs directory '{self.configs_dir}' does not exist") + + # auto set docker-compose project directory + if docker_compose_project_dir is None: + caller_project_dir = os.path.join(caller_dir, "docker-compose") + if os.path.exists(caller_project_dir): + docker_compose_project_dir = caller_project_dir + + docker_compose_file_path = os.path.join(docker_compose_project_dir or "", docker_compose_file) + + if not os.path.exists(docker_compose_file_path): + raise TypeError("docker compose file '{docker_compose_file_path}' does not exist") + + self.docker_compose += f" --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\"" + self.lock = threading.Lock() + + def bash(self, node, timeout=60): + """Returns thread-local bash terminal + to a specific node. + + :param node: name of the service + """ + current_thread = threading.current_thread() + id = f"{current_thread.ident}-{node}" + with self.lock: + if self._bash.get(id) is None: + if node is None: + self._bash[id] = Shell().__enter__() + else: + self._bash[id] = Shell(command=[ + "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting" + ], name=node).__enter__() + self._bash[id].timeout = timeout + return self._bash[id] + + def __enter__(self): + with Given("docker-compose cluster"): + self.up() + return self + + def __exit__(self, type, value, traceback): + try: + with Finally("I clean up"): + self.down() + finally: + with self.lock: + for shell in self._bash.values(): + shell.__exit__(type, value, traceback) + + def node(self, name): + """Get object with node bound methods. + + :param name: name of service name + """ + if name.startswith("clickhouse"): + return ClickHouseNode(self, name) + return Node(self, name) + + def down(self, timeout=120): + """Bring cluster down by executing docker-compose down.""" + try: + bash = self.bash(None) + with self.lock: + # remove and close all not None node terminals + for id in list(self._bash.keys()): + shell = self._bash.pop(id) + if shell is not bash: + shell.__exit__(None, None, None) + else: + self._bash[id] = shell + finally: + return self.command(None, f"{self.docker_compose} down", timeout=timeout) + + def up(self): + if self.local: + with Given("I am running in local mode"): + with Then("check --clickhouse-binary-path is specified"): + assert self.clickhouse_binary_path, "when running in local mode then --clickhouse-binary-path must be specified" + with And("path should exist"): + assert os.path.exists(self.clickhouse_binary_path) + + os.environ["CLICKHOUSE_TESTS_SERVER_BIN_PATH"] = self.clickhouse_binary_path + os.environ["CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"] = os.path.join(os.path.dirname(self.clickhouse_binary_path), + "clickhouse-odbc-bridge") + os.environ["CLICKHOUSE_TESTS_DIR"] = self.configs_dir + + with Given("docker-compose"): + self.command(None, "env | grep CLICKHOUSE") + cmd = self.command(None, f'{self.docker_compose} up -d 2>&1 | tee', timeout=30 * 60) + else: + with Given("docker-compose"): + cmd = self.command(None, f'{self.docker_compose} up -d --no-recreate 2>&1 | tee') + + with Then("check there are no unhealthy containers"): + assert "is unhealthy" not in cmd.output, error() + + with Then("wait all nodes report healhy"): + for name in self.nodes["clickhouse"]: + self.node(name).wait_healthy() + + def command(self, node, command, message=None, exitcode=None, steps=True, *args, **kwargs): + """Execute and check command. + + :param node: name of the service + :param command: command + :param message: expected message that should be in the output, default: None + :param exitcode: expected exitcode, default: None + :param steps: don't break command into steps, default: True + """ + debug(f"command() {node}, {command}") + with By("executing command", description=command) if steps else NullStep(): + r = self.bash(node)(command, *args, **kwargs) + if exitcode is not None: + with Then(f"exitcode should be {exitcode}") if steps else NullStep(): + assert r.exitcode == exitcode, error(r.output) + if message is not None: + with Then(f"output should contain message", description=message) if steps else NullStep(): + assert message in r.output, error(r.output) + return r diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py new file mode 100755 index 00000000000..0f74b6e82cd --- /dev/null +++ b/tests/testflows/regression.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import sys +from testflows.core import * + +append_path(sys.path, "."), + +from helpers.argparser import argparser + +@TestModule +@Name("clickhouse") +@ArgumentParser(argparser) +def regression(self, local, clickhouse_binary_path): + """ClickHouse regression. + """ + Feature(test=load("example.regression", "regression"))( + local=local, clickhouse_binary_path=clickhouse_binary_path) + +if main(): + regression() From 321bad14eeb40f542644825d2ba65771d16a38ff Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 2 Jul 2020 20:41:28 +0200 Subject: [PATCH 03/12] Adding a template of the SRS with the notes how to use it. --- .../requirements/CH_SRS001_ClickHouse.html | 1439 +++++++++++++++++ .../requirements/CH_SRS001_ClickHouse.md | 136 ++ .../example/requirements/requirements.py | 65 +- tests/testflows/example/tests/example.py | 2 +- 4 files changed, 1637 insertions(+), 5 deletions(-) create mode 100644 tests/testflows/example/requirements/CH_SRS001_ClickHouse.html create mode 100644 tests/testflows/example/requirements/CH_SRS001_ClickHouse.md diff --git a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html new file mode 100644 index 00000000000..5aedad25979 --- /dev/null +++ b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html @@ -0,0 +1,1439 @@ + + + + + + + + +

CH-SRS001 ClickHouse Software Requirements Specification Template

+ +

Author: [name of the author]

+ +

Date: [date]

+ +

Table of Contents

+ + + +

Revision History

+ +

This document is stored in an electronic form using Git source control management software.

+ +

Introduction

+ +

This section provides an introduction to the project or the feature. +All SRS documents must be uniquely identified by a number. In this +case this document is identified by the number

+ +
CH-SRS001
+
+ +

The document number must always be used as a prefix to the document title. For example,

+ +
CH-SRSxxx Name of the document
+
+ +

All the requirements must be specified in the Requirements section.

+ +

Table of Contents

+ +

Note that currently the table of contents is generated manually using

+ +
cat CH_SRS001_ClickHouse.md | tfs document toc
+
+ +

command and needs to be updated any time requirement name is changed +or a new requirement is added.

+ +

Generating HTML version

+ +

You can easily generate a pretty HTML version of this document using the command.

+ +
cat CH_SRS001_ClickHouse.md | tfs document convert > CH_SRS001_ClickHouse.html
+
+ +

Generating Python Requirements

+ +

You can convert this SRS into the requirements.py by using the command.

+ +
cat CH_SRS001_ClickHouse.md | tfs requirements generate > requirements.py
+
+ +

Terminology

+ +

You can define terminolgy using the examples below and make you can make them +linkable as SRS by defining the links in the References section.

+ +

SRS

+ +

Software Requirements Specification

+ +

Some term that you will use

+ +

Some description of the term that you would like to use.

+ +

Requirements

+ +

This section includes all the requirements. This section can be structured in any way one sees fit.

+ +

Each requirement is defined by the section that starts with +the following prefix:

+ +
RQ.[document id].[requirement name]
+
+ +

then immediately followed by a one-line block that contains the +the version of the requirement.

+ +

RQ.CH-SRS001.Example short description of the requirement

+ +

version: 1.0

+ +

This is a long description of the requirement that can include any +relevant information.

+ +

The one-line block that follows the requirement defines the version +of the requirement. The version is controlled manually and is used +to indicate material changes to the requirement that would +require tests that cover this requirement to be updated.

+ +

It is a good practice to use requirement names that are broken +up into groups. It is not recommended to use only numbers +because if the requirement must be moved the numbering will not match. +Therefore, the requirement name should start with the group +name which is then followed by a number if any. For example,

+ +
RQ.SRS001.Group.Subgroup.1
+
+ +

To keep names short, try to use abbreviations for the requirement's group name.

+ +

RQ.CH-SRS001.Example.Subgroup short description of the sub-requirement

+ +

version: 1.0

+ +

This an example of a sub-requirement of the RQ.CH-SRS001.Example. +CH_SRS001_Software_Requirements_Template.md

+ +

RQ.CH-SRS001.Example.Select.1

+ +

version: 1.0

+ +

ClickHouse SHALL return 1 when user executes query

+ +
SELECT 1
+
+ +

References

+ + + + + \ No newline at end of file diff --git a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md new file mode 100644 index 00000000000..370ff14c2cb --- /dev/null +++ b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md @@ -0,0 +1,136 @@ +# CH-SRS001 ClickHouse Software Requirements Specification Template + +**Author:** [name of the author] + +**Date:** [date] + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) + * 2.1 [Table of Contents](#table-of-contents) + * 2.2 [Generating HTML version](#generating-html-version) + * 2.3 [Generating Python Requirements](#generating-python-requirements) +* 3 [Terminology](#terminology) + * 3.1 [SRS](#srs) + * 3.2 [Some term that you will use](#some-term-that-you-will-use) +* 4 [Requirements](#requirements) + * 4.1 [RQ.CH-SRS001.Example short description of the requirement ](#rqch-srs001example-short-description-of-the-requirement-) + * 4.2 [RQ.CH-SRS001.Example.Subgroup short description of the sub-requirement](#rqch-srs001examplesubgroup-short-description-of-the-sub-requirement) + * 4.3 [RQ.CH-SRS001.Example.Select.1](#rqch-srs001exampleselect1) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software. + +## Introduction + +This section provides an introduction to the project or the feature. +All [SRS] documents must be uniquely identified by a number. In this +case this document is identified by the number + + CH-SRS001 + +The document number must always be used as a prefix to the document title. For example, + + CH-SRSxxx Name of the document + +All the requirements must be specified in the [Requirements](#requirements) section. + +### Table of Contents + +Note that currently the table of contents is generated manually using + +```bash +cat CH_SRS001_ClickHouse.md | tfs document toc +``` + +command and needs to be updated any time requirement name is changed +or a new requirement is added. + +### Generating HTML version + +You can easily generate a pretty HTML version of this document using the command. + +```bash +cat CH_SRS001_ClickHouse.md | tfs document convert > CH_SRS001_ClickHouse.html +``` + +### Generating Python Requirements + +You can convert this [SRS] into the `requirements.py` by using the command. + +```bash +cat CH_SRS001_ClickHouse.md | tfs requirements generate > requirements.py +``` + +## Terminology + +You can define terminolgy using the examples below and make you can make them +linkable as [SRS] by defining the links in the [References](#References) section. + +### SRS + +Software Requirements Specification + +### Some term that you will use + +Some description of the term that you would like to use. + +## Requirements + +This section includes all the requirements. This section can be structured in any way one sees fit. + +Each requirement is defined by the section that starts with +the following prefix: + + RQ.[document id].[requirement name] + +then immediately followed by a one-line block that contains the +the `version` of the requirement. + +### RQ.CH-SRS001.Example short description of the requirement +version: 1.0 + +This is a long description of the requirement that can include any +relevant information. + +The one-line block that follows the requirement defines the `version` +of the requirement. The version is controlled manually and is used +to indicate material changes to the requirement that would +require tests that cover this requirement to be updated. + +It is a good practice to use requirement names that are broken +up into groups. It is not recommended to use only numbers +because if the requirement must be moved the numbering will not match. +Therefore, the requirement name should start with the group +name which is then followed by a number if any. For example, + + RQ.SRS001.Group.Subgroup.1 + +To keep names short, try to use abbreviations for the requirement's group name. + +### RQ.CH-SRS001.Example.Subgroup short description of the sub-requirement +version: 1.0 + +This an example of a sub-requirement of the [RQ.CH-SRS001.Example](#rqch-srs001example). +CH_SRS001_Software_Requirements_Template.md + +### RQ.CH-SRS001.Example.Select.1 +version: 1.0 + +[ClickHouse] SHALL return `1` when user executes query + +```sql +SELECT 1 +``` + +## References + +* **ClickHouse:** https://clickhouse.tech + +[SRS]: #SRS +[Some term that you will use]: #Sometermthatyouwilluse +[ClickHouse]: https://clickhouse.tech +[Git]: https://git-scm.com/ diff --git a/tests/testflows/example/requirements/requirements.py b/tests/testflows/example/requirements/requirements.py index 9a0b4922f19..f890b9a1daf 100644 --- a/tests/testflows/example/requirements/requirements.py +++ b/tests/testflows/example/requirements/requirements.py @@ -1,9 +1,66 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.200627.1211752. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. from testflows.core import Requirement -RQ_ClickHouse_Select_1 = Requirement( - name='RQ.ClickHouse.Select.1', +RQ_CH_SRS001_Example_short_description_of_the_requirement_ = Requirement( + name='RQ.CH-SRS001.Example short description of the requirement ', version='1.0', + priority=None, + group=None, + type=None, + uid=None, description=( - '[ClickHouse] SHALL return `1` when user sends a simple `SELECT 1` query.\n' - ) + 'This is a long description of the requirement that can include any\n' + 'relevant information. \n' + '\n' + 'The one-line block that follows the requirement defines the `version` \n' + 'of the requirement. The version is controlled manually and is used\n' + 'to indicate material changes to the requirement that would \n' + 'require tests that cover this requirement to be updated.\n' + '\n' + 'It is a good practice to use requirement names that are broken\n' + 'up into groups. It is not recommended to use only numbers\n' + 'because if the requirement must be moved the numbering will not match.\n' + 'Therefore, the requirement name should start with the group\n' + 'name which is then followed by a number if any. For example,\n' + '\n' + ' RQ.SRS001.Group.Subgroup.1\n' + '\n' + "To keep names short, try to use abbreviations for the requirement's group name.\n" + ), + link=None + ) + +RQ_CH_SRS001_Example_Subgroup_short_description_of_the_sub_requirement = Requirement( + name='RQ.CH-SRS001.Example.Subgroup short description of the sub-requirement', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'This an example of a sub-requirement of the [RQ.CH-SRS001.Example](#rqch-srs001example).\n' + 'CH_SRS001_Software_Requirements_Template.md\n' + ), + link=None + ) + +RQ_CH_SRS001_Example_Select_1 = Requirement( + name='RQ.CH-SRS001.Example.Select.1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return `1` when user executes query\n' + '\n' + '```sql\n' + 'SELECT 1\n' + '```\n' + ), + link=None ) diff --git a/tests/testflows/example/tests/example.py b/tests/testflows/example/tests/example.py index 8559f290cc4..6586fa80e41 100644 --- a/tests/testflows/example/tests/example.py +++ b/tests/testflows/example/tests/example.py @@ -6,7 +6,7 @@ from example.requirements import * @TestScenario @Name("select 1") @Requirements( - RQ_ClickHouse_Select_1("1.0") + RQ_CH_SRS001_Example_Select_1("1.0") ) def scenario(self, node="clickhouse1"): """Check that ClickHouse returns 1 when user executes `SELECT 1` query. From a233aa257d6bb7a3a1249c25ca0f90703e6aa642 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 2 Jul 2020 20:48:17 +0200 Subject: [PATCH 04/12] Small changes to the SRS template. --- .../example/requirements/CH_SRS001_ClickHouse.html | 8 ++++---- .../example/requirements/CH_SRS001_ClickHouse.md | 8 ++++---- tests/testflows/example/requirements/requirements.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html index 5aedad25979..ec466d16888 100644 --- a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html +++ b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html @@ -1313,8 +1313,8 @@ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  • 4 Requirements
  • 5 References
    • @@ -1390,7 +1390,7 @@ the following prefix:

    then immediately followed by a one-line block that contains the the version of the requirement.

    -

    RQ.CH-SRS001.Example short description of the requirement

    +

    RQ.CH-SRS001.Example

    version: 1.0

    @@ -1413,7 +1413,7 @@ name which is then followed by a number if any. For example,

    To keep names short, try to use abbreviations for the requirement's group name.

    -

    RQ.CH-SRS001.Example.Subgroup short description of the sub-requirement

    +

    RQ.CH-SRS001.Example.Subgroup

    version: 1.0

    diff --git a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md index 370ff14c2cb..284c6810958 100644 --- a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md +++ b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md @@ -15,8 +15,8 @@ * 3.1 [SRS](#srs) * 3.2 [Some term that you will use](#some-term-that-you-will-use) * 4 [Requirements](#requirements) - * 4.1 [RQ.CH-SRS001.Example short description of the requirement ](#rqch-srs001example-short-description-of-the-requirement-) - * 4.2 [RQ.CH-SRS001.Example.Subgroup short description of the sub-requirement](#rqch-srs001examplesubgroup-short-description-of-the-sub-requirement) + * 4.1 [RQ.CH-SRS001.Example](#rqch-srs001example) + * 4.2 [RQ.CH-SRS001.Example.Subgroup](#rqch-srs001examplesubgroup) * 4.3 [RQ.CH-SRS001.Example.Select.1](#rqch-srs001exampleselect1) * 5 [References](#references) @@ -90,7 +90,7 @@ the following prefix: then immediately followed by a one-line block that contains the the `version` of the requirement. -### RQ.CH-SRS001.Example short description of the requirement +### RQ.CH-SRS001.Example version: 1.0 This is a long description of the requirement that can include any @@ -111,7 +111,7 @@ name which is then followed by a number if any. For example, To keep names short, try to use abbreviations for the requirement's group name. -### RQ.CH-SRS001.Example.Subgroup short description of the sub-requirement +### RQ.CH-SRS001.Example.Subgroup version: 1.0 This an example of a sub-requirement of the [RQ.CH-SRS001.Example](#rqch-srs001example). diff --git a/tests/testflows/example/requirements/requirements.py b/tests/testflows/example/requirements/requirements.py index f890b9a1daf..b42a73c293e 100644 --- a/tests/testflows/example/requirements/requirements.py +++ b/tests/testflows/example/requirements/requirements.py @@ -5,8 +5,8 @@ # using 'tfs requirements generate' command. from testflows.core import Requirement -RQ_CH_SRS001_Example_short_description_of_the_requirement_ = Requirement( - name='RQ.CH-SRS001.Example short description of the requirement ', +RQ_CH_SRS001_Example = Requirement( + name='RQ.CH-SRS001.Example', version='1.0', priority=None, group=None, @@ -34,8 +34,8 @@ RQ_CH_SRS001_Example_short_description_of_the_requirement_ = Requirement( link=None ) -RQ_CH_SRS001_Example_Subgroup_short_description_of_the_sub_requirement = Requirement( - name='RQ.CH-SRS001.Example.Subgroup short description of the sub-requirement', +RQ_CH_SRS001_Example_Subgroup = Requirement( + name='RQ.CH-SRS001.Example.Subgroup', version='1.0', priority=None, group=None, From 464eb70ca4d4602116d15b694a6483cf591ed3dc Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 2 Jul 2020 20:52:17 +0200 Subject: [PATCH 05/12] Removing unnecessary text. --- tests/testflows/example/requirements/CH_SRS001_ClickHouse.html | 3 +-- tests/testflows/example/requirements/CH_SRS001_ClickHouse.md | 1 - tests/testflows/example/requirements/requirements.py | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html index ec466d16888..9cfb3564288 100644 --- a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html +++ b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html @@ -1417,8 +1417,7 @@ name which is then followed by a number if any. For example,

    version: 1.0

    -

    This an example of a sub-requirement of the RQ.CH-SRS001.Example. -CH_SRS001_Software_Requirements_Template.md

    +

    This an example of a sub-requirement of the RQ.CH-SRS001.Example.

    RQ.CH-SRS001.Example.Select.1

    diff --git a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md index 284c6810958..e4eb2e38c79 100644 --- a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md +++ b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md @@ -115,7 +115,6 @@ To keep names short, try to use abbreviations for the requirement's group name. version: 1.0 This an example of a sub-requirement of the [RQ.CH-SRS001.Example](#rqch-srs001example). -CH_SRS001_Software_Requirements_Template.md ### RQ.CH-SRS001.Example.Select.1 version: 1.0 diff --git a/tests/testflows/example/requirements/requirements.py b/tests/testflows/example/requirements/requirements.py index b42a73c293e..ef28a5fe149 100644 --- a/tests/testflows/example/requirements/requirements.py +++ b/tests/testflows/example/requirements/requirements.py @@ -43,7 +43,6 @@ RQ_CH_SRS001_Example_Subgroup = Requirement( uid=None, description=( 'This an example of a sub-requirement of the [RQ.CH-SRS001.Example](#rqch-srs001example).\n' - 'CH_SRS001_Software_Requirements_Template.md\n' ), link=None ) From 3f9dd7f62bae54ad6a8292bbbf511ae3f66c72e3 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 2 Jul 2020 21:02:15 +0200 Subject: [PATCH 06/12] Fix small typo. --- tests/testflows/example/requirements/CH_SRS001_ClickHouse.html | 2 +- tests/testflows/example/requirements/CH_SRS001_ClickHouse.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html index 9cfb3564288..8c40d7484cd 100644 --- a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html +++ b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.html @@ -1366,7 +1366,7 @@ or a new requirement is added.

    Terminology

    -

    You can define terminolgy using the examples below and make you can make them +

    You can define terminolgy using the examples below and you can make them linkable as SRS by defining the links in the References section.

    SRS

    diff --git a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md index e4eb2e38c79..36445fab6cf 100644 --- a/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md +++ b/tests/testflows/example/requirements/CH_SRS001_ClickHouse.md @@ -67,7 +67,7 @@ cat CH_SRS001_ClickHouse.md | tfs requirements generate > requirements.py ## Terminology -You can define terminolgy using the examples below and make you can make them +You can define terminolgy using the examples below and you can make them linkable as [SRS] by defining the links in the [References](#References) section. ### SRS From a449f3e9ff950e88de9269ae01c61e0bde99e6b2 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Tue, 7 Jul 2020 12:03:37 +0300 Subject: [PATCH 07/12] A test for UInt8 as bool --- .../00552_logical_functions_uint8_as_bool.reference | 8 ++++++++ .../00552_logical_functions_uint8_as_bool.sql | 11 +++++++++++ 2 files changed, 19 insertions(+) create mode 100644 tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference create mode 100644 tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql diff --git a/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference new file mode 100644 index 00000000000..9a65a2603dc --- /dev/null +++ b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference @@ -0,0 +1,8 @@ +0 0 0 0 0 +0 0 1 0 1 +0 2 0 0 1 +0 2 1 0 1 +4 0 0 0 1 +4 0 1 0 1 +4 2 0 0 1 +4 2 1 1 1 diff --git a/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql new file mode 100644 index 00000000000..46a5fa39d47 --- /dev/null +++ b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql @@ -0,0 +1,11 @@ + +-- Test that UInt8 type is processed correctly as bool + +SELECT + toUInt8(bitAnd(number, 4)) AS a, + toUInt8(bitAnd(number, 2)) AS b, + toUInt8(bitAnd(number, 1)) AS c, + a AND b AND c AS AND, + a OR b OR c AS OR +FROM numbers(8) +; \ No newline at end of file From 66128acd113bbcaf8c43bac9de13b39093f6775b Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Tue, 7 Jul 2020 12:17:35 +0300 Subject: [PATCH 08/12] Included const uint8 values in test --- .../00552_logical_functions_uint8_as_bool.reference | 1 + .../00552_logical_functions_uint8_as_bool.sql | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference index 9a65a2603dc..2a1151dc692 100644 --- a/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference +++ b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.reference @@ -1,3 +1,4 @@ +1 1 1 1 1 1 0 0 0 0 0 0 0 1 0 1 0 2 0 0 1 diff --git a/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql index 46a5fa39d47..feee33add1c 100644 --- a/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql +++ b/tests/queries/0_stateless/00552_logical_functions_uint8_as_bool.sql @@ -1,6 +1,15 @@ -- Test that UInt8 type is processed correctly as bool +SELECT + 1 AND 2, + 2 AND 4, + 1 AND 2 AND 4, + 1 OR 2, + 2 OR 4, + 1 OR 2 OR 4 +; + SELECT toUInt8(bitAnd(number, 4)) AS a, toUInt8(bitAnd(number, 2)) AS b, @@ -8,4 +17,4 @@ SELECT a AND b AND c AS AND, a OR b OR c AS OR FROM numbers(8) -; \ No newline at end of file +; From 2906ae37f50bfb5504ad80ec56585d245caa6f54 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Tue, 7 Jul 2020 13:26:11 +0300 Subject: [PATCH 09/12] Simple (and fast) inplace fix for UInt8 -> bool --- src/Functions/FunctionsLogical.cpp | 14 ++++++++------ src/Functions/FunctionsLogical.h | 9 ++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 07351f6f2c9..5443df33d79 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -71,7 +71,7 @@ bool tryConvertColumnToBool(const IColumn * column, UInt8Container & res) std::transform( col->getData().cbegin(), col->getData().cend(), res.begin(), - [](const auto x) { return x != 0; }); + [](const auto x) { return !!x; }); return true; } @@ -145,6 +145,7 @@ inline bool extractConstColumnsAsTernary(ColumnRawPtrs & in, UInt8 & res_3v) } +/// N.B. This class calculates result only for non-nullable types template class AssociativeApplierImpl { @@ -158,7 +159,7 @@ public: /// Returns a combination of values in the i-th row of all columns stored in the constructor. inline ResultValueType apply(const size_t i) const { - const auto & a = vec[i]; + const auto a = !!vec[i]; if constexpr (Op::isSaturable()) return Op::isSaturatedValue(a) ? a : Op::apply(a, next.apply(i)); else @@ -179,7 +180,7 @@ public: explicit AssociativeApplierImpl(const UInt8ColumnPtrs & in) : vec(in[in.size() - 1]->getData()) {} - inline ResultValueType apply(const size_t i) const { return vec[i]; } + inline ResultValueType apply(const size_t i) const { return !!vec[i]; } private: const UInt8Container & vec; @@ -247,7 +248,7 @@ public: { const auto a = val_getter(i); if constexpr (Op::isSaturable()) - return Op::isSaturatedValue(a) ? a : Op::apply(a, next.apply(i)); + return Op::isSaturatedValueTernary(a) ? a : Op::apply(a, next.apply(i)); else return Op::apply(a, next.apply(i)); } @@ -332,7 +333,7 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn { /// Combine all constant columns into a single constant value. UInt8 const_3v_value = 0; - const bool has_consts = extractConstColumnsTernary(arguments, const_3v_value); + const bool has_consts = extractConstColumnsAsTernary(arguments, const_3v_value); /// If the constant value uniquely determines the result, return it. if (has_consts && (arguments.empty() || Op::isSaturatedValue(const_3v_value))) @@ -402,12 +403,13 @@ struct TypedExecutorInvoker }; +/// Types of all of the arguments are guaranteed to be non-nullable here template static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & result_info, size_t input_rows_count) { /// Combine all constant columns into a single constant value. UInt8 const_val = 0; - bool has_consts = extractConstColumns(arguments, const_val); + bool has_consts = extractConstColumnsAsBool(arguments, const_val); /// If the constant value uniquely determines the result, return it. if (has_consts && (arguments.empty() || Op::apply(const_val, 0) == Op::apply(const_val, 1))) diff --git a/src/Functions/FunctionsLogical.h b/src/Functions/FunctionsLogical.h index 520a2418890..4751f0b7548 100644 --- a/src/Functions/FunctionsLogical.h +++ b/src/Functions/FunctionsLogical.h @@ -61,7 +61,8 @@ struct AndImpl using ResultType = UInt8; static inline constexpr bool isSaturable() { return true; } - static inline constexpr bool isSaturatedValue(UInt8 a) { return a == Ternary::False; } + static inline constexpr bool isSaturatedValue(bool a) { return !a; } + static inline constexpr bool isSaturatedValueTernary(UInt8 a) { return a == Ternary::False; } static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a & b; } static inline constexpr bool specialImplementationForNulls() { return true; } }; @@ -71,7 +72,8 @@ struct OrImpl using ResultType = UInt8; static inline constexpr bool isSaturable() { return true; } - static inline constexpr bool isSaturatedValue(UInt8 a) { return a == Ternary::True; } + static inline constexpr bool isSaturatedValue(bool a) { return a; } + static inline constexpr bool isSaturatedValueTernary(UInt8 a) { return a == Ternary::True; } static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a | b; } static inline constexpr bool specialImplementationForNulls() { return true; } }; @@ -82,7 +84,8 @@ struct XorImpl static inline constexpr bool isSaturable() { return false; } static inline constexpr bool isSaturatedValue(bool) { return false; } - static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return !!a != !!b; } + static inline constexpr bool isSaturatedValueTernary(UInt8) { return false; } + static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return a != b; } static inline constexpr bool specialImplementationForNulls() { return false; } #if USE_EMBEDDED_COMPILER From f5a7d8a5c73254d2457b04f87b4f1fc80effc28f Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 7 Jul 2020 15:35:06 +0300 Subject: [PATCH 10/12] Add runner for testflows --- docker/test/testflows/runner/Dockerfile | 76 +++++++++++ .../testflows/runner/dockerd-entrypoint.sh | 26 ++++ docker/test/testflows/runner/modprobe.sh | 20 +++ tests/testflows/runner | 121 ++++++++++++++++++ 4 files changed, 243 insertions(+) create mode 100644 docker/test/testflows/runner/Dockerfile create mode 100755 docker/test/testflows/runner/dockerd-entrypoint.sh create mode 100755 docker/test/testflows/runner/modprobe.sh create mode 100755 tests/testflows/runner diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile new file mode 100644 index 00000000000..7dd73b0e0bb --- /dev/null +++ b/docker/test/testflows/runner/Dockerfile @@ -0,0 +1,76 @@ +# docker build -t yandex/clickhouse-testflows-runner . +FROM ubuntu:20.04 + +RUN apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + ca-certificates \ + bash \ + btrfs-progs \ + e2fsprogs \ + iptables \ + xfsprogs \ + tar \ + pigz \ + wget \ + git \ + iproute2 \ + cgroupfs-mount \ + python3-pip \ + tzdata \ + libreadline-dev \ + libicu-dev \ + bsdutils \ + curl \ + liblua5.1-dev \ + luajit \ + libssl-dev \ + libcurl4-openssl-dev \ + gdb \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/debconf \ + /tmp/* \ + && apt-get clean + +ENV TZ=Europe/Moscow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN pip3 install urllib3 testflows==1.6.24 docker-compose docker dicttoxml kazoo tzlocal + +ENV DOCKER_CHANNEL stable +ENV DOCKER_VERSION 17.09.1-ce + +RUN set -eux; \ + \ +# this "case" statement is generated via "update.sh" + \ + if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \ + echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \ + exit 1; \ + fi; \ + \ + tar --extract \ + --file docker.tgz \ + --strip-components 1 \ + --directory /usr/local/bin/ \ + ; \ + rm docker.tgz; \ + \ + dockerd --version; \ + docker --version + +COPY modprobe.sh /usr/local/bin/modprobe +COPY dockerd-entrypoint.sh /usr/local/bin/ + +RUN set -x \ + && addgroup --system dockremap \ + && adduser --system dockremap \ + && adduser dockremap dockremap \ + && echo 'dockremap:165536:65536' >> /etc/subuid \ + && echo 'dockremap:165536:65536' >> /etc/subgid + +VOLUME /var/lib/docker +EXPOSE 2375 +ENTRYPOINT ["dockerd-entrypoint.sh"] +CMD ["sh", "-c", "python3 regression.py --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS} && cat test.log | tfs report results --format json > results.json"] + diff --git a/docker/test/testflows/runner/dockerd-entrypoint.sh b/docker/test/testflows/runner/dockerd-entrypoint.sh new file mode 100755 index 00000000000..b10deaded08 --- /dev/null +++ b/docker/test/testflows/runner/dockerd-entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &>/var/log/somefile & + +set +e +reties=0 +while true; do + docker info &>/dev/null && break + reties=$[$reties+1] + if [[ $reties -ge 100 ]]; then # 10 sec max + echo "Can't start docker daemon, timeout exceeded." >&2 + exit 1; + fi + sleep 0.1 +done +set -e + +echo "Start tests" +export CLICKHOUSE_TESTS_SERVER_BIN_PATH=/clickhouse +export CLICKHOUSE_TESTS_CLIENT_BIN_PATH=/clickhouse +export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=/clickhouse-config +export CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH=/clickhouse-odbc-bridge + +cd /ClickHouse/tests/testflows +exec "$@" diff --git a/docker/test/testflows/runner/modprobe.sh b/docker/test/testflows/runner/modprobe.sh new file mode 100755 index 00000000000..cb6a527736b --- /dev/null +++ b/docker/test/testflows/runner/modprobe.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu + +# "modprobe" without modprobe +# https://twitter.com/lucabruno/status/902934379835662336 + +# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe + +# Docker often uses "modprobe -va foo bar baz" +# so we ignore modules that start with "-" +for module; do + if [ "${module#-}" = "$module" ]; then + ip link show "$module" || true + lsmod | grep "$module" || true + fi +done + +# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort +export PATH='/usr/sbin:/usr/bin:/sbin:/bin' +exec modprobe "$@" diff --git a/tests/testflows/runner b/tests/testflows/runner new file mode 100755 index 00000000000..eef51b3c20f --- /dev/null +++ b/tests/testflows/runner @@ -0,0 +1,121 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- +import subprocess +import os +import getpass +import argparse +import logging +import signal +import subprocess +import sys + +CUR_FILE_DIR = os.path.dirname(os.path.realpath(__file__)) +DEFAULT_CLICKHOUSE_ROOT = os.path.abspath(os.path.join(CUR_FILE_DIR, "../../")) +CURRENT_WORK_DIR = os.getcwd() +CONTAINER_NAME = "clickhouse_testflows_tests" + +DIND_TESTFLOWS_TESTS_IMAGE_NAME = "yandex/clickhouse-testflows-runner" + +def check_args_and_update_paths(args): + if not os.path.isabs(args.binary): + args.binary = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.binary)) + + if not args.bridge_binary: + args.bridge_binary = os.path.join(os.path.dirname(args.binary), 'clickhouse-odbc-bridge') + elif not os.path.isabs(args.bridge_binary): + args.bridge_binary = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.bridge_binary)) + + if not os.path.isabs(args.configs_dir): + args.configs_dir = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.configs_dir)) + + if not os.path.isabs(args.clickhouse_root): + args.clickhouse_root = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.clickhouse_root)) + + for path in [args.binary, args.configs_dir, args.clickhouse_root]: + if not os.path.exists(path): + raise Exception("Path {} doesn't exists".format(path)) + +def docker_kill_handler_handler(signum, frame): + subprocess.check_call('docker kill $(docker ps -a -q --filter name={name} --format="{{{{.ID}}}}")'.format(name=CONTAINER_NAME), shell=True) + raise KeyboardInterrupt("Killed by Ctrl+C") + +signal.signal(signal.SIGINT, docker_kill_handler_handler) + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + parser = argparse.ArgumentParser(description="ClickHouse testflows runner") + + parser.add_argument( + "--binary", + default=os.environ.get("CLICKHOUSE_TESTS_SERVER_BIN_PATH", os.environ.get("CLICKHOUSE_TESTS_CLIENT_BIN_PATH", "/usr/bin/clickhouse")), + help="Path to clickhouse binary") + + parser.add_argument( + "--bridge-binary", + default=os.environ.get("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH", ""), + help="Path to clickhouse-odbc-bridge binary. Defaults to clickhouse-odbc-bridge in the same dir as clickhouse.") + + parser.add_argument( + "--configs-dir", + default=os.environ.get("CLICKHOUSE_TESTS_BASE_CONFIG_DIR", os.path.join(DEFAULT_CLICKHOUSE_ROOT, "programs/server")), + help="Path to clickhouse configs directory") + + parser.add_argument( + "--clickhouse-root", + default=DEFAULT_CLICKHOUSE_ROOT, + help="Path to repository root folder") + + parser.add_argument( + "--command", + default='', + help="Set it to run some other command in container (for example bash)") + + parser.add_argument( + "--disable-net-host", + action='store_true', + default=False, + help="Don't use net host in parent docker container") + + parser.add_argument( + "--docker-image-version", + default="latest", + help="Version of docker image which runner will use to run tests") + + + parser.add_argument('testflows_args', nargs='*', help="args for testflows command") + + args = parser.parse_args() + + check_args_and_update_paths(args) + + net = "" + if not args.disable_net_host: + net = "--net=host" + + # create named volume which will be used inside to store images and other docker related files, + # to avoid redownloading it every time + # + # should be removed manually when not needed + subprocess.check_call('docker volume create {name}_volume'.format(name=CONTAINER_NAME), shell=True) + + # enable tty mode & interactive for docker if we have real tty + tty = "" + if sys.stdout.isatty() and sys.stdin.isatty(): + tty = "-it" + + cmd = "docker run {net} {tty} --rm --name {name} --privileged --volume={bridge_bin}:/clickhouse-odbc-bridge --volume={bin}:/clickhouse \ + --volume={cfg}:/clickhouse-config --volume={pth}:/ClickHouse --volume={name}_volume:/var/lib/docker -e TESTFLOWS_OPTS='{opts}' {img} {command}".format( + net=net, + tty=tty, + bin=args.binary, + bridge_bin=args.bridge_binary, + cfg=args.configs_dir, + pth=args.clickhouse_root, + opts=' '.join(args.testflows_args), + img=DIND_TESTFLOWS_TESTS_IMAGE_NAME + ":" + args.docker_image_version, + name=CONTAINER_NAME, + command=args.command + ) + + print("Running testflows container as: '" + cmd + "'.") + subprocess.check_call(cmd, shell=True) From e89ccfd5bf99e5c98abe18b0290990d47cd350f6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 7 Jul 2020 16:11:19 +0300 Subject: [PATCH 11/12] No color --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 7dd73b0e0bb..71dca40bca6 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -72,5 +72,5 @@ RUN set -x \ VOLUME /var/lib/docker EXPOSE 2375 ENTRYPOINT ["dockerd-entrypoint.sh"] -CMD ["sh", "-c", "python3 regression.py --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS} && cat test.log | tfs report results --format json > results.json"] +CMD ["sh", "-c", "python3 regression.py --no-color --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS} && cat test.log | tfs report results --format json > results.json"] From a7348ed2c8fdc6b4ec67b82ee071fea8086b50e6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 7 Jul 2020 18:48:47 +0300 Subject: [PATCH 12/12] Add to images.json --- docker/images.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/images.json b/docker/images.json index 23f8cc0d9fd..45209fc775f 100644 --- a/docker/images.json +++ b/docker/images.json @@ -79,5 +79,9 @@ "docker/test/integration/runner": { "name": "yandex/clickhouse-integration-tests-runner", "dependent": [] + }, + "docker/test/testflows/runner": { + "name": "yandex/clickhouse-testflows-runner", + "dependent": [] } }