diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 6948aeb3b18..fccae66b66b 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -42,15 +42,12 @@ RUN apt-get update \ clang-tidy-10 \ clang-tidy-11 \ cmake \ - cmake \ curl \ g++-9 \ gcc-9 \ gdb \ git \ gperf \ - gperf \ - intel-opencl-icd \ libicu-dev \ libreadline-dev \ lld-10 \ @@ -61,10 +58,7 @@ RUN apt-get update \ llvm-11-dev \ moreutils \ ninja-build \ - ocl-icd-libopencl1 \ - opencl-headers \ pigz \ - pixz \ rename \ tzdata \ --yes --no-install-recommends diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile index f640c595f14..4dd6dbc61d8 100644 --- a/docker/packager/unbundled/Dockerfile +++ b/docker/packager/unbundled/Dockerfile @@ -35,9 +35,6 @@ RUN apt-get update \ libjemalloc-dev \ libmsgpack-dev \ libcurl4-openssl-dev \ - opencl-headers \ - ocl-icd-libopencl1 \ - intel-opencl-icd \ unixodbc-dev \ odbcinst \ tzdata \ diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index 9efe08dbf23..c5463ac447d 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -14,9 +14,7 @@ RUN apt-get --allow-unauthenticated update -y \ expect \ gdb \ gperf \ - gperf \ heimdal-multidev \ - intel-opencl-icd \ libboost-filesystem-dev \ libboost-iostreams-dev \ libboost-program-options-dev \ @@ -50,9 +48,7 @@ RUN apt-get --allow-unauthenticated update -y \ moreutils \ ncdu \ netcat-openbsd \ - ocl-icd-libopencl1 \ odbcinst \ - opencl-headers \ openssl \ perl \ pigz \ diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 3181f26800d..852b9de4fb3 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -27,53 +27,20 @@ Or cmake3 instead of cmake on older systems. On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) -```bash +```bash sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html). -#### Use clang-11 for Builds {#use-gcc-10-for-builds} +#### Use clang-11 for Builds ``` bash $ export CC=clang-11 $ export CXX=clang++-11 ``` -### Install GCC 10 {#install-gcc-10} - -We recommend building ClickHouse with clang-11, GCC-10 also supported, but it is not used for production builds. - -If you want to use GCC-10 there are several ways to install it. - -#### Install from Repository {#install-from-repository} - -On Ubuntu 19.10 or newer: - - $ sudo apt-get update - $ sudo apt-get install gcc-10 g++-10 - -#### Install from a PPA Package {#install-from-a-ppa-package} - -On older Ubuntu: - -``` bash -$ sudo apt-get install software-properties-common -$ sudo apt-add-repository ppa:ubuntu-toolchain-r/test -$ sudo apt-get update -$ sudo apt-get install gcc-10 g++-10 -``` - -#### Install from Sources {#install-from-sources} - -See [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) - -#### Use GCC 10 for Builds {#use-gcc-10-for-builds} - -``` bash -$ export CC=gcc-10 -$ export CXX=g++-10 -``` +Gcc can also be used though it is discouraged. ### Checkout ClickHouse Sources {#checkout-clickhouse-sources} diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index 5511e8e19c7..35ca4725af8 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -131,17 +131,18 @@ ClickHouse uses several external libraries for building. All of them do not need ## C++ Compiler {#c-compiler} -Compilers GCC starting from version 10 and Clang version 8 or above are supported for building ClickHouse. +Compilers Clang starting from version 11 is supported for building ClickHouse. -Official Yandex builds currently use GCC because it generates machine code of slightly better performance (yielding a difference of up to several percent according to our benchmarks). And Clang is more convenient for development usually. Though, our continuous integration (CI) platform runs checks for about a dozen of build combinations. +Clang should be used instead of gcc. Though, our continuous integration (CI) platform runs checks for about a dozen of build combinations. -To install GCC on Ubuntu run: `sudo apt install gcc g++` +On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) -Check the version of gcc: `gcc --version`. If it is below 10, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/#install-gcc-10. +```bash +sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +``` -Mac OS X build is supported only for Clang. Just run `brew install llvm` +Mac OS X build is also supported. Just run `brew install llvm` -If you decide to use Clang, you can also install `libc++` and `lld`, if you know what it is. Using `ccache` is also recommended. ## The Building Process {#the-building-process} @@ -152,14 +153,7 @@ Now that you are ready to build ClickHouse we recommend you to create a separate You can have several different directories (build_release, build_debug, etc.) for different types of build. -While inside the `build` directory, configure your build by running CMake. Before the first run, you need to define environment variables that specify compiler (version 10 gcc compiler in this example). - -Linux: - - export CC=gcc-10 CXX=g++-10 - cmake .. - -Mac OS X: +While inside the `build` directory, configure your build by running CMake. Before the first run, you need to define environment variables that specify compiler. export CC=clang CXX=clang++ cmake .. diff --git a/docs/en/development/style.md b/docs/en/development/style.md index 4c620c44aef..b27534d9890 100644 --- a/docs/en/development/style.md +++ b/docs/en/development/style.md @@ -701,7 +701,7 @@ But other things being equal, cross-platform or portable code is preferred. **2.** Language: C++20 (see the list of available [C++20 features](https://en.cppreference.com/w/cpp/compiler_support#C.2B.2B20_features)). -**3.** Compiler: `gcc`. At this time (August 2020), the code is compiled using version 9.3. (It can also be compiled using `clang 8`.) +**3.** Compiler: `clang`. At this time (April 2021), the code is compiled using clang version 11. (It can also be compiled using `gcc` version 10, but it's untested and not suitable for production usage). The standard library is used (`libc++`). @@ -711,7 +711,7 @@ The standard library is used (`libc++`). The CPU instruction set is the minimum supported set among our servers. Currently, it is SSE 4.2. -**6.** Use `-Wall -Wextra -Werror` compilation flags. +**6.** Use `-Wall -Wextra -Werror` compilation flags. Also `-Weverything` is used with few exceptions. **7.** Use static linking with all libraries except those that are difficult to connect to statically (see the output of the `ldd` command). diff --git a/docs/en/operations/performance-test.md b/docs/en/operations/performance-test.md index ca805923ba9..a808ffd0a85 100644 --- a/docs/en/operations/performance-test.md +++ b/docs/en/operations/performance-test.md @@ -12,6 +12,7 @@ With this instruction you can run basic ClickHouse performance test on any serve 3. Copy the link to `clickhouse` binary for amd64 or aarch64. 4. ssh to the server and download it with wget: ```bash +# These links are outdated, please obtain the fresh link from the "commits" page. # For amd64: wget https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse # For aarch64: diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 6a1b25982a0..db17fa905b0 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -854,8 +854,6 @@ For example, when reading from a table, if it is possible to evaluate expression Default value: the number of physical CPU cores. -If less than one SELECT query is normally run on a server at a time, set this parameter to a value slightly less than the actual number of processor cores. - For queries that are completed quickly because of a LIMIT, you can set a lower ‘max_threads’. For example, if the necessary number of entries are located in every block and max_threads = 8, then 8 blocks are retrieved, although it would have been enough to read just one. The smaller the `max_threads` value, the less memory is consumed. diff --git a/docs/en/operations/system-tables/replication_queue.md b/docs/en/operations/system-tables/replication_queue.md index f3e3a35f13b..539a29432ac 100644 --- a/docs/en/operations/system-tables/replication_queue.md +++ b/docs/en/operations/system-tables/replication_queue.md @@ -15,16 +15,16 @@ Columns: - `node_name` ([String](../../sql-reference/data-types/string.md)) — Node name in ZooKeeper. - `type` ([String](../../sql-reference/data-types/string.md)) — Type of the task in the queue, one of: - - `GET_PART` - Get the part from another replica. - - `ATTACH_PART` - Attach the part, possibly from our own replica (if found in `detached` folder). - You may think of it as a `GET_PART` with some optimisations as they're nearly identical. - - `MERGE_PARTS` - Merge the parts. - - `DROP_RANGE` - Delete the parts in the specified partition in the specified number range. - - `CLEAR_COLUMN` - NOTE: Deprecated. Drop specific column from specified partition. - - `CLEAR_INDEX` - NOTE: Deprecated. Drop specific index from specified partition. - - `REPLACE_RANGE` - Drop certain range of partitions and replace them by new ones - - `MUTATE_PART` - Apply one or several mutations to the part. - - `ALTER_METADATA` - Apply alter modification according to global /metadata and /columns paths + + - `GET_PART` — Get the part from another replica. + - `ATTACH_PART` — Attach the part, possibly from our own replica (if found in the `detached` folder). You may think of it as a `GET_PART` with some optimizations as they're nearly identical. + - `MERGE_PARTS` — Merge the parts. + - `DROP_RANGE` — Delete the parts in the specified partition in the specified number range. + - `CLEAR_COLUMN` — NOTE: Deprecated. Drop specific column from specified partition. + - `CLEAR_INDEX` — NOTE: Deprecated. Drop specific index from specified partition. + - `REPLACE_RANGE` — Drop a certain range of parts and replace them with new ones. + - `MUTATE_PART` — Apply one or several mutations to the part. + - `ALTER_METADATA` — Apply alter modification according to global /metadata and /columns paths. - `create_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — Date and time when the task was submitted for execution. diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 948711e6d9e..b22f89928b9 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -88,12 +88,10 @@ Read more about setting the partition expression in a section [How to specify th This query is replicated. The replica-initiator checks whether there is data in the `detached` directory. If data exists, the query checks its integrity. If everything is correct, the query adds the data to the table. -If the non-initiator replica, receiving the attach command, finds the part with the correct checksums in its own -`detached` folder, it attaches the data without fetching it from other replicas. +If the non-initiator replica, receiving the attach command, finds the part with the correct checksums in its own `detached` folder, it attaches the data without fetching it from other replicas. If there is no part with the correct checksums, the data is downloaded from any replica having the part. -You can put data to the `detached` directory on one replica and use the `ALTER ... ATTACH` query to add it to the -table on all replicas. +You can put data to the `detached` directory on one replica and use the `ALTER ... ATTACH` query to add it to the table on all replicas. ## ATTACH PARTITION FROM {#alter_attach-partition-from} @@ -101,8 +99,8 @@ table on all replicas. ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1 ``` -This query copies the data partition from the `table1` to `table2`. -Note that data won't be deleted neither from `table1` nor from `table2`. +This query copies the data partition from `table1` to `table2`. +Note that data will be deleted neither from `table1` nor from `table2`. For the query to run successfully, the following conditions must be met: diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 919bd65d56b..7871894ccac 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -264,9 +264,7 @@ Wait until a `ReplicatedMergeTree` table will be synced with other replicas in a SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ``` -After running this statement the `[db.]replicated_merge_tree_family_table_name` fetches commands from -the common replicated log into its own replication queue, and then the query waits till the replica processes all -of the fetched commands. +After running this statement the `[db.]replicated_merge_tree_family_table_name` fetches commands from the common replicated log into its own replication queue, and then the query waits till the replica processes all of the fetched commands. ### RESTART REPLICA {#query_language-system-restart-replica} diff --git a/docs/ja/development/build.md b/docs/ja/development/build.md index e44ba45485e..191fa665ccd 100644 --- a/docs/ja/development/build.md +++ b/docs/ja/development/build.md @@ -19,28 +19,17 @@ $ sudo apt-get install git cmake python ninja-build 古いシステムではcmakeの代わりにcmake3。 -## GCC9のインストール {#install-gcc-10} +## Clang 11 のインストール -これを行うにはいくつかの方法があります。 +On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) -### PPAパッケージからインストール {#install-from-a-ppa-package} - -``` bash -$ sudo apt-get install software-properties-common -$ sudo apt-add-repository ppa:ubuntu-toolchain-r/test -$ sudo apt-get update -$ sudo apt-get install gcc-10 g++-10 +```bash +sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` -### ソースからインスト {#install-from-sources} - -見て [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) - -## ビルドにGCC9を使用する {#use-gcc-10-for-builds} - ``` bash -$ export CC=gcc-10 -$ export CXX=g++-10 +$ export CC=clang +$ export CXX=clang++ ``` ## ﾂつｨﾂ姪"ﾂ債ﾂつｹ {#checkout-clickhouse-sources} @@ -76,7 +65,7 @@ $ cd .. - Git（ソースをチェックアウトするためにのみ使用され、ビルドには必要ありません) - CMake3.10以降 - 忍者（推奨）または作る -- C++コンパイラ:gcc9またはclang8以降 +- C++コンパイラ:clang11以降 - リンカ:lldまたはgold(古典的なGNU ldは動作しません) - Python(LLVMビルド内でのみ使用され、オプションです) diff --git a/docs/ja/development/developer-instruction.md b/docs/ja/development/developer-instruction.md index ccc3a177d1f..d7e5217b3b6 100644 --- a/docs/ja/development/developer-instruction.md +++ b/docs/ja/development/developer-instruction.md @@ -133,19 +133,19 @@ ArchまたはGentooを使用する場合は、おそらくCMakeのインスト ClickHouseはビルドに複数の外部ライブラリを使用します。 それらのすべては、サブモジュールにあるソースからClickHouseと一緒に構築されているので、別々にインストールする必要はありません。 リストは次の場所で確認できます `contrib`. -# C++コンパイラ {#c-compiler} +## C++ Compiler {#c-compiler} -ClickHouseのビルドには、バージョン9以降のGCCとClangバージョン8以降のコンパイラがサポートされます。 +Compilers Clang starting from version 11 is supported for building ClickHouse. -公式のYandexビルドは、わずかに優れたパフォーマンスのマシンコードを生成するため、GCCを使用しています（私たちのベンチマークに応じて最大数パーセントの そしてClangは開発のために通常より便利です。 が、当社の継続的インテグレーション（CI)プラットフォームを運チェックのための十数の組み合わせとなります。 +Clang should be used instead of gcc. Though, our continuous integration (CI) platform runs checks for about a dozen of build combinations. -UBUNTUにGCCをインストールするには: `sudo apt install gcc g++` +On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) -Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/ja/development/build/#install-gcc-10. +```bash +sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +``` -Mac OS XのビルドはClangでのみサポートされています。 ちょうど実行 `brew install llvm` - -Clangを使用する場合は、次のものもインストールできます `libc++` と `lld` あなたがそれが何であるか知っていれば。 を使用して `ccache` また、推奨されます。 +Mac OS X build is also supported. Just run `brew install llvm` # 建築プロセス {#the-building-process} @@ -158,13 +158,6 @@ ClickHouseを構築する準備ができたので、別のディレクトリを 中の間 `build` cmakeを実行してビルドを構成します。 最初の実行の前に、コンパイラ（この例ではバージョン9gccコンパイラ）を指定する環境変数を定義する必要があります。 -Linux: - - export CC=gcc-10 CXX=g++-10 - cmake .. - -Mac OS X: - export CC=clang CXX=clang++ cmake .. diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 66bbb9030cf..463d38a44fb 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -136,18 +136,18 @@ ClickHouse использует для сборки некоторое коли ## Компилятор C++ {#kompiliator-c} -В качестве компилятора C++ поддерживается GCC начиная с версии 9 или Clang начиная с версии 8. +В качестве компилятора C++ поддерживается Clang начиная с версии 11. -Официальные сборки от Яндекса, на данный момент, используют GCC, так как он генерирует слегка более производительный машинный код (разница в среднем до нескольких процентов по нашим бенчмаркам). Clang обычно более удобен для разработки. Впрочем, наша среда continuous integration проверяет около десятка вариантов сборки. +Впрочем, наша среда continuous integration проверяет около десятка вариантов сборки, включая gcc, но сборка с помощью gcc непригодна для использования в продакшене. -Для установки GCC под Ubuntu, выполните: `sudo apt install gcc g++`. +On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) -Проверьте версию gcc: `gcc --version`. Если версия меньше 10, то следуйте инструкции: https://clickhouse.tech/docs/ru/development/build/#install-gcc-10. +```bash +sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +``` Сборка под Mac OS X поддерживается только для компилятора Clang. Чтобы установить его выполните `brew install llvm` -Если вы решили использовать Clang, вы также можете установить `libc++` и `lld`, если вы знаете, что это такое. При желании, установите `ccache`. - ## Процесс сборки {#protsess-sborki} Теперь вы готовы к сборке ClickHouse. Для размещения собранных файлов, рекомендуется создать отдельную директорию build внутри директории ClickHouse: @@ -158,14 +158,7 @@ ClickHouse использует для сборки некоторое коли Вы можете иметь несколько разных директорий (build_release, build_debug) для разных вариантов сборки. Находясь в директории build, выполните конфигурацию сборки с помощью CMake. -Перед первым запуском необходимо выставить переменные окружения, отвечающие за выбор компилятора (в данном примере это - gcc версии 9). - -Linux: - - export CC=gcc-10 CXX=g++-10 - cmake .. - -Mac OS X: +Перед первым запуском необходимо выставить переменные окружения, отвечающие за выбор компилятора. export CC=clang CXX=clang++ cmake .. diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md index f08ecc3c4c7..de29e629ceb 100644 --- a/docs/ru/development/style.md +++ b/docs/ru/development/style.md @@ -747,7 +747,7 @@ The dictionary is configured incorrectly. Есть два основных варианта проверки на такие ошибки: * Исключение с кодом `LOGICAL_ERROR`. Его можно использовать для важных проверок, которые делаются в том числе в релизной сборке. -* `assert`. Такие условия не проверяются в релизной сборке, можно использовать для тяжёлых и опциональных проверок. +* `assert`. Такие условия не проверяются в релизной сборке, можно использовать для тяжёлых и опциональных проверок. Пример сообщения, у которого должен быть код `LOGICAL_ERROR`: `Block header is inconsistent with Chunk in ICompicatedProcessor::munge(). It is a bug!` @@ -780,7 +780,7 @@ The dictionary is configured incorrectly. **2.** Язык - C++20 (см. список доступных [C++20 фич](https://en.cppreference.com/w/cpp/compiler_support#C.2B.2B20_features)). -**3.** Компилятор - `gcc`. На данный момент (август 2020), код собирается версией 9.3. (Также код может быть собран `clang` версий 10 и 9) +**3.** Компилятор - `clang`. На данный момент (апрель 2021), код собирается версией 11. (Также код может быть собран `gcc` версии 10, но такая сборка не тестируется и непригодна для продакшена). Используется стандартная библиотека (реализация `libc++`). diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 7acdd65051b..4951be49629 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -844,8 +844,6 @@ SELECT type, query FROM system.query_log WHERE log_comment = 'log_comment test' Значение по умолчанию: количество процессорных ядер без учёта Hyper-Threading. -Если на сервере обычно исполняется менее одного запроса SELECT одновременно, то выставите этот параметр в значение чуть меньше количества реальных процессорных ядер. - Для запросов, которые быстро завершаются из-за LIMIT-а, имеет смысл выставить max_threads поменьше. Например, если нужное количество записей находится в каждом блоке, то при max_threads = 8 будет считано 8 блоков, хотя достаточно было прочитать один. Чем меньше `max_threads`, тем меньше будет использоваться оперативки. diff --git a/docs/ru/operations/system-tables/replication_queue.md b/docs/ru/operations/system-tables/replication_queue.md index 56e8c695a21..2f9d80be16f 100644 --- a/docs/ru/operations/system-tables/replication_queue.md +++ b/docs/ru/operations/system-tables/replication_queue.md @@ -14,7 +14,17 @@ - `node_name` ([String](../../sql-reference/data-types/string.md)) — имя узла в ZooKeeper. -- `type` ([String](../../sql-reference/data-types/string.md)) — тип задачи в очереди: `GET_PARTS`, `MERGE_PARTS`, `DETACH_PARTS`, `DROP_PARTS` или `MUTATE_PARTS`. +- `type` ([String](../../sql-reference/data-types/string.md)) — тип задачи в очереди: + + - `GET_PART` — скачать кусок с другой реплики. + - `ATTACH_PART` — присоединить кусок. Задача может быть выполнена и с куском из нашей собственной реплики (если он находится в папке `detached`). Эта задача практически идентична задаче `GET_PART`, лишь немного оптимизирована. + - `MERGE_PARTS` — выполнить слияние кусков. + - `DROP_RANGE` — удалить куски в партициях из указнного диапазона. + - `CLEAR_COLUMN` — удалить указанный столбец из указанной партиции. Примечание: не используется с 20.4. + - `CLEAR_INDEX` — удалить указанный индекс из указанной партиции. Примечание: не используется с 20.4. + - `REPLACE_RANGE` — удалить указанный диапазон кусков и заменить их на новые. + - `MUTATE_PART` — применить одну или несколько мутаций к куску. + - `ALTER_METADATA` — применить изменения структуры таблицы в результате запросов с выражением `ALTER`. - `create_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — дата и время отправки задачи на выполнение. @@ -77,4 +87,3 @@ last_postpone_time: 1970-01-01 03:00:00 **Смотрите также** - [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system.md#query-language-system-replicated) - diff --git a/docs/ru/operations/system-tables/trace_log.md b/docs/ru/operations/system-tables/trace_log.md index 3d22e4eabfd..6d8130c1d00 100644 --- a/docs/ru/operations/system-tables/trace_log.md +++ b/docs/ru/operations/system-tables/trace_log.md @@ -18,10 +18,12 @@ ClickHouse создает эту таблицу когда утсановлен Во время соединения с сервером через `clickhouse-client`, вы видите строку похожую на `Connected to ClickHouse server version 19.18.1 revision 54429.`. Это поле содержит номер после `revision`, но не содержит строку после `version`. -- `timer_type`([Enum8](../../sql-reference/data-types/enum.md)) — тип таймера: +- `trace_type`([Enum8](../../sql-reference/data-types/enum.md)) — тип трассировки: - - `Real` означает wall-clock время. - - `CPU` означает относительное CPU время. + - `Real` — сбор трассировок стека адресов вызова по времени wall-clock. + - `CPU` — сбор трассировок стека адресов вызова по времени CPU. + - `Memory` — сбор выделенной памяти, когда ее размер превышает относительный инкремент. + - `MemorySample` — сбор случайно выделенной памяти. - `thread_number`([UInt32](../../sql-reference/data-types/int-uint.md)) — идентификатор треда. diff --git a/docs/ru/sql-reference/statements/alter/partition.md b/docs/ru/sql-reference/statements/alter/partition.md index 3e7b069b066..02a87406e86 100644 --- a/docs/ru/sql-reference/statements/alter/partition.md +++ b/docs/ru/sql-reference/statements/alter/partition.md @@ -38,7 +38,7 @@ ALTER TABLE mt DETACH PART 'all_2_2_0'; После того как запрос будет выполнен, вы сможете производить любые операции с данными в директории `detached`. Например, можно удалить их из файловой системы. -Запрос реплицируется — данные будут перенесены в директорию `detached` и забыты на всех репликах. Обратите внимание, запрос может быть отправлен только на реплику-лидер. Чтобы узнать, является ли реплика лидером, выполните запрос `SELECT` к системной таблице [system.replicas](../../../operations/system-tables/replicas.md#system_tables-replicas). Либо можно выполнить запрос `DETACH` на всех репликах — тогда на всех репликах, кроме реплики-лидера, запрос вернет ошибку. +Запрос реплицируется — данные будут перенесены в директорию `detached` и забыты на всех репликах. Обратите внимание, запрос может быть отправлен только на реплику-лидер. Чтобы узнать, является ли реплика лидером, выполните запрос `SELECT` к системной таблице [system.replicas](../../../operations/system-tables/replicas.md#system_tables-replicas). Либо можно выполнить запрос `DETACH` на всех репликах — тогда на всех репликах, кроме реплик-лидеров (поскольку допускается несколько лидеров), запрос вернет ошибку. ## DROP PARTITION\|PART {#alter_drop-partition} @@ -83,9 +83,13 @@ ALTER TABLE visits ATTACH PART 201901_2_2_0; Как корректно задать имя партиции или куска, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). -Этот запрос реплицируется. Реплика-иницатор проверяет, есть ли данные в директории `detached`. Если данные есть, то запрос проверяет их целостность. В случае успеха данные добавляются в таблицу. Все остальные реплики загружают данные с реплики-инициатора запроса. +Этот запрос реплицируется. Реплика-иницатор проверяет, есть ли данные в директории `detached`. +Если данные есть, то запрос проверяет их целостность. В случае успеха данные добавляются в таблицу. -Это означает, что вы можете разместить данные в директории `detached` на одной реплике и с помощью запроса `ALTER ... ATTACH` добавить их в таблицу на всех репликах. +Если реплика, не являющаяся инициатором запроса, получив команду присоединения, находит кусок с правильными контрольными суммами в своей собственной папке `detached`, она присоединяет данные, не скачивая их с других реплик. +Если нет куска с правильными контрольными суммами, данные загружаются из любой реплики, имеющей этот кусок. + +Вы можете поместить данные в директорию `detached` на одной реплике и с помощью запроса `ALTER ... ATTACH` добавить их в таблицу на всех репликах. ## ATTACH PARTITION FROM {#alter_attach-partition-from} @@ -93,7 +97,8 @@ ALTER TABLE visits ATTACH PART 201901_2_2_0; ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1 ``` -Копирует партицию из таблицы `table1` в таблицу `table2` и добавляет к существующим данным `table2`. Данные из `table1` не удаляются. +Копирует партицию из таблицы `table1` в таблицу `table2`. +Обратите внимание, что данные не удаляются ни из `table1`, ни из `table2`. Следует иметь в виду: @@ -305,4 +310,3 @@ OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; `IN PARTITION` указывает на партицию, для которой применяются выражения [UPDATE](../../../sql-reference/statements/alter/update.md#alter-table-update-statements) или [DELETE](../../../sql-reference/statements/alter/delete.md#alter-mutations) в результате запроса `ALTER TABLE`. Новые куски создаются только в указанной партиции. Таким образом, `IN PARTITION` помогает снизить нагрузку, когда таблица разбита на множество партиций, а вам нужно обновить данные лишь точечно. Примеры запросов `ALTER ... PARTITION` можно посмотреть в тестах: [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) и [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). - diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index ab68033d4f3..f0f9b77b5ba 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -204,6 +204,7 @@ SYSTEM STOP MOVES [[db.]merge_tree_family_table_name] ClickHouse может управлять фоновыми процессами связанными c репликацией в таблицах семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md). ### STOP FETCHES {#query_language-system-stop-fetches} + Позволяет остановить фоновые процессы синхронизации новыми вставленными кусками данных с другими репликами в кластере для таблиц семейства `ReplicatedMergeTree`: Всегда возвращает `Ok.` вне зависимости от типа таблицы и даже если таблица или база данных не существет. @@ -212,6 +213,7 @@ SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name] ``` ### START FETCHES {#query_language-system-start-fetches} + Позволяет запустить фоновые процессы синхронизации новыми вставленными кусками данных с другими репликами в кластере для таблиц семейства `ReplicatedMergeTree`: Всегда возвращает `Ok.` вне зависимости от типа таблицы и даже если таблица или база данных не существет. @@ -220,6 +222,7 @@ SYSTEM START FETCHES [[db.]replicated_merge_tree_family_table_name] ``` ### STOP REPLICATED SENDS {#query_language-system-start-replicated-sends} + Позволяет остановить фоновые процессы отсылки новых вставленных кусков данных другим репликам в кластере для таблиц семейства `ReplicatedMergeTree`: ``` sql @@ -227,6 +230,7 @@ SYSTEM STOP REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name] ``` ### START REPLICATED SENDS {#query_language-system-start-replicated-sends} + Позволяет запустить фоновые процессы отсылки новых вставленных кусков данных другим репликам в кластере для таблиц семейства `ReplicatedMergeTree`: ``` sql @@ -234,6 +238,7 @@ SYSTEM START REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name] ``` ### STOP REPLICATION QUEUES {#query_language-system-stop-replication-queues} + Останавливает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER: ``` sql @@ -241,6 +246,7 @@ SYSTEM STOP REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name] ``` ### START REPLICATION QUEUES {#query_language-system-start-replication-queues} + Запускает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER: ``` sql @@ -248,20 +254,24 @@ SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name] ``` ### SYNC REPLICA {#query_language-system-sync-replica} + Ждет когда таблица семейства `ReplicatedMergeTree` будет синхронизирована с другими репликами в кластере, будет работать до достижения `receive_timeout`, если синхронизация для таблицы отключена в настоящий момент времени: ``` sql SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ``` +После выполнения этого запроса таблица `[db.]replicated_merge_tree_family_table_name` синхронизирует команды из общего реплицированного лога в свою собственную очередь репликации. Затем запрос ждет, пока реплика не обработает все синхронизированные команды. + ### RESTART REPLICA {#query_language-system-restart-replica} -Реинициализация состояния Zookeeper сессий для таблицы семейства `ReplicatedMergeTree`, сравнивает текущее состояние с тем что хранится в Zookeeper как источник правды и добавляет задачи Zookeeper очередь если необходимо -Инициализация очереди репликации на основе данных ZooKeeper, происходит так же как при attach table. На короткое время таблица станет недоступной для любых операций. + +Реинициализация состояния Zookeeper-сессий для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с тем, что хранится в Zookeeper, как источник правды, и добавляет задачи в очередь репликации в Zookeeper, если необходимо. +Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при attach table. На короткое время таблица станет недоступной для любых операций. ``` sql SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name ``` ### RESTART REPLICAS {#query_language-system-restart-replicas} -Реинициализация состояния Zookeeper сессий для всех `ReplicatedMergeTree` таблиц, сравнивает текущее состояние с тем что хранится в Zookeeper как источник правды и добавляет задачи Zookeeper очередь если необходимо +Реинициализация состояния ZooKeeper-сессий для всех `ReplicatedMergeTree` таблиц. Сравнивает текущее состояние реплики с тем, что хранится в ZooKeeper, как c источником правды, и добавляет задачи в очередь репликации в ZooKeeper, если необходимо. diff --git a/docs/zh/development/build.md b/docs/zh/development/build.md index 1aa5c1c97b7..01e0740bfa4 100644 --- a/docs/zh/development/build.md +++ b/docs/zh/development/build.md @@ -35,28 +35,12 @@ sudo apt-get install git cmake ninja-build 或cmake3而不是旧系统上的cmake。 或者在早期版本的系统中用 cmake3 替代 cmake -## 安装 GCC 10 {#an-zhuang-gcc-10} +## 安装 Clang -有几种方法可以做到这一点。 +On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) -### 安装 PPA 包 {#an-zhuang-ppa-bao} - -``` bash -sudo apt-get install software-properties-common -sudo apt-add-repository ppa:ubuntu-toolchain-r/test -sudo apt-get update -sudo apt-get install gcc-10 g++-10 -``` - -### 源码安装 gcc {#yuan-ma-an-zhuang-gcc} - -请查看 [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh) - -## 使用 GCC 10 来编译 {#shi-yong-gcc-10-lai-bian-yi} - -``` bash -export CC=gcc-10 -export CXX=g++-10 +```bash +sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` ## 拉取 ClickHouse 源码 {#la-qu-clickhouse-yuan-ma-1} diff --git a/docs/zh/development/developer-instruction.md b/docs/zh/development/developer-instruction.md index 53aab5dc086..04950c11521 100644 --- a/docs/zh/development/developer-instruction.md +++ b/docs/zh/development/developer-instruction.md @@ -123,17 +123,13 @@ ClickHouse使用多个外部库进行构建。大多数外部库不需要单独 # C++ 编译器 {#c-bian-yi-qi} -GCC编译器从版本9开始，以及Clang版本\>=8都可支持构建ClickHouse。 +We support clang starting from version 11. -Yandex官方当前使用GCC构建ClickHouse，因为它生成的机器代码性能较好（根据测评，最多可以相差几个百分点）。Clang通常可以更加便捷的开发。我们的持续集成（CI）平台会运行大约十二种构建组合的检查。 +On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) -在Ubuntu上安装GCC，请执行：`sudo apt install gcc g++` - -请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.tech/docs/zh/development/build/#an-zhuang-gcc-10 。 - -在Mac OS X上安装GCC，请执行：`brew install gcc` - -如果您决定使用Clang，还可以同时安装 `libc++`以及`lld`，前提是您也熟悉它们。此外，也推荐使用`ccache`。 +```bash +sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +``` # 构建的过程 {#gou-jian-de-guo-cheng} @@ -146,7 +142,7 @@ Yandex官方当前使用GCC构建ClickHouse，因为它生成的机器代码性 在`build`目录下，通过运行CMake配置构建。 在第一次运行之前，请定义用于指定编译器的环境变量（本示例中为gcc 9 编译器）。 - export CC=gcc-10 CXX=g++-10 + export CC=clang CXX=clang++ cmake .. `CC`变量指代C的编译器（C Compiler的缩写），而`CXX`变量指代要使用哪个C++编译器进行编译。 diff --git a/docs/zh/development/style.md b/docs/zh/development/style.md index c8e883920dd..bb9bfde7b9b 100644 --- a/docs/zh/development/style.md +++ b/docs/zh/development/style.md @@ -696,7 +696,7 @@ auto s = std::string{"Hello"}; **2.** 语言： C++20. -**3.** 编译器： `gcc`。 此时（2020年08月），代码使用9.3版编译。（它也可以使用`clang 8` 编译） +**3.** 编译器： `clang`。 此时（2021年03月），代码使用11版编译。（它也可以使用`gcc` 编译 but it is not suitable for production） 使用标准库 (`libc++`)。 diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 9b762cfa4c5..5a676819b77 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -360,7 +360,8 @@ inline bool isEnum(const DataTypePtr & data_type) { return WhichDataType(data_ty inline bool isDecimal(const DataTypePtr & data_type) { return WhichDataType(data_type).isDecimal(); } inline bool isTuple(const DataTypePtr & data_type) { return WhichDataType(data_type).isTuple(); } inline bool isArray(const DataTypePtr & data_type) { return WhichDataType(data_type).isArray(); } -inline bool isMap(const DataTypePtr & data_type) {return WhichDataType(data_type).isMap(); } +inline bool isMap(const DataTypePtr & data_type) { return WhichDataType(data_type).isMap(); } +inline bool isNothing(const DataTypePtr & data_type) { return WhichDataType(data_type).isNothing(); } template inline bool isUInt8(const T & data_type) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 9b40be58862..1cd308c774d 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -2496,7 +2496,7 @@ private: } } - WrapperType createArrayWrapper(const DataTypePtr & from_type_untyped, const DataTypeArray * to_type) const + WrapperType createArrayWrapper(const DataTypePtr & from_type_untyped, const DataTypeArray & to_type) const { /// Conversion from String through parsing. if (checkAndGetDataType(from_type_untyped.get())) @@ -2507,24 +2507,23 @@ private: }; } - DataTypePtr from_nested_type; - DataTypePtr to_nested_type; const auto * from_type = checkAndGetDataType(from_type_untyped.get()); - - /// get the most nested type - if (from_type && to_type) + if (!from_type) { - from_nested_type = from_type->getNestedType(); - to_nested_type = to_type->getNestedType(); - - from_type = checkAndGetDataType(from_nested_type.get()); - to_type = checkAndGetDataType(to_nested_type.get()); + throw Exception(ErrorCodes::TYPE_MISMATCH, + "CAST AS Array can only be perforamed between same-dimensional Array or String types"); } - /// both from_type and to_type should be nullptr now is array types had same dimensions - if ((from_type == nullptr) != (to_type == nullptr)) - throw Exception{"CAST AS Array can only be performed between same-dimensional array types or from String", - ErrorCodes::TYPE_MISMATCH}; + DataTypePtr from_nested_type = from_type->getNestedType(); + + /// In query SELECT CAST([] AS Array(Array(String))) from type is Array(Nothing) + bool from_empty_array = isNothing(from_nested_type); + + if (from_type->getNumberOfDimensions() != to_type.getNumberOfDimensions() && !from_empty_array) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "CAST AS Array can only be perforamed between same-dimensional array types"); + + const DataTypePtr & to_nested_type = to_type.getNestedType(); /// Prepare nested type conversion const auto nested_function = prepareUnpackDictionaries(from_nested_type, to_nested_type); @@ -3090,14 +3089,12 @@ private: return createStringWrapper(from_type); case TypeIndex::FixedString: return createFixedStringWrapper(from_type, checkAndGetDataType(to_type.get())->getN()); - case TypeIndex::Array: - return createArrayWrapper(from_type, checkAndGetDataType(to_type.get())); + return createArrayWrapper(from_type, static_cast(*to_type)); case TypeIndex::Tuple: return createTupleWrapper(from_type, checkAndGetDataType(to_type.get())); case TypeIndex::Map: return createMapWrapper(from_type, checkAndGetDataType(to_type.get())); - case TypeIndex::AggregateFunction: return createAggregateFunctionWrapper(from_type, checkAndGetDataType(to_type.get())); default: diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index 1e6ffa57e2d..52d2cf98c25 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -24,7 +24,6 @@ namespace ErrorCodes ArrowBlockInputFormat::ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_, bool stream_) : IInputFormat(header_, in_), stream{stream_} { - prepareReader(); } Chunk ArrowBlockInputFormat::generate() @@ -35,12 +34,18 @@ Chunk ArrowBlockInputFormat::generate() if (stream) { + if (!stream_reader) + prepareReader(); + batch_result = stream_reader->Next(); if (batch_result.ok() && !(*batch_result)) return res; } else { + if (!file_reader) + prepareReader(); + if (record_batch_current >= record_batch_total) return res; @@ -71,7 +76,7 @@ void ArrowBlockInputFormat::resetParser() stream_reader.reset(); else file_reader.reset(); - prepareReader(); + record_batch_current = 0; } void ArrowBlockInputFormat::prepareReader() diff --git a/src/Storages/System/IStorageSystemOneBlock.h b/src/Storages/System/IStorageSystemOneBlock.h index 36e92d5a1f4..fdc966130ad 100644 --- a/src/Storages/System/IStorageSystemOneBlock.h +++ b/src/Storages/System/IStorageSystemOneBlock.h @@ -12,7 +12,16 @@ namespace DB class Context; -/** Base class for system tables whose all columns have String type. +/** IStorageSystemOneBlock is base class for system tables whose all columns can be synchronously fetched. + * + * Client class need to provide static method static NamesAndTypesList getNamesAndTypes() that will return list of column names and + * their types. IStorageSystemOneBlock during read will create result columns in same order as result of getNamesAndTypes + * and pass it with fillData method. + * + * Client also must override fillData and fill result columns. + * + * If subclass want to support virtual columns, it should override getVirtuals method of IStorage interface. + * IStorageSystemOneBlock will add virtuals columns at the end of result columns of fillData method. */ template class IStorageSystemOneBlock : public IStorage @@ -41,9 +50,10 @@ public: size_t /*max_block_size*/, unsigned /*num_streams*/) override { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + auto virtuals_names_and_types = getVirtuals(); + metadata_snapshot->check(column_names, virtuals_names_and_types, getStorageID()); - Block sample_block = metadata_snapshot->getSampleBlock(); + Block sample_block = metadata_snapshot->getSampleBlockWithVirtuals(virtuals_names_and_types); MutableColumns res_columns = sample_block.cloneEmptyColumns(); fillData(res_columns, context, query_info); diff --git a/src/Storages/System/StorageSystemDictionaries.cpp b/src/Storages/System/StorageSystemDictionaries.cpp index a362e6a44e3..c76dba9df58 100644 --- a/src/Storages/System/StorageSystemDictionaries.cpp +++ b/src/Storages/System/StorageSystemDictionaries.cpp @@ -50,6 +50,13 @@ NamesAndTypesList StorageSystemDictionaries::getNamesAndTypes() }; } +NamesAndTypesList StorageSystemDictionaries::getVirtuals() const +{ + return { + {"key", std::make_shared()} + }; +} + void StorageSystemDictionaries::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & /*query_info*/) const { const auto access = context->getAccess(); @@ -128,6 +135,9 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, ContextPt else res_columns[i++]->insertDefault(); + /// Start fill virtual columns + + res_columns[i++]->insert(dictionary_structure.getKeyDescription()); } } diff --git a/src/Storages/System/StorageSystemDictionaries.h b/src/Storages/System/StorageSystemDictionaries.h index 63b33adaaff..aa65a946127 100644 --- a/src/Storages/System/StorageSystemDictionaries.h +++ b/src/Storages/System/StorageSystemDictionaries.h @@ -18,6 +18,8 @@ public: static NamesAndTypesList getNamesAndTypes(); + NamesAndTypesList getVirtuals() const override; + protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 99617367bb1..04a78c5f2c4 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -637,49 +637,21 @@ def test_kafka_formats(kafka_cluster): avro_message({'id': 0, 'blockNo': 0, 'val1': str('AM'), 'val2': 0.5, "val3": 1}), ], 'supports_empty_value': False, - } - # 'Arrow' : { - # # Not working at all: DB::Exception: Error while opening a table: Invalid: File is too small: 0, Stack trace (when copying this message, always include the lines below): - # # /src/Common/Exception.cpp:37: DB::Exception::Exception(std::__1::basic_string, std::__1::allocator > const&, int) @ 0x15c2d2a3 in /usr/bin/clickhouse - # # /src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp:88: DB::ArrowBlockInputFormat::prepareReader() @ 0x1ddff1c3 in /usr/bin/clickhouse - # # /src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp:26: DB::ArrowBlockInputFormat::ArrowBlockInputFormat(DB::ReadBuffer&, DB::Block const&, bool) @ 0x1ddfef63 in /usr/bin/clickhouse - # # /contrib/libcxx/include/memory:2214: std::__1::__compressed_pair_elem::__compressed_pair_elem(std::__1::piecewise_construct_t, std::__1::tuple, std::__1::__tuple_indices<0ul, 1ul, 2ul>) @ 0x1de0470f in /usr/bin/clickhouse - # # /contrib/libcxx/include/memory:2299: std::__1::__compressed_pair, DB::ArrowBlockInputFormat>::__compressed_pair&, DB::ReadBuffer&, DB::Block const&, bool&&>(std::__1::piecewise_construct_t, std::__1::tuple&>, std::__1::tuple) @ 0x1de04375 in /usr/bin/clickhouse - # # /contrib/libcxx/include/memory:3569: std::__1::__shared_ptr_emplace >::__shared_ptr_emplace(std::__1::allocator, DB::ReadBuffer&, DB::Block const&, bool&&) @ 0x1de03f97 in /usr/bin/clickhouse - # # /contrib/libcxx/include/memory:4400: std::__1::enable_if::value), std::__1::shared_ptr >::type std::__1::make_shared(DB::ReadBuffer&, DB::Block const&, bool&&) @ 0x1de03d4c in /usr/bin/clickhouse - # # /src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp:107: DB::registerInputFormatProcessorArrow(DB::FormatFactory&)::$_0::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) const @ 0x1de010df in /usr/bin/clickhouse - # 'data_sample' : [ - # '\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', - # '\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x06\x00\x00\x00\x08\x00\x00\x00\x0a\x00\x00\x00\x0c\x00\x00\x00\x0e\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x14\x00\x00\x00\x16\x00\x00\x00\x18\x00\x00\x00\x1a\x00\x00\x00\x1c\x00\x00\x00\x1e\x00\x00\x00\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', - # '\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', - # ], - # }, - # 'ArrowStream' : { - # # Not working at all: - # # Error while opening a table: Invalid: Tried reading schema message, was null or length 0, Stack trace (when copying this message, always include the lines below): - # # /src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp:88: DB::ArrowBlockInputFormat::prepareReader() @ 0x1ddff1c3 in /usr/bin/clickhouse - # # /src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp:26: DB::ArrowBlockInputFormat::ArrowBlockInputFormat(DB::ReadBuffer&, DB::Block const&, bool) @ 0x1ddfef63 in /usr/bin/clickhouse - # # /contrib/libcxx/include/memory:2214: std::__1::__compressed_pair_elem::__compressed_pair_elem(std::__1::piecewise_construct_t, std::__1::tuple, std::__1::__tuple_indices<0ul, 1ul, 2ul>) @ 0x1de0470f in /usr/bin/clickhouse - # # /contrib/libcxx/include/memory:2299: std::__1::__compressed_pair, DB::ArrowBlockInputFormat>::__compressed_pair&, DB::ReadBuffer&, DB::Block const&, bool&&>(std::__1::piecewise_construct_t, std::__1::tuple&>, std::__1::tuple) @ 0x1de04375 in /usr/bin/clickhouse - # # /contrib/libcxx/include/memory:3569: std::__1::__shared_ptr_emplace >::__shared_ptr_emplace(std::__1::allocator, DB::ReadBuffer&, DB::Block const&, bool&&) @ 0x1de03f97 in /usr/bin/clickhouse - # # /contrib/libcxx/include/memory:4400: std::__1::enable_if::value), std::__1::shared_ptr >::type std::__1::make_shared(DB::ReadBuffer&, DB::Block const&, bool&&) @ 0x1de03d4c in /usr/bin/clickhouse - # # /src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp:117: DB::registerInputFormatProcessorArrow(DB::FormatFactory&)::$_1::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) const @ 0x1de0273f in /usr/bin/clickhouse - # # /contrib/libcxx/include/type_traits:3519: decltype(std::__1::forward(fp)(std::__1::forward(fp0), std::__1::forward(fp0), std::__1::forward(fp0), std::__1::forward(fp0))) std::__1::__invoke(DB::registerInputFormatProcessorArrow(DB::FormatFactory&)::$_1&, DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) @ 0x1de026da in /usr/bin/clickhouse - # # /contrib/libcxx/include/__functional_base:317: std::__1::shared_ptr std::__1::__invoke_void_return_wrapper >::__call(DB::registerInputFormatProcessorArrow(DB::FormatFactory&)::$_1&, DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) @ 0x1de025ed in /usr/bin/clickhouse - # # /contrib/libcxx/include/functional:1540: std::__1::__function::__alloc_func, std::__1::shared_ptr (DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&)>::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) @ 0x1de0254a in /usr/bin/clickhouse - # # /contrib/libcxx/include/functional:1714: std::__1::__function::__func, std::__1::shared_ptr (DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&)>::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) @ 0x1de0165c in /usr/bin/clickhouse - # # /contrib/libcxx/include/functional:1867: std::__1::__function::__value_func (DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&)>::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) const @ 0x1dd14dbd in /usr/bin/clickhouse - # # /contrib/libcxx/include/functional:2473: std::__1::function (DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&)>::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) const @ 0x1dd07035 in /usr/bin/clickhouse - # # /src/Formats/FormatFactory.cpp:258: DB::FormatFactory::getInputFormat(std::__1::basic_string, std::__1::allocator > const&, DB::ReadBuffer&, DB::Block const&, DB::Context const&, unsigned long, std::__1::function) const @ 0x1dd04007 in /usr/bin/clickhouse - # # /src/Storages/Kafka/KafkaBlockInputStream.cpp:76: DB::KafkaBlockInputStream::readImpl() @ 0x1d8f6559 in /usr/bin/clickhouse - # # /src/DataStreams/IBlockInputStream.cpp:60: DB::IBlockInputStream::read() @ 0x1c9c92fd in /usr/bin/clickhouse - # # /src/DataStreams/copyData.cpp:26: void DB::copyDataImpl*)::$_0&, void (&)(DB::Block const&)>(DB::IBlockInputStream&, DB::IBlockOutputStream&, DB::copyData(DB::IBlockInputStream&, DB::IBlockOutputStream&, std::__1::atomic*)::$_0&, void (&)(DB::Block const&)) @ 0x1c9ea01c in /usr/bin/clickhouse - # 'data_sample' : [ - # '\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00', - # '\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x06\x00\x00\x00\x08\x00\x00\x00\x0a\x00\x00\x00\x0c\x00\x00\x00\x0e\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x14\x00\x00\x00\x16\x00\x00\x00\x18\x00\x00\x00\x1a\x00\x00\x00\x1c\x00\x00\x00\x1e\x00\x00\x00\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\xff\xff\xff\xff\x00\x00\x00\x00', - # '\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00', - # ], - # }, + }, + 'Arrow' : { + 'data_sample' : [ + b'\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', + b'\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x06\x00\x00\x00\x08\x00\x00\x00\x0a\x00\x00\x00\x0c\x00\x00\x00\x0e\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x14\x00\x00\x00\x16\x00\x00\x00\x18\x00\x00\x00\x1a\x00\x00\x00\x1c\x00\x00\x00\x1e\x00\x00\x00\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', + b'\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', + ], + }, + 'ArrowStream' : { + 'data_sample' : [ + b'\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00', + b'\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x06\x00\x00\x00\x08\x00\x00\x00\x0a\x00\x00\x00\x0c\x00\x00\x00\x0e\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x14\x00\x00\x00\x16\x00\x00\x00\x18\x00\x00\x00\x1a\x00\x00\x00\x1c\x00\x00\x00\x1e\x00\x00\x00\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\xff\xff\xff\xff\x00\x00\x00\x00', + b'\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00', + ], + }, } for format_name, format_opts in list(all_formats.items()): diff --git a/tests/queries/0_stateless/01837_cast_to_array_from_empty_array.reference b/tests/queries/0_stateless/01837_cast_to_array_from_empty_array.reference new file mode 100644 index 00000000000..c71bf50e82f --- /dev/null +++ b/tests/queries/0_stateless/01837_cast_to_array_from_empty_array.reference @@ -0,0 +1,2 @@ +[] +[] diff --git a/tests/queries/0_stateless/01837_cast_to_array_from_empty_array.sql b/tests/queries/0_stateless/01837_cast_to_array_from_empty_array.sql new file mode 100644 index 00000000000..f3aa595f6d5 --- /dev/null +++ b/tests/queries/0_stateless/01837_cast_to_array_from_empty_array.sql @@ -0,0 +1,2 @@ +SELECT CAST([] AS Array(Array(String))); +SELECT CAST([] AS Array(Array(Array(String)))); diff --git a/tests/queries/0_stateless/01838_system_dictionaries_virtual_key_column.reference b/tests/queries/0_stateless/01838_system_dictionaries_virtual_key_column.reference new file mode 100644 index 00000000000..f0543d9221e --- /dev/null +++ b/tests/queries/0_stateless/01838_system_dictionaries_virtual_key_column.reference @@ -0,0 +1,4 @@ +simple key +example_simple_key_dictionary UInt64 +complex key +example_complex_key_dictionary (UInt64, String) diff --git a/tests/queries/0_stateless/01838_system_dictionaries_virtual_key_column.sql b/tests/queries/0_stateless/01838_system_dictionaries_virtual_key_column.sql new file mode 100644 index 00000000000..97d96f643cf --- /dev/null +++ b/tests/queries/0_stateless/01838_system_dictionaries_virtual_key_column.sql @@ -0,0 +1,26 @@ +DROP DICTIONARY IF EXISTS example_simple_key_dictionary; +CREATE DICTIONARY example_simple_key_dictionary ( + id UInt64, + value UInt64 +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE '' DATABASE currentDatabase())) +LAYOUT(DIRECT()); + +SELECT 'simple key'; + +SELECT name, key FROM system.dictionaries WHERE name='example_simple_key_dictionary' AND database=currentDatabase(); + +DROP DICTIONARY IF EXISTS example_complex_key_dictionary; +CREATE DICTIONARY example_complex_key_dictionary ( + id UInt64, + id_key String, + value UInt64 +) +PRIMARY KEY id, id_key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE '' DATABASE currentDatabase())) +LAYOUT(COMPLEX_KEY_DIRECT()); + +SELECT 'complex key'; + +SELECT name, key FROM system.dictionaries WHERE name='example_complex_key_dictionary' AND database=currentDatabase(); diff --git a/tests/testflows/map_type/__init__.py b/tests/testflows/map_type/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml b/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..e5077af3f49 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,16 @@ + + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse/config.d/macros.xml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml b/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..b7d02ceeec1 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,42 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.xml b/tests/testflows/map_type/configs/clickhouse/config.xml new file mode 100644 index 00000000000..4ec12232539 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.xml @@ -0,0 +1,448 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + 0.0.0.0 + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + /var/lib/clickhouse/access/ + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 + + + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/users.xml b/tests/testflows/map_type/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml new file mode 100644 index 00000000000..a114a9ce4ab --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse2 + 01 + 02 + + diff --git a/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml new file mode 100644 index 00000000000..904a27b0172 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse3 + 01 + 03 + + diff --git a/tests/testflows/map_type/docker-compose/clickhouse-service.yml b/tests/testflows/map_type/docker-compose/clickhouse-service.yml new file mode 100755 index 00000000000..fdd4a8057a9 --- /dev/null +++ b/tests/testflows/map_type/docker-compose/clickhouse-service.yml @@ -0,0 +1,27 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/map_type/docker-compose/docker-compose.yml b/tests/testflows/map_type/docker-compose/docker-compose.yml new file mode 100755 index 00000000000..29f2ef52470 --- /dev/null +++ b/tests/testflows/map_type/docker-compose/docker-compose.yml @@ -0,0 +1,60 @@ +version: '2.3' + +services: + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy diff --git a/tests/testflows/map_type/docker-compose/zookeeper-service.yml b/tests/testflows/map_type/docker-compose/zookeeper-service.yml new file mode 100755 index 00000000000..f3df33358be --- /dev/null +++ b/tests/testflows/map_type/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py new file mode 100755 index 00000000000..54d713347c6 --- /dev/null +++ b/tests/testflows/map_type/regression.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +import sys + +from testflows.core import * + +append_path(sys.path, "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from map_type.requirements import SRS018_ClickHouse_Map_Data_Type + +xfails = { + "tests/table map with key integer/Int:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], + "tests/table map with value integer/Int:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], + "tests/table map with key integer/UInt256": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], + "tests/table map with value integer/UInt256": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], + "tests/select map with key integer/Int64": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030")], + "tests/select map with value integer/Int64": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030")], + "tests/cast tuple of two arrays to map/string -> int": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21029")], + "tests/mapcontains/null key in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapcontains/null key not in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapkeys/null key not in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapkeys/null key in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapcontains/select nullable key": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], + "tests/mapkeys/select keys from column": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], + "tests/table map select key with value string/LowCardinality:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key string/FixedString": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key string/Nullable": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key string/Nullable(NULL)": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], + "tests/table map select key with key string/LowCardinality:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key integer/Int:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], + "tests/table map select key with key integer/UInt256": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], + "tests/table map select key with key integer/toNullable": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key integer/toNullable(NULL)": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], + "tests/select map with key integer/Int128": + [(Fail, "large Int128 as key not supported")], + "tests/select map with key integer/Int256": + [(Fail, "large Int256 as key not supported")], + "tests/select map with key integer/UInt256": + [(Fail, "large UInt256 as key not supported")], + "tests/select map with key integer/toNullable": + [(Fail, "Nullable type as key not supported")], + "tests/select map with key integer/toNullable(NULL)": + [(Fail, "Nullable type as key not supported")], + "tests/select map with key string/Nullable": + [(Fail, "Nullable type as key not supported")], + "tests/select map with key string/Nullable(NULL)": + [(Fail, "Nullable type as key not supported")], + "tests/table map queries/select map with nullable value": + [(Fail, "Nullable value not supported")], + "tests/table map with key integer/toNullable": + [(Fail, "Nullable type as key not supported")], + "tests/table map with key integer/toNullable(NULL)": + [(Fail, "Nullable type as key not supported")], + "tests/table map with key string/Nullable": + [(Fail, "Nullable type as key not supported")], + "tests/table map with key string/Nullable(NULL)": + [(Fail, "Nullable type as key not supported")], + "tests/table map with key string/LowCardinality(String)": + [(Fail, "LowCardinality(String) as key not supported")], + "tests/table map with key string/LowCardinality(String) cast from String": + [(Fail, "LowCardinality(String) as key not supported")], + "tests/table map with key string/LowCardinality(String) for key and value": + [(Fail, "LowCardinality(String) as key not supported")], + "tests/table map with key string/LowCardinality(FixedString)": + [(Fail, "LowCardinality(FixedString) as key not supported")], + "tests/table map with value string/LowCardinality(String) for key and value": + [(Fail, "LowCardinality(String) as key not supported")], +} + +xflags = { +} + +@TestModule +@ArgumentParser(argparser) +@XFails(xfails) +@XFlags(xflags) +@Name("map type") +@Specifications( + SRS018_ClickHouse_Map_Data_Type +) +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): + """Map type regression. + """ + nodes = { + "clickhouse": + ("clickhouse1", "clickhouse2", "clickhouse3") + } + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + self.context.stress = stress + + if parallel is not None: + self.context.parallel = parallel + + Feature(run=load("map_type.tests.feature", "feature")) + +if main(): + regression() diff --git a/tests/testflows/map_type/requirements/__init__.py b/tests/testflows/map_type/requirements/__init__.py new file mode 100644 index 00000000000..02f7d430154 --- /dev/null +++ b/tests/testflows/map_type/requirements/__init__.py @@ -0,0 +1 @@ +from .requirements import * diff --git a/tests/testflows/map_type/requirements/requirements.md b/tests/testflows/map_type/requirements/requirements.md new file mode 100644 index 00000000000..f19f5a7f7bd --- /dev/null +++ b/tests/testflows/map_type/requirements/requirements.md @@ -0,0 +1,512 @@ +# SRS018 ClickHouse Map Data Type +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Requirements](#requirements) + * 3.1 [General](#general) + * 3.1.1 [RQ.SRS-018.ClickHouse.Map.DataType](#rqsrs-018clickhousemapdatatype) + * 3.2 [Performance](#performance) + * 3.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples](#rqsrs-018clickhousemapdatatypeperformancevsarrayoftuples) + * 3.2.2 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays](#rqsrs-018clickhousemapdatatypeperformancevstupleofarrays) + * 3.3 [Key Types](#key-types) + * 3.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Key.String](#rqsrs-018clickhousemapdatatypekeystring) + * 3.3.2 [RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer](#rqsrs-018clickhousemapdatatypekeyinteger) + * 3.4 [Value Types](#value-types) + * 3.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.String](#rqsrs-018clickhousemapdatatypevaluestring) + * 3.4.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer](#rqsrs-018clickhousemapdatatypevalueinteger) + * 3.4.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Array](#rqsrs-018clickhousemapdatatypevaluearray) + * 3.5 [Invalid Types](#invalid-types) + * 3.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable](#rqsrs-018clickhousemapdatatypeinvalidnullable) + * 3.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing](#rqsrs-018clickhousemapdatatypeinvalidnothingnothing) + * 3.6 [Duplicated Keys](#duplicated-keys) + * 3.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys](#rqsrs-018clickhousemapdatatypeduplicatedkeys) + * 3.7 [Array of Maps](#array-of-maps) + * 3.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps](#rqsrs-018clickhousemapdatatypearrayofmaps) + * 3.8 [Nested With Maps](#nested-with-maps) + * 3.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps](#rqsrs-018clickhousemapdatatypenestedwithmaps) + * 3.9 [Value Retrieval](#value-retrieval) + * 3.9.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval](#rqsrs-018clickhousemapdatatypevalueretrieval) + * 3.9.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid](#rqsrs-018clickhousemapdatatypevalueretrievalkeyinvalid) + * 3.9.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound](#rqsrs-018clickhousemapdatatypevalueretrievalkeynotfound) + * 3.10 [Converting Tuple(Array, Array) to Map](#converting-tuplearray-array-to-map) + * 3.10.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraystomap) + * 3.10.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraysmapinvalid) + * 3.11 [Converting Array(Tuple(K,V)) to Map](#converting-arraytuplekv-to-map) + * 3.11.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomap) + * 3.11.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomapinvalid) + * 3.12 [Keys and Values Subcolumns](#keys-and-values-subcolumns) + * 3.12.1 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys](#rqsrs-018clickhousemapdatatypesubcolumnskeys) + * 3.12.2 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnskeysarrayfunctions) + * 3.12.3 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnskeysinlinedefinedmap) + * 3.12.4 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values](#rqsrs-018clickhousemapdatatypesubcolumnsvalues) + * 3.12.5 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesarrayfunctions) + * 3.12.6 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesinlinedefinedmap) + * 3.13 [Functions](#functions) + * 3.13.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap](#rqsrs-018clickhousemapdatatypefunctionsinlinedefinedmap) + * 3.13.2 [`length`](#length) + * 3.13.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length](#rqsrs-018clickhousemapdatatypefunctionslength) + * 3.13.3 [`empty`](#empty) + * 3.13.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty](#rqsrs-018clickhousemapdatatypefunctionsempty) + * 3.13.4 [`notEmpty`](#notempty) + * 3.13.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty](#rqsrs-018clickhousemapdatatypefunctionsnotempty) + * 3.13.5 [`map`](#map) + * 3.13.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map](#rqsrs-018clickhousemapdatatypefunctionsmap) + * 3.13.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments](#rqsrs-018clickhousemapdatatypefunctionsmapinvalidnumberofarguments) + * 3.13.5.3 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes](#rqsrs-018clickhousemapdatatypefunctionsmapmixedkeyorvaluetypes) + * 3.13.5.4 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd](#rqsrs-018clickhousemapdatatypefunctionsmapmapadd) + * 3.13.5.5 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract](#rqsrs-018clickhousemapdatatypefunctionsmapmapsubstract) + * 3.13.5.6 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries](#rqsrs-018clickhousemapdatatypefunctionsmapmappopulateseries) + * 3.13.6 [`mapContains`](#mapcontains) + * 3.13.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains](#rqsrs-018clickhousemapdatatypefunctionsmapcontains) + * 3.13.7 [`mapKeys`](#mapkeys) + * 3.13.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys](#rqsrs-018clickhousemapdatatypefunctionsmapkeys) + * 3.13.8 [`mapValues`](#mapvalues) + * 3.13.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues](#rqsrs-018clickhousemapdatatypefunctionsmapvalues) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +This software requirements specification covers requirements for `Map(key, value)` data type in [ClickHouse]. + +## Requirements + +### General + +#### RQ.SRS-018.ClickHouse.Map.DataType +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs. + +### Performance + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Array(Tuple(K,V))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Tuple(Array(String), Array(String))` data type where the first +array defines an array of keys and the second array defines an array of values. + +### Key Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type. + +### Value Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Array +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type. + +### Invalid Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type. + +### Duplicated Keys + +#### RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys +version: 1.0 + +[ClickHouse] MAY support `Map(key, value)` data type with duplicated keys. + +### Array of Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps +version: 1.0 + +[ClickHouse] SHALL support `Array(Map(key, value))` data type. + +### Nested With Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps +version: 1.0 + +[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type. + +### Value Retrieval + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval +version: 1.0 + +[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax. +If `key` has duplicates then the first `key:value` pair MAY be returned. + +For example, + +```sql +SELECT a['key2'] FROM table_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid +version: 1.0 + +[ClickHouse] SHALL return an error when key does not match the key type. + +For example, + +```sql +SELECT map(1,2) AS m, m[1024] +``` + +Exceptions: + +* when key is `NULL` the return value MAY be `NULL` +* when key value is not valid for the key type, for example it is out of range for [Integer] type, + when reading from a table column it MAY return the default value for key data type + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound +version: 1.0 + +[ClickHouse] SHALL return default value for the data type of the value +when there's no corresponding `key` defined in the `Map(key, value)` data type. + + +### Converting Tuple(Array, Array) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function. + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid +version: 1.0 + +[ClickHouse] MAY return an error when casting [Tuple(Array, Array)] to `Map(key, value)` + +* when arrays are not of equal size + + For example, + + ```sql + SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] + ``` + +### Converting Array(Tuple(K,V)) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid +version: 1.0 + +[ClickHouse] MAY return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` + +* when element is not a [Tuple] + + ```sql + SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map + ``` + +* when [Tuple] does not contain two elements + + ```sql + SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map + ``` + +### Keys and Values Subcolumns + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys +version: 1.0 + +[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map keys. + +```sql +SELECT m.keys FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.keys, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap +version: 1.0 + +[ClickHouse] MAY not support using inline defined map to get `keys` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.keys +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values +version: 1.0 + +[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map values. + +```sql +SELECT m.values FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.values, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap +version: 1.0 + +[ClickHouse] MAY not support using inline defined map to get `values` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.values +``` + +### Functions + +#### RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL support using inline defined maps as an argument to map functions. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c) +SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c) +``` + +#### `length` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function +that SHALL return number of keys in the map. + +For example, + +```sql +SELECT length(map(1,2,3,4)) +SELECT length(map()) +``` + +#### `empty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function +that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 0. + +For example, + +```sql +SELECT empty(map(1,2,3,4)) +SELECT empty(map()) +``` + +#### `notEmpty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function +that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 1. + +For example, + +```sql +SELECT notEmpty(map(1,2,3,4)) +SELECT notEmpty(map()) +``` + +#### `map` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map +version: 1.0 + +[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type +using `map` function. + +**Syntax** + +``` sql +map(key1, value1[, key2, value2, ...]) +``` + +For example, + +``` sql +SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); + +┌─map('key1', number, 'key2', multiply(number, 2))─┐ +│ {'key1':0,'key2':0} │ +│ {'key1':1,'key2':2} │ +│ {'key1':2,'key2':4} │ +└──────────────────────────────────────────────────┘ +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments. + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types. + + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type. + +For example, + +``` sql +SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)") +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)") +``` +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)") +``` + +#### `mapContains` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains +version: 1.0 + +[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`. + +For example, + +```sql +SELECT mapContains(a, 'abc') from table_map; +``` + +#### `mapKeys` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys +version: 1.0 + +[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format. + +For example, + +```sql +SELECT mapKeys(a) from table_map; +``` + +#### `mapValues` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues +version: 1.0 + +[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format. + +For example, + +```sql +SELECT mapValues(a) from table_map; +``` + +[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/ +[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length +[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty +[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty +[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast +[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ +[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/ +[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/ +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py new file mode 100644 index 00000000000..24e8abdf15f --- /dev/null +++ b/tests/testflows/map_type/requirements/requirements.py @@ -0,0 +1,1427 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.210226.1200017. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Specification +from testflows.core import Requirement + +Heading = Specification.Heading + +RQ_SRS_018_ClickHouse_Map_DataType = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs.\n' + '\n' + ), + link=None, + level=3, + num='3.1.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n' + 'compared to `Array(Tuple(K,V))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.2.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n' + 'compared to `Tuple(Array(String), Array(String))` data type where the first\n' + 'array defines an array of keys and the second array defines an array of values.\n' + '\n' + ), + link=None, + level=3, + num='3.2.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Key_String = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Key.String', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type.\n' + '\n' + ), + link=None, + level=3, + num='3.3.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type.\n' + '\n' + ), + link=None, + level=3, + num='3.3.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_String = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.String', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Array = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Array', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.5.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.5.2') + +RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] MAY support `Map(key, value)` data type with duplicated keys.\n' + '\n' + ), + link=None, + level=3, + num='3.6.1') + +RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Array(Map(key, value))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.7.1') + +RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type.\n' + '\n' + ), + link=None, + level=3, + num='3.8.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax.\n' + 'If `key` has duplicates then the first `key:value` pair MAY be returned. \n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT a['key2'] FROM table_map;\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.9.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when key does not match the key type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT map(1,2) AS m, m[1024]\n' + '```\n' + '\n' + 'Exceptions:\n' + '\n' + '* when key is `NULL` the return value MAY be `NULL`\n' + '* when key value is not valid for the key type, for example it is out of range for [Integer] type, \n' + ' when reading from a table column it MAY return the default value for key data type\n' + '\n' + ), + link=None, + level=3, + num='3.9.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return default value for the data type of the value\n' + "when there's no corresponding `key` defined in the `Map(key, value)` data type. \n" + '\n' + '\n' + ), + link=None, + level=3, + num='3.9.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function.\n' + '\n' + '``` sql\n' + "SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map;\n" + '```\n' + '\n' + '``` text\n' + '┌─map───────────────────────────┐\n' + "│ {1:'Ready',2:'Steady',3:'Go'} │\n" + '└───────────────────────────────┘\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.10.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] MAY return an error when casting [Tuple(Array, Array)] to `Map(key, value)`\n' + '\n' + '* when arrays are not of equal size\n' + '\n' + ' For example,\n' + '\n' + ' ```sql\n' + " SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10]\n" + ' ```\n' + '\n' + ), + link=None, + level=3, + num='3.10.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.11.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] MAY return an error when casting [Array(Tuple(K, V))] to `Map(key, value)`\n' + '\n' + '* when element is not a [Tuple]\n' + '\n' + ' ```sql\n' + " SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" + ' ```\n' + '\n' + '* when [Tuple] does not contain two elements\n' + '\n' + ' ```sql\n' + " SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map\n" + ' ```\n' + '\n' + ), + link=None, + level=3, + num='3.11.2') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used \n' + 'to retrieve an [Array] of map keys.\n' + '\n' + '```sql\n' + 'SELECT m.keys FROM t_map;\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.1') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT * FROM t_map WHERE has(m.keys, 'a');\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.2') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] MAY not support using inline defined map to get `keys` subcolumn.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, c.keys\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.3') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used \n' + 'to retrieve an [Array] of map values.\n' + '\n' + '```sql\n' + 'SELECT m.values FROM t_map;\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.4') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT * FROM t_map WHERE has(m.values, 'a');\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.5') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] MAY not support using inline defined map to get `values` subcolumn.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, c.values\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.6') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support using inline defined maps as an argument to map functions.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c)\n" + "SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c)\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.13.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function\n' + 'that SHALL return number of keys in the map.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT length(map(1,2,3,4))\n' + 'SELECT length(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.2.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function\n' + 'that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is \n' + 'greater or equal to 1 it SHALL return 0.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT empty(map(1,2,3,4))\n' + 'SELECT empty(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.3.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function\n' + 'that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is\n' + 'greater or equal to 1 it SHALL return 1.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT notEmpty(map(1,2,3,4))\n' + 'SELECT notEmpty(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.4.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type\n' + 'using `map` function.\n' + '\n' + '**Syntax** \n' + '\n' + '``` sql\n' + 'map(key1, value1[, key2, value2, ...])\n' + '```\n' + '\n' + 'For example,\n' + '\n' + '``` sql\n' + "SELECT map('key1', number, 'key2', number * 2) FROM numbers(3);\n" + '\n' + "┌─map('key1', number, 'key2', multiply(number, 2))─┐\n" + "│ {'key1':0,'key2':0} │\n" + "│ {'key1':1,'key2':2} │\n" + "│ {'key1':2,'key2':4} │\n" + '└──────────────────────────────────────────────────┘\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments.\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types.\n' + '\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '``` sql\n' + 'SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)")\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.4') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)")\n' + '```\n' + ), + link=None, + level=4, + num='3.13.5.5') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)")\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.6') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT mapContains(a, 'abc') from table_map;\n" + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.6.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT mapKeys(a) from table_map;\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.7.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT mapValues(a) from table_map;\n' + '```\n' + '\n' + '[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/\n' + '[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length\n' + '[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty\n' + '[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty\n' + '[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast\n' + '[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/\n' + '[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/\n' + '[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ \n' + '[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/\n' + '[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/\n' + '[ClickHouse]: https://clickhouse.tech\n' + '[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md \n' + '[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md\n' + '[Git]: https://git-scm.com/\n' + '[GitHub]: https://github.com\n' + ), + link=None, + level=4, + num='3.13.8.1') + +SRS018_ClickHouse_Map_Data_Type = Specification( + name='SRS018 ClickHouse Map Data Type', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + headings=( + Heading(name='Revision History', level=1, num='1'), + Heading(name='Introduction', level=1, num='2'), + Heading(name='Requirements', level=1, num='3'), + Heading(name='General', level=2, num='3.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType', level=3, num='3.1.1'), + Heading(name='Performance', level=2, num='3.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples', level=3, num='3.2.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays', level=3, num='3.2.2'), + Heading(name='Key Types', level=2, num='3.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Key.String', level=3, num='3.3.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer', level=3, num='3.3.2'), + Heading(name='Value Types', level=2, num='3.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.String', level=3, num='3.4.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer', level=3, num='3.4.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Array', level=3, num='3.4.3'), + Heading(name='Invalid Types', level=2, num='3.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable', level=3, num='3.5.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing', level=3, num='3.5.2'), + Heading(name='Duplicated Keys', level=2, num='3.6'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys', level=3, num='3.6.1'), + Heading(name='Array of Maps', level=2, num='3.7'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps', level=3, num='3.7.1'), + Heading(name='Nested With Maps', level=2, num='3.8'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps', level=3, num='3.8.1'), + Heading(name='Value Retrieval', level=2, num='3.9'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval', level=3, num='3.9.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid', level=3, num='3.9.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound', level=3, num='3.9.3'), + Heading(name='Converting Tuple(Array, Array) to Map', level=2, num='3.10'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap', level=3, num='3.10.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid', level=3, num='3.10.2'), + Heading(name='Converting Array(Tuple(K,V)) to Map', level=2, num='3.11'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap', level=3, num='3.11.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid', level=3, num='3.11.2'), + Heading(name='Keys and Values Subcolumns', level=2, num='3.12'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys', level=3, num='3.12.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions', level=3, num='3.12.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap', level=3, num='3.12.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values', level=3, num='3.12.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions', level=3, num='3.12.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap', level=3, num='3.12.6'), + Heading(name='Functions', level=2, num='3.13'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap', level=3, num='3.13.1'), + Heading(name='`length`', level=3, num='3.13.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length', level=4, num='3.13.2.1'), + Heading(name='`empty`', level=3, num='3.13.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty', level=4, num='3.13.3.1'), + Heading(name='`notEmpty`', level=3, num='3.13.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty', level=4, num='3.13.4.1'), + Heading(name='`map`', level=3, num='3.13.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map', level=4, num='3.13.5.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments', level=4, num='3.13.5.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes', level=4, num='3.13.5.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd', level=4, num='3.13.5.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract', level=4, num='3.13.5.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries', level=4, num='3.13.5.6'), + Heading(name='`mapContains`', level=3, num='3.13.6'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains', level=4, num='3.13.6.1'), + Heading(name='`mapKeys`', level=3, num='3.13.7'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys', level=4, num='3.13.7.1'), + Heading(name='`mapValues`', level=3, num='3.13.8'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues', level=4, num='3.13.8.1'), + ), + requirements=( + RQ_SRS_018_ClickHouse_Map_DataType, + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples, + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays, + RQ_SRS_018_ClickHouse_Map_DataType_Key_String, + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer, + RQ_SRS_018_ClickHouse_Map_DataType_Value_String, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array, + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable, + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing, + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys, + RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps, + RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues, + ), + content=''' +# SRS018 ClickHouse Map Data Type +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Requirements](#requirements) + * 3.1 [General](#general) + * 3.1.1 [RQ.SRS-018.ClickHouse.Map.DataType](#rqsrs-018clickhousemapdatatype) + * 3.2 [Performance](#performance) + * 3.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples](#rqsrs-018clickhousemapdatatypeperformancevsarrayoftuples) + * 3.2.2 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays](#rqsrs-018clickhousemapdatatypeperformancevstupleofarrays) + * 3.3 [Key Types](#key-types) + * 3.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Key.String](#rqsrs-018clickhousemapdatatypekeystring) + * 3.3.2 [RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer](#rqsrs-018clickhousemapdatatypekeyinteger) + * 3.4 [Value Types](#value-types) + * 3.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.String](#rqsrs-018clickhousemapdatatypevaluestring) + * 3.4.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer](#rqsrs-018clickhousemapdatatypevalueinteger) + * 3.4.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Array](#rqsrs-018clickhousemapdatatypevaluearray) + * 3.5 [Invalid Types](#invalid-types) + * 3.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable](#rqsrs-018clickhousemapdatatypeinvalidnullable) + * 3.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing](#rqsrs-018clickhousemapdatatypeinvalidnothingnothing) + * 3.6 [Duplicated Keys](#duplicated-keys) + * 3.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys](#rqsrs-018clickhousemapdatatypeduplicatedkeys) + * 3.7 [Array of Maps](#array-of-maps) + * 3.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps](#rqsrs-018clickhousemapdatatypearrayofmaps) + * 3.8 [Nested With Maps](#nested-with-maps) + * 3.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps](#rqsrs-018clickhousemapdatatypenestedwithmaps) + * 3.9 [Value Retrieval](#value-retrieval) + * 3.9.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval](#rqsrs-018clickhousemapdatatypevalueretrieval) + * 3.9.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid](#rqsrs-018clickhousemapdatatypevalueretrievalkeyinvalid) + * 3.9.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound](#rqsrs-018clickhousemapdatatypevalueretrievalkeynotfound) + * 3.10 [Converting Tuple(Array, Array) to Map](#converting-tuplearray-array-to-map) + * 3.10.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraystomap) + * 3.10.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraysmapinvalid) + * 3.11 [Converting Array(Tuple(K,V)) to Map](#converting-arraytuplekv-to-map) + * 3.11.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomap) + * 3.11.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomapinvalid) + * 3.12 [Keys and Values Subcolumns](#keys-and-values-subcolumns) + * 3.12.1 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys](#rqsrs-018clickhousemapdatatypesubcolumnskeys) + * 3.12.2 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnskeysarrayfunctions) + * 3.12.3 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnskeysinlinedefinedmap) + * 3.12.4 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values](#rqsrs-018clickhousemapdatatypesubcolumnsvalues) + * 3.12.5 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesarrayfunctions) + * 3.12.6 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesinlinedefinedmap) + * 3.13 [Functions](#functions) + * 3.13.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap](#rqsrs-018clickhousemapdatatypefunctionsinlinedefinedmap) + * 3.13.2 [`length`](#length) + * 3.13.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length](#rqsrs-018clickhousemapdatatypefunctionslength) + * 3.13.3 [`empty`](#empty) + * 3.13.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty](#rqsrs-018clickhousemapdatatypefunctionsempty) + * 3.13.4 [`notEmpty`](#notempty) + * 3.13.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty](#rqsrs-018clickhousemapdatatypefunctionsnotempty) + * 3.13.5 [`map`](#map) + * 3.13.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map](#rqsrs-018clickhousemapdatatypefunctionsmap) + * 3.13.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments](#rqsrs-018clickhousemapdatatypefunctionsmapinvalidnumberofarguments) + * 3.13.5.3 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes](#rqsrs-018clickhousemapdatatypefunctionsmapmixedkeyorvaluetypes) + * 3.13.5.4 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd](#rqsrs-018clickhousemapdatatypefunctionsmapmapadd) + * 3.13.5.5 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract](#rqsrs-018clickhousemapdatatypefunctionsmapmapsubstract) + * 3.13.5.6 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries](#rqsrs-018clickhousemapdatatypefunctionsmapmappopulateseries) + * 3.13.6 [`mapContains`](#mapcontains) + * 3.13.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains](#rqsrs-018clickhousemapdatatypefunctionsmapcontains) + * 3.13.7 [`mapKeys`](#mapkeys) + * 3.13.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys](#rqsrs-018clickhousemapdatatypefunctionsmapkeys) + * 3.13.8 [`mapValues`](#mapvalues) + * 3.13.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues](#rqsrs-018clickhousemapdatatypefunctionsmapvalues) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +This software requirements specification covers requirements for `Map(key, value)` data type in [ClickHouse]. + +## Requirements + +### General + +#### RQ.SRS-018.ClickHouse.Map.DataType +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs. + +### Performance + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Array(Tuple(K,V))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Tuple(Array(String), Array(String))` data type where the first +array defines an array of keys and the second array defines an array of values. + +### Key Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type. + +### Value Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Array +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type. + +### Invalid Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type. + +### Duplicated Keys + +#### RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys +version: 1.0 + +[ClickHouse] MAY support `Map(key, value)` data type with duplicated keys. + +### Array of Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps +version: 1.0 + +[ClickHouse] SHALL support `Array(Map(key, value))` data type. + +### Nested With Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps +version: 1.0 + +[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type. + +### Value Retrieval + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval +version: 1.0 + +[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax. +If `key` has duplicates then the first `key:value` pair MAY be returned. + +For example, + +```sql +SELECT a['key2'] FROM table_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid +version: 1.0 + +[ClickHouse] SHALL return an error when key does not match the key type. + +For example, + +```sql +SELECT map(1,2) AS m, m[1024] +``` + +Exceptions: + +* when key is `NULL` the return value MAY be `NULL` +* when key value is not valid for the key type, for example it is out of range for [Integer] type, + when reading from a table column it MAY return the default value for key data type + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound +version: 1.0 + +[ClickHouse] SHALL return default value for the data type of the value +when there's no corresponding `key` defined in the `Map(key, value)` data type. + + +### Converting Tuple(Array, Array) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function. + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid +version: 1.0 + +[ClickHouse] MAY return an error when casting [Tuple(Array, Array)] to `Map(key, value)` + +* when arrays are not of equal size + + For example, + + ```sql + SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] + ``` + +### Converting Array(Tuple(K,V)) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid +version: 1.0 + +[ClickHouse] MAY return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` + +* when element is not a [Tuple] + + ```sql + SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map + ``` + +* when [Tuple] does not contain two elements + + ```sql + SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map + ``` + +### Keys and Values Subcolumns + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys +version: 1.0 + +[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map keys. + +```sql +SELECT m.keys FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.keys, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap +version: 1.0 + +[ClickHouse] MAY not support using inline defined map to get `keys` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.keys +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values +version: 1.0 + +[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map values. + +```sql +SELECT m.values FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.values, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap +version: 1.0 + +[ClickHouse] MAY not support using inline defined map to get `values` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.values +``` + +### Functions + +#### RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL support using inline defined maps as an argument to map functions. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c) +SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c) +``` + +#### `length` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function +that SHALL return number of keys in the map. + +For example, + +```sql +SELECT length(map(1,2,3,4)) +SELECT length(map()) +``` + +#### `empty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function +that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 0. + +For example, + +```sql +SELECT empty(map(1,2,3,4)) +SELECT empty(map()) +``` + +#### `notEmpty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function +that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 1. + +For example, + +```sql +SELECT notEmpty(map(1,2,3,4)) +SELECT notEmpty(map()) +``` + +#### `map` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map +version: 1.0 + +[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type +using `map` function. + +**Syntax** + +``` sql +map(key1, value1[, key2, value2, ...]) +``` + +For example, + +``` sql +SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); + +┌─map('key1', number, 'key2', multiply(number, 2))─┐ +│ {'key1':0,'key2':0} │ +│ {'key1':1,'key2':2} │ +│ {'key1':2,'key2':4} │ +└──────────────────────────────────────────────────┘ +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments. + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types. + + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type. + +For example, + +``` sql +SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)") +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)") +``` +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)") +``` + +#### `mapContains` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains +version: 1.0 + +[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`. + +For example, + +```sql +SELECT mapContains(a, 'abc') from table_map; +``` + +#### `mapKeys` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys +version: 1.0 + +[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format. + +For example, + +```sql +SELECT mapKeys(a) from table_map; +``` + +#### `mapValues` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues +version: 1.0 + +[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format. + +For example, + +```sql +SELECT mapValues(a) from table_map; +``` + +[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/ +[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length +[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty +[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty +[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast +[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ +[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/ +[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/ +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com +''') diff --git a/tests/testflows/map_type/tests/__init__.py b/tests/testflows/map_type/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/tests/common.py b/tests/testflows/map_type/tests/common.py new file mode 100644 index 00000000000..a3a0d0ef0b1 --- /dev/null +++ b/tests/testflows/map_type/tests/common.py @@ -0,0 +1,49 @@ +import uuid +from collections import namedtuple + +from testflows.core import * +from testflows.core.name import basename, parentname +from testflows._core.testtype import TestSubType + +def getuid(): + if current().subtype == TestSubType.Example: + testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + else: + testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" + return testname + "_" + str(uuid.uuid1()).replace('-', '_') + +@TestStep(Given) +def allow_experimental_map_type(self): + """Set allow_experimental_map_type = 1 + """ + setting = ("allow_experimental_map_type", 1) + default_query_settings = None + + try: + with By("adding allow_experimental_map_type to the default query settings"): + default_query_settings = getsattr(current().context, "default_query_settings", []) + default_query_settings.append(setting) + yield + finally: + with Finally("I remove allow_experimental_map_type from the default query settings"): + if default_query_settings: + try: + default_query_settings.pop(default_query_settings.index(setting)) + except ValueError: + pass + +@TestStep(Given) +def create_table(self, name, statement, on_cluster=False): + """Create table. + """ + node = current().context.node + try: + with Given(f"I have a {name} table"): + node.query(statement.format(name=name)) + yield name + finally: + with Finally("I drop the table"): + if on_cluster: + node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster}") + else: + node.query(f"DROP TABLE IF EXISTS {name}") diff --git a/tests/testflows/map_type/tests/feature.py b/tests/testflows/map_type/tests/feature.py new file mode 100755 index 00000000000..5fd48844825 --- /dev/null +++ b/tests/testflows/map_type/tests/feature.py @@ -0,0 +1,1195 @@ +# -*- coding: utf-8 -*- +import time + +from testflows.core import * +from testflows.asserts import error + +from map_type.requirements import * +from map_type.tests.common import * + +@TestOutline +def select_map(self, map, output, exitcode=0, message=None): + """Create a map using select statement. + """ + node = self.context.node + + with When("I create a map using select", description=map): + r = node.query(f"SELECT {map}", exitcode=exitcode, message=message) + + with Then("I expect output to match", description=output): + assert r.output == output, error() + +@TestOutline +def table_map(self, type, data, select, filter, exitcode, message, check_insert=False, order_by=None): + """Check using a map column in a table. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + if check_insert: + node.query(f"INSERT INTO {table} VALUES {data}", exitcode=exitcode, message=message) + else: + node.query(f"INSERT INTO {table} VALUES {data}") + + if not check_insert: + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +) +@Examples("map output", [ + ("map('',1)", "{'':1}", Name("empty string")), + ("map('hello',1)", "{'hello':1}", Name("non-empty string")), + ("map('Gãńdåłf_Thê_Gręât',1)", "{'Gãńdåłf_Thê_Gręât':1}", Name("utf-8 string")), + ("map('hello there',1)", "{'hello there':1}", Name("multi word string")), + ("map('hello',1,'there',2)", "{'hello':1,'there':2}", Name("multiple keys")), + ("map(toString(1),1)", "{'1':1}", Name("toString")), + ("map(toFixedString('1',1),1)", "{'1':1}", Name("toFixedString")), + ("map(toNullable('1'),1)", "{'1':1}", Name("Nullable")), + ("map(toNullable(NULL),1)", "{NULL:1}", Name("Nullable(NULL)")), + ("map(toLowCardinality('1'),1)", "{'1':1}", Name("LowCardinality(String)")), + ("map(toLowCardinality(toFixedString('1',1)),1)", "{'1':1}", Name("LowCardinality(FixedString)")), +], row_format="%20s,%20s") +def select_map_with_key_string(self, map, output): + """Create a map using select that has key string type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +) +@Examples("map output", [ + ("map('key','')", "{'key':''}", Name("empty string")), + ("map('key','hello')", "{'key':'hello'}", Name("non-empty string")), + ("map('key','Gãńdåłf_Thê_Gręât')", "{'key':'Gãńdåłf_Thê_Gręât'}", Name("utf-8 string")), + ("map('key','hello there')", "{'key':'hello there'}", Name("multi word string")), + ("map('key','hello','key2','there')", "{'key':'hello','key2':'there'}", Name("multiple keys")), + ("map('key',toString(1))", "{'key':'1'}", Name("toString")), + ("map('key',toFixedString('1',1))", "{'key':'1'}", Name("toFixedString")), + ("map('key',toNullable('1'))", "{'key':'1'}", Name("Nullable")), + ("map('key',toNullable(NULL))", "{'key':NULL}", Name("Nullable(NULL)")), + ("map('key',toLowCardinality('1'))", "{'key':'1'}", Name("LowCardinality(String)")), + ("map('key',toLowCardinality(toFixedString('1',1)))", "{'key':'1'}", Name("LowCardinality(FixedString)")), +], row_format="%20s,%20s") +def select_map_with_value_string(self, map, output): + """Create a map using select that has value string type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0") +) +@Examples("map output", [ + ("map('key',[])", "{'key':[]}", Name("empty Array")), + ("map('key',[1,2,3])", "{'key':[1,2,3]}", Name("non-empty array of ints")), + ("map('key',['1','2','3'])", "{'key':['1','2','3']}", Name("non-empty array of strings")), + ("map('key',[map(1,2),map(2,3)])", "{'key':[{1:2},{2:3}]}", Name("non-empty array of maps")), + ("map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])])", "{'key':[{1:[{1:[1]}]},{2:[{2:[3]}]}]}", Name("non-empty array of maps of array of maps")), +]) +def select_map_with_value_array(self, map, output): + """Create a map using select that has value array type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") +) +@Examples("map output", [ + ("(map(1,127,2,0,3,-128))", '{1:127,2:0,3:-128}', Name("Int8")), + ("(map(1,0,2,255))", '{1:0,2:255}', Name("UInt8")), + ("(map(1,32767,2,0,3,-32768))", '{1:32767,2:0,3:-32768}', Name("Int16")), + ("(map(1,0,2,65535))", '{1:0,2:65535}', Name("UInt16")), + ("(map(1,2147483647,2,0,3,-2147483648))", '{1:2147483647,2:0,3:-2147483648}', Name("Int32")), + ("(map(1,0,2,4294967295))", '{1:0,2:4294967295}', Name("UInt32")), + ("(map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}', Name("Int64")), + ("(map(1,0,2,18446744073709551615))", '{1:0,2:18446744073709551615}', Name("UInt64")), + ("(map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{1:1.7014118346046923e38,2:0,3:-1.7014118346046923e38}', Name("Int128")), + ("(map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{1:5.78960446186581e76,2:0,3:-5.78960446186581e76}', Name("Int256")), + ("(map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{1:0,2:1.157920892373162e77}', Name("UInt256")), + ("(map(1,toNullable(1)))", '{1:1}', Name("toNullable")), + ("(map(1,toNullable(NULL)))", '{1:NULL}', Name("toNullable(NULL)")), +]) +def select_map_with_value_integer(self, map, output): + """Create a map using select that has value integer type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +) +@Examples("map output", [ + ("(map(127,1,0,1,-128,1))", '{127:1,0:1,-128:1}', Name("Int8")), + ("(map(0,1,255,1))", '{0:1,255:1}', Name("UInt8")), + ("(map(32767,1,0,1,-32768,1))", '{32767:1,0:1,-32768:1}', Name("Int16")), + ("(map(0,1,65535,1))", '{0:1,65535:1}', Name("UInt16")), + ("(map(2147483647,1,0,1,-2147483648,1))", '{2147483647:1,0:1,-2147483648:1}', Name("Int32")), + ("(map(0,1,4294967295,1))", '{0:1,4294967295:1}', Name("UInt32")), + ("(map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"9223372036854775807":1,"0":1,"-9223372036854775808":1}', Name("Int64")), + ("(map(0,1,18446744073709551615,1))", '{0:1,18446744073709551615:1}', Name("UInt64")), + ("(map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{1.7014118346046923e38:1,0:1,-1.7014118346046923e38:1}', Name("Int128")), + ("(map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{5.78960446186581e76:1,0:1,-5.78960446186581e76:1}', Name("Int256")), + ("(map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{0:1,1.157920892373162e77:1}', Name("UInt256")), + ("(map(toNullable(1),1))", '{1:1}', Name("toNullable")), + ("(map(toNullable(NULL),1))", '{NULL:1}', Name("toNullable(NULL)")), +]) +def select_map_with_key_integer(self, map, output): + """Create a map using select that has key integer type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +) +@Examples("type data output", [ + ("Map(String, Int8)", "('2020-01-01', map('',1))", '{"d":"2020-01-01","m":{"":1}}', Name("empty string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1))", '{"d":"2020-01-01","m":{"hello":1}}', Name("non-empty string")), + ("Map(String, Int8)", "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", '{"d":"2020-01-01","m":{"Gãńdåłf_Thê_Gręât":1}}', Name("utf-8 string")), + ("Map(String, Int8)", "('2020-01-01', map('hello there',1))", '{"d":"2020-01-01","m":{"hello there":1}}', Name("multi word string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1,'there',2))", '{"d":"2020-01-01","m":{"hello":1,"there":2}}', Name("multiple keys")), + ("Map(String, Int8)", "('2020-01-01', map(toString(1),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("toString")), + ("Map(FixedString(1), Int8)", "('2020-01-01', map(toFixedString('1',1),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("FixedString")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("Nullable")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("Nullable(NULL)")), + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map(toLowCardinality('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(String)")), + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map('1',1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(String) cast from String")), + ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"d":"2020-01-01","m":{"1":"1"}}', Name("LowCardinality(String) for key and value")), + ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(FixedString)")), +]) +def table_map_with_key_string(self, type, data, output): + """Check what values we can insert into map type column with key string. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +) +@Examples("type data output select", [ + ("Map(String, Int8)", "('2020-01-01', map('',1))", '{"m":1}', "m[''] AS m", Name("empty string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1))", '{"m":1}', "m['hello'] AS m", Name("non-empty string")), + ("Map(String, Int8)", "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", '{"m":1}', "m['Gãńdåłf_Thê_Gręât'] AS m", Name("utf-8 string")), + ("Map(String, Int8)", "('2020-01-01', map('hello there',1))", '{"m":1}', "m['hello there'] AS m", Name("multi word string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1,'there',2))", '{"m":1}', "m['hello'] AS m", Name("multiple keys")), + ("Map(String, Int8)", "('2020-01-01', map(toString(1),1))", '{"m":1}', "m['1'] AS m", Name("toString")), + ("Map(FixedString(1), Int8)", "('2020-01-01', map(toFixedString('1',1),1))", '{"m":1}', "m['1'] AS m", Name("FixedString")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable('1'),1))", '{"m":1}}', "m['1'] AS m", Name("Nullable")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"m":1}', "m[null] AS m", Name("Nullable(NULL)")), + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map(toLowCardinality('1'),1))", '{"m":1}}', "m['1'] AS m", Name("LowCardinality(String)")), + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map('1',1))", '{"m":1}', "m['1'] AS m", Name("LowCardinality(String) cast from String")), + ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"m":"1"}', "m['1'] AS m", Name("LowCardinality(String) for key and value")), + ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"m":1}', "m['1'] AS m", Name("LowCardinality(FixedString)")), +]) +def table_map_select_key_with_key_string(self, type, data, output, select): + """Check what values we can insert into map type column with key string and if key can be selected. + """ + insert_into_table(type=type, data=data, output=output, select=select) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +) +@Examples("type data output", [ + ("Map(String, String)", "('2020-01-01', map('key',''))", '{"d":"2020-01-01","m":{"key":""}}', Name("empty string")), + ("Map(String, String)", "('2020-01-01', map('key','hello'))", '{"d":"2020-01-01","m":{"key":"hello"}}', Name("non-empty string")), + ("Map(String, String)", "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", '{"d":"2020-01-01","m":{"key":"Gãńdåłf_Thê_Gręât"}}', Name("utf-8 string")), + ("Map(String, String)", "('2020-01-01', map('key', 'hello there'))", '{"d":"2020-01-01","m":{"key":"hello there"}}', Name("multi word string")), + ("Map(String, String)", "('2020-01-01', map('key','hello','key2','there'))", '{"d":"2020-01-01","m":{"key":"hello","key2":"there"}}', Name("multiple keys")), + ("Map(String, String)", "('2020-01-01', map('key', toString(1)))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("toString")), + ("Map(String, FixedString(1))", "('2020-01-01', map('key',toFixedString('1',1)))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("FixedString")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("Nullable")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable(NULL)))", '{"d":"2020-01-01","m":{"key":null}}', Name("Nullable(NULL)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key',toLowCardinality('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(String)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(String) cast from String")), + ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"d":"2020-01-01","m":{"1":"1"}}', Name("LowCardinality(String) for key and value")), + ("Map(String, LowCardinality(FixedString(1)))", "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(FixedString)")) +]) +def table_map_with_value_string(self, type, data, output): + """Check what values we can insert into map type column with value string. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +) +@Examples("type data output", [ + ("Map(String, String)", "('2020-01-01', map('key',''))", '{"m":""}', Name("empty string")), + ("Map(String, String)", "('2020-01-01', map('key','hello'))", '{"m":"hello"}', Name("non-empty string")), + ("Map(String, String)", "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", '{"m":"Gãńdåłf_Thê_Gręât"}', Name("utf-8 string")), + ("Map(String, String)", "('2020-01-01', map('key', 'hello there'))", '{"m":"hello there"}', Name("multi word string")), + ("Map(String, String)", "('2020-01-01', map('key','hello','key2','there'))", '{"m":"hello"}', Name("multiple keys")), + ("Map(String, String)", "('2020-01-01', map('key', toString(1)))", '{"m":"1"}', Name("toString")), + ("Map(String, FixedString(1))", "('2020-01-01', map('key',toFixedString('1',1)))", '{"m":"1"}', Name("FixedString")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable('1')))", '{"m":"1"}', Name("Nullable")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable(NULL)))", '{"m":null}', Name("Nullable(NULL)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key',toLowCardinality('1')))", '{"m":"1"}', Name("LowCardinality(String)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"m":"1"}', Name("LowCardinality(String) cast from String")), + ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"m":"1"}', Name("LowCardinality(String) for key and value")), + ("Map(String, LowCardinality(FixedString(1)))", "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", '{"m":"1"}', Name("LowCardinality(FixedString)")) +]) +def table_map_select_key_with_value_string(self, type, data, output): + """Check what values we can insert into map type column with value string and if it can be selected by key. + """ + insert_into_table(type=type, data=data, output=output, select="m['key'] AS m") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") +) +@Examples("type data output", [ + ("Map(Int8, Int8)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:127,2:0,3:-128}}', Name("Int8")), + ("Map(Int8, UInt8)", "('2020-01-01', map(1,0,2,255))", '{"d":"2020-01-01","m":{1:0,2:255}}', Name("UInt8")), + ("Map(Int8, Int16)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:32767,2:0,3:-32768}}', Name("Int16")), + ("Map(Int8, UInt16)", "('2020-01-01', map(1,0,2,65535))", '{"d":"2020-01-01","m":{1:0,2:65535}}', Name("UInt16")), + ("Map(Int8, Int32)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:2147483647,2:0,3:-2147483648}}', Name("Int32")), + ("Map(Int8, UInt32)", "('2020-01-01', map(1,0,2,4294967295))", '{"d":"2020-01-01","m":{1:0,2:4294967295}}', Name("UInt32")), + ("Map(Int8, Int64)", "('2020-01-01', map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{"d":"2020-01-01","m":{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}}', Name("Int64")), + ("Map(Int8, UInt64)", "('2020-01-01', map(1,0,2,18446744073709551615))", '{"d":"2020-01-01","m":{1:"0",2:"18446744073709551615"}}', Name("UInt64")), + ("Map(Int8, Int128)", "('2020-01-01', map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{"d":"2020-01-01","m":{1:"170141183460469231731687303715884105727",2:"0",3:"-170141183460469231731687303715884105728"}}', Name("Int128")), + ("Map(Int8, Int256)", "('2020-01-01', map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{"d":"2020-01-01","m":{1:"57896044618658097711785492504343953926634992332820282019728792003956564819967",2:"0",3:"-57896044618658097711785492504343953926634992332820282019728792003956564819968"}}', Name("Int256")), + ("Map(Int8, UInt256)", "('2020-01-01', map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{"d":"2020-01-01","m":{1:"0",2:"115792089237316195423570985008687907853269984665640564039457584007913129639935"}}', Name("UInt256")), + ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(1)))", '{"d":"2020-01-01","m":{1:1}}', Name("toNullable")), + ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(NULL)))", '{"d":"2020-01-01","m":{1:null}}', Name("toNullable(NULL)")), +]) +def table_map_with_value_integer(self, type, data, output): + """Check what values we can insert into map type column with value integer. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0") +) +@Examples("type data output", [ + ("Map(String, Array(Int8))", "('2020-01-01', map('key',[]))", '{"d":"2020-01-01","m":{"key":[]}}', Name("empty array")), + ("Map(String, Array(Int8))", "('2020-01-01', map('key',[1,2,3]))", '{"d":"2020-01-01","m":{"key":[1,2,3]}}', Name("non-empty array of ints")), + ("Map(String, Array(String))", "('2020-01-01', map('key',['1','2','3']))", '{"d":"2020-01-01","m":{"key":["1","2","3"]}}', Name("non-empty array of strings")), + ("Map(String, Array(Map(Int8, Int8)))", "('2020-01-01', map('key',[map(1,2),map(2,3)]))", '{"d":"2020-01-01","m":{"key":[{1:2},{2:3}]}}', Name("non-empty array of maps")), + ("Map(String, Array(Map(Int8, Array(Map(Int8, Array(Int8))))))", "('2020-01-01', map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])]))", '{"d":"2020-01-01","m":{"key":[{1:[{1:[1]}]},{2:[{2:[3]}]}]}}', Name("non-empty array of maps of array of maps")), +]) +def table_map_with_value_array(self, type, data, output): + """Check what values we can insert into map type column with value Array. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +) +@Examples("type data output", [ + ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{127:1,0:1,-128:1}}', Name("Int8")), + ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"d":"2020-01-01","m":{0:1,255:1}}', Name("UInt8")), + ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{32767:1,0:1,-32768:1}}', Name("Int16")), + ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"d":"2020-01-01","m":{0:1,65535:1}}', Name("UInt16")), + ("Map(Int32, Int8)", "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", '{"d":"2020-01-01","m":{2147483647:1,0:1,-2147483648:1}}', Name("Int32")), + ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"d":"2020-01-01","m":{0:1,4294967295:1}}', Name("UInt32")), + ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"d":"2020-01-01","m":{"9223372036854775807":1,"0":1,"-9223372036854775808":1}}', Name("Int64")), + ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"d":"2020-01-01","m":{"0":1,"18446744073709551615":1}}', Name("UInt64")), + ("Map(Int128, Int8)", "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{"d":"2020-01-01","m":{170141183460469231731687303715884105727:1,0:1,"-170141183460469231731687303715884105728":1}}', Name("Int128")), + ("Map(Int256, Int8)", "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{"d":"2020-01-01","m":{"57896044618658097711785492504343953926634992332820282019728792003956564819967":1,"0":1,"-57896044618658097711785492504343953926634992332820282019728792003956564819968":1}}', Name("Int256")), + ("Map(UInt256, Int8)", "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{"d":"2020-01-01","m":{"0":1,"115792089237316195423570985008687907853269984665640564039457584007913129639935":1}}', Name("UInt256")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(1),1))", '{"d":"2020-01-01","m":{1:1}}', Name("toNullable")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("toNullable(NULL)")), +]) +def table_map_with_key_integer(self, type, data, output): + """Check what values we can insert into map type column with key integer. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +) +@Examples("type data output select", [ + ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"m":1}', "m[127] AS m", Name("Int8")), + ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"m":2}', "(m[255] + m[0]) AS m", Name("UInt8")), + ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"m":3}', "(m[-128] + m[0] + m[-128]) AS m", Name("Int16")), + ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"m":2}', "(m[0] + m[65535]) AS m", Name("UInt16")), + ("Map(Int32, Int8)", "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", '{"m":3}', "(m[2147483647] + m[0] + m[-2147483648]) AS m", Name("Int32")), + ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"m":2}', "(m[0] + m[4294967295]) AS m", Name("UInt32")), + ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"m":3}', "(m[9223372036854775807] + m[0] + m[-9223372036854775808]) AS m", Name("Int64")), + ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"m":2}', "(m[0] + m[18446744073709551615]) AS m", Name("UInt64")), + ("Map(Int128, Int8)", "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{"m":3}', "(m[170141183460469231731687303715884105727] + m[0] + m[-170141183460469231731687303715884105728]) AS m", Name("Int128")), + ("Map(Int256, Int8)", "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{"m":3}', "(m[57896044618658097711785492504343953926634992332820282019728792003956564819967] + m[0] + m[-57896044618658097711785492504343953926634992332820282019728792003956564819968]) AS m", Name("Int256")), + ("Map(UInt256, Int8)", "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{"m":2}', "(m[0] + m[115792089237316195423570985008687907853269984665640564039457584007913129639935]) AS m", Name("UInt256")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(1),1))", '{"m":1}', "m[1] AS m", Name("toNullable")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"m":1}', "m[null] AS m", Name("toNullable(NULL)")), +]) +def table_map_select_key_with_key_integer(self, type, data, output, select): + """Check what values we can insert into map type column with key integer and if we can use the key to select the value. + """ + insert_into_table(type=type, data=data, output=output, select=select) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps("1.0") +) +@Examples("type data output partition_by", [ + ("Array(Map(String, Int8))", + "('2020-01-01', [map('hello',1),map('hello',1,'there',2)])", + '{"d":"2020-01-01","m":[{"hello":1},{"hello":1,"there":2}]}', + "m", + Name("Array(Map(String, Int8))")), + ("Nested(x Map(String, Int8))", + "('2020-01-01', [map('hello',1)])", + '{"d":"2020-01-01","m.x":[{"hello":1}]}', + "m.x", + Name("Nested(x Map(String, Int8)")) +]) +def table_with_map_inside_another_type(self, type, data, output, partition_by): + """Check what values we can insert into a type that has map type. + """ + insert_into_table(type=type, data=data, output=output, partition_by=partition_by) + +@TestOutline +def insert_into_table(self, type, data, output, partition_by="m", select="*"): + """Check we can insert data into a table. + """ + uid = getuid() + node = self.context.node + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (d DATE, m " + type + ") ENGINE = MergeTree() PARTITION BY " + partition_by + " ORDER BY d" + + with Given(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data", description=data): + sql = f"INSERT INTO {table} VALUES {data}" + node.query(sql) + + with And("I select rows from the table"): + r = node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow") + + with Then("I expect output to match", description=output): + assert r.output == output, error() + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes("1.0") +) +def select_map_with_invalid_mixed_key_and_value_types(self): + """Check that creating a map with mixed key types fails. + """ + node = self.context.node + exitcode = 130 + message = "DB::Exception: There is no supertype for types String, UInt8 because some of them are String/FixedString and some of them are not" + + with Check("attempt to create a map using SELECT with mixed key types then it fails"): + node.query("SELECT map('hello',1,2,3)", exitcode=exitcode, message=message) + + with Check("attempt to create a map using SELECT with mixed value types then it fails"): + node.query("SELECT map(1,'hello',2,2)", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments("1.0") +) +def select_map_with_invalid_number_of_arguments(self): + """Check that creating a map with invalid number of arguments fails. + """ + node = self.context.node + exitcode = 42 + message = "DB::Exception: Function map requires even number of arguments" + + with When("I create a map using SELECT with invalid number of arguments"): + node.query("SELECT map(1,2,3)", exitcode=exitcode, message=message) + +@TestScenario +def select_map_empty(self): + """Check that we can can create a empty map by not passing any arguments. + """ + node = self.context.node + + with When("I create a map using SELECT with no arguments"): + r = node.query("SELECT map()") + + with Then("it should create an empty map"): + assert r.output == "{}", error() + +@TestScenario +def insert_invalid_mixed_key_and_value_types(self): + """Check that inserting a map with mixed key or value types fails. + """ + uid = getuid() + node = self.context.node + exitcode = 130 + message = "DB::Exception: There is no supertype for types String, UInt8 because some of them are String/FixedString and some of them are not" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (d DATE, m Map(String, Int8)) ENGINE = MergeTree() PARTITION BY m ORDER BY d" + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert a map with mixed key types then it should fail"): + sql = f"INSERT INTO {table} VALUES ('2020-01-01', map('hello',1,2,3))" + node.query(sql, exitcode=exitcode, message=message) + + with When("I insert a map with mixed value types then it should fail"): + sql = f"INSERT INTO {table} VALUES ('2020-01-01', map(1,'hello',2,2))" + node.query(sql, exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0") +) +@Examples("type data output", [ + ("Map(String, String)", + "('2020-01-01', map('hello','there','hello','over there'))", + '{"d":"2020-01-01","m":{"hello":"there","hello":"over there"}}', + Name("Map(String, String))")), + ("Map(Int64, String)", + "('2020-01-01', map(12345,'there',12345,'over there'))", + '{"d":"2020-01-01","m":{"12345":"there","12345":"over there"}}', + Name("Map(Int64, String))")), +]) +def table_map_with_duplicated_keys(self, type, data, output): + """Check that map supports duplicated keys. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0") +) +@Examples("map output", [ + ("map('hello','there','hello','over there')", "{'hello':'there','hello':'over there'}", Name("String")), + ("map(12345,'there',12345,'over there')", "{12345:'there',12345:'over there'}", Name("Integer")) +]) +def select_map_with_duplicated_keys(self, map, output): + """Check creating a map with duplicated keys. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0") +) +def select_map_key_not_found(self): + node = self.context.node + + with When("map is empty"): + node.query("SELECT map() AS m, m[1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("map has integer values"): + r = node.query("SELECT map(1,2) AS m, m[2] FORMAT Values") + with Then("zero should be returned for key that is not found"): + assert r.output == "({1:2},0)", error() + + with When("map has string values"): + r = node.query("SELECT map(1,'2') AS m, m[2] FORMAT Values") + with Then("empty string should be returned for key that is not found"): + assert r.output == "({1:'2'},'')", error() + + with When("map has array values"): + r = node.query("SELECT map(1,[2]) AS m, m[2] FORMAT Values") + with Then("empty array be returned for key that is not found"): + assert r.output == "({1:[2]},[])", error() + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0") +) +@Examples("type data select exitcode message", [ + ("Map(UInt8, UInt8), y Int8", "(y) VALUES (1)", "m[1] AS v", 0, '{"v":0}', Name("empty map")), + ("Map(UInt8, UInt8)", "VALUES (map(1,2))", "m[2] AS v", 0, '{"v":0}', Name("map has integer values")), + ("Map(UInt8, String)", "VALUES (map(1,'2'))", "m[2] AS v", 0, '{"v":""}', Name("map has string values")), + ("Map(UInt8, Array(Int8))", "VALUES (map(1,[2]))", "m[2] AS v", 0, '{"v":[]}', Name("map has array values")), +]) +def table_map_key_not_found(self, type, data, select, exitcode, message, order_by=None): + """Check values returned from a map column when key is not found. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0") +) +def invalid_key(self): + """Check when key is not valid. + """ + node = self.context.node + + with When("I try to use an integer key that is too large"): + node.query("SELECT map(1,2) AS m, m[256]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an integer key that is negative when key is unsigned"): + node.query("SELECT map(1,2) AS m, m[-1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use a string key when key is an integer"): + node.query("SELECT map(1,2) AS m, m['1']", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an integer key when key is a string"): + r = node.query("SELECT map('1',2) AS m, m[1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an empty key when key is a string"): + r = node.query("SELECT map('1',2) AS m, m[]", exitcode=62, message="DB::Exception: Syntax error: failed at position") + + with When("I try to use wrong type conversion in key"): + r = node.query("SELECT map(1,2) AS m, m[toInt8('1')]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("in array of maps I try to use an integer key that is negative when key is unsigned"): + node.query("SELECT [map(1,2)] AS m, m[1][-1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use a NULL key when key is not nullable"): + r = node.query("SELECT map(1,2) AS m, m[NULL] FORMAT Values") + with Then("it should return NULL"): + assert r.output == "({1:2},NULL)", error() + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0") +) +@Examples("type data select exitcode message order_by", [ + ("Map(UInt8, UInt8)", "(map(1,2))", "m[256] AS v", 0, '{"v":0}', "m", Name("key too large)")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[-1] AS v", 0, '{"v":0}', "m", Name("key is negative")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("string when key is integer")), + ("Map(String, UInt8)", "(map('1',2))", "m[1] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("integer when key is string")), + ("Map(String, UInt8)", "(map('1',2))", "m[] AS v", 62, "DB::Exception: Syntax error: failed at position", "m", Name("empty when key is string")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[toInt8('1')] AS v", 0, '{"v":2}', "m", Name("wrong type conversion when key is integer")), + ("Map(String, UInt8)", "(map('1',2))", "m[toFixedString('1',1)] AS v", 0, '{"v":2}', "m", Name("wrong type conversion when key is string")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[NULL] AS v", 0, '{"v":null}', "m", Name("NULL key when key is not nullable")), + ("Array(Map(UInt8, UInt8))", "([map(1,2)])", "m[1]['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("string when key is integer in array of maps")), + ("Nested(x Map(UInt8, UInt8))", "([map(1,2)])", "m.x[1]['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m.x", Name("string when key is integer in nested map")), +]) +def table_map_invalid_key(self, type, data, select, exitcode, message, order_by="m"): + """Check selecting values from a map column using an invalid key. + """ + uid = getuid() + node = self.context.node + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} VALUES {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval("1.0") +) +@Examples("type data select filter exitcode message order_by", [ + ("Map(UInt8, UInt8)", "(map(1,1)),(map(1,2)),(map(2,3))", "m[1] AS v", "1=1 ORDER BY m[1]", 0, '{"v":0}\n{"v":1}\n{"v":2}', None, + Name("select the same key from all the rows")), + ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('e','f','c','d')", 0, '', None, + Name("filter rows by map having different pair order")), + ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('c','d','e','f')", 0, '{"m":{"c":"d","e":"f"}}', None, + Name("filter rows by map having the same pair order")), + ("Map(String, String)", "(map('a','b')),(map('e','f'))", "m", "m = map()", 0, '', None, + Name("filter rows by empty map")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1", 0, '{"m":{"a":1,"b":2}}', None, + Name("filter rows by map key value")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1 AND m['b'] = 2", 0, '{"m":{"a":1,"b":2}}', None, + Name("filter rows by map multiple key value combined with AND")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1 OR m['b'] = 3", 0, '{"m":{"a":1,"b":2}}\n{"m":{"b":3}}', None, + Name("filter rows by map multiple key value combined with OR")), + ("Map(String, Array(Int8))", "(map('a',[])),(map('b',[1])),(map('c',[2]))", "m['b'] AS v", "m['b'] IN ([1],[2])", 0, '{"v":[1]}', None, + Name("filter rows by map array value using IN")), + ("Map(String, Nullable(String))", "(map('a',NULL)),(map('a',1))", "m", "isNull(m['a']) = 1", 0, '{"m":{"a":null}}', None, + Name("select map with nullable value")) +]) +def table_map_queries(self, type, data, select, filter, exitcode, message, order_by=None): + """Check retrieving map values and using maps in queries. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} VALUES {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing("1.0") +) +@Examples("type exitcode message", [ + ("Nullable(Map(String, String))", + 43, "DB::Exception: Nested type Map(String,String) cannot be inside Nullable type", + Name("nullable map")), + ("Map(Nothing, Nothing)", + 37, "DB::Exception: Column `m` with type Map(Nothing,Nothing) is not allowed in key expression, it's not comparable", + Name("map with nothing type for key and value")) +]) +def table_map_unsupported_types(self, type, exitcode, message): + """Check creating a table with unsupported map column types. + """ + uid = getuid() + node = self.context.node + + try: + with When(f"I create a table definition with {type}"): + sql = f"CREATE TABLE {uid} (m " + type + ") ENGINE = MergeTree() ORDER BY m" + node.query(sql, exitcode=exitcode, message=message) + finally: + with Finally("drop table if any"): + node.query(f"DROP TABLE IF EXISTS {uid}") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0") +) +@Examples("tuple type exitcode message", [ + ("([1, 2, 3], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 0, "{1:'Ready',2:'Steady',3:'Go'}", Name("int -> int")), + ("([1, 2, 3], ['Ready', 'Steady', 'Go'])", "Map(String, String)", + 0, "{'1':'Ready','2':'Steady','3':'Go'}", Name("int -> string")), + ("(['1', '2', '3'], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 0, "{1:'Ready',187:'Steady',143:'Go'}", Name("string -> int")), + ("([],[])", "Map(String, String)", + 0, "{}", Name("empty arrays to map str:str")), + ("([],[])", "Map(UInt8, Array(Int8))", + 0, "{}", Name("empty arrays to map uint8:array")), + ("([[1]],['hello'])", "Map(String, String)", + 0, "{'[1]':'hello'}", Name("array -> string")), + ("([(1,2),(3,4)])", "Map(UInt8, UInt8)", + 0, "{1:2,3:4}", Name("array of two tuples")), + ("([1, 2], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", + Name("unequal array sizes")), +]) +def cast_tuple_of_two_arrays_to_map(self, tuple, type, exitcode, message): + """Check casting Tuple(Array, Array) to a map type. + """ + node = self.context.node + + with When("I try to cast tuple", description=tuple): + node.query(f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0") +) +@Examples("tuple type exitcode message check_insert", [ + ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 0, '{"m":{1:"Ready",2:"Steady",3:"Go"}}', False, Name("int -> int")), + ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(String, String)", + 0, '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', False, Name("int -> string")), + ("((['1', '2', '3'], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 0, '', True, Name("string -> int")), + ("(([],[]))", "Map(String, String)", + 0, '{"m":{}}', False, Name("empty arrays to map str:str")), + ("(([],[]))", "Map(UInt8, Array(Int8))", + 0, '{"m":{}}', False, Name("empty arrays to map uint8:array")), + ("(([[1]],['hello']))", "Map(String, String)", + 53, 'DB::Exception: Type mismatch in IN or VALUES section', True, Name("array -> string")), + ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", + 0, '{"m":{1:2,3:4}}', False, Name("array of two tuples")), + ("(([1, 2], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", True, + Name("unequal array sizes")), +]) +def table_map_cast_tuple_of_arrays_to_map(self, tuple, type, exitcode, message, check_insert): + """Check converting Tuple(Array, Array) into map on insert into a map type column. + """ + table_map(type=type, data=tuple, select="*", filter="1=1", exitcode=exitcode, message=message, check_insert=check_insert) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") +) +@Examples("tuple type exitcode message", [ + ("([(1,2),(3,4)])", "Map(UInt8, UInt8)", 0, "{1:2,3:4}", + Name("array of two tuples")), + ("([(1,2),(3)])", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", + Name("not a tuple")), + ("([(1,2),(3,)])", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", + Name("invalid tuple")), +]) +def cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message): + """Check casting Array(Tuple(K,V)) to a map type. + """ + node = self.context.node + + with When("I try to cast tuple", description=tuple): + node.query(f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") +) +@Examples("tuple type exitcode message check_insert", [ + ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", 0, '{"m":{1:2,3:4}}', False, + Name("array of two tuples")), + ("(([(1,2),(3)]))", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", True, + Name("not a tuple")), + ("(([(1,2),(3,)]))", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", True, + Name("invalid tuple")), +]) +def table_map_cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message, check_insert): + """Check converting Array(Tuple(K,V),...) into map on insert into a map type column. + """ + table_map(type=type, data=tuple, select="*", filter="1=1", exitcode=exitcode, message=message, check_insert=check_insert) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap("1.0") +) +def subcolumns_keys_using_inline_defined_map(self): + node = self.context.node + exitcode = 47 + message = "DB::Exception: Missing columns: 'c.keys'" + + with When("I try to access keys sub-column using an inline defined map"): + node.query("SELECT map( 'aa', 4, '44' , 5) as c, c.keys", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap("1.0") +) +def subcolumns_values_using_inline_defined_map(self): + node = self.context.node + exitcode = 47 + message = "DB::Exception: Missing columns: 'c.values'" + + with When("I try to access values sub-column using an inline defined map"): + node.query("SELECT map( 'aa', 4, '44' , 5) as c, c.values", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions("1.0") +) +@Examples("type data select filter exitcode message", [ + # keys + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.keys AS keys", "1=1", + 0, '{"keys":["a","c"]}\n{"keys":["e"]}', Name("select keys")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.keys AS keys", "has(m.keys, 'e')", + 0, '{"keys":["e"]}', Name("filter by using keys in an array function")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "has(m.keys, 'e') AS r", "1=1", + 0, '{"r":0}\n{"r":1}', Name("column that uses keys in an array function")), + # values + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.values AS values", "1=1", + 0, '{"values":["b","d"]}\n{"values":["f"]}', Name("select values")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.values AS values", "has(m.values, 'f')", + 0, '{"values":["f"]}', Name("filter by using values in an array function")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "has(m.values, 'f') AS r", "1=1", + 0, '{"r":0}\n{"r":1}', Name("column that uses values in an array function")) +]) +def subcolumns(self, type, data, select, filter, exitcode, message, order_by=None): + """Check usage of sub-columns in queries. + """ + table_map(type=type, data=data, select=select, filter=filter, exitcode=exitcode, message=message, order_by=order_by) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length("1.0") +) +def length(self): + """Check usage of length function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('a','b','c','d')),(map('e','f'))", + select="length(m) AS len, m", + filter="length(m) = 1", + exitcode=0, message='{"len":"1","m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty("1.0") +) +def empty(self): + """Check usage of empty function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('e','f'))", + select="empty(m) AS em, m", + filter="empty(m) <> 1", + exitcode=0, message='{"em":0,"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty("1.0") +) +def notempty(self): + """Check usage of notEmpty function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('e','f'))", + select="notEmpty(m) AS em, m", + filter="notEmpty(m) = 1", + exitcode=0, message='{"em":1,"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd("1.0") +) +def cast_from_mapadd(self): + """Check converting the result of mapAdd function to a map data type. + """ + select_map(map="CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), 'Map(Int8, Int8)')", output="{1:2,2:2}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract("1.0") +) +def cast_from_mapsubstract(self): + """Check converting the result of mapSubstract function to a map data type. + """ + select_map(map="CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), 'Map(Int8, Int8)')", output="{1:-1,2:0}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries("1.0") +) +def cast_from_mappopulateseries(self): + """Check converting the result of mapPopulateSeries function to a map data type. + """ + select_map(map="CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), 'Map(Int8, Int8)')", output="{1:11,2:22,3:0,4:44,5:0}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains("1.0") +) +def mapcontains(self): + """Check usages of mapContains function with map data type. + """ + node = self.context.node + + with Example("key in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="mapContains(m, 'a')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("key not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT mapContains(m, 'a')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null key not in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="mapContains(m, NULL)", + exitcode=0, message='') + + with Example("null key in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="m", + filter="mapContains(m, NULL)", + exitcode=0, message='{null:"c"}') + + with Example("select nullable key"): + node.query("SELECT map(NULL, 1, 2, 3) AS m, mapContains(m, toNullable(toUInt8(2)))", exitcode=0, message="{2:3}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys("1.0") +) +def mapkeys(self): + """Check usages of mapKeys function with map data type. + """ + with Example("key in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapKeys(m), 'a')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("key not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT has(mapKeys(m), 'a')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null key not in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapKeys(m), NULL)", + exitcode=0, message='') + + with Example("null key in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="m", + filter="has(mapKeys(m), NULL)", + exitcode=0, message='{"m":{null:"c"}}') + + with Example("select keys from column"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="mapKeys(m) AS keys", + filter="1 = 1", + exitcode=0, message='{"keys":["a"]}\n{"keys":["e"]}\n{"keys":[null]}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues("1.0") +) +def mapvalues(self): + """Check usages of mapValues function with map data type. + """ + with Example("value in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapValues(m), 'b')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("value not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT has(mapValues(m), 'b')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null value not in map"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapValues(m), NULL)", + exitcode=0, message='') + + with Example("null value in map"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b')),(map('c',NULL))", + select="m", + filter="has(mapValues(m), NULL)", + exitcode=0, message='{"m":{"c":null}}') + + with Example("select values from column"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b')),(map('c',NULL))", + select="mapValues(m) AS values", + filter="1 = 1", + exitcode=0, message='{"values":["b"]}\n{"values":[null]}\n{"values":["f"]}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap("1.0") +) +def functions_with_inline_defined_map(self): + """Check that a map defined inline inside the select statement + can be used with functions that work with maps. + """ + with Example("mapKeys"): + select_map(map="map(1,2,3,4) as map, mapKeys(map) AS keys", output="{1:2,3:4}\t[1,3]") + + with Example("mapValyes"): + select_map(map="map(1,2,3,4) as map, mapValues(map) AS values", output="{1:2,3:4}\t[2,4]") + + with Example("mapContains"): + select_map(map="map(1,2,3,4) as map, mapContains(map, 1) AS contains", output="{1:2,3:4}\t1") + +@TestScenario +def empty_map(self): + """Check creating of an empty map `{}` using the map() function + when inserting data into a map type table column. + """ + table_map(type="Map(String, String)", + data="(map('e','f')),(map())", + select="m", + filter="1=1", + exitcode=0, message='{"m":{}}\n{"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays("1.0") +) +def performance_vs_two_tuple_of_arrays(self, len=10, rows=6000000): + """Check performance of using map data type vs Tuple(Array, Array). + """ + uid = getuid() + node = self.context.node + + with Given(f"table with Tuple(Array(Int8),Array(Int8))"): + sql = "CREATE TABLE {name} (pairs Tuple(Array(Int8),Array(Int8))) ENGINE = MergeTree() ORDER BY pairs" + tuple_table = create_table(name=f"tuple_{uid}", statement=sql) + + with And(f"table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with tuples"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {tuple_table} SELECT ({keys},{values}) FROM numbers({rows})") + tuple_insert_time = time.time() - start_time + metric("tuple insert time", tuple_insert_time, "sec") + + with When("I insert data into table with a map"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with tuples"): + start_time = time.time() + node.query(f"SELECT sum(arrayFirst((v, k) -> k = {len-1}, tupleElement(pairs, 2), tupleElement(pairs, 1))) AS sum FROM {tuple_table}", + exitcode=0, message=f"{rows*(len-1)}") + tuple_select_time = time.time() - start_time + metric("tuple(array, array) select time", tuple_select_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + + metric("insert difference", (1 - map_insert_time/tuple_insert_time) * 100, "%") + metric("select difference", (1 - map_select_time/tuple_select_time) * 100, "%") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples("1.0") +) +def performance_vs_array_of_tuples(self, len=10, rows=6000000): + """Check performance of using map data type vs Array(Tuple(K,V)). + """ + uid = getuid() + node = self.context.node + + with Given(f"table with Array(Tuple(K,V))"): + sql = "CREATE TABLE {name} (pairs Array(Tuple(Int8, Int8))) ENGINE = MergeTree() ORDER BY pairs" + array_table = create_table(name=f"tuple_{uid}", statement=sql) + + with And(f"table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with an array of tuples"): + pairs = list(zip(range(len),range(len))) + start_time = time.time() + node.query(f"INSERT INTO {array_table} SELECT ({pairs}) FROM numbers({rows})") + array_insert_time = time.time() - start_time + metric("array insert time", array_insert_time, "sec") + + with When("I insert data into table with a map"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with an array of tuples"): + start_time = time.time() + node.query(f"SELECT sum(arrayFirst((v) -> v.1 = {len-1}, pairs).2) AS sum FROM {array_table}", + exitcode=0, message=f"{rows*(len-1)}") + array_select_time = time.time() - start_time + metric("array(tuple(k,v)) select time", array_select_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + + metric("insert difference", (1 - map_insert_time/array_insert_time) * 100, "%") + metric("select difference", (1 - map_select_time/array_select_time) * 100, "%") + +@TestScenario +def performance(self, len=10, rows=6000000): + """Check insert and select performance of using map data type. + """ + uid = getuid() + node = self.context.node + + with Given("table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with a map"): + values = [x for pair in zip(range(len),range(len)) for x in pair] + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT (map({','.join([str(v) for v in values])})) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + +# FIXME: add tests for different table engines + +@TestFeature +@Name("tests") +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map("1.0") +) +def feature(self, node="clickhouse1"): + self.context.node = self.context.cluster.node(node) + + with Given("I allow experimental map type"): + allow_experimental_map_type() + + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index 45f1ed64a6c..13a24f97f9f 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -18,6 +18,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): # Feature(test=load("ldap.regression", "regression"))(**args) # Feature(test=load("rbac.regression", "regression"))(**args) # Feature(test=load("aes_encryption.regression", "regression"))(**args) + Feature(test=load("map_type.regression", "regression"))(**args) # Feature(test=load("kerberos.regression", "regression"))(**args) if main():