From d1f8f52f160e4fd5724a5b314e3d96197d6046b7 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 22 Feb 2021 15:04:34 -0500 Subject: [PATCH 01/12] Adding Map type tests in TestFlows --- tests/testflows/map_type/__init__.py | 0 .../map_type/configs/clickhouse/common.xml | 6 + .../configs/clickhouse/config.d/logs.xml | 17 + .../configs/clickhouse/config.d/macros.xml | 0 .../configs/clickhouse/config.d/ports.xml | 5 + .../configs/clickhouse/config.d/remote.xml | 211 +++ .../configs/clickhouse/config.d/ssl.xml | 18 + .../configs/clickhouse/config.d/storage.xml | 20 + .../configs/clickhouse/config.d/zookeeper.xml | 10 + .../map_type/configs/clickhouse/config.xml | 448 +++++ .../configs/clickhouse/ssl/dhparam.pem | 8 + .../configs/clickhouse/ssl/server.crt | 19 + .../configs/clickhouse/ssl/server.key | 28 + .../map_type/configs/clickhouse/users.xml | 133 ++ .../configs/clickhouse1/config.d/macros.xml | 8 + .../configs/clickhouse2/config.d/macros.xml | 8 + .../configs/clickhouse3/config.d/macros.xml | 8 + .../docker-compose/clickhouse-service.yml | 28 + .../docker-compose/docker-compose.yml | 60 + .../docker-compose/zookeeper-service.yml | 18 + tests/testflows/map_type/regression.py | 61 + .../map_type/requirements/__init__.py | 1 + .../map_type/requirements/requirements.md | 523 ++++++ .../map_type/requirements/requirements.py | 1449 +++++++++++++++++ tests/testflows/map_type/tests/__init__.py | 0 tests/testflows/map_type/tests/common.py | 89 + tests/testflows/map_type/tests/feature.py | 1119 +++++++++++++ 27 files changed, 4295 insertions(+) create mode 100644 tests/testflows/map_type/__init__.py create mode 100644 tests/testflows/map_type/configs/clickhouse/common.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/logs.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/macros.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/ports.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/remote.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/storage.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem create mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/server.crt create mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/server.key create mode 100644 tests/testflows/map_type/configs/clickhouse/users.xml create mode 100644 tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml create mode 100644 tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml create mode 100644 tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml create mode 100755 tests/testflows/map_type/docker-compose/clickhouse-service.yml create mode 100755 tests/testflows/map_type/docker-compose/docker-compose.yml create mode 100755 tests/testflows/map_type/docker-compose/zookeeper-service.yml create mode 100755 tests/testflows/map_type/regression.py create mode 100644 tests/testflows/map_type/requirements/__init__.py create mode 100644 tests/testflows/map_type/requirements/requirements.md create mode 100644 tests/testflows/map_type/requirements/requirements.py create mode 100644 tests/testflows/map_type/tests/__init__.py create mode 100644 tests/testflows/map_type/tests/common.py create mode 100755 tests/testflows/map_type/tests/feature.py diff --git a/tests/testflows/map_type/__init__.py b/tests/testflows/map_type/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/configs/clickhouse/common.xml b/tests/testflows/map_type/configs/clickhouse/common.xml new file mode 100644 index 00000000000..567c8c05e32 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + :: + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml b/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 +
+
diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse/config.d/macros.xml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml b/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml new file mode 100644 index 00000000000..fbc6cea74c0 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,5 @@ + + + 8443 + 9440 + \ No newline at end of file diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml b/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..a7ed0d6e2b4 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,211 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + true + + clickhouse1 + 9440 + 1 + + + clickhouse2 + 9440 + 1 + + + clickhouse3 + 9440 + 1 + + + + + + + clickhouse1 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9440 + 1 + + + + + clickhouse2 + 9440 + 1 + + + + + clickhouse3 + 9440 + 1 + + + + + + true + + clickhouse1 + 9440 + 1 + + user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end + + + + clickhouse2 + 9440 + 1 + user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end + + + + clickhouse3 + 9440 + 1 + user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end + + + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml b/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 00000000000..768d2250b79 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,18 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + /etc/clickhouse-server/ssl/dhparam.pem + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml b/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml new file mode 100644 index 00000000000..618fd6b6d24 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,20 @@ + + + + + + 1024 + + + + + + + default + + + + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.xml b/tests/testflows/map_type/configs/clickhouse/config.xml new file mode 100644 index 00000000000..4ec12232539 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.xml @@ -0,0 +1,448 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + 0.0.0.0 + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + /var/lib/clickhouse/access/ + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 +
+ + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + +
diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 00000000000..2e6cee0798d --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/server.crt b/tests/testflows/map_type/configs/clickhouse/ssl/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/server.key b/tests/testflows/map_type/configs/clickhouse/ssl/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/testflows/map_type/configs/clickhouse/users.xml b/tests/testflows/map_type/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml new file mode 100644 index 00000000000..a114a9ce4ab --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse2 + 01 + 02 + + diff --git a/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml new file mode 100644 index 00000000000..904a27b0172 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse3 + 01 + 03 + + diff --git a/tests/testflows/map_type/docker-compose/clickhouse-service.yml b/tests/testflows/map_type/docker-compose/clickhouse-service.yml new file mode 100755 index 00000000000..2d79443dcbb --- /dev/null +++ b/tests/testflows/map_type/docker-compose/clickhouse-service.yml @@ -0,0 +1,28 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/map_type/docker-compose/docker-compose.yml b/tests/testflows/map_type/docker-compose/docker-compose.yml new file mode 100755 index 00000000000..a3f5144c9ed --- /dev/null +++ b/tests/testflows/map_type/docker-compose/docker-compose.yml @@ -0,0 +1,60 @@ +version: '2.3' + +services: + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy \ No newline at end of file diff --git a/tests/testflows/map_type/docker-compose/zookeeper-service.yml b/tests/testflows/map_type/docker-compose/zookeeper-service.yml new file mode 100755 index 00000000000..f3df33358be --- /dev/null +++ b/tests/testflows/map_type/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py new file mode 100755 index 00000000000..78b0ac74b15 --- /dev/null +++ b/tests/testflows/map_type/regression.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import sys + +from testflows.core import * + +append_path(sys.path, "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from map_type.requirements import QA_SRS018_ClickHouse_Map_Data_Type + +xfails = { + "tests/table map with key integer/Int:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], + "tests/table map with value integer/Int:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], + "tests/table map with key integer/UInt256": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], + "tests/table map with value integer/UInt256": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], + "tests/select map with key integer/Int64": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030")], + "tests/select map with value integer/Int64": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030")], + "tests/cast tuple of two arrays to map/string -> int": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21029")], + "tests/mapcontains/null key in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapcontains/select nullable key": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")] +} + +xflags = { +} + +@TestModule +@ArgumentParser(argparser) +@XFails(xfails) +@XFlags(xflags) +@Name("map type") +@Specifications( + QA_SRS018_ClickHouse_Map_Data_Type +) +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): + """Map type regression. + """ + nodes = { + "clickhouse": + ("clickhouse1", "clickhouse2", "clickhouse3") + } + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + self.context.stress = stress + + if parallel is not None: + self.context.parallel = parallel + + Feature(run=load("map_type.tests.feature", "feature")) + +if main(): + regression() diff --git a/tests/testflows/map_type/requirements/__init__.py b/tests/testflows/map_type/requirements/__init__.py new file mode 100644 index 00000000000..02f7d430154 --- /dev/null +++ b/tests/testflows/map_type/requirements/__init__.py @@ -0,0 +1 @@ +from .requirements import * diff --git a/tests/testflows/map_type/requirements/requirements.md b/tests/testflows/map_type/requirements/requirements.md new file mode 100644 index 00000000000..becdeaecbb2 --- /dev/null +++ b/tests/testflows/map_type/requirements/requirements.md @@ -0,0 +1,523 @@ +# SRS018 ClickHouse Map Data Type +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Requirements](#requirements) + * 3.1 [General](#general) + * 3.1.1 [RQ.SRS-018.ClickHouse.Map.DataType](#rqsrs-018clickhousemapdatatype) + * 3.2 [Performance](#performance) + * 3.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples](#rqsrs-018clickhousemapdatatypeperformancevsarrayoftuples) + * 3.2.2 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays](#rqsrs-018clickhousemapdatatypeperformancevstupleofarrays) + * 3.3 [Key Types](#key-types) + * 3.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Key.String](#rqsrs-018clickhousemapdatatypekeystring) + * 3.3.2 [RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer](#rqsrs-018clickhousemapdatatypekeyinteger) + * 3.4 [Value Types](#value-types) + * 3.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.String](#rqsrs-018clickhousemapdatatypevaluestring) + * 3.4.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer](#rqsrs-018clickhousemapdatatypevalueinteger) + * 3.4.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Array](#rqsrs-018clickhousemapdatatypevaluearray) + * 3.5 [Invalid Types](#invalid-types) + * 3.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable](#rqsrs-018clickhousemapdatatypeinvalidnullable) + * 3.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing](#rqsrs-018clickhousemapdatatypeinvalidnothingnothing) + * 3.6 [Duplicated Keys](#duplicated-keys) + * 3.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys](#rqsrs-018clickhousemapdatatypeduplicatedkeys) + * 3.7 [Array of Maps](#array-of-maps) + * 3.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps](#rqsrs-018clickhousemapdatatypearrayofmaps) + * 3.8 [Nested With Maps](#nested-with-maps) + * 3.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps](#rqsrs-018clickhousemapdatatypenestedwithmaps) + * 3.9 [Value Retrieval](#value-retrieval) + * 3.9.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval](#rqsrs-018clickhousemapdatatypevalueretrieval) + * 3.9.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid](#rqsrs-018clickhousemapdatatypevalueretrievalkeyinvalid) + * 3.9.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound](#rqsrs-018clickhousemapdatatypevalueretrievalkeynotfound) + * 3.10 [Converting Tuple(Array, Array) to Map](#converting-tuplearray-array-to-map) + * 3.10.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraystomap) + * 3.10.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraysmapinvalid) + * 3.11 [Converting Array(Tuple(K,V)) to Map](#converting-arraytuplekv-to-map) + * 3.11.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomap) + * 3.11.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomapinvalid) + * 3.12 [Keys and Values Subcolumns](#keys-and-values-subcolumns) + * 3.12.1 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys](#rqsrs-018clickhousemapdatatypesubcolumnskeys) + * 3.12.2 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnskeysarrayfunctions) + * 3.12.3 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnskeysinlinedefinedmap) + * 3.12.4 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values](#rqsrs-018clickhousemapdatatypesubcolumnsvalues) + * 3.12.5 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesarrayfunctions) + * 3.12.6 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesinlinedefinedmap) + * 3.13 [Functions](#functions) + * 3.13.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap](#rqsrs-018clickhousemapdatatypefunctionsinlinedefinedmap) + * 3.13.2 [`length`](#length) + * 3.13.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length](#rqsrs-018clickhousemapdatatypefunctionslength) + * 3.13.3 [`empty`](#empty) + * 3.13.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty](#rqsrs-018clickhousemapdatatypefunctionsempty) + * 3.13.4 [`notEmpty`](#notempty) + * 3.13.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty](#rqsrs-018clickhousemapdatatypefunctionsnotempty) + * 3.13.5 [`map`](#map) + * 3.13.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map](#rqsrs-018clickhousemapdatatypefunctionsmap) + * 3.13.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments](#rqsrs-018clickhousemapdatatypefunctionsmapinvalidnumberofarguments) + * 3.13.5.3 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes](#rqsrs-018clickhousemapdatatypefunctionsmapmixedkeyorvaluetypes) + * 3.13.5.4 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd](#rqsrs-018clickhousemapdatatypefunctionsmapmapadd) + * 3.13.5.5 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract](#rqsrs-018clickhousemapdatatypefunctionsmapmapsubstract) + * 3.13.5.6 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries](#rqsrs-018clickhousemapdatatypefunctionsmapmappopulateseries) + * 3.13.6 [`mapContains`](#mapcontains) + * 3.13.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains](#rqsrs-018clickhousemapdatatypefunctionsmapcontains) + * 3.13.7 [`mapKeys`](#mapkeys) + * 3.13.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys](#rqsrs-018clickhousemapdatatypefunctionsmapkeys) + * 3.13.8 [`mapValues`](#mapvalues) + * 3.13.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues](#rqsrs-018clickhousemapdatatypefunctionsmapvalues) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +This software requirements specification covers requirements for `Map(key, value)` data type in [ClickHouse]. + +## Requirements + +### General + +#### RQ.SRS-018.ClickHouse.Map.DataType +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs. + +### Performance + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Array(Tuple(K,V))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Tuple(Array(String), Array(String))` data type where the first +array defines an array of keys and the second array defines an array of values. + +### Key Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type. + +### Value Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Array +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type. + +### Invalid Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type. + +### Duplicated Keys + +#### RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys. + +### Array of Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps +version: 1.0 + +[ClickHouse] SHALL support `Array(Map(key, value))` data type. + +### Nested With Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps +version: 1.0 + +[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type. + +### Value Retrieval + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval +version: 1.0 + +[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax. +If `key` has duplicates then the first `key:value` pair SHALL be returned. + +For example, + +```sql +SELECT a['key2'] FROM table_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid +version: 1.0 + +[ClickHouse] SHALL return an error when key does not match the key type. + +For example, + +```sql +SELECT map(1,2) AS m, m[1024] +``` + +SHALL have the following output + +```bash +Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. +``` + +Exceptions: + +* when key is `NULL` the return value SHALL be `NULL` +* when key has [Integer] type and key is either too large or negative then + when reading from a table column it SHALL return `zero`. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound +version: 1.0 + +[ClickHouse] SHALL return + +* zeros for [Integer] values +* empty strings for [String] values +* empty for [Array] values + +when there's no corresponding `key` defined in the `Map(key, value)` data type. + + +### Converting Tuple(Array, Array) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function. + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid +such as when arrays are not of equal size. + +For example, + +```sql +SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] +``` + +### Converting Array(Tuple(K,V)) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` +when [Tuple] is invalid such as when it is not a [Tuple] with two elements. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +or + +```sql +SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map +``` + +### Keys and Values Subcolumns + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys +version: 1.0 + +[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map keys. + +```sql +SELECT m.keys FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.keys, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.keys +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values +version: 1.0 + +[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map values. + +```sql +SELECT m.values FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.values, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.values +``` + +### Functions + +#### RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL support using inline defined maps as an argument to map functions. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c) +SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c) +``` + +#### `length` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function +that SHALL return number of keys in the map. + +For example, + +```sql +SELECT length(map(1,2,3,4)) +SELECT length(map()) +``` + +#### `empty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function +that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 0. + +For example, + +```sql +SELECT empty(map(1,2,3,4)) +SELECT empty(map()) +``` + +#### `notEmpty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function +that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 1. + +For example, + +```sql +SELECT notEmpty(map(1,2,3,4)) +SELECT notEmpty(map()) +``` + +#### `map` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map +version: 1.0 + +[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type +using `map` function. + +**Syntax** + +``` sql +map(key1, value1[, key2, value2, ...]) +``` + +For example, + +``` sql +SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); + +┌─map('key1', number, 'key2', multiply(number, 2))─┐ +│ {'key1':0,'key2':0} │ +│ {'key1':1,'key2':2} │ +│ {'key1':2,'key2':4} │ +└──────────────────────────────────────────────────┘ +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments. + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types. + + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type. + +For example, + +``` sql +SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)") +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)") +``` +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)") +``` + +#### `mapContains` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains +version: 1.0 + +[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`. + +For example, + +```sql +SELECT mapContains(a, 'abc') from table_map; +``` + +#### `mapKeys` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys +version: 1.0 + +[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format. + +For example, + +```sql +SELECT mapKeys(a) from table_map; +``` + +#### `mapValues` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues +version: 1.0 + +[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format. + +For example, + +```sql +SELECT mapValues(a) from table_map; +``` + +[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/ +[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length +[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty +[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty +[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast +[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ +[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/ +[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/ +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py new file mode 100644 index 00000000000..6042512abd0 --- /dev/null +++ b/tests/testflows/map_type/requirements/requirements.py @@ -0,0 +1,1449 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.210214.1233241. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Specification +from testflows.core import Requirement + +Heading = Specification.Heading + +RQ_SRS_018_ClickHouse_Map_DataType = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs.\n' + '\n' + ), + link=None, + level=3, + num='3.1.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n' + 'compared to `Array(Tuple(K,V))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.2.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n' + 'compared to `Tuple(Array(String), Array(String))` data type where the first\n' + 'array defines an array of keys and the second array defines an array of values.\n' + '\n' + ), + link=None, + level=3, + num='3.2.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Key_String = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Key.String', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type.\n' + '\n' + ), + link=None, + level=3, + num='3.3.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type.\n' + '\n' + ), + link=None, + level=3, + num='3.3.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_String = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.String', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Array = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Array', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.5.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.5.2') + +RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys.\n' + '\n' + ), + link=None, + level=3, + num='3.6.1') + +RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Array(Map(key, value))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.7.1') + +RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type.\n' + '\n' + ), + link=None, + level=3, + num='3.8.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax.\n' + 'If `key` has duplicates then the first `key:value` pair SHALL be returned. \n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT a['key2'] FROM table_map;\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.9.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when key does not match the key type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT map(1,2) AS m, m[1024]\n' + '```\n' + '\n' + 'SHALL have the following output \n' + '\n' + '```bash\n' + 'Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. \n' + '```\n' + '\n' + 'Exceptions:\n' + '\n' + '* when key is `NULL` the return value SHALL be `NULL`\n' + '* when key has [Integer] type and key is either too large or negative then\n' + ' when reading from a table column it SHALL return `zero`.\n' + '\n' + ), + link=None, + level=3, + num='3.9.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return\n' + '\n' + '* zeros for [Integer] values\n' + '* empty strings for [String] values\n' + '* empty for [Array] values\n' + '\n' + "when there's no corresponding `key` defined in the `Map(key, value)` data type. \n" + '\n' + '\n' + ), + link=None, + level=3, + num='3.9.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function.\n' + '\n' + '``` sql\n' + "SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map;\n" + '```\n' + '\n' + '``` text\n' + '┌─map───────────────────────────┐\n' + "│ {1:'Ready',2:'Steady',3:'Go'} │\n" + '└───────────────────────────────┘\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.10.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid\n' + 'such as when arrays are not of equal size.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10]\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.10.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.11.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)`\n' + 'when [Tuple] is invalid such as when it is not a [Tuple] with two elements.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" + '```\n' + '\n' + 'or \n' + '\n' + '```sql\n' + "SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.11.2') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used \n' + 'to retrieve an [Array] of map keys.\n' + '\n' + '```sql\n' + 'SELECT m.keys FROM t_map;\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.1') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT * FROM t_map WHERE has(m.keys, 'a');\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.2') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, c.keys\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.3') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used \n' + 'to retrieve an [Array] of map values.\n' + '\n' + '```sql\n' + 'SELECT m.values FROM t_map;\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.4') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT * FROM t_map WHERE has(m.values, 'a');\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.5') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, c.values\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.6') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support using inline defined maps as an argument to map functions.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c)\n" + "SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c)\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.13.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function\n' + 'that SHALL return number of keys in the map.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT length(map(1,2,3,4))\n' + 'SELECT length(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.2.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function\n' + 'that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is \n' + 'greater or equal to 1 it SHALL return 0.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT empty(map(1,2,3,4))\n' + 'SELECT empty(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.3.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function\n' + 'that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is\n' + 'greater or equal to 1 it SHALL return 1.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT notEmpty(map(1,2,3,4))\n' + 'SELECT notEmpty(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.4.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type\n' + 'using `map` function.\n' + '\n' + '**Syntax** \n' + '\n' + '``` sql\n' + 'map(key1, value1[, key2, value2, ...])\n' + '```\n' + '\n' + 'For example,\n' + '\n' + '``` sql\n' + "SELECT map('key1', number, 'key2', number * 2) FROM numbers(3);\n" + '\n' + "┌─map('key1', number, 'key2', multiply(number, 2))─┐\n" + "│ {'key1':0,'key2':0} │\n" + "│ {'key1':1,'key2':2} │\n" + "│ {'key1':2,'key2':4} │\n" + '└──────────────────────────────────────────────────┘\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments.\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types.\n' + '\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '``` sql\n' + 'SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)")\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.4') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)")\n' + '```\n' + ), + link=None, + level=4, + num='3.13.5.5') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)")\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.6') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT mapContains(a, 'abc') from table_map;\n" + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.6.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT mapKeys(a) from table_map;\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.7.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT mapValues(a) from table_map;\n' + '```\n' + '\n' + '[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/\n' + '[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length\n' + '[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty\n' + '[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty\n' + '[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast\n' + '[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/\n' + '[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/\n' + '[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ \n' + '[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/\n' + '[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/\n' + '[ClickHouse]: https://clickhouse.tech\n' + '[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md \n' + '[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md\n' + '[Git]: https://git-scm.com/\n' + '[GitHub]: https://github.com\n' + ), + link=None, + level=4, + num='3.13.8.1') + +SRS018_ClickHouse_Map_Data_Type = Specification( + name='SRS018 ClickHouse Map Data Type', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + headings=( + Heading(name='Revision History', level=1, num='1'), + Heading(name='Introduction', level=1, num='2'), + Heading(name='Requirements', level=1, num='3'), + Heading(name='General', level=2, num='3.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType', level=3, num='3.1.1'), + Heading(name='Performance', level=2, num='3.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples', level=3, num='3.2.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays', level=3, num='3.2.2'), + Heading(name='Key Types', level=2, num='3.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Key.String', level=3, num='3.3.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer', level=3, num='3.3.2'), + Heading(name='Value Types', level=2, num='3.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.String', level=3, num='3.4.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer', level=3, num='3.4.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Array', level=3, num='3.4.3'), + Heading(name='Invalid Types', level=2, num='3.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable', level=3, num='3.5.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing', level=3, num='3.5.2'), + Heading(name='Duplicated Keys', level=2, num='3.6'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys', level=3, num='3.6.1'), + Heading(name='Array of Maps', level=2, num='3.7'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps', level=3, num='3.7.1'), + Heading(name='Nested With Maps', level=2, num='3.8'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps', level=3, num='3.8.1'), + Heading(name='Value Retrieval', level=2, num='3.9'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval', level=3, num='3.9.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid', level=3, num='3.9.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound', level=3, num='3.9.3'), + Heading(name='Converting Tuple(Array, Array) to Map', level=2, num='3.10'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap', level=3, num='3.10.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid', level=3, num='3.10.2'), + Heading(name='Converting Array(Tuple(K,V)) to Map', level=2, num='3.11'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap', level=3, num='3.11.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid', level=3, num='3.11.2'), + Heading(name='Keys and Values Subcolumns', level=2, num='3.12'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys', level=3, num='3.12.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions', level=3, num='3.12.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap', level=3, num='3.12.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values', level=3, num='3.12.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions', level=3, num='3.12.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap', level=3, num='3.12.6'), + Heading(name='Functions', level=2, num='3.13'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap', level=3, num='3.13.1'), + Heading(name='`length`', level=3, num='3.13.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length', level=4, num='3.13.2.1'), + Heading(name='`empty`', level=3, num='3.13.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty', level=4, num='3.13.3.1'), + Heading(name='`notEmpty`', level=3, num='3.13.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty', level=4, num='3.13.4.1'), + Heading(name='`map`', level=3, num='3.13.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map', level=4, num='3.13.5.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments', level=4, num='3.13.5.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes', level=4, num='3.13.5.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd', level=4, num='3.13.5.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract', level=4, num='3.13.5.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries', level=4, num='3.13.5.6'), + Heading(name='`mapContains`', level=3, num='3.13.6'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains', level=4, num='3.13.6.1'), + Heading(name='`mapKeys`', level=3, num='3.13.7'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys', level=4, num='3.13.7.1'), + Heading(name='`mapValues`', level=3, num='3.13.8'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues', level=4, num='3.13.8.1'), + ), + requirements=( + RQ_SRS_018_ClickHouse_Map_DataType, + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples, + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays, + RQ_SRS_018_ClickHouse_Map_DataType_Key_String, + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer, + RQ_SRS_018_ClickHouse_Map_DataType_Value_String, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array, + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable, + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing, + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys, + RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps, + RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues, + ), + content=''' +# SRS018 ClickHouse Map Data Type +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Requirements](#requirements) + * 3.1 [General](#general) + * 3.1.1 [RQ.SRS-018.ClickHouse.Map.DataType](#rqsrs-018clickhousemapdatatype) + * 3.2 [Performance](#performance) + * 3.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples](#rqsrs-018clickhousemapdatatypeperformancevsarrayoftuples) + * 3.2.2 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays](#rqsrs-018clickhousemapdatatypeperformancevstupleofarrays) + * 3.3 [Key Types](#key-types) + * 3.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Key.String](#rqsrs-018clickhousemapdatatypekeystring) + * 3.3.2 [RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer](#rqsrs-018clickhousemapdatatypekeyinteger) + * 3.4 [Value Types](#value-types) + * 3.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.String](#rqsrs-018clickhousemapdatatypevaluestring) + * 3.4.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer](#rqsrs-018clickhousemapdatatypevalueinteger) + * 3.4.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Array](#rqsrs-018clickhousemapdatatypevaluearray) + * 3.5 [Invalid Types](#invalid-types) + * 3.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable](#rqsrs-018clickhousemapdatatypeinvalidnullable) + * 3.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing](#rqsrs-018clickhousemapdatatypeinvalidnothingnothing) + * 3.6 [Duplicated Keys](#duplicated-keys) + * 3.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys](#rqsrs-018clickhousemapdatatypeduplicatedkeys) + * 3.7 [Array of Maps](#array-of-maps) + * 3.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps](#rqsrs-018clickhousemapdatatypearrayofmaps) + * 3.8 [Nested With Maps](#nested-with-maps) + * 3.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps](#rqsrs-018clickhousemapdatatypenestedwithmaps) + * 3.9 [Value Retrieval](#value-retrieval) + * 3.9.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval](#rqsrs-018clickhousemapdatatypevalueretrieval) + * 3.9.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid](#rqsrs-018clickhousemapdatatypevalueretrievalkeyinvalid) + * 3.9.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound](#rqsrs-018clickhousemapdatatypevalueretrievalkeynotfound) + * 3.10 [Converting Tuple(Array, Array) to Map](#converting-tuplearray-array-to-map) + * 3.10.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraystomap) + * 3.10.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraysmapinvalid) + * 3.11 [Converting Array(Tuple(K,V)) to Map](#converting-arraytuplekv-to-map) + * 3.11.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomap) + * 3.11.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomapinvalid) + * 3.12 [Keys and Values Subcolumns](#keys-and-values-subcolumns) + * 3.12.1 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys](#rqsrs-018clickhousemapdatatypesubcolumnskeys) + * 3.12.2 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnskeysarrayfunctions) + * 3.12.3 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnskeysinlinedefinedmap) + * 3.12.4 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values](#rqsrs-018clickhousemapdatatypesubcolumnsvalues) + * 3.12.5 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesarrayfunctions) + * 3.12.6 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesinlinedefinedmap) + * 3.13 [Functions](#functions) + * 3.13.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap](#rqsrs-018clickhousemapdatatypefunctionsinlinedefinedmap) + * 3.13.2 [`length`](#length) + * 3.13.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length](#rqsrs-018clickhousemapdatatypefunctionslength) + * 3.13.3 [`empty`](#empty) + * 3.13.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty](#rqsrs-018clickhousemapdatatypefunctionsempty) + * 3.13.4 [`notEmpty`](#notempty) + * 3.13.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty](#rqsrs-018clickhousemapdatatypefunctionsnotempty) + * 3.13.5 [`map`](#map) + * 3.13.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map](#rqsrs-018clickhousemapdatatypefunctionsmap) + * 3.13.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments](#rqsrs-018clickhousemapdatatypefunctionsmapinvalidnumberofarguments) + * 3.13.5.3 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes](#rqsrs-018clickhousemapdatatypefunctionsmapmixedkeyorvaluetypes) + * 3.13.5.4 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd](#rqsrs-018clickhousemapdatatypefunctionsmapmapadd) + * 3.13.5.5 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract](#rqsrs-018clickhousemapdatatypefunctionsmapmapsubstract) + * 3.13.5.6 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries](#rqsrs-018clickhousemapdatatypefunctionsmapmappopulateseries) + * 3.13.6 [`mapContains`](#mapcontains) + * 3.13.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains](#rqsrs-018clickhousemapdatatypefunctionsmapcontains) + * 3.13.7 [`mapKeys`](#mapkeys) + * 3.13.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys](#rqsrs-018clickhousemapdatatypefunctionsmapkeys) + * 3.13.8 [`mapValues`](#mapvalues) + * 3.13.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues](#rqsrs-018clickhousemapdatatypefunctionsmapvalues) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +This software requirements specification covers requirements for `Map(key, value)` data type in [ClickHouse]. + +## Requirements + +### General + +#### RQ.SRS-018.ClickHouse.Map.DataType +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs. + +### Performance + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Array(Tuple(K,V))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Tuple(Array(String), Array(String))` data type where the first +array defines an array of keys and the second array defines an array of values. + +### Key Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type. + +### Value Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Array +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type. + +### Invalid Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type. + +### Duplicated Keys + +#### RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys. + +### Array of Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps +version: 1.0 + +[ClickHouse] SHALL support `Array(Map(key, value))` data type. + +### Nested With Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps +version: 1.0 + +[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type. + +### Value Retrieval + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval +version: 1.0 + +[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax. +If `key` has duplicates then the first `key:value` pair SHALL be returned. + +For example, + +```sql +SELECT a['key2'] FROM table_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid +version: 1.0 + +[ClickHouse] SHALL return an error when key does not match the key type. + +For example, + +```sql +SELECT map(1,2) AS m, m[1024] +``` + +SHALL have the following output + +```bash +Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. +``` + +Exceptions: + +* when key is `NULL` the return value SHALL be `NULL` +* when key has [Integer] type and key is either too large or negative then + when reading from a table column it SHALL return `zero`. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound +version: 1.0 + +[ClickHouse] SHALL return + +* zeros for [Integer] values +* empty strings for [String] values +* empty for [Array] values + +when there's no corresponding `key` defined in the `Map(key, value)` data type. + + +### Converting Tuple(Array, Array) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function. + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid +such as when arrays are not of equal size. + +For example, + +```sql +SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] +``` + +### Converting Array(Tuple(K,V)) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` +when [Tuple] is invalid such as when it is not a [Tuple] with two elements. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +or + +```sql +SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map +``` + +### Keys and Values Subcolumns + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys +version: 1.0 + +[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map keys. + +```sql +SELECT m.keys FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.keys, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.keys +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values +version: 1.0 + +[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map values. + +```sql +SELECT m.values FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.values, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.values +``` + +### Functions + +#### RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL support using inline defined maps as an argument to map functions. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c) +SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c) +``` + +#### `length` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function +that SHALL return number of keys in the map. + +For example, + +```sql +SELECT length(map(1,2,3,4)) +SELECT length(map()) +``` + +#### `empty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function +that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 0. + +For example, + +```sql +SELECT empty(map(1,2,3,4)) +SELECT empty(map()) +``` + +#### `notEmpty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function +that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 1. + +For example, + +```sql +SELECT notEmpty(map(1,2,3,4)) +SELECT notEmpty(map()) +``` + +#### `map` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map +version: 1.0 + +[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type +using `map` function. + +**Syntax** + +``` sql +map(key1, value1[, key2, value2, ...]) +``` + +For example, + +``` sql +SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); + +┌─map('key1', number, 'key2', multiply(number, 2))─┐ +│ {'key1':0,'key2':0} │ +│ {'key1':1,'key2':2} │ +│ {'key1':2,'key2':4} │ +└──────────────────────────────────────────────────┘ +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments. + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types. + + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type. + +For example, + +``` sql +SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)") +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)") +``` +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)") +``` + +#### `mapContains` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains +version: 1.0 + +[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`. + +For example, + +```sql +SELECT mapContains(a, 'abc') from table_map; +``` + +#### `mapKeys` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys +version: 1.0 + +[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format. + +For example, + +```sql +SELECT mapKeys(a) from table_map; +``` + +#### `mapValues` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues +version: 1.0 + +[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format. + +For example, + +```sql +SELECT mapValues(a) from table_map; +``` + +[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/ +[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length +[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty +[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty +[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast +[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ +[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/ +[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/ +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com +''') diff --git a/tests/testflows/map_type/tests/__init__.py b/tests/testflows/map_type/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/tests/common.py b/tests/testflows/map_type/tests/common.py new file mode 100644 index 00000000000..d4461062256 --- /dev/null +++ b/tests/testflows/map_type/tests/common.py @@ -0,0 +1,89 @@ +import uuid +from collections import namedtuple + +from testflows.core import * +from testflows.core.name import basename, parentname +from testflows._core.testtype import TestSubType + +table_tuple = namedtuple("table_tuple", "create_statement cluster") + +table_types = { + "MergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, m Map(String, Int8)) ENGINE = MergeTree() PARTITION BY m ORDER BY d", None), + # "ReplacingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = ReplacingMergeTree() PARTITION BY y ORDER BY d", None), + # "SummingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) ENGINE = SummingMergeTree() PARTITION BY y ORDER BY d", None), + # "AggregatingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = AggregatingMergeTree() PARTITION BY y ORDER BY d", None), + # "CollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(sign) PARTITION BY y ORDER BY d", None), + # "VersionedCollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY y ORDER BY d", None), + # "GraphiteMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) ENGINE = GraphiteMergeTree('graphite_rollup_example') PARTITION BY y ORDER by d", None), + # "ReplicatedMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedReplacingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedReplacingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedSummingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ + # ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedSummingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ + # ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedAggregatingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedAggregatingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ + # ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ + # ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedVersionedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ + # ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedVersionedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ + # ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedGraphiteMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ + # ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedGraphiteMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ + # ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "one_shard_cluster"), +} + +def getuid(): + if current().subtype == TestSubType.Example: + testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + else: + testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" + return testname + "_" + str(uuid.uuid1()).replace('-', '_') + +@TestStep(Given) +def allow_experimental_map_type(self): + """Set allow_experimental_map_type = 1 + """ + setting = ("allow_experimental_map_type", 1) + default_query_settings = None + + try: + with By("adding allow_experimental_map_type to the default query settings"): + default_query_settings = getsattr(current().context, "default_query_settings", []) + default_query_settings.append(setting) + yield + finally: + with Finally("I remove allow_experimental_map_type from the default query settings"): + if default_query_settings: + try: + default_query_settings.pop(default_query_settings.index(setting)) + except ValueError: + pass + +@TestStep(Given) +def create_table(self, name, statement, on_cluster=False): + """Create table. + """ + node = current().context.node + try: + with Given(f"I have a {name} table"): + node.query(statement.format(name=name)) + yield name + finally: + with Finally("I drop the table"): + if on_cluster: + node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster}") + else: + node.query(f"DROP TABLE IF EXISTS {name}") \ No newline at end of file diff --git a/tests/testflows/map_type/tests/feature.py b/tests/testflows/map_type/tests/feature.py new file mode 100755 index 00000000000..07ae7cc2159 --- /dev/null +++ b/tests/testflows/map_type/tests/feature.py @@ -0,0 +1,1119 @@ +# -*- coding: utf-8 -*- +import time + +from testflows.core import * +from testflows.asserts import error + +from map_type.requirements import * +from map_type.tests.common import * + +@TestOutline +def select_map(self, map, output, exitcode=0, message=None): + """Create a map using select statement. + """ + node = self.context.node + + with When("I create a map using select", description=map): + r = node.query(f"SELECT {map}", exitcode=exitcode, message=message) + + with Then("I expect output to match", description=output): + assert r.output == output, error() + +@TestOutline +def table_map(self, type, data, select, filter, exitcode, message, check_insert=False, order_by=None): + """Check using a map column in a table. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + if check_insert: + node.query(f"INSERT INTO {table} VALUES {data}", exitcode=exitcode, message=message) + else: + node.query(f"INSERT INTO {table} VALUES {data}") + + if not check_insert: + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +) +@Examples("map output", [ + ("map('',1)", "{'':1}", Name("empty string")), + ("map('hello',1)", "{'hello':1}", Name("non-empty string")), + ("map('Gãńdåłf_Thê_Gręât',1)", "{'Gãńdåłf_Thê_Gręât':1}", Name("utf-8 string")), + ("map('hello there',1)", "{'hello there':1}", Name("multi word string")), + ("map('hello',1,'there',2)", "{'hello':1,'there':2}", Name("multiple keys")), + ("map(toString(1),1)", "{'1':1}", Name("toString")), + ("map(toFixedString('1',1),1)", "{'1':1}", Name("toFixedString")), + ("map(toNullable('1'),1)", "{'1':1}", Name("Nullable")), + ("map(toNullable(NULL),1)", "{NULL:1}", Name("Nullable(NULL)")), + ("map(toLowCardinality('1'),1)", "{'1':1}", Name("LowCardinality(String)")), + ("map(toLowCardinality(toFixedString('1',1)),1)", "{'1':1}", Name("LowCardinality(FixedString)")), +], row_format="%20s,%20s") +def select_map_with_key_string(self, map, output): + """Create a map using select that has key string type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +) +@Examples("map output", [ + ("map('key','')", "{'key':''}", Name("empty string")), + ("map('key','hello')", "{'key':'hello'}", Name("non-empty string")), + ("map('key','Gãńdåłf_Thê_Gręât')", "{'key':'Gãńdåłf_Thê_Gręât'}", Name("utf-8 string")), + ("map('key','hello there')", "{'key':'hello there'}", Name("multi word string")), + ("map('key','hello','key2','there')", "{'key':'hello','key2':'there'}", Name("multiple keys")), + ("map('key',toString(1))", "{'key':'1'}", Name("toString")), + ("map('key',toFixedString('1',1))", "{'key':'1'}", Name("toFixedString")), + ("map('key',toNullable('1'))", "{'key':'1'}", Name("Nullable")), + ("map('key',toNullable(NULL))", "{'key':NULL}", Name("Nullable(NULL)")), + ("map('key',toLowCardinality('1'))", "{'key':'1'}", Name("LowCardinality(String)")), + ("map('key',toLowCardinality(toFixedString('1',1)))", "{'key':'1'}", Name("LowCardinality(FixedString)")), +], row_format="%20s,%20s") +def select_map_with_value_string(self, map, output): + """Create a map using select that has value string type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0") +) +@Examples("map output", [ + ("map('key',[])", "{'key':[]}", Name("empty Array")), + ("map('key',[1,2,3])", "{'key':[1,2,3]}", Name("non-empty array of ints")), + ("map('key',['1','2','3'])", "{'key':['1','2','3']}", Name("non-empty array of strings")), + ("map('key',[map(1,2),map(2,3)])", "{'key':[{1:2},{2:3}]}", Name("non-empty array of maps")), + ("map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])])", "{'key':[{1:[{1:[1]}]},{2:[{2:[3]}]}]}", Name("non-empty array of maps of array of maps")), +]) +def select_map_with_value_array(self, map, output): + """Create a map using select that has value array type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") +) +@Examples("map output", [ + ("(map(1,127,2,0,3,-128))", '{1:127,2:0,3:-128}', Name("Int8")), + ("(map(1,0,2,255))", '{1:0,2:255}', Name("UInt8")), + ("(map(1,32767,2,0,3,-32768))", '{1:32767,2:0,3:-32768}', Name("Int16")), + ("(map(1,0,2,65535))", '{1:0,2:65535}', Name("UInt16")), + ("(map(1,2147483647,2,0,3,-2147483648))", '{1:2147483647,2:0,3:-2147483648}', Name("Int32")), + ("(map(1,0,2,4294967295))", '{1:0,2:4294967295}', Name("UInt32")), + ("(map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}', Name("Int64")), + ("(map(1,0,2,18446744073709551615))", '{1:0,2:18446744073709551615}', Name("UInt64")), + ("(map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{1:1.7014118346046923e38,2:0,3:-1.7014118346046923e38}', Name("Int128")), + ("(map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{1:5.78960446186581e76,2:0,3:-5.78960446186581e76}', Name("Int256")), + ("(map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{1:0,2:1.157920892373162e77}', Name("UInt256")), + ("(map(1,toNullable(1)))", '{1:1}', Name("toNullable")), + ("(map(1,toNullable(NULL)))", '{1:NULL}', Name("toNullable(NULL)")), +]) +def select_map_with_value_integer(self, map, output): + """Create a map using select that has value integer type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +) +@Examples("map output", [ + ("(map(127,1,0,1,-128,1))", '{127:1,0:1,-128:1}', Name("Int8")), + ("(map(0,1,255,1))", '{0:1,255:1}', Name("UInt8")), + ("(map(32767,1,0,1,-32768,1))", '{32767:1,0:1,-32768:1}', Name("Int16")), + ("(map(0,1,65535,1))", '{0:1,65535:1}', Name("UInt16")), + ("(map(2147483647,1,0,1,-2147483648,1))", '{2147483647:1,0:1,-2147483648:1}', Name("Int32")), + ("(map(0,1,4294967295,1))", '{0:1,4294967295:1}', Name("UInt32")), + ("(map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"9223372036854775807":1,"0":1,"-9223372036854775808":1}', Name("Int64")), + ("(map(0,1,18446744073709551615,1))", '{0:1,18446744073709551615:1}', Name("UInt64")), + ("(map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{1.7014118346046923e38:1,0:1,-1.7014118346046923e38:1}', Name("Int128")), + ("(map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{5.78960446186581e76:1,0:1,-5.78960446186581e76:1}', Name("Int256")), + ("(map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{0:1,1.157920892373162e77:1}', Name("UInt256")), + ("(map(toNullable(1),1))", '{1:1}', Name("toNullable")), + ("(map(toNullable(NULL),1))", '{NULL:1}', Name("toNullable(NULL)")), +]) +def select_map_with_key_integer(self, map, output): + """Create a map using select that has key integer type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +) +@Examples("type data output", [ + ("Map(String, Int8)", "('2020-01-01', map('',1))", '{"d":"2020-01-01","m":{"":1}}', Name("empty string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1))", '{"d":"2020-01-01","m":{"hello":1}}', Name("non-empty string")), + ("Map(String, Int8)", "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", '{"d":"2020-01-01","m":{"Gãńdåłf_Thê_Gręât":1}}', Name("utf-8 string")), + ("Map(String, Int8)", "('2020-01-01', map('hello there',1))", '{"d":"2020-01-01","m":{"hello there":1}}', Name("multi word string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1,'there',2))", '{"d":"2020-01-01","m":{"hello":1,"there":2}}', Name("multiple keys")), + ("Map(String, Int8)", "('2020-01-01', map(toString(1),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("toString")), + ("Map(FixedString(1), Int8)", "('2020-01-01', map(toFixedString('1',1),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("FixedString")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("Nullable")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("Nullable(NULL)")), + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map(toLowCardinality('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(String)")), + ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(FixedString)")) +]) +def table_map_with_key_string(self, type, data, output): + """Check what values we can insert into map type column with key string. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +) +@Examples("type data output", [ + ("Map(String, String)", "('2020-01-01', map('key',''))", '{"d":"2020-01-01","m":{"key":""}}', Name("empty string")), + ("Map(String, String)", "('2020-01-01', map('key','hello'))", '{"d":"2020-01-01","m":{"key":"hello"}}', Name("non-empty string")), + ("Map(String, String)", "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", '{"d":"2020-01-01","m":{"key":"Gãńdåłf_Thê_Gręât"}}', Name("utf-8 string")), + ("Map(String, String)", "('2020-01-01', map('key', 'hello there'))", '{"d":"2020-01-01","m":{"key":"hello there"}}', Name("multi word string")), + ("Map(String, String)", "('2020-01-01', map('key','hello','key2','there'))", '{"d":"2020-01-01","m":{"key":"hello","key2":"there"}}', Name("multiple keys")), + ("Map(String, String)", "('2020-01-01', map('key', toString(1)))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("toString")), + ("Map(String, FixedString(1))", "('2020-01-01', map('key',toFixedString('1',1)))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("FixedString")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("Nullable")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable(NULL)))", '{"d":"2020-01-01","m":{"key":null}}', Name("Nullable(NULL)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key',toLowCardinality('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(String)")), + ("Map(String, LowCardinality(FixedString(1)))", "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(FixedString)")) +]) +def table_map_with_value_string(self, type, data, output): + """Check what values we can insert into map type column with value string. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") +) +@Examples("type data output", [ + ("Map(Int8, Int8)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:127,2:0,3:-128}}', Name("Int8")), + ("Map(Int8, UInt8)", "('2020-01-01', map(1,0,2,255))", '{"d":"2020-01-01","m":{1:0,2:255}}', Name("UInt8")), + ("Map(Int8, Int16)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:32767,2:0,3:-32768}}', Name("Int16")), + ("Map(Int8, UInt16)", "('2020-01-01', map(1,0,2,65535))", '{"d":"2020-01-01","m":{1:0,2:65535}}', Name("UInt16")), + ("Map(Int8, Int32)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:2147483647,2:0,3:-2147483648}}', Name("Int32")), + ("Map(Int8, UInt32)", "('2020-01-01', map(1,0,2,4294967295))", '{"d":"2020-01-01","m":{1:0,2:4294967295}}', Name("UInt32")), + ("Map(Int8, Int64)", "('2020-01-01', map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{"d":"2020-01-01","m":{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}}', Name("Int64")), + ("Map(Int8, UInt64)", "('2020-01-01', map(1,0,2,18446744073709551615))", '{"d":"2020-01-01","m":{1:"0",2:"18446744073709551615"}}', Name("UInt64")), + ("Map(Int8, Int128)", "('2020-01-01', map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{"d":"2020-01-01","m":{1:"170141183460469231731687303715884105727",2:"0",3:"-170141183460469231731687303715884105728"}}', Name("Int128")), + ("Map(Int8, Int256)", "('2020-01-01', map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{"d":"2020-01-01","m":{1:"57896044618658097711785492504343953926634992332820282019728792003956564819967",2:"0",3:"-57896044618658097711785492504343953926634992332820282019728792003956564819968"}}', Name("Int256")), + ("Map(Int8, UInt256)", "('2020-01-01', map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{"d":"2020-01-01","m":{1:"0",2:"115792089237316195423570985008687907853269984665640564039457584007913129639935"}}', Name("UInt256")), + ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(1)))", '{"d":"2020-01-01","m":{1:1}}', Name("toNullable")), + ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(NULL)))", '{"d":"2020-01-01","m":{1:null}}', Name("toNullable(NULL)")), +]) +def table_map_with_value_integer(self, type, data, output): + """Check what values we can insert into map type column with value integer. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0") +) +@Examples("type data output", [ + ("Map(String, Array(Int8))", "('2020-01-01', map('key',[]))", '{"d":"2020-01-01","m":{"key":[]}}', Name("empty array")), + ("Map(String, Array(Int8))", "('2020-01-01', map('key',[1,2,3]))", '{"d":"2020-01-01","m":{"key":[1,2,3]}}', Name("non-empty array of ints")), + ("Map(String, Array(String))", "('2020-01-01', map('key',['1','2','3']))", '{"d":"2020-01-01","m":{"key":["1","2","3"]}}', Name("non-empty array of strings")), + ("Map(String, Array(Map(Int8, Int8)))", "('2020-01-01', map('key',[map(1,2),map(2,3)]))", '{"d":"2020-01-01","m":{"key":[{1:2},{2:3}]}}', Name("non-empty array of maps")), + ("Map(String, Array(Map(Int8, Array(Map(Int8, Array(Int8))))))", "('2020-01-01', map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])]))", '{"d":"2020-01-01","m":{"key":[{1:[{1:[1]}]},{2:[{2:[3]}]}]}}', Name("non-empty array of maps of array of maps")), +]) +def table_map_with_value_array(self, type, data, output): + """Check what values we can insert into map type column with value Array. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +) +@Examples("type data output", [ + ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{127:1,0:1,-128:1}}', Name("Int8")), + ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"d":"2020-01-01","m":{0:1,255:1}}', Name("UInt8")), + ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{32767:1,0:1,-32768:1}}', Name("Int16")), + ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"d":"2020-01-01","m":{0:1,65535:1}}', Name("UInt16")), + ("Map(Int32, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{2147483647:1,0:1,-2147483648:1}}', Name("Int32")), + ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"d":"2020-01-01","m":{0:1,4294967295:1}}', Name("UInt32")), + ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"d":"2020-01-01","m":{"9223372036854775807":1,"0":1,"-9223372036854775808":1}}', Name("Int64")), + ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"d":"2020-01-01","m":{"0":1,"18446744073709551615":1}}', Name("UInt64")), + ("Map(Int128, Int8)", "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{"d":"2020-01-01","m":{170141183460469231731687303715884105727:1,0:1,"-170141183460469231731687303715884105728":1}}', Name("Int128")), + ("Map(Int256, Int8)", "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{"d":"2020-01-01","m":{"57896044618658097711785492504343953926634992332820282019728792003956564819967":1,"0":1,"-57896044618658097711785492504343953926634992332820282019728792003956564819968":1}}', Name("Int256")), + ("Map(UInt256, Int8)", "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{"d":"2020-01-01","m":{"0":1,"115792089237316195423570985008687907853269984665640564039457584007913129639935":1}}', Name("UInt256")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(1),1))", '{"d":"2020-01-01","m":{1:1}}', Name("toNullable")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("toNullable(NULL)")), +]) +def table_map_with_key_integer(self, type, data, output): + """Check what values we can insert into map type column with key integer. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps("1.0") +) +@Examples("type data output partition_by", [ + ("Array(Map(String, Int8))", + "('2020-01-01', [map('hello',1),map('hello',1,'there',2)])", + '{"d":"2020-01-01","m":[{"hello":1},{"hello":1,"there":2}]}', + "m", + Name("Array(Map(String, Int8))")), + ("Nested(x Map(String, Int8))", + "('2020-01-01', [map('hello',1)])", + '{"d":"2020-01-01","m.x":[{"hello":1}]}', + "m.x", + Name("Nested(x Map(String, Int8)")) +]) +def table_with_map_inside_another_type(self, type, data, output, partition_by): + """Check what values we can insert into a type that has map type. + """ + insert_into_table(type=type, data=data, output=output, partition_by=partition_by) + +@TestOutline +def insert_into_table(self, type, data, output, partition_by="m"): + """Check we can insert data into a table. + """ + uid = getuid() + node = self.context.node + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (d DATE, m " + type + ") ENGINE = MergeTree() PARTITION BY " + partition_by + " ORDER BY d" + + with Given(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data", description=data): + sql = f"INSERT INTO {table} VALUES {data}" + node.query(sql) + + with And("I select all rows from the table"): + r = node.query(f"SELECT * FROM {table} FORMAT JSONEachRow") + + with Then("I expect output to match", description=output): + assert r.output == output, error() + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes("1.0") +) +def select_map_with_invalid_mixed_key_and_value_types(self): + """Check that creating a map with mixed key types fails. + """ + node = self.context.node + exitcode = 130 + message = "DB::Exception: There is no supertype for types String, UInt8 because some of them are String/FixedString and some of them are not" + + with Check("attempt to create a map using SELECT with mixed key types then it fails"): + node.query("SELECT map('hello',1,2,3)", exitcode=exitcode, message=message) + + with Check("attempt to create a map using SELECT with mixed value types then it fails"): + node.query("SELECT map(1,'hello',2,2)", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments("1.0") +) +def select_map_with_invalid_number_of_arguments(self): + """Check that creating a map with invalid number of arguments fails. + """ + node = self.context.node + exitcode = 42 + message = "DB::Exception: Function map even number of arguments" + + with When("I create a map using SELECT with invalid number of arguments"): + node.query("SELECT map(1,2,3)", exitcode=exitcode, message=message) + +@TestScenario +def select_map_empty(self): + """Check that we can can create a empty map by not passing any arguments. + """ + node = self.context.node + + with When("I create a map using SELECT with no arguments"): + r = node.query("SELECT map()") + + with Then("it should create an empty map"): + assert r.output == "{}", error() + +@TestScenario +def insert_invalid_mixed_key_and_value_types(self): + """Check that inserting a map with mixed key or value types fails. + """ + uid = getuid() + node = self.context.node + exitcode = 130 + message = "DB::Exception: There is no supertype for types String, UInt8 because some of them are String/FixedString and some of them are not" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (d DATE, m Map(String, Int8)) ENGINE = MergeTree() PARTITION BY m ORDER BY d" + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert a map with mixed key types then it should fail"): + sql = f"INSERT INTO {table} VALUES ('2020-01-01', map('hello',1,2,3))" + node.query(sql, exitcode=exitcode, message=message) + + with When("I insert a map with mixed value types then it should fail"): + sql = f"INSERT INTO {table} VALUES ('2020-01-01', map(1,'hello',2,2))" + node.query(sql, exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0") +) +@Examples("type data output", [ + ("Map(String, String)", + "('2020-01-01', map('hello','there','hello','over there'))", + '{"d":"2020-01-01","m":{"hello":"there","hello":"over there"}}', + Name("Map(String, String))")), + ("Map(Int64, String)", + "('2020-01-01', map(12345,'there',12345,'over there'))", + '{"d":"2020-01-01","m":{"12345":"there","12345":"over there"}}', + Name("Map(Int64, String))")), +]) +def table_map_with_duplicated_keys(self, type, data, output): + """Check that map supports duplicated keys. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0") +) +@Examples("map output", [ + ("map('hello','there','hello','over there')", "{'hello':'there','hello':'over there'}", Name("String")), + ("map(12345,'there',12345,'over there')", "{12345:'there',12345:'over there'}", Name("Integer")) +]) +def select_map_with_duplicated_keys(self, map, output): + """Check creating a map with duplicated keys. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0") +) +def select_map_key_not_found(self): + node = self.context.node + + with When("map is empty"): + node.query("SELECT map() AS m, m[1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("map has integer values"): + r = node.query("SELECT map(1,2) AS m, m[2] FORMAT Values") + with Then("zero should be returned for key that is not found"): + assert r.output == "({1:2},0)", error() + + with When("map has string values"): + r = node.query("SELECT map(1,'2') AS m, m[2] FORMAT Values") + with Then("empty string should be returned for key that is not found"): + assert r.output == "({1:'2'},'')", error() + + with When("map has array values"): + r = node.query("SELECT map(1,[2]) AS m, m[2] FORMAT Values") + with Then("empty array be returned for key that is not found"): + assert r.output == "({1:[2]},[])", error() + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0") +) +@Examples("type data select exitcode message", [ + ("Map(UInt8, UInt8), y Int8", "(y) VALUES (1)", "m[1] AS v", 0, '{"v":0}', Name("empty map")), + ("Map(UInt8, UInt8)", "VALUES (map(1,2))", "m[2] AS v", 0, '{"v":0}', Name("map has integer values")), + ("Map(UInt8, String)", "VALUES (map(1,'2'))", "m[2] AS v", 0, '{"v":""}', Name("map has string values")), + ("Map(UInt8, Array(Int8))", "VALUES (map(1,[2]))", "m[2] AS v", 0, '{"v":[]}', Name("map has array values")), +]) +def table_map_key_not_found(self, type, data, select, exitcode, message, order_by=None): + """Check values returned from a map column when key is not found. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0") +) +def invalid_key(self): + """Check when key is not valid. + """ + node = self.context.node + + with When("I try to use an integer key that is too large"): + node.query("SELECT map(1,2) AS m, m[256]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an integer key that is negative when key is unsigned"): + node.query("SELECT map(1,2) AS m, m[-1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use a string key when key is an integer"): + node.query("SELECT map(1,2) AS m, m['1']", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an integer key when key is a string"): + r = node.query("SELECT map('1',2) AS m, m[1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an empty key when key is a string"): + r = node.query("SELECT map('1',2) AS m, m[]", exitcode=62, message="DB::Exception: Syntax error: failed at position") + + with When("I try to use wrong type conversion in key"): + r = node.query("SELECT map(1,2) AS m, m[toInt8('1')]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("in array of maps I try to use an integer key that is negative when key is unsigned"): + node.query("SELECT [map(1,2)] AS m, m[1][-1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use a NULL key when key is not nullable"): + r = node.query("SELECT map(1,2) AS m, m[NULL] FORMAT Values") + with Then("it should return NULL"): + assert r.output == "({1:2},NULL)", error() + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0") +) +@Examples("type data select exitcode message order_by", [ + ("Map(UInt8, UInt8)", "(map(1,2))", "m[256] AS v", 0, '{"v":0}', "m", Name("key too large)")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[-1] AS v", 0, '{"v":0}', "m", Name("key is negative")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("string when key is integer")), + ("Map(String, UInt8)", "(map('1',2))", "m[1] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("integer when key is string")), + ("Map(String, UInt8)", "(map('1',2))", "m[] AS v", 62, "DB::Exception: Syntax error: failed at position", "m", Name("empty when key is string")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[toInt8('1')] AS v", 0, '{"v":2}', "m", Name("wrong type conversion when key is integer")), + ("Map(String, UInt8)", "(map('1',2))", "m[toFixedString('1',1)] AS v", 0, '{"v":2}', "m", Name("wrong type conversion when key is string")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[NULL] AS v", 0, '{"v":null}', "m", Name("NULL key when key is not nullable")), + ("Array(Map(UInt8, UInt8))", "([map(1,2)])", "m[1]['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("string when key is integer in array of maps")), + ("Nested(x Map(UInt8, UInt8))", "([map(1,2)])", "m.x[1]['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m.x", Name("string when key is integer in nested map")), +]) +def table_map_invalid_key(self, type, data, select, exitcode, message, order_by="m"): + """Check selecting values from a map column using an invalid key. + """ + uid = getuid() + node = self.context.node + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} VALUES {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval("1.0") +) +@Examples("type data select filter exitcode message order_by", [ + ("Map(UInt8, UInt8)", "(map(1,1)),(map(1,2)),(map(2,3))", "m[1] AS v", "1=1 ORDER BY m[1]", 0, '{"v":0}\n{"v":1}\n{"v":2}', None, + Name("select the same key from all the rows")), + ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('e','f','c','d')", 0, '', None, + Name("filter rows by map having different pair order")), + ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('c','d','e','f')", 0, '{"m":{"c":"d","e":"f"}}', None, + Name("filter rows by map having the same pair order")), + ("Map(String, String)", "(map('a','b')),(map('e','f'))", "m", "m = map()", 43, 'DB::Exception: Illegal types of arguments ', None, + Name("filter rows by empty map")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1", 0, '{"m":{"a":1,"b":2}}', None, + Name("filter rows by map key value")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1 AND m['b'] = 2", 0, '{"m":{"a":1,"b":2}}', None, + Name("filter rows by map multiple key value combined with AND")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1 OR m['b'] = 3", 0, '{"m":{"a":1,"b":2}}\n{"m":{"b":3}}', None, + Name("filter rows by map multiple key value combined with OR")), + ("Map(String, Array(Int8))", "(map('a',[])),(map('b',[1])),(map('c',[2]))", "m['b'] AS v", "m['b'] IN ([1],[2])", 0, '{"v":[1]}', None, + Name("filter rows by map array value using IN")), + ("Map(String, Nullable(String))", "(map('a',NULL)),(map('a',1))", "m", "isNull(m['a']) = 1", 0, '{"m":{"a":null}}', None, + Name("select map with nullable value")) +]) +def table_map_queries(self, type, data, select, filter, exitcode, message, order_by=None): + """Check retrieving map values and using maps in queries. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} VALUES {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing("1.0") +) +@Examples("type exitcode message", [ + ("Nullable(Map(String, String))", + 43, "DB::Exception: Nested type Map(String,String) cannot be inside Nullable type", + Name("nullable map")), + ("Map(Nothing, Nothing)", + 37, "DB::Exception: Column `m` with type Map(Nothing,Nothing) is not allowed in key expression, it's not comparable", + Name("map with nothing type for key and value")) +]) +def table_map_unsupported_types(self, type, exitcode, message): + """Check creating a table with unsupported map column types. + """ + uid = getuid() + node = self.context.node + + try: + with When(f"I create a table definition with {type}"): + sql = f"CREATE TABLE {uid} (m " + type + ") ENGINE = MergeTree() ORDER BY m" + node.query(sql, exitcode=exitcode, message=message) + finally: + with Finally("drop table if any"): + node.query(f"DROP TABLE IF EXISTS {uid}") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0") +) +@Examples("tuple type exitcode message", [ + ("([1, 2, 3], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 0, "{1:'Ready',2:'Steady',3:'Go'}", Name("int -> int")), + ("([1, 2, 3], ['Ready', 'Steady', 'Go'])", "Map(String, String)", + 0, "{'1':'Ready','2':'Steady','3':'Go'}", Name("int -> string")), + ("(['1', '2', '3'], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 0, "{1:'Ready',187:'Steady',143:'Go'}", Name("string -> int")), + ("([],[])", "Map(String, String)", + 0, "{}", Name("empty arrays to map str:str")), + ("([],[])", "Map(UInt8, Array(Int8))", + 0, "{}", Name("empty arrays to map uint8:array")), + ("([[1]],['hello'])", "Map(String, String)", + 0, "{'[1]':'hello'}", Name("array -> string")), + ("([(1,2),(3,4)])", "Map(UInt8, UInt8)", + 0, "{1:2,3:4}", Name("array of two tuples")), + ("([1, 2], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", + Name("unequal array sizes")), +]) +def cast_tuple_of_two_arrays_to_map(self, tuple, type, exitcode, message): + """Check casting Tuple(Array, Array) to a map type. + """ + node = self.context.node + + with When("I try to cast tuple", description=tuple): + node.query(f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0") +) +@Examples("tuple type exitcode message check_insert", [ + ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 0, '{"m":{1:"Ready",2:"Steady",3:"Go"}}', False, Name("int -> int")), + ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(String, String)", + 0, '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', False, Name("int -> string")), + ("((['1', '2', '3'], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 176, 'DB::Exception: Unknown codec family code', True, Name("string -> int")), + ("(([],[]))", "Map(String, String)", + 0, '{"m":{}}', False, Name("empty arrays to map str:str")), + ("(([],[]))", "Map(UInt8, Array(Int8))", + 0, '{"m":{}}', False, Name("empty arrays to map uint8:array")), + ("(([[1]],['hello']))", "Map(String, String)", + 53, 'DB::Exception: Type mismatch in IN or VALUES section', True, Name("array -> string")), + ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", + 0, '{"m":{1:2,3:4}}', False, Name("array of two tuples")), + ("(([1, 2], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", True, + Name("unequal array sizes")), +]) +def table_map_cast_tuple_of_arrays_to_map(self, tuple, type, exitcode, message, check_insert): + """Check converting Tuple(Array, Array) into map on insert into a map type column. + """ + table_map(type=type, data=tuple, select="*", filter="1=1", exitcode=exitcode, message=message, check_insert=check_insert) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") +) +@Examples("tuple type exitcode message", [ + ("([(1,2),(3,4)])", "Map(UInt8, UInt8)", 0, "{1:2,3:4}", + Name("array of two tuples")), + ("([(1,2),(3)])", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", + Name("not a tuple")), + ("([(1,2),(3,)])", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", + Name("invalid tuple")), +]) +def cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message): + """Check casting Array(Tuple(K,V)) to a map type. + """ + node = self.context.node + + with When("I try to cast tuple", description=tuple): + node.query(f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") +) +@Examples("tuple type exitcode message check_insert", [ + ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", 0, '{"m":{1:2,3:4}}', False, + Name("array of two tuples")), + ("(([(1,2),(3)]))", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", True, + Name("not a tuple")), + ("(([(1,2),(3,)]))", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", True, + Name("invalid tuple")), +]) +def table_map_cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message, check_insert): + """Check converting Array(Tuple(K,V),...) into map on insert into a map type column. + """ + table_map(type=type, data=tuple, select="*", filter="1=1", exitcode=exitcode, message=message, check_insert=check_insert) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap("1.0") +) +def subcolumns_keys_using_inline_defined_map(self): + node = self.context.node + exitcode = 47 + message = "DB::Exception: Missing columns: 'c.keys'" + + with When("I try to access keys sub-column using an inline defined map"): + node.query("SELECT map( 'aa', 4, '44' , 5) as c, c.keys", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap("1.0") +) +def subcolumns_values_using_inline_defined_map(self): + node = self.context.node + exitcode = 47 + message = "DB::Exception: Missing columns: 'c.values'" + + with When("I try to access values sub-column using an inline defined map"): + node.query("SELECT map( 'aa', 4, '44' , 5) as c, c.values", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions("1.0") +) +@Examples("type data select filter exitcode message", [ + # keys + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.keys AS keys", "1=1", + 0, '{"keys":["a","c"]}\n{"keys":["e"]}', Name("select keys")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.keys AS keys", "has(m.keys, 'e')", + 0, '{"keys":["e"]}', Name("filter by using keys in an array function")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "has(m.keys, 'e') AS r", "1=1", + 0, '{"r":0}\n{"r":1}', Name("column that uses keys in an array function")), + # values + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.values AS values", "1=1", + 0, '{"values":["b","d"]}\n{"values":["f"]}', Name("select values")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.values AS values", "has(m.values, 'f')", + 0, '{"values":["f"]}', Name("filter by using values in an array function")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "has(m.values, 'f') AS r", "1=1", + 0, '{"r":0}\n{"r":1}', Name("column that uses values in an array function")) +]) +def subcolumns(self, type, data, select, filter, exitcode, message, order_by=None): + """Check usage of sub-columns in queries. + """ + table_map(type=type, data=data, select=select, filter=filter, exitcode=exitcode, message=message, order_by=order_by) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length("1.0") +) +def length(self): + """Check usage of length function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('a','b','c','d')),(map('e','f'))", + select="length(m) AS len, m", + filter="length(m) = 1", + exitcode=0, message='{"len":"1","m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty("1.0") +) +def empty(self): + """Check usage of empty function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('e','f'))", + select="empty(m) AS em, m", + filter="empty(m) <> 1", + exitcode=0, message='{"em":0,"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty("1.0") +) +def notempty(self): + """Check usage of notEmpty function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('e','f'))", + select="notEmpty(m) AS em, m", + filter="notEmpty(m) = 1", + exitcode=0, message='{"em":1,"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd("1.0") +) +def cast_from_mapadd(self): + """Check converting the result of mapAdd function to a map data type. + """ + select_map(map="CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), 'Map(Int8, Int8)')", output="{1:2,2:2}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract("1.0") +) +def cast_from_mapsubstract(self): + """Check converting the result of mapSubstract function to a map data type. + """ + select_map(map="CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), 'Map(Int8, Int8)')", output="{1:-1,2:0}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries("1.0") +) +def cast_from_mappopulateseries(self): + """Check converting the result of mapPopulateSeries function to a map data type. + """ + select_map(map="CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), 'Map(Int8, Int8)')", output="{1:11,2:22,3:0,4:44,5:0}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains("1.0") +) +def mapcontains(self): + """Check usages of mapContains function with map data type. + """ + node = self.context.node + + with Example("key in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="mapContains(m, 'a')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("key not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT mapContains(m, 'a')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null key not in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="mapContains(m, NULL)", + exitcode=0, message='') + + with Example("null key in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="m", + filter="mapContains(m, NULL)", + exitcode=0, message='{null:"c"}') + + with Example("select nullable key"): + node.query("SELECT map(NULL, 1, 2, 3) AS m, mapContains(m, toNullable(toUInt8(2)))", exitcode=0, message="{2:3}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys("1.0") +) +def mapkeys(self): + """Check usages of mapKeys function with map data type. + """ + with Example("key in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapKeys(m), 'a')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("key not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT has(mapKeys(m), 'a')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null key not in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapKeys(m), NULL)", + exitcode=0, message='') + + with Example("null key in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="m", + filter="has(mapKeys(m), NULL)", + exitcode=0, message='{"m":{null:"c"}}') + + with Example("select keys from column"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="mapKeys(m) AS keys", + filter="1 = 1", + exitcode=0, message='{"keys":["a"]}\n{"keys":["e"]}\n{"keys":[null]}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues("1.0") +) +def mapvalues(self): + """Check usages of mapValues function with map data type. + """ + with Example("value in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapValues(m), 'b')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("value not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT has(mapValues(m), 'b')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null value not in map"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapValues(m), NULL)", + exitcode=0, message='') + + with Example("null value in map"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b')),(map('c',NULL))", + select="m", + filter="has(mapValues(m), NULL)", + exitcode=0, message='{"m":{"c":null}}') + + with Example("select values from column"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b')),(map('c',NULL))", + select="mapValues(m) AS values", + filter="1 = 1", + exitcode=0, message='{"values":["b"]}\n{"values":[null]}\n{"values":["f"]}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap("1.0") +) +def functions_with_inline_defined_map(self): + """Check that a map defined inline inside the select statement + can be used with functions that work with maps. + """ + with Example("mapKeys"): + select_map(map="map(1,2,3,4) as map, mapKeys(map) AS keys", output="{1:2,3:4}\t[1,3]") + + with Example("mapValyes"): + select_map(map="map(1,2,3,4) as map, mapValues(map) AS values", output="{1:2,3:4}\t[2,4]") + + with Example("mapContains"): + select_map(map="map(1,2,3,4) as map, mapContains(map, 1) AS contains", output="{1:2,3:4}\t1") + +@TestScenario +def empty_map(self): + """Check creating of an empty map `{}` using the map() function + when inserting data into a map type table column. + """ + table_map(type="Map(String, String)", + data="(map('e','f')),(map())", + select="m", + filter="1=1", + exitcode=0, message='{"m":{}}\n{"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays("1.0") +) +def performance_vs_two_tuple_of_arrays(self, len=10, rows=6000000): + """Check performance of using map data type vs Tuple(Array, Array). + """ + uid = getuid() + node = self.context.node + + with Given(f"table with Tuple(Array(Int8),Array(Int8))"): + sql = "CREATE TABLE {name} (pairs Tuple(Array(Int8),Array(Int8))) ENGINE = MergeTree() ORDER BY pairs" + tuple_table = create_table(name=f"tuple_{uid}", statement=sql) + + with And(f"table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with tuples"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {tuple_table} SELECT ({keys},{values}) FROM numbers({rows})") + tuple_insert_time = time.time() - start_time + metric("tuple insert time", tuple_insert_time, "sec") + + with When("I insert data into table with a map"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with tuples"): + start_time = time.time() + node.query(f"SELECT sum(arrayFirst((v, k) -> k = {len-1}, tupleElement(pairs, 2), tupleElement(pairs, 1))) AS sum FROM {tuple_table}", + exitcode=0, message=f"{rows*(len-1)}") + tuple_select_time = time.time() - start_time + metric("tuple(array, array) select time", tuple_select_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + + metric("insert difference", (1 - map_insert_time/tuple_insert_time) * 100, "%") + metric("select difference", (1 - map_select_time/tuple_select_time) * 100, "%") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples("1.0") +) +def performance_vs_array_of_tuples(self, len=10, rows=6000000): + """Check performance of using map data type vs Array(Tuple(K,V)). + """ + uid = getuid() + node = self.context.node + + with Given(f"table with Array(Tuple(K,V))"): + sql = "CREATE TABLE {name} (pairs Array(Tuple(Int8, Int8))) ENGINE = MergeTree() ORDER BY pairs" + array_table = create_table(name=f"tuple_{uid}", statement=sql) + + with And(f"table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with an array of tuples"): + pairs = list(zip(range(len),range(len))) + start_time = time.time() + node.query(f"INSERT INTO {array_table} SELECT ({pairs}) FROM numbers({rows})") + array_insert_time = time.time() - start_time + metric("array insert time", array_insert_time, "sec") + + with When("I insert data into table with a map"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with an array of tuples"): + start_time = time.time() + node.query(f"SELECT sum(arrayFirst((v) -> v.1 = {len-1}, pairs).2) AS sum FROM {array_table}", + exitcode=0, message=f"{rows*(len-1)}") + array_select_time = time.time() - start_time + metric("array(tuple(k,v)) select time", array_select_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + + metric("insert difference", (1 - map_insert_time/array_insert_time) * 100, "%") + metric("select difference", (1 - map_select_time/array_select_time) * 100, "%") + +@TestScenario +def performance(self, len=10, rows=6000000): + """Check insert and select performance of using map data type. + """ + uid = getuid() + node = self.context.node + + with Given("table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with a map"): + values = [x for pair in zip(range(len),range(len)) for x in pair] + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT (map({','.join([str(v) for v in values])})) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + +# FIXME: add tests for different table engines + +@TestFeature +@Name("tests") +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map("1.0") +) +def feature(self, node="clickhouse1"): + self.context.node = self.context.cluster.node(node) + + with Given("I allow experimental map type"): + allow_experimental_map_type() + + for scenario in loads(current_module(), Scenario): + scenario() From 4a085d7c2183794fc04dde6fac071f3826ce9de1 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 22 Feb 2021 15:06:45 -0500 Subject: [PATCH 02/12] Removing unused table types. --- tests/testflows/map_type/tests/common.py | 42 +----------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/tests/testflows/map_type/tests/common.py b/tests/testflows/map_type/tests/common.py index d4461062256..a3a0d0ef0b1 100644 --- a/tests/testflows/map_type/tests/common.py +++ b/tests/testflows/map_type/tests/common.py @@ -5,46 +5,6 @@ from testflows.core import * from testflows.core.name import basename, parentname from testflows._core.testtype import TestSubType -table_tuple = namedtuple("table_tuple", "create_statement cluster") - -table_types = { - "MergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, m Map(String, Int8)) ENGINE = MergeTree() PARTITION BY m ORDER BY d", None), - # "ReplacingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = ReplacingMergeTree() PARTITION BY y ORDER BY d", None), - # "SummingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) ENGINE = SummingMergeTree() PARTITION BY y ORDER BY d", None), - # "AggregatingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = AggregatingMergeTree() PARTITION BY y ORDER BY d", None), - # "CollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(sign) PARTITION BY y ORDER BY d", None), - # "VersionedCollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY y ORDER BY d", None), - # "GraphiteMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) ENGINE = GraphiteMergeTree('graphite_rollup_example') PARTITION BY y ORDER by d", None), - # "ReplicatedMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedReplacingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedReplacingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedSummingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ - # ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedSummingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ - # ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedAggregatingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedAggregatingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ - # ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ - # ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedVersionedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ - # ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedVersionedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ - # ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedGraphiteMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ - # ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedGraphiteMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ - # ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "one_shard_cluster"), -} - def getuid(): if current().subtype == TestSubType.Example: testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" @@ -86,4 +46,4 @@ def create_table(self, name, statement, on_cluster=False): if on_cluster: node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster}") else: - node.query(f"DROP TABLE IF EXISTS {name}") \ No newline at end of file + node.query(f"DROP TABLE IF EXISTS {name}") From 8e770a850100ec1ae4a17a2883424461669723aa Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 23 Feb 2021 16:41:26 +0300 Subject: [PATCH 03/12] Update docker-compose.yml --- tests/testflows/map_type/docker-compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testflows/map_type/docker-compose/docker-compose.yml b/tests/testflows/map_type/docker-compose/docker-compose.yml index a3f5144c9ed..29f2ef52470 100755 --- a/tests/testflows/map_type/docker-compose/docker-compose.yml +++ b/tests/testflows/map_type/docker-compose/docker-compose.yml @@ -57,4 +57,4 @@ services: clickhouse3: condition: service_healthy zookeeper: - condition: service_healthy \ No newline at end of file + condition: service_healthy From b1d043b7dbe679a3c7a29855c570381eab91897e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 3 Mar 2021 14:09:04 -0500 Subject: [PATCH 04/12] Fixing configs based on review comments. --- .../map_type/configs/clickhouse/common.xml | 6 - .../configs/clickhouse/config.d/logs.xml | 1 - .../configs/clickhouse/config.d/ports.xml | 5 - .../configs/clickhouse/config.d/remote.xml | 169 ------------------ .../configs/clickhouse/config.d/ssl.xml | 18 -- .../configs/clickhouse/config.d/storage.xml | 20 --- .../configs/clickhouse/ssl/dhparam.pem | 8 - .../configs/clickhouse/ssl/server.crt | 19 -- .../configs/clickhouse/ssl/server.key | 28 --- .../docker-compose/clickhouse-service.yml | 1 - tests/testflows/map_type/regression.py | 4 +- .../map_type/requirements/requirements.py | 2 +- tests/testflows/regression.py | 1 + 13 files changed, 4 insertions(+), 278 deletions(-) delete mode 100644 tests/testflows/map_type/configs/clickhouse/common.xml delete mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/ports.xml delete mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml delete mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/storage.xml delete mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem delete mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/server.crt delete mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/server.key diff --git a/tests/testflows/map_type/configs/clickhouse/common.xml b/tests/testflows/map_type/configs/clickhouse/common.xml deleted file mode 100644 index 567c8c05e32..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - :: - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml b/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml index bdf1bbc11c1..e5077af3f49 100644 --- a/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml +++ b/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml @@ -1,5 +1,4 @@ - 3 trace /var/log/clickhouse-server/log.log diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml b/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml deleted file mode 100644 index fbc6cea74c0..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - 8443 - 9440 - \ No newline at end of file diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml b/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml index a7ed0d6e2b4..b7d02ceeec1 100644 --- a/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml +++ b/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml @@ -18,109 +18,6 @@ - - - - true - - clickhouse1 - 9440 - 1 - - - clickhouse2 - 9440 - 1 - - - clickhouse3 - 9440 - 1 - - - - - - - clickhouse1 - 9440 - 1 - - - - - - - clickhouse1 - 9000 - - - - - clickhouse2 - 9000 - - - - - - - clickhouse1 - 9000 - - - clickhouse2 - 9000 - - - - - - - clickhouse2 - 9000 - - - - - clickhouse3 - 9000 - - - - - - - clickhouse2 - 9000 - - - clickhouse3 - 9000 - - - @@ -141,71 +38,5 @@ - - - - clickhouse1 - 9000 - - - clickhouse2 - 9000 - - - clickhouse3 - 9000 - - - - - - - clickhouse1 - 9440 - 1 - - - - - clickhouse2 - 9440 - 1 - - - - - clickhouse3 - 9440 - 1 - - - - - - true - - clickhouse1 - 9440 - 1 - - user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end - - - - clickhouse2 - 9440 - 1 - user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end - - - - clickhouse3 - 9440 - 1 - user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end - - - - diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml b/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml deleted file mode 100644 index 768d2250b79..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - /etc/clickhouse-server/ssl/server.crt - /etc/clickhouse-server/ssl/server.key - /etc/clickhouse-server/ssl/dhparam.pem - none - true - - - true - none - - AcceptCertificateHandler - - - - diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml b/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml deleted file mode 100644 index 618fd6b6d24..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - 1024 - - - - - - - default - - - - - - - diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem deleted file mode 100644 index 2e6cee0798d..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN DH PARAMETERS----- -MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR -XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 -Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb -9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf -slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb -dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== ------END DH PARAMETERS----- diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/server.crt b/tests/testflows/map_type/configs/clickhouse/ssl/server.crt deleted file mode 100644 index 7ade2d96273..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/ssl/server.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow -FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI -4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T -4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU -7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj -sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg -pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC -kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j -4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr -85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C -L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD -+UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L -P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp -0g== ------END CERTIFICATE----- diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/server.key b/tests/testflows/map_type/configs/clickhouse/ssl/server.key deleted file mode 100644 index f0fb61ac443..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/ssl/server.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn -DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L -OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus -TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 -JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F -KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 -ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP -TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f -7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN -MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 -212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu -plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr -/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu -Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa -CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g -1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 -bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq -cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez -on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K -GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 -nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU -2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh -vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa -dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr -vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 -KCBtmIrQLqzMpnBpLNuSY+Q= ------END PRIVATE KEY----- diff --git a/tests/testflows/map_type/docker-compose/clickhouse-service.yml b/tests/testflows/map_type/docker-compose/clickhouse-service.yml index 2d79443dcbb..fdd4a8057a9 100755 --- a/tests/testflows/map_type/docker-compose/clickhouse-service.yml +++ b/tests/testflows/map_type/docker-compose/clickhouse-service.yml @@ -10,7 +10,6 @@ services: volumes: - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d:/etc/clickhouse-server/users.d" - - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py index 78b0ac74b15..df529d7c07b 100755 --- a/tests/testflows/map_type/regression.py +++ b/tests/testflows/map_type/regression.py @@ -7,7 +7,7 @@ append_path(sys.path, "..") from helpers.cluster import Cluster from helpers.argparser import argparser -from map_type.requirements import QA_SRS018_ClickHouse_Map_Data_Type +from map_type.requirements import SRS018_ClickHouse_Map_Data_Type xfails = { "tests/table map with key integer/Int:": @@ -39,7 +39,7 @@ xflags = { @XFlags(xflags) @Name("map type") @Specifications( - QA_SRS018_ClickHouse_Map_Data_Type + SRS018_ClickHouse_Map_Data_Type ) def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """Map type regression. diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py index 6042512abd0..e923f9ed85b 100644 --- a/tests/testflows/map_type/requirements/requirements.py +++ b/tests/testflows/map_type/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.210214.1233241. +# document by TestFlows v1.6.210223.1172503. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index 0e9a821cae0..b111fa241d0 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -18,6 +18,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): Feature(test=load("ldap.regression", "regression"))(**args) Feature(test=load("rbac.regression", "regression"))(**args) Feature(test=load("aes_encryption.regression", "regression"))(**args) + Feature(test=load("map_type.regression", "regression"))(**args) if main(): regression() From 3b7adc8373aec2f061ca6b3d8454ba4b48f71d13 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 5 Mar 2021 08:48:18 -0500 Subject: [PATCH 05/12] Updating requirements based on the review. --- .../map_type/requirements/requirements.md | 30 +++------ .../map_type/requirements/requirements.py | 62 ++++++------------- 2 files changed, 28 insertions(+), 64 deletions(-) diff --git a/tests/testflows/map_type/requirements/requirements.md b/tests/testflows/map_type/requirements/requirements.md index becdeaecbb2..e8f8a0d67d3 100644 --- a/tests/testflows/map_type/requirements/requirements.md +++ b/tests/testflows/map_type/requirements/requirements.md @@ -187,12 +187,6 @@ For example, SELECT map(1,2) AS m, m[1024] ``` -SHALL have the following output - -```bash -Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. -``` - Exceptions: * when key is `NULL` the return value SHALL be `NULL` @@ -202,12 +196,7 @@ Exceptions: #### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound version: 1.0 -[ClickHouse] SHALL return - -* zeros for [Integer] values -* empty strings for [String] values -* empty for [Array] values - +[ClickHouse] SHALL return default value for the data type of the value when there's no corresponding `key` defined in the `Map(key, value)` data type. @@ -257,19 +246,18 @@ SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map version: 1.0 [ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` -when [Tuple] is invalid such as when it is not a [Tuple] with two elements. -For example, +* when element is not a [Tuple] -```sql -SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map -``` + ```sql + SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map + ``` -or +* when [Tuple] does not contain two elements -```sql -SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map -``` + ```sql + SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map + ``` ### Keys and Values Subcolumns diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py index e923f9ed85b..0dfde7d50b3 100644 --- a/tests/testflows/map_type/requirements/requirements.py +++ b/tests/testflows/map_type/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.210223.1172503. +# document by TestFlows v1.6.210226.1200017. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification @@ -244,12 +244,6 @@ RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid = Requirement( 'SELECT map(1,2) AS m, m[1024]\n' '```\n' '\n' - 'SHALL have the following output \n' - '\n' - '```bash\n' - 'Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. \n' - '```\n' - '\n' 'Exceptions:\n' '\n' '* when key is `NULL` the return value SHALL be `NULL`\n' @@ -269,12 +263,7 @@ RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL return\n' - '\n' - '* zeros for [Integer] values\n' - '* empty strings for [String] values\n' - '* empty for [Array] values\n' - '\n' + '[ClickHouse] SHALL return default value for the data type of the value\n' "when there's no corresponding `key` defined in the `Map(key, value)` data type. \n" '\n' '\n' @@ -360,19 +349,18 @@ RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid = uid=None, description=( '[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)`\n' - 'when [Tuple] is invalid such as when it is not a [Tuple] with two elements.\n' '\n' - 'For example,\n' + '* when element is not a [Tuple]\n' '\n' - '```sql\n' - "SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" - '```\n' + ' ```sql\n' + " SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" + ' ```\n' '\n' - 'or \n' + '* when [Tuple] does not contain two elements\n' '\n' - '```sql\n' - "SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map\n" - '```\n' + ' ```sql\n' + " SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map\n" + ' ```\n' '\n' ), link=None, @@ -1112,12 +1100,6 @@ For example, SELECT map(1,2) AS m, m[1024] ``` -SHALL have the following output - -```bash -Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. -``` - Exceptions: * when key is `NULL` the return value SHALL be `NULL` @@ -1127,12 +1109,7 @@ Exceptions: #### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound version: 1.0 -[ClickHouse] SHALL return - -* zeros for [Integer] values -* empty strings for [String] values -* empty for [Array] values - +[ClickHouse] SHALL return default value for the data type of the value when there's no corresponding `key` defined in the `Map(key, value)` data type. @@ -1182,19 +1159,18 @@ SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map version: 1.0 [ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` -when [Tuple] is invalid such as when it is not a [Tuple] with two elements. -For example, +* when element is not a [Tuple] -```sql -SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map -``` + ```sql + SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map + ``` -or +* when [Tuple] does not contain two elements -```sql -SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map -``` + ```sql + SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map + ``` ### Keys and Values Subcolumns From 4db7d47e1c6352f54f24e203530c3e5007619ac9 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 7 Mar 2021 09:52:42 -0500 Subject: [PATCH 06/12] Updating requirements to use the word MAY instead of SHALL where behavior is not fully defined. --- .../map_type/requirements/requirements.md | 12 +++++----- .../map_type/requirements/requirements.py | 24 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/testflows/map_type/requirements/requirements.md b/tests/testflows/map_type/requirements/requirements.md index e8f8a0d67d3..7093243f581 100644 --- a/tests/testflows/map_type/requirements/requirements.md +++ b/tests/testflows/map_type/requirements/requirements.md @@ -146,7 +146,7 @@ version: 1.0 #### RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys version: 1.0 -[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys. +[ClickHouse] MAY support `Map(key, value)` data type with duplicated keys. ### Array of Maps @@ -168,7 +168,7 @@ version: 1.0 version: 1.0 [ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax. -If `key` has duplicates then the first `key:value` pair SHALL be returned. +If `key` has duplicates then the first `key:value` pair MAY be returned. For example, @@ -189,9 +189,9 @@ SELECT map(1,2) AS m, m[1024] Exceptions: -* when key is `NULL` the return value SHALL be `NULL` -* when key has [Integer] type and key is either too large or negative then - when reading from a table column it SHALL return `zero`. +* when key is `NULL` the return value MAY be `NULL` +* when key value is not valid for the key type, for example it is out of range for [Integer] type, + when reading from a table column it MAY return the default value for key data type #### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound version: 1.0 @@ -245,7 +245,7 @@ SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map #### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid version: 1.0 -[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` +[ClickHouse] MAY return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` * when element is not a [Tuple] diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py index 0dfde7d50b3..2e47a15a6d6 100644 --- a/tests/testflows/map_type/requirements/requirements.py +++ b/tests/testflows/map_type/requirements/requirements.py @@ -169,7 +169,7 @@ RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys.\n' + '[ClickHouse] MAY support `Map(key, value)` data type with duplicated keys.\n' '\n' ), link=None, @@ -215,7 +215,7 @@ RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval = Requirement( uid=None, description=( '[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax.\n' - 'If `key` has duplicates then the first `key:value` pair SHALL be returned. \n' + 'If `key` has duplicates then the first `key:value` pair MAY be returned. \n' '\n' 'For example,\n' '\n' @@ -246,9 +246,9 @@ RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid = Requirement( '\n' 'Exceptions:\n' '\n' - '* when key is `NULL` the return value SHALL be `NULL`\n' - '* when key has [Integer] type and key is either too large or negative then\n' - ' when reading from a table column it SHALL return `zero`.\n' + '* when key is `NULL` the return value MAY be `NULL`\n' + '* when key value is not valid for the key type, for example it is out of range for [Integer] type, \n' + ' when reading from a table column it MAY return the default value for key data type\n' '\n' ), link=None, @@ -348,7 +348,7 @@ RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid = type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)`\n' + '[ClickHouse] MAY return an error when casting [Array(Tuple(K, V))] to `Map(key, value)`\n' '\n' '* when element is not a [Tuple]\n' '\n' @@ -1059,7 +1059,7 @@ version: 1.0 #### RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys version: 1.0 -[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys. +[ClickHouse] MAY support `Map(key, value)` data type with duplicated keys. ### Array of Maps @@ -1081,7 +1081,7 @@ version: 1.0 version: 1.0 [ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax. -If `key` has duplicates then the first `key:value` pair SHALL be returned. +If `key` has duplicates then the first `key:value` pair MAY be returned. For example, @@ -1102,9 +1102,9 @@ SELECT map(1,2) AS m, m[1024] Exceptions: -* when key is `NULL` the return value SHALL be `NULL` -* when key has [Integer] type and key is either too large or negative then - when reading from a table column it SHALL return `zero`. +* when key is `NULL` the return value MAY be `NULL` +* when key value is not valid for the key type, for example it is out of range for [Integer] type, + when reading from a table column it MAY return the default value for key data type #### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound version: 1.0 @@ -1158,7 +1158,7 @@ SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map #### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid version: 1.0 -[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` +[ClickHouse] MAY return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` * when element is not a [Tuple] From 2ad584d9ab9b120b26b8f4439d9a270e20b7b480 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 7 Mar 2021 09:56:46 -0500 Subject: [PATCH 07/12] Updating requirement to use MAY --- .../map_type/requirements/requirements.md | 13 +++++----- .../map_type/requirements/requirements.py | 26 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/testflows/map_type/requirements/requirements.md b/tests/testflows/map_type/requirements/requirements.md index 7093243f581..0d206a5a456 100644 --- a/tests/testflows/map_type/requirements/requirements.md +++ b/tests/testflows/map_type/requirements/requirements.md @@ -220,14 +220,15 @@ SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map #### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid version: 1.0 -[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid -such as when arrays are not of equal size. +[ClickHouse] MAY return an error when casting [Tuple(Array, Array)] to `Map(key, value)` -For example, +* when arrays are not of equal size -```sql -SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] -``` + For example, + + ```sql + SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] + ``` ### Converting Array(Tuple(K,V)) to Map diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py index 2e47a15a6d6..6b2b919367c 100644 --- a/tests/testflows/map_type/requirements/requirements.py +++ b/tests/testflows/map_type/requirements/requirements.py @@ -305,14 +305,15 @@ RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid = Re type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid\n' - 'such as when arrays are not of equal size.\n' + '[ClickHouse] MAY return an error when casting [Tuple(Array, Array)] to `Map(key, value)`\n' '\n' - 'For example,\n' + '* when arrays are not of equal size\n' '\n' - '```sql\n' - "SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10]\n" - '```\n' + ' For example,\n' + '\n' + ' ```sql\n' + " SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10]\n" + ' ```\n' '\n' ), link=None, @@ -1133,14 +1134,15 @@ SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map #### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid version: 1.0 -[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid -such as when arrays are not of equal size. +[ClickHouse] MAY return an error when casting [Tuple(Array, Array)] to `Map(key, value)` -For example, +* when arrays are not of equal size -```sql -SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] -``` + For example, + + ```sql + SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] + ``` ### Converting Array(Tuple(K,V)) to Map From 80ba5c996483bf56719858b0df81542867ae03fe Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 7 Mar 2021 10:02:15 -0500 Subject: [PATCH 08/12] Updated requirements for using keys and values subcolums with inline defined map to use the word MAY instead of SHALL. --- tests/testflows/map_type/requirements/requirements.md | 4 ++-- tests/testflows/map_type/requirements/requirements.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/testflows/map_type/requirements/requirements.md b/tests/testflows/map_type/requirements/requirements.md index 0d206a5a456..f19f5a7f7bd 100644 --- a/tests/testflows/map_type/requirements/requirements.md +++ b/tests/testflows/map_type/requirements/requirements.md @@ -286,7 +286,7 @@ SELECT * FROM t_map WHERE has(m.keys, 'a'); #### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap version: 1.0 -[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn. +[ClickHouse] MAY not support using inline defined map to get `keys` subcolumn. For example, @@ -318,7 +318,7 @@ SELECT * FROM t_map WHERE has(m.values, 'a'); #### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap version: 1.0 -[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn. +[ClickHouse] MAY not support using inline defined map to get `values` subcolumn. For example, diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py index 6b2b919367c..24e8abdf15f 100644 --- a/tests/testflows/map_type/requirements/requirements.py +++ b/tests/testflows/map_type/requirements/requirements.py @@ -417,7 +417,7 @@ RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap = Requiremen type=None, uid=None, description=( - '[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn.\n' + '[ClickHouse] MAY not support using inline defined map to get `keys` subcolumn.\n' '\n' 'For example,\n' '\n' @@ -479,7 +479,7 @@ RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap = Requirem type=None, uid=None, description=( - '[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn.\n' + '[ClickHouse] MAY not support using inline defined map to get `values` subcolumn.\n' '\n' 'For example,\n' '\n' @@ -1200,7 +1200,7 @@ SELECT * FROM t_map WHERE has(m.keys, 'a'); #### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap version: 1.0 -[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn. +[ClickHouse] MAY not support using inline defined map to get `keys` subcolumn. For example, @@ -1232,7 +1232,7 @@ SELECT * FROM t_map WHERE has(m.values, 'a'); #### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap version: 1.0 -[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn. +[ClickHouse] MAY not support using inline defined map to get `values` subcolumn. For example, From d3fdb153681d691dfb1ffb35bcb334612cf5a917 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 7 Mar 2021 11:24:03 -0500 Subject: [PATCH 09/12] Adding select by key tests for value and key with string type. --- tests/testflows/map_type/regression.py | 12 ++++- tests/testflows/map_type/tests/feature.py | 62 +++++++++++++++++++++-- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py index df529d7c07b..40cbb42a106 100755 --- a/tests/testflows/map_type/regression.py +++ b/tests/testflows/map_type/regression.py @@ -27,7 +27,17 @@ xfails = { "tests/mapcontains/null key in map": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], "tests/mapcontains/select nullable key": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")] + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], + "tests/table map select key with value string/LowCardinality:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key string/FixedString": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key string/Nullable": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key string/Nullable(NULL)": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], + "tests/table map select key with key string/LowCardinality:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")] } xflags = { diff --git a/tests/testflows/map_type/tests/feature.py b/tests/testflows/map_type/tests/feature.py index 07ae7cc2159..182891ae7da 100755 --- a/tests/testflows/map_type/tests/feature.py +++ b/tests/testflows/map_type/tests/feature.py @@ -168,13 +168,39 @@ def select_map_with_key_integer(self, map, output): ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("Nullable")), ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("Nullable(NULL)")), ("Map(LowCardinality(String), Int8)", "('2020-01-01', map(toLowCardinality('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(String)")), - ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(FixedString)")) + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map('1',1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(String) cast from String")), + ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"d":"2020-01-01","m":{"1":"1"}}', Name("LowCardinality(String) for key and value")), + ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(FixedString)")), ]) def table_map_with_key_string(self, type, data, output): """Check what values we can insert into map type column with key string. """ insert_into_table(type=type, data=data, output=output) +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +) +@Examples("type data output select", [ + ("Map(String, Int8)", "('2020-01-01', map('',1))", '{"m":1}', "m[''] AS m", Name("empty string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1))", '{"m":1}', "m['hello'] AS m", Name("non-empty string")), + ("Map(String, Int8)", "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", '{"m":1}', "m['Gãńdåłf_Thê_Gręât'] AS m", Name("utf-8 string")), + ("Map(String, Int8)", "('2020-01-01', map('hello there',1))", '{"m":1}', "m['hello there'] AS m", Name("multi word string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1,'there',2))", '{"m":1}', "m['hello'] AS m", Name("multiple keys")), + ("Map(String, Int8)", "('2020-01-01', map(toString(1),1))", '{"m":1}', "m['1'] AS m", Name("toString")), + ("Map(FixedString(1), Int8)", "('2020-01-01', map(toFixedString('1',1),1))", '{"m":1}', "m['1'] AS m", Name("FixedString")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable('1'),1))", '{"m":1}}', "m['1'] AS m", Name("Nullable")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"m":1}', "m[null] AS m", Name("Nullable(NULL)")), + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map(toLowCardinality('1'),1))", '{"m":1}}', "m['1'] AS m", Name("LowCardinality(String)")), + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map('1',1))", '{"m":1}', "m['1'] AS m", Name("LowCardinality(String) cast from String")), + ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"m":"1"}', "m['1'] AS m", Name("LowCardinality(String) for key and value")), + ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"m":1}', "m['1'] AS m", Name("LowCardinality(FixedString)")), +]) +def table_map_select_key_with_key_string(self, type, data, output, select): + """Check what values we can insert into map type column with key string and if key can be selected. + """ + insert_into_table(type=type, data=data, output=output, select=select) + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") @@ -190,6 +216,8 @@ def table_map_with_key_string(self, type, data, output): ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("Nullable")), ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable(NULL)))", '{"d":"2020-01-01","m":{"key":null}}', Name("Nullable(NULL)")), ("Map(String, LowCardinality(String))", "('2020-01-01', map('key',toLowCardinality('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(String)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(String) cast from String")), + ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"d":"2020-01-01","m":{"1":"1"}}', Name("LowCardinality(String) for key and value")), ("Map(String, LowCardinality(FixedString(1)))", "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(FixedString)")) ]) def table_map_with_value_string(self, type, data, output): @@ -197,6 +225,30 @@ def table_map_with_value_string(self, type, data, output): """ insert_into_table(type=type, data=data, output=output) +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +) +@Examples("type data output", [ + ("Map(String, String)", "('2020-01-01', map('key',''))", '{"m":""}', Name("empty string")), + ("Map(String, String)", "('2020-01-01', map('key','hello'))", '{"m":"hello"}', Name("non-empty string")), + ("Map(String, String)", "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", '{"m":"Gãńdåłf_Thê_Gręât"}', Name("utf-8 string")), + ("Map(String, String)", "('2020-01-01', map('key', 'hello there'))", '{"m":"hello there"}', Name("multi word string")), + ("Map(String, String)", "('2020-01-01', map('key','hello','key2','there'))", '{"m":"hello"}', Name("multiple keys")), + ("Map(String, String)", "('2020-01-01', map('key', toString(1)))", '{"m":"1"}', Name("toString")), + ("Map(String, FixedString(1))", "('2020-01-01', map('key',toFixedString('1',1)))", '{"m":"1"}', Name("FixedString")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable('1')))", '{"m":"1"}', Name("Nullable")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable(NULL)))", '{"m":null}', Name("Nullable(NULL)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key',toLowCardinality('1')))", '{"m":"1"}', Name("LowCardinality(String)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"m":"1"}', Name("LowCardinality(String) cast from String")), + ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"m":"1"}', Name("LowCardinality(String) for key and value")), + ("Map(String, LowCardinality(FixedString(1)))", "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", '{"m":"1"}', Name("LowCardinality(FixedString)")) +]) +def table_map_select_key_with_value_string(self, type, data, output): + """Check what values we can insert into map type column with value string and if it can be selected by key. + """ + insert_into_table(type=type, data=data, output=output, select="m['key'] AS m") + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") @@ -284,7 +336,7 @@ def table_with_map_inside_another_type(self, type, data, output, partition_by): insert_into_table(type=type, data=data, output=output, partition_by=partition_by) @TestOutline -def insert_into_table(self, type, data, output, partition_by="m"): +def insert_into_table(self, type, data, output, partition_by="m", select="*"): """Check we can insert data into a table. """ uid = getuid() @@ -300,8 +352,8 @@ def insert_into_table(self, type, data, output, partition_by="m"): sql = f"INSERT INTO {table} VALUES {data}" node.query(sql) - with And("I select all rows from the table"): - r = node.query(f"SELECT * FROM {table} FORMAT JSONEachRow") + with And("I select rows from the table"): + r = node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow") with Then("I expect output to match", description=output): assert r.output == output, error() @@ -861,7 +913,7 @@ def mapcontains(self): exitcode=0, message='{null:"c"}') with Example("select nullable key"): - node.query("SELECT map(NULL, 1, 2, 3) AS m, mapContains(m, toNullable(toUInt8(2)))", exitcode=0, message="{2:3}") + node.query("SELECT map(NULL, 1, 2, 3) AS m, mapContains(m, toNullable(toUInt8(2)))", exitcode=0, message="{2:3}") @TestScenario @Requirements( From 3a5f4abd33d79e60ad5a2ea109ef5952a3e72110 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 7 Mar 2021 11:53:45 -0500 Subject: [PATCH 10/12] Adding test to check selecting key when key has integer type. --- tests/testflows/map_type/regression.py | 10 ++++++++- tests/testflows/map_type/tests/feature.py | 26 ++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py index 40cbb42a106..930f0365678 100755 --- a/tests/testflows/map_type/regression.py +++ b/tests/testflows/map_type/regression.py @@ -37,7 +37,15 @@ xfails = { "tests/table map select key with key string/Nullable(NULL)": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], "tests/table map select key with key string/LowCardinality:": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")] + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key integer/Int:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], + "tests/table map select key with key integer/UInt256": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], + "tests/table map select key with key integer/toNullable": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], + "tests/table map select key with key integer/toNullable(NULL)": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")] } xflags = { diff --git a/tests/testflows/map_type/tests/feature.py b/tests/testflows/map_type/tests/feature.py index 182891ae7da..ea8fb64d0eb 100755 --- a/tests/testflows/map_type/tests/feature.py +++ b/tests/testflows/map_type/tests/feature.py @@ -298,7 +298,7 @@ def table_map_with_value_array(self, type, data, output): ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"d":"2020-01-01","m":{0:1,255:1}}', Name("UInt8")), ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{32767:1,0:1,-32768:1}}', Name("Int16")), ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"d":"2020-01-01","m":{0:1,65535:1}}', Name("UInt16")), - ("Map(Int32, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{2147483647:1,0:1,-2147483648:1}}', Name("Int32")), + ("Map(Int32, Int8)", "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", '{"d":"2020-01-01","m":{2147483647:1,0:1,-2147483648:1}}', Name("Int32")), ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"d":"2020-01-01","m":{0:1,4294967295:1}}', Name("UInt32")), ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"d":"2020-01-01","m":{"9223372036854775807":1,"0":1,"-9223372036854775808":1}}', Name("Int64")), ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"d":"2020-01-01","m":{"0":1,"18446744073709551615":1}}', Name("UInt64")), @@ -313,6 +313,30 @@ def table_map_with_key_integer(self, type, data, output): """ insert_into_table(type=type, data=data, output=output) +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +) +@Examples("type data output select", [ + ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"m":1}', "m[127] AS m", Name("Int8")), + ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"m":2}', "(m[255] + m[0]) AS m", Name("UInt8")), + ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"m":3}', "(m[-128] + m[0] + m[-128]) AS m", Name("Int16")), + ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"m":2}', "(m[0] + m[65535]) AS m", Name("UInt16")), + ("Map(Int32, Int8)", "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", '{"m":3}', "(m[2147483647] + m[0] + m[-2147483648]) AS m", Name("Int32")), + ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"m":2}', "(m[0] + m[4294967295]) AS m", Name("UInt32")), + ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"m":3}', "(m[9223372036854775807] + m[0] + m[-9223372036854775808]) AS m", Name("Int64")), + ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"m":2}', "(m[0] + m[18446744073709551615]) AS m", Name("UInt64")), + ("Map(Int128, Int8)", "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{"m":3}', "(m[170141183460469231731687303715884105727] + m[0] + m[-170141183460469231731687303715884105728]) AS m", Name("Int128")), + ("Map(Int256, Int8)", "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{"m":3}', "(m[57896044618658097711785492504343953926634992332820282019728792003956564819967] + m[0] + m[-57896044618658097711785492504343953926634992332820282019728792003956564819968]) AS m", Name("Int256")), + ("Map(UInt256, Int8)", "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{"m":2}', "(m[0] + m[115792089237316195423570985008687907853269984665640564039457584007913129639935]) AS m", Name("UInt256")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(1),1))", '{"m":1}', "m[1] AS m", Name("toNullable")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"m":1}', "m[null] AS m", Name("toNullable(NULL)")), +]) +def table_map_select_key_with_key_integer(self, type, data, output, select): + """Check what values we can insert into map type column with key integer and if we can use the key to select the value. + """ + insert_into_table(type=type, data=data, output=output, select=select) + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps("1.0"), From 5942e293f143db29206c84da789e7741044b1544 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 21 Apr 2021 16:29:41 -0400 Subject: [PATCH 11/12] Crossing out failing map type tests. --- tests/testflows/map_type/regression.py | 44 ++++++++++++++++++++++- tests/testflows/map_type/tests/feature.py | 6 ++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py index 930f0365678..54d713347c6 100755 --- a/tests/testflows/map_type/regression.py +++ b/tests/testflows/map_type/regression.py @@ -26,8 +26,16 @@ xfails = { [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21029")], "tests/mapcontains/null key in map": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapcontains/null key not in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapkeys/null key not in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapkeys/null key in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], "tests/mapcontains/select nullable key": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], + "tests/mapkeys/select keys from column": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], "tests/table map select key with value string/LowCardinality:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], "tests/table map select key with key string/FixedString": @@ -45,7 +53,41 @@ xfails = { "tests/table map select key with key integer/toNullable": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], "tests/table map select key with key integer/toNullable(NULL)": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")] + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], + "tests/select map with key integer/Int128": + [(Fail, "large Int128 as key not supported")], + "tests/select map with key integer/Int256": + [(Fail, "large Int256 as key not supported")], + "tests/select map with key integer/UInt256": + [(Fail, "large UInt256 as key not supported")], + "tests/select map with key integer/toNullable": + [(Fail, "Nullable type as key not supported")], + "tests/select map with key integer/toNullable(NULL)": + [(Fail, "Nullable type as key not supported")], + "tests/select map with key string/Nullable": + [(Fail, "Nullable type as key not supported")], + "tests/select map with key string/Nullable(NULL)": + [(Fail, "Nullable type as key not supported")], + "tests/table map queries/select map with nullable value": + [(Fail, "Nullable value not supported")], + "tests/table map with key integer/toNullable": + [(Fail, "Nullable type as key not supported")], + "tests/table map with key integer/toNullable(NULL)": + [(Fail, "Nullable type as key not supported")], + "tests/table map with key string/Nullable": + [(Fail, "Nullable type as key not supported")], + "tests/table map with key string/Nullable(NULL)": + [(Fail, "Nullable type as key not supported")], + "tests/table map with key string/LowCardinality(String)": + [(Fail, "LowCardinality(String) as key not supported")], + "tests/table map with key string/LowCardinality(String) cast from String": + [(Fail, "LowCardinality(String) as key not supported")], + "tests/table map with key string/LowCardinality(String) for key and value": + [(Fail, "LowCardinality(String) as key not supported")], + "tests/table map with key string/LowCardinality(FixedString)": + [(Fail, "LowCardinality(FixedString) as key not supported")], + "tests/table map with value string/LowCardinality(String) for key and value": + [(Fail, "LowCardinality(String) as key not supported")], } xflags = { diff --git a/tests/testflows/map_type/tests/feature.py b/tests/testflows/map_type/tests/feature.py index ea8fb64d0eb..5fd48844825 100755 --- a/tests/testflows/map_type/tests/feature.py +++ b/tests/testflows/map_type/tests/feature.py @@ -408,7 +408,7 @@ def select_map_with_invalid_number_of_arguments(self): """ node = self.context.node exitcode = 42 - message = "DB::Exception: Function map even number of arguments" + message = "DB::Exception: Function map requires even number of arguments" with When("I create a map using SELECT with invalid number of arguments"): node.query("SELECT map(1,2,3)", exitcode=exitcode, message=message) @@ -616,7 +616,7 @@ def table_map_invalid_key(self, type, data, select, exitcode, message, order_by= Name("filter rows by map having different pair order")), ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('c','d','e','f')", 0, '{"m":{"c":"d","e":"f"}}', None, Name("filter rows by map having the same pair order")), - ("Map(String, String)", "(map('a','b')),(map('e','f'))", "m", "m = map()", 43, 'DB::Exception: Illegal types of arguments ', None, + ("Map(String, String)", "(map('a','b')),(map('e','f'))", "m", "m = map()", 0, '', None, Name("filter rows by empty map")), ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1", 0, '{"m":{"a":1,"b":2}}', None, Name("filter rows by map key value")), @@ -720,7 +720,7 @@ def cast_tuple_of_two_arrays_to_map(self, tuple, type, exitcode, message): ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(String, String)", 0, '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', False, Name("int -> string")), ("((['1', '2', '3'], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", - 176, 'DB::Exception: Unknown codec family code', True, Name("string -> int")), + 0, '', True, Name("string -> int")), ("(([],[]))", "Map(String, String)", 0, '{"m":{}}', False, Name("empty arrays to map str:str")), ("(([],[]))", "Map(UInt8, Array(Int8))", From e3af501de239c134afe84d5d4ca95ac77dfb6d59 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 22 Apr 2021 01:17:26 +0300 Subject: [PATCH 12/12] Update regression.py --- tests/testflows/regression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index 31cd056bbd8..13a24f97f9f 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -14,10 +14,10 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path, "stress": stress, "parallel": parallel} - Feature(test=load("example.regression", "regression"))(**args) + # Feature(test=load("example.regression", "regression"))(**args) # Feature(test=load("ldap.regression", "regression"))(**args) # Feature(test=load("rbac.regression", "regression"))(**args) - Feature(test=load("aes_encryption.regression", "regression"))(**args) + # Feature(test=load("aes_encryption.regression", "regression"))(**args) Feature(test=load("map_type.regression", "regression"))(**args) # Feature(test=load("kerberos.regression", "regression"))(**args)