From d1f8f52f160e4fd5724a5b314e3d96197d6046b7 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 22 Feb 2021 15:04:34 -0500 Subject: [PATCH 001/512] Adding Map type tests in TestFlows --- tests/testflows/map_type/__init__.py | 0 .../map_type/configs/clickhouse/common.xml | 6 + .../configs/clickhouse/config.d/logs.xml | 17 + .../configs/clickhouse/config.d/macros.xml | 0 .../configs/clickhouse/config.d/ports.xml | 5 + .../configs/clickhouse/config.d/remote.xml | 211 +++ .../configs/clickhouse/config.d/ssl.xml | 18 + .../configs/clickhouse/config.d/storage.xml | 20 + .../configs/clickhouse/config.d/zookeeper.xml | 10 + .../map_type/configs/clickhouse/config.xml | 448 +++++ .../configs/clickhouse/ssl/dhparam.pem | 8 + .../configs/clickhouse/ssl/server.crt | 19 + .../configs/clickhouse/ssl/server.key | 28 + .../map_type/configs/clickhouse/users.xml | 133 ++ .../configs/clickhouse1/config.d/macros.xml | 8 + .../configs/clickhouse2/config.d/macros.xml | 8 + .../configs/clickhouse3/config.d/macros.xml | 8 + .../docker-compose/clickhouse-service.yml | 28 + .../docker-compose/docker-compose.yml | 60 + .../docker-compose/zookeeper-service.yml | 18 + tests/testflows/map_type/regression.py | 61 + .../map_type/requirements/__init__.py | 1 + .../map_type/requirements/requirements.md | 523 ++++++ .../map_type/requirements/requirements.py | 1449 +++++++++++++++++ tests/testflows/map_type/tests/__init__.py | 0 tests/testflows/map_type/tests/common.py | 89 + tests/testflows/map_type/tests/feature.py | 1119 +++++++++++++ 27 files changed, 4295 insertions(+) create mode 100644 tests/testflows/map_type/__init__.py create mode 100644 tests/testflows/map_type/configs/clickhouse/common.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/logs.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/macros.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/ports.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/remote.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/storage.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/config.xml create mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem create mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/server.crt create mode 100644 tests/testflows/map_type/configs/clickhouse/ssl/server.key create mode 100644 tests/testflows/map_type/configs/clickhouse/users.xml create mode 100644 tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml create mode 100644 tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml create mode 100644 tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml create mode 100755 tests/testflows/map_type/docker-compose/clickhouse-service.yml create mode 100755 tests/testflows/map_type/docker-compose/docker-compose.yml create mode 100755 tests/testflows/map_type/docker-compose/zookeeper-service.yml create mode 100755 tests/testflows/map_type/regression.py create mode 100644 tests/testflows/map_type/requirements/__init__.py create mode 100644 tests/testflows/map_type/requirements/requirements.md create mode 100644 tests/testflows/map_type/requirements/requirements.py create mode 100644 tests/testflows/map_type/tests/__init__.py create mode 100644 tests/testflows/map_type/tests/common.py create mode 100755 tests/testflows/map_type/tests/feature.py diff --git a/tests/testflows/map_type/__init__.py b/tests/testflows/map_type/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/configs/clickhouse/common.xml b/tests/testflows/map_type/configs/clickhouse/common.xml new file mode 100644 index 00000000000..567c8c05e32 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + :: + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml b/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse/config.d/macros.xml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml b/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml new file mode 100644 index 00000000000..fbc6cea74c0 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,5 @@ + + + 8443 + 9440 + \ No newline at end of file diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml b/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..a7ed0d6e2b4 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,211 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + true + + clickhouse1 + 9440 + 1 + + + clickhouse2 + 9440 + 1 + + + clickhouse3 + 9440 + 1 + + + + + + + clickhouse1 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9440 + 1 + + + + + clickhouse2 + 9440 + 1 + + + + + clickhouse3 + 9440 + 1 + + + + + + true + + clickhouse1 + 9440 + 1 + + user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end + + + + clickhouse2 + 9440 + 1 + user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end + + + + clickhouse3 + 9440 + 1 + user_with_a_very_very_very_very_long_name_we_will_use_him_to_simulate_the_problem_with_inserting_to_distributed_when_folder_name_is_too_long_i_hope_that_length_is_enough_username_end + + + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml b/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 00000000000..768d2250b79 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,18 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + /etc/clickhouse-server/ssl/dhparam.pem + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml b/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml new file mode 100644 index 00000000000..618fd6b6d24 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,20 @@ + + + + + + 1024 + + + + + + + default + + + + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/map_type/configs/clickhouse/config.xml b/tests/testflows/map_type/configs/clickhouse/config.xml new file mode 100644 index 00000000000..4ec12232539 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/config.xml @@ -0,0 +1,448 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + 0.0.0.0 + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + /var/lib/clickhouse/access/ + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 + + + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 00000000000..2e6cee0798d --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/server.crt b/tests/testflows/map_type/configs/clickhouse/ssl/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/testflows/map_type/configs/clickhouse/ssl/server.key b/tests/testflows/map_type/configs/clickhouse/ssl/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/testflows/map_type/configs/clickhouse/users.xml b/tests/testflows/map_type/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml new file mode 100644 index 00000000000..a114a9ce4ab --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse2/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse2 + 01 + 02 + + diff --git a/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml b/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml new file mode 100644 index 00000000000..904a27b0172 --- /dev/null +++ b/tests/testflows/map_type/configs/clickhouse3/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse3 + 01 + 03 + + diff --git a/tests/testflows/map_type/docker-compose/clickhouse-service.yml b/tests/testflows/map_type/docker-compose/clickhouse-service.yml new file mode 100755 index 00000000000..2d79443dcbb --- /dev/null +++ b/tests/testflows/map_type/docker-compose/clickhouse-service.yml @@ -0,0 +1,28 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/map_type/docker-compose/docker-compose.yml b/tests/testflows/map_type/docker-compose/docker-compose.yml new file mode 100755 index 00000000000..a3f5144c9ed --- /dev/null +++ b/tests/testflows/map_type/docker-compose/docker-compose.yml @@ -0,0 +1,60 @@ +version: '2.3' + +services: + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy \ No newline at end of file diff --git a/tests/testflows/map_type/docker-compose/zookeeper-service.yml b/tests/testflows/map_type/docker-compose/zookeeper-service.yml new file mode 100755 index 00000000000..f3df33358be --- /dev/null +++ b/tests/testflows/map_type/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py new file mode 100755 index 00000000000..78b0ac74b15 --- /dev/null +++ b/tests/testflows/map_type/regression.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import sys + +from testflows.core import * + +append_path(sys.path, "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from map_type.requirements import QA_SRS018_ClickHouse_Map_Data_Type + +xfails = { + "tests/table map with key integer/Int:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], + "tests/table map with value integer/Int:": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], + "tests/table map with key integer/UInt256": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], + "tests/table map with value integer/UInt256": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], + "tests/select map with key integer/Int64": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030")], + "tests/select map with value integer/Int64": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030")], + "tests/cast tuple of two arrays to map/string -> int": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21029")], + "tests/mapcontains/null key in map": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], + "tests/mapcontains/select nullable key": + [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")] +} + +xflags = { +} + +@TestModule +@ArgumentParser(argparser) +@XFails(xfails) +@XFlags(xflags) +@Name("map type") +@Specifications( + QA_SRS018_ClickHouse_Map_Data_Type +) +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): + """Map type regression. + """ + nodes = { + "clickhouse": + ("clickhouse1", "clickhouse2", "clickhouse3") + } + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + self.context.stress = stress + + if parallel is not None: + self.context.parallel = parallel + + Feature(run=load("map_type.tests.feature", "feature")) + +if main(): + regression() diff --git a/tests/testflows/map_type/requirements/__init__.py b/tests/testflows/map_type/requirements/__init__.py new file mode 100644 index 00000000000..02f7d430154 --- /dev/null +++ b/tests/testflows/map_type/requirements/__init__.py @@ -0,0 +1 @@ +from .requirements import * diff --git a/tests/testflows/map_type/requirements/requirements.md b/tests/testflows/map_type/requirements/requirements.md new file mode 100644 index 00000000000..becdeaecbb2 --- /dev/null +++ b/tests/testflows/map_type/requirements/requirements.md @@ -0,0 +1,523 @@ +# SRS018 ClickHouse Map Data Type +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Requirements](#requirements) + * 3.1 [General](#general) + * 3.1.1 [RQ.SRS-018.ClickHouse.Map.DataType](#rqsrs-018clickhousemapdatatype) + * 3.2 [Performance](#performance) + * 3.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples](#rqsrs-018clickhousemapdatatypeperformancevsarrayoftuples) + * 3.2.2 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays](#rqsrs-018clickhousemapdatatypeperformancevstupleofarrays) + * 3.3 [Key Types](#key-types) + * 3.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Key.String](#rqsrs-018clickhousemapdatatypekeystring) + * 3.3.2 [RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer](#rqsrs-018clickhousemapdatatypekeyinteger) + * 3.4 [Value Types](#value-types) + * 3.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.String](#rqsrs-018clickhousemapdatatypevaluestring) + * 3.4.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer](#rqsrs-018clickhousemapdatatypevalueinteger) + * 3.4.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Array](#rqsrs-018clickhousemapdatatypevaluearray) + * 3.5 [Invalid Types](#invalid-types) + * 3.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable](#rqsrs-018clickhousemapdatatypeinvalidnullable) + * 3.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing](#rqsrs-018clickhousemapdatatypeinvalidnothingnothing) + * 3.6 [Duplicated Keys](#duplicated-keys) + * 3.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys](#rqsrs-018clickhousemapdatatypeduplicatedkeys) + * 3.7 [Array of Maps](#array-of-maps) + * 3.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps](#rqsrs-018clickhousemapdatatypearrayofmaps) + * 3.8 [Nested With Maps](#nested-with-maps) + * 3.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps](#rqsrs-018clickhousemapdatatypenestedwithmaps) + * 3.9 [Value Retrieval](#value-retrieval) + * 3.9.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval](#rqsrs-018clickhousemapdatatypevalueretrieval) + * 3.9.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid](#rqsrs-018clickhousemapdatatypevalueretrievalkeyinvalid) + * 3.9.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound](#rqsrs-018clickhousemapdatatypevalueretrievalkeynotfound) + * 3.10 [Converting Tuple(Array, Array) to Map](#converting-tuplearray-array-to-map) + * 3.10.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraystomap) + * 3.10.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraysmapinvalid) + * 3.11 [Converting Array(Tuple(K,V)) to Map](#converting-arraytuplekv-to-map) + * 3.11.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomap) + * 3.11.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomapinvalid) + * 3.12 [Keys and Values Subcolumns](#keys-and-values-subcolumns) + * 3.12.1 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys](#rqsrs-018clickhousemapdatatypesubcolumnskeys) + * 3.12.2 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnskeysarrayfunctions) + * 3.12.3 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnskeysinlinedefinedmap) + * 3.12.4 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values](#rqsrs-018clickhousemapdatatypesubcolumnsvalues) + * 3.12.5 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesarrayfunctions) + * 3.12.6 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesinlinedefinedmap) + * 3.13 [Functions](#functions) + * 3.13.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap](#rqsrs-018clickhousemapdatatypefunctionsinlinedefinedmap) + * 3.13.2 [`length`](#length) + * 3.13.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length](#rqsrs-018clickhousemapdatatypefunctionslength) + * 3.13.3 [`empty`](#empty) + * 3.13.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty](#rqsrs-018clickhousemapdatatypefunctionsempty) + * 3.13.4 [`notEmpty`](#notempty) + * 3.13.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty](#rqsrs-018clickhousemapdatatypefunctionsnotempty) + * 3.13.5 [`map`](#map) + * 3.13.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map](#rqsrs-018clickhousemapdatatypefunctionsmap) + * 3.13.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments](#rqsrs-018clickhousemapdatatypefunctionsmapinvalidnumberofarguments) + * 3.13.5.3 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes](#rqsrs-018clickhousemapdatatypefunctionsmapmixedkeyorvaluetypes) + * 3.13.5.4 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd](#rqsrs-018clickhousemapdatatypefunctionsmapmapadd) + * 3.13.5.5 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract](#rqsrs-018clickhousemapdatatypefunctionsmapmapsubstract) + * 3.13.5.6 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries](#rqsrs-018clickhousemapdatatypefunctionsmapmappopulateseries) + * 3.13.6 [`mapContains`](#mapcontains) + * 3.13.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains](#rqsrs-018clickhousemapdatatypefunctionsmapcontains) + * 3.13.7 [`mapKeys`](#mapkeys) + * 3.13.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys](#rqsrs-018clickhousemapdatatypefunctionsmapkeys) + * 3.13.8 [`mapValues`](#mapvalues) + * 3.13.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues](#rqsrs-018clickhousemapdatatypefunctionsmapvalues) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +This software requirements specification covers requirements for `Map(key, value)` data type in [ClickHouse]. + +## Requirements + +### General + +#### RQ.SRS-018.ClickHouse.Map.DataType +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs. + +### Performance + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Array(Tuple(K,V))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Tuple(Array(String), Array(String))` data type where the first +array defines an array of keys and the second array defines an array of values. + +### Key Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type. + +### Value Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Array +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type. + +### Invalid Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type. + +### Duplicated Keys + +#### RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys. + +### Array of Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps +version: 1.0 + +[ClickHouse] SHALL support `Array(Map(key, value))` data type. + +### Nested With Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps +version: 1.0 + +[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type. + +### Value Retrieval + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval +version: 1.0 + +[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax. +If `key` has duplicates then the first `key:value` pair SHALL be returned. + +For example, + +```sql +SELECT a['key2'] FROM table_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid +version: 1.0 + +[ClickHouse] SHALL return an error when key does not match the key type. + +For example, + +```sql +SELECT map(1,2) AS m, m[1024] +``` + +SHALL have the following output + +```bash +Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. +``` + +Exceptions: + +* when key is `NULL` the return value SHALL be `NULL` +* when key has [Integer] type and key is either too large or negative then + when reading from a table column it SHALL return `zero`. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound +version: 1.0 + +[ClickHouse] SHALL return + +* zeros for [Integer] values +* empty strings for [String] values +* empty for [Array] values + +when there's no corresponding `key` defined in the `Map(key, value)` data type. + + +### Converting Tuple(Array, Array) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function. + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid +such as when arrays are not of equal size. + +For example, + +```sql +SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] +``` + +### Converting Array(Tuple(K,V)) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` +when [Tuple] is invalid such as when it is not a [Tuple] with two elements. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +or + +```sql +SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map +``` + +### Keys and Values Subcolumns + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys +version: 1.0 + +[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map keys. + +```sql +SELECT m.keys FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.keys, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.keys +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values +version: 1.0 + +[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map values. + +```sql +SELECT m.values FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.values, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.values +``` + +### Functions + +#### RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL support using inline defined maps as an argument to map functions. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c) +SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c) +``` + +#### `length` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function +that SHALL return number of keys in the map. + +For example, + +```sql +SELECT length(map(1,2,3,4)) +SELECT length(map()) +``` + +#### `empty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function +that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 0. + +For example, + +```sql +SELECT empty(map(1,2,3,4)) +SELECT empty(map()) +``` + +#### `notEmpty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function +that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 1. + +For example, + +```sql +SELECT notEmpty(map(1,2,3,4)) +SELECT notEmpty(map()) +``` + +#### `map` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map +version: 1.0 + +[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type +using `map` function. + +**Syntax** + +``` sql +map(key1, value1[, key2, value2, ...]) +``` + +For example, + +``` sql +SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); + +┌─map('key1', number, 'key2', multiply(number, 2))─┐ +│ {'key1':0,'key2':0} │ +│ {'key1':1,'key2':2} │ +│ {'key1':2,'key2':4} │ +└──────────────────────────────────────────────────┘ +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments. + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types. + + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type. + +For example, + +``` sql +SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)") +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)") +``` +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)") +``` + +#### `mapContains` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains +version: 1.0 + +[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`. + +For example, + +```sql +SELECT mapContains(a, 'abc') from table_map; +``` + +#### `mapKeys` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys +version: 1.0 + +[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format. + +For example, + +```sql +SELECT mapKeys(a) from table_map; +``` + +#### `mapValues` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues +version: 1.0 + +[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format. + +For example, + +```sql +SELECT mapValues(a) from table_map; +``` + +[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/ +[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length +[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty +[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty +[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast +[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ +[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/ +[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/ +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py new file mode 100644 index 00000000000..6042512abd0 --- /dev/null +++ b/tests/testflows/map_type/requirements/requirements.py @@ -0,0 +1,1449 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.210214.1233241. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Specification +from testflows.core import Requirement + +Heading = Specification.Heading + +RQ_SRS_018_ClickHouse_Map_DataType = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs.\n' + '\n' + ), + link=None, + level=3, + num='3.1.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n' + 'compared to `Array(Tuple(K,V))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.2.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n' + 'compared to `Tuple(Array(String), Array(String))` data type where the first\n' + 'array defines an array of keys and the second array defines an array of values.\n' + '\n' + ), + link=None, + level=3, + num='3.2.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Key_String = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Key.String', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type.\n' + '\n' + ), + link=None, + level=3, + num='3.3.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type.\n' + '\n' + ), + link=None, + level=3, + num='3.3.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_String = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.String', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Array = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Array', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type.\n' + '\n' + ), + link=None, + level=3, + num='3.4.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.5.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.5.2') + +RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys.\n' + '\n' + ), + link=None, + level=3, + num='3.6.1') + +RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Array(Map(key, value))` data type.\n' + '\n' + ), + link=None, + level=3, + num='3.7.1') + +RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type.\n' + '\n' + ), + link=None, + level=3, + num='3.8.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax.\n' + 'If `key` has duplicates then the first `key:value` pair SHALL be returned. \n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT a['key2'] FROM table_map;\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.9.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when key does not match the key type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT map(1,2) AS m, m[1024]\n' + '```\n' + '\n' + 'SHALL have the following output \n' + '\n' + '```bash\n' + 'Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. \n' + '```\n' + '\n' + 'Exceptions:\n' + '\n' + '* when key is `NULL` the return value SHALL be `NULL`\n' + '* when key has [Integer] type and key is either too large or negative then\n' + ' when reading from a table column it SHALL return `zero`.\n' + '\n' + ), + link=None, + level=3, + num='3.9.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return\n' + '\n' + '* zeros for [Integer] values\n' + '* empty strings for [String] values\n' + '* empty for [Array] values\n' + '\n' + "when there's no corresponding `key` defined in the `Map(key, value)` data type. \n" + '\n' + '\n' + ), + link=None, + level=3, + num='3.9.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function.\n' + '\n' + '``` sql\n' + "SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map;\n" + '```\n' + '\n' + '``` text\n' + '┌─map───────────────────────────┐\n' + "│ {1:'Ready',2:'Steady',3:'Go'} │\n" + '└───────────────────────────────┘\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.10.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid\n' + 'such as when arrays are not of equal size.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10]\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.10.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.11.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)`\n' + 'when [Tuple] is invalid such as when it is not a [Tuple] with two elements.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" + '```\n' + '\n' + 'or \n' + '\n' + '```sql\n' + "SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.11.2') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used \n' + 'to retrieve an [Array] of map keys.\n' + '\n' + '```sql\n' + 'SELECT m.keys FROM t_map;\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.1') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT * FROM t_map WHERE has(m.keys, 'a');\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.2') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, c.keys\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.3') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used \n' + 'to retrieve an [Array] of map values.\n' + '\n' + '```sql\n' + 'SELECT m.values FROM t_map;\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.4') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT * FROM t_map WHERE has(m.values, 'a');\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.5') + +RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, c.values\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.12.6') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support using inline defined maps as an argument to map functions.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c)\n" + "SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c)\n" + '```\n' + '\n' + ), + link=None, + level=3, + num='3.13.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function\n' + 'that SHALL return number of keys in the map.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT length(map(1,2,3,4))\n' + 'SELECT length(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.2.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function\n' + 'that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is \n' + 'greater or equal to 1 it SHALL return 0.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT empty(map(1,2,3,4))\n' + 'SELECT empty(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.3.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function\n' + 'that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is\n' + 'greater or equal to 1 it SHALL return 1.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT notEmpty(map(1,2,3,4))\n' + 'SELECT notEmpty(map())\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.4.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type\n' + 'using `map` function.\n' + '\n' + '**Syntax** \n' + '\n' + '``` sql\n' + 'map(key1, value1[, key2, value2, ...])\n' + '```\n' + '\n' + 'For example,\n' + '\n' + '``` sql\n' + "SELECT map('key1', number, 'key2', number * 2) FROM numbers(3);\n" + '\n' + "┌─map('key1', number, 'key2', multiply(number, 2))─┐\n" + "│ {'key1':0,'key2':0} │\n" + "│ {'key1':1,'key2':2} │\n" + "│ {'key1':2,'key2':4} │\n" + '└──────────────────────────────────────────────────┘\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments.\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.2') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types.\n' + '\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.3') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '``` sql\n' + 'SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)")\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.4') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)")\n' + '```\n' + ), + link=None, + level=4, + num='3.13.5.5') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)")\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.5.6') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + "SELECT mapContains(a, 'abc') from table_map;\n" + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.6.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT mapKeys(a) from table_map;\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='3.13.7.1') + +RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues = Requirement( + name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format.\n' + '\n' + 'For example,\n' + '\n' + '```sql\n' + 'SELECT mapValues(a) from table_map;\n' + '```\n' + '\n' + '[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/\n' + '[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length\n' + '[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty\n' + '[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty\n' + '[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast\n' + '[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/\n' + '[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/\n' + '[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ \n' + '[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/\n' + '[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/\n' + '[ClickHouse]: https://clickhouse.tech\n' + '[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md \n' + '[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md\n' + '[Git]: https://git-scm.com/\n' + '[GitHub]: https://github.com\n' + ), + link=None, + level=4, + num='3.13.8.1') + +SRS018_ClickHouse_Map_Data_Type = Specification( + name='SRS018 ClickHouse Map Data Type', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + headings=( + Heading(name='Revision History', level=1, num='1'), + Heading(name='Introduction', level=1, num='2'), + Heading(name='Requirements', level=1, num='3'), + Heading(name='General', level=2, num='3.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType', level=3, num='3.1.1'), + Heading(name='Performance', level=2, num='3.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples', level=3, num='3.2.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays', level=3, num='3.2.2'), + Heading(name='Key Types', level=2, num='3.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Key.String', level=3, num='3.3.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer', level=3, num='3.3.2'), + Heading(name='Value Types', level=2, num='3.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.String', level=3, num='3.4.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer', level=3, num='3.4.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Array', level=3, num='3.4.3'), + Heading(name='Invalid Types', level=2, num='3.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable', level=3, num='3.5.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing', level=3, num='3.5.2'), + Heading(name='Duplicated Keys', level=2, num='3.6'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys', level=3, num='3.6.1'), + Heading(name='Array of Maps', level=2, num='3.7'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps', level=3, num='3.7.1'), + Heading(name='Nested With Maps', level=2, num='3.8'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps', level=3, num='3.8.1'), + Heading(name='Value Retrieval', level=2, num='3.9'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval', level=3, num='3.9.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid', level=3, num='3.9.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound', level=3, num='3.9.3'), + Heading(name='Converting Tuple(Array, Array) to Map', level=2, num='3.10'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap', level=3, num='3.10.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid', level=3, num='3.10.2'), + Heading(name='Converting Array(Tuple(K,V)) to Map', level=2, num='3.11'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap', level=3, num='3.11.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid', level=3, num='3.11.2'), + Heading(name='Keys and Values Subcolumns', level=2, num='3.12'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys', level=3, num='3.12.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions', level=3, num='3.12.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap', level=3, num='3.12.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values', level=3, num='3.12.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions', level=3, num='3.12.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap', level=3, num='3.12.6'), + Heading(name='Functions', level=2, num='3.13'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap', level=3, num='3.13.1'), + Heading(name='`length`', level=3, num='3.13.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length', level=4, num='3.13.2.1'), + Heading(name='`empty`', level=3, num='3.13.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty', level=4, num='3.13.3.1'), + Heading(name='`notEmpty`', level=3, num='3.13.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty', level=4, num='3.13.4.1'), + Heading(name='`map`', level=3, num='3.13.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map', level=4, num='3.13.5.1'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments', level=4, num='3.13.5.2'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes', level=4, num='3.13.5.3'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd', level=4, num='3.13.5.4'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract', level=4, num='3.13.5.5'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries', level=4, num='3.13.5.6'), + Heading(name='`mapContains`', level=3, num='3.13.6'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains', level=4, num='3.13.6.1'), + Heading(name='`mapKeys`', level=3, num='3.13.7'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys', level=4, num='3.13.7.1'), + Heading(name='`mapValues`', level=3, num='3.13.8'), + Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues', level=4, num='3.13.8.1'), + ), + requirements=( + RQ_SRS_018_ClickHouse_Map_DataType, + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples, + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays, + RQ_SRS_018_ClickHouse_Map_DataType_Key_String, + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer, + RQ_SRS_018_ClickHouse_Map_DataType_Value_String, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array, + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable, + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing, + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys, + RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps, + RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid, + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap, + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions, + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys, + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues, + ), + content=''' +# SRS018 ClickHouse Map Data Type +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Requirements](#requirements) + * 3.1 [General](#general) + * 3.1.1 [RQ.SRS-018.ClickHouse.Map.DataType](#rqsrs-018clickhousemapdatatype) + * 3.2 [Performance](#performance) + * 3.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples](#rqsrs-018clickhousemapdatatypeperformancevsarrayoftuples) + * 3.2.2 [RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays](#rqsrs-018clickhousemapdatatypeperformancevstupleofarrays) + * 3.3 [Key Types](#key-types) + * 3.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Key.String](#rqsrs-018clickhousemapdatatypekeystring) + * 3.3.2 [RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer](#rqsrs-018clickhousemapdatatypekeyinteger) + * 3.4 [Value Types](#value-types) + * 3.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.String](#rqsrs-018clickhousemapdatatypevaluestring) + * 3.4.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer](#rqsrs-018clickhousemapdatatypevalueinteger) + * 3.4.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Array](#rqsrs-018clickhousemapdatatypevaluearray) + * 3.5 [Invalid Types](#invalid-types) + * 3.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable](#rqsrs-018clickhousemapdatatypeinvalidnullable) + * 3.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing](#rqsrs-018clickhousemapdatatypeinvalidnothingnothing) + * 3.6 [Duplicated Keys](#duplicated-keys) + * 3.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys](#rqsrs-018clickhousemapdatatypeduplicatedkeys) + * 3.7 [Array of Maps](#array-of-maps) + * 3.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps](#rqsrs-018clickhousemapdatatypearrayofmaps) + * 3.8 [Nested With Maps](#nested-with-maps) + * 3.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps](#rqsrs-018clickhousemapdatatypenestedwithmaps) + * 3.9 [Value Retrieval](#value-retrieval) + * 3.9.1 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval](#rqsrs-018clickhousemapdatatypevalueretrieval) + * 3.9.2 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid](#rqsrs-018clickhousemapdatatypevalueretrievalkeyinvalid) + * 3.9.3 [RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound](#rqsrs-018clickhousemapdatatypevalueretrievalkeynotfound) + * 3.10 [Converting Tuple(Array, Array) to Map](#converting-tuplearray-array-to-map) + * 3.10.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraystomap) + * 3.10.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromtupleofarraysmapinvalid) + * 3.11 [Converting Array(Tuple(K,V)) to Map](#converting-arraytuplekv-to-map) + * 3.11.1 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomap) + * 3.11.2 [RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid](#rqsrs-018clickhousemapdatatypeconversionfromarrayoftuplestomapinvalid) + * 3.12 [Keys and Values Subcolumns](#keys-and-values-subcolumns) + * 3.12.1 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys](#rqsrs-018clickhousemapdatatypesubcolumnskeys) + * 3.12.2 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnskeysarrayfunctions) + * 3.12.3 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnskeysinlinedefinedmap) + * 3.12.4 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values](#rqsrs-018clickhousemapdatatypesubcolumnsvalues) + * 3.12.5 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesarrayfunctions) + * 3.12.6 [RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap](#rqsrs-018clickhousemapdatatypesubcolumnsvaluesinlinedefinedmap) + * 3.13 [Functions](#functions) + * 3.13.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap](#rqsrs-018clickhousemapdatatypefunctionsinlinedefinedmap) + * 3.13.2 [`length`](#length) + * 3.13.2.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length](#rqsrs-018clickhousemapdatatypefunctionslength) + * 3.13.3 [`empty`](#empty) + * 3.13.3.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty](#rqsrs-018clickhousemapdatatypefunctionsempty) + * 3.13.4 [`notEmpty`](#notempty) + * 3.13.4.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty](#rqsrs-018clickhousemapdatatypefunctionsnotempty) + * 3.13.5 [`map`](#map) + * 3.13.5.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map](#rqsrs-018clickhousemapdatatypefunctionsmap) + * 3.13.5.2 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments](#rqsrs-018clickhousemapdatatypefunctionsmapinvalidnumberofarguments) + * 3.13.5.3 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes](#rqsrs-018clickhousemapdatatypefunctionsmapmixedkeyorvaluetypes) + * 3.13.5.4 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd](#rqsrs-018clickhousemapdatatypefunctionsmapmapadd) + * 3.13.5.5 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract](#rqsrs-018clickhousemapdatatypefunctionsmapmapsubstract) + * 3.13.5.6 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries](#rqsrs-018clickhousemapdatatypefunctionsmapmappopulateseries) + * 3.13.6 [`mapContains`](#mapcontains) + * 3.13.6.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains](#rqsrs-018clickhousemapdatatypefunctionsmapcontains) + * 3.13.7 [`mapKeys`](#mapkeys) + * 3.13.7.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys](#rqsrs-018clickhousemapdatatypefunctionsmapkeys) + * 3.13.8 [`mapValues`](#mapvalues) + * 3.13.8.1 [RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues](#rqsrs-018clickhousemapdatatypefunctionsmapvalues) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +This software requirements specification covers requirements for `Map(key, value)` data type in [ClickHouse]. + +## Requirements + +### General + +#### RQ.SRS-018.ClickHouse.Map.DataType +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs. + +### Performance + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Array(Tuple(K,V))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays +version:1.0 + +[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as +compared to `Tuple(Array(String), Array(String))` data type where the first +array defines an array of keys and the second array defines an array of values. + +### Key Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type. + +### Value Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.String +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Array +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type. + +### Invalid Types + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing +version: 1.0 + +[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type. + +### Duplicated Keys + +#### RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type with duplicated keys. + +### Array of Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps +version: 1.0 + +[ClickHouse] SHALL support `Array(Map(key, value))` data type. + +### Nested With Maps + +#### RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps +version: 1.0 + +[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type. + +### Value Retrieval + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval +version: 1.0 + +[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax. +If `key` has duplicates then the first `key:value` pair SHALL be returned. + +For example, + +```sql +SELECT a['key2'] FROM table_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid +version: 1.0 + +[ClickHouse] SHALL return an error when key does not match the key type. + +For example, + +```sql +SELECT map(1,2) AS m, m[1024] +``` + +SHALL have the following output + +```bash +Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Illegal types of arguments: Map(UInt8,UInt8), UInt16 for function : While processing {1, 2} AS m, m[1024]. +``` + +Exceptions: + +* when key is `NULL` the return value SHALL be `NULL` +* when key has [Integer] type and key is either too large or negative then + when reading from a table column it SHALL return `zero`. + +#### RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound +version: 1.0 + +[ClickHouse] SHALL return + +* zeros for [Integer] values +* empty strings for [String] values +* empty for [Array] values + +when there's no corresponding `key` defined in the `Map(key, value)` data type. + + +### Converting Tuple(Array, Array) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function. + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error when casting [Tuple(Array, Array)] when arguments are invalid +such as when arrays are not of equal size. + +For example, + +```sql +SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10] +``` + +### Converting Array(Tuple(K,V)) to Map + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap +version: 1.0 + +[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error when casting [Array(Tuple(K, V))] to `Map(key, value)` +when [Tuple] is invalid such as when it is not a [Tuple] with two elements. + +For example, + +```sql +SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map +``` + +or + +```sql +SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map +``` + +### Keys and Values Subcolumns + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys +version: 1.0 + +[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map keys. + +```sql +SELECT m.keys FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.keys, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL not support using inline defined map to get `keys` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.keys +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values +version: 1.0 + +[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used +to retrieve an [Array] of map values. + +```sql +SELECT m.values FROM t_map; +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions +version: 1.0 + +[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type. + +For example, + +```sql +SELECT * FROM t_map WHERE has(m.values, 'a'); +``` + +#### RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL not support using inline defined map to get `values` subcolumn. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, c.values +``` + +### Functions + +#### RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap +version: 1.0 + +[ClickHouse] SHALL support using inline defined maps as an argument to map functions. + +For example, + +```sql +SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c) +SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c) +``` + +#### `length` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function +that SHALL return number of keys in the map. + +For example, + +```sql +SELECT length(map(1,2,3,4)) +SELECT length(map()) +``` + +#### `empty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function +that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 0. + +For example, + +```sql +SELECT empty(map(1,2,3,4)) +SELECT empty(map()) +``` + +#### `notEmpty` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty +version: 1.0 + +[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function +that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is +greater or equal to 1 it SHALL return 1. + +For example, + +```sql +SELECT notEmpty(map(1,2,3,4)) +SELECT notEmpty(map()) +``` + +#### `map` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map +version: 1.0 + +[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type +using `map` function. + +**Syntax** + +``` sql +map(key1, value1[, key2, value2, ...]) +``` + +For example, + +``` sql +SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); + +┌─map('key1', number, 'key2', multiply(number, 2))─┐ +│ {'key1':0,'key2':0} │ +│ {'key1':1,'key2':2} │ +│ {'key1':2,'key2':4} │ +└──────────────────────────────────────────────────┘ +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments. + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes +version: 1.0 + +[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types. + + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type. + +For example, + +``` sql +SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)") +``` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)") +``` +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries +version: 1.0 + +[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type. + +For example, + +```sql +SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)") +``` + +#### `mapContains` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains +version: 1.0 + +[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`. + +For example, + +```sql +SELECT mapContains(a, 'abc') from table_map; +``` + +#### `mapKeys` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys +version: 1.0 + +[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format. + +For example, + +```sql +SELECT mapKeys(a) from table_map; +``` + +#### `mapValues` + +##### RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues +version: 1.0 + +[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format. + +For example, + +```sql +SELECT mapValues(a) from table_map; +``` + +[Nested]: https://clickhouse.tech/docs/en/sql-reference/data-types/nested-data-structures/nested/ +[length]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#array_functions-length +[empty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-empty +[notEmpty]: https://clickhouse.tech/docs/en/sql-reference/functions/array-functions/#function-notempty +[CAST]: https://clickhouse.tech/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast +[Tuple]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Tuple(Array,Array)]: https://clickhouse.tech/docs/en/sql-reference/data-types/tuple/ +[Array]: https://clickhouse.tech/docs/en/sql-reference/data-types/array/ +[String]: https://clickhouse.tech/docs/en/sql-reference/data-types/string/ +[Integer]: https://clickhouse.tech/docs/en/sql-reference/data-types/int-uint/ +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com +''') diff --git a/tests/testflows/map_type/tests/__init__.py b/tests/testflows/map_type/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/map_type/tests/common.py b/tests/testflows/map_type/tests/common.py new file mode 100644 index 00000000000..d4461062256 --- /dev/null +++ b/tests/testflows/map_type/tests/common.py @@ -0,0 +1,89 @@ +import uuid +from collections import namedtuple + +from testflows.core import * +from testflows.core.name import basename, parentname +from testflows._core.testtype import TestSubType + +table_tuple = namedtuple("table_tuple", "create_statement cluster") + +table_types = { + "MergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, m Map(String, Int8)) ENGINE = MergeTree() PARTITION BY m ORDER BY d", None), + # "ReplacingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = ReplacingMergeTree() PARTITION BY y ORDER BY d", None), + # "SummingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) ENGINE = SummingMergeTree() PARTITION BY y ORDER BY d", None), + # "AggregatingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = AggregatingMergeTree() PARTITION BY y ORDER BY d", None), + # "CollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(sign) PARTITION BY y ORDER BY d", None), + # "VersionedCollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY y ORDER BY d", None), + # "GraphiteMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) ENGINE = GraphiteMergeTree('graphite_rollup_example') PARTITION BY y ORDER by d", None), + # "ReplicatedMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedReplacingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedReplacingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedSummingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ + # ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedSummingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ + # ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedAggregatingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedAggregatingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + # ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ + # ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ + # ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedVersionedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ + # ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedVersionedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ + # ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "one_shard_cluster"), + # "ReplicatedGraphiteMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ + # ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "sharded_cluster"), + # "ReplicatedGraphiteMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ + # ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "one_shard_cluster"), +} + +def getuid(): + if current().subtype == TestSubType.Example: + testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + else: + testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" + return testname + "_" + str(uuid.uuid1()).replace('-', '_') + +@TestStep(Given) +def allow_experimental_map_type(self): + """Set allow_experimental_map_type = 1 + """ + setting = ("allow_experimental_map_type", 1) + default_query_settings = None + + try: + with By("adding allow_experimental_map_type to the default query settings"): + default_query_settings = getsattr(current().context, "default_query_settings", []) + default_query_settings.append(setting) + yield + finally: + with Finally("I remove allow_experimental_map_type from the default query settings"): + if default_query_settings: + try: + default_query_settings.pop(default_query_settings.index(setting)) + except ValueError: + pass + +@TestStep(Given) +def create_table(self, name, statement, on_cluster=False): + """Create table. + """ + node = current().context.node + try: + with Given(f"I have a {name} table"): + node.query(statement.format(name=name)) + yield name + finally: + with Finally("I drop the table"): + if on_cluster: + node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster}") + else: + node.query(f"DROP TABLE IF EXISTS {name}") \ No newline at end of file diff --git a/tests/testflows/map_type/tests/feature.py b/tests/testflows/map_type/tests/feature.py new file mode 100755 index 00000000000..07ae7cc2159 --- /dev/null +++ b/tests/testflows/map_type/tests/feature.py @@ -0,0 +1,1119 @@ +# -*- coding: utf-8 -*- +import time + +from testflows.core import * +from testflows.asserts import error + +from map_type.requirements import * +from map_type.tests.common import * + +@TestOutline +def select_map(self, map, output, exitcode=0, message=None): + """Create a map using select statement. + """ + node = self.context.node + + with When("I create a map using select", description=map): + r = node.query(f"SELECT {map}", exitcode=exitcode, message=message) + + with Then("I expect output to match", description=output): + assert r.output == output, error() + +@TestOutline +def table_map(self, type, data, select, filter, exitcode, message, check_insert=False, order_by=None): + """Check using a map column in a table. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + if check_insert: + node.query(f"INSERT INTO {table} VALUES {data}", exitcode=exitcode, message=message) + else: + node.query(f"INSERT INTO {table} VALUES {data}") + + if not check_insert: + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +) +@Examples("map output", [ + ("map('',1)", "{'':1}", Name("empty string")), + ("map('hello',1)", "{'hello':1}", Name("non-empty string")), + ("map('Gãńdåłf_Thê_Gręât',1)", "{'Gãńdåłf_Thê_Gręât':1}", Name("utf-8 string")), + ("map('hello there',1)", "{'hello there':1}", Name("multi word string")), + ("map('hello',1,'there',2)", "{'hello':1,'there':2}", Name("multiple keys")), + ("map(toString(1),1)", "{'1':1}", Name("toString")), + ("map(toFixedString('1',1),1)", "{'1':1}", Name("toFixedString")), + ("map(toNullable('1'),1)", "{'1':1}", Name("Nullable")), + ("map(toNullable(NULL),1)", "{NULL:1}", Name("Nullable(NULL)")), + ("map(toLowCardinality('1'),1)", "{'1':1}", Name("LowCardinality(String)")), + ("map(toLowCardinality(toFixedString('1',1)),1)", "{'1':1}", Name("LowCardinality(FixedString)")), +], row_format="%20s,%20s") +def select_map_with_key_string(self, map, output): + """Create a map using select that has key string type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +) +@Examples("map output", [ + ("map('key','')", "{'key':''}", Name("empty string")), + ("map('key','hello')", "{'key':'hello'}", Name("non-empty string")), + ("map('key','Gãńdåłf_Thê_Gręât')", "{'key':'Gãńdåłf_Thê_Gręât'}", Name("utf-8 string")), + ("map('key','hello there')", "{'key':'hello there'}", Name("multi word string")), + ("map('key','hello','key2','there')", "{'key':'hello','key2':'there'}", Name("multiple keys")), + ("map('key',toString(1))", "{'key':'1'}", Name("toString")), + ("map('key',toFixedString('1',1))", "{'key':'1'}", Name("toFixedString")), + ("map('key',toNullable('1'))", "{'key':'1'}", Name("Nullable")), + ("map('key',toNullable(NULL))", "{'key':NULL}", Name("Nullable(NULL)")), + ("map('key',toLowCardinality('1'))", "{'key':'1'}", Name("LowCardinality(String)")), + ("map('key',toLowCardinality(toFixedString('1',1)))", "{'key':'1'}", Name("LowCardinality(FixedString)")), +], row_format="%20s,%20s") +def select_map_with_value_string(self, map, output): + """Create a map using select that has value string type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0") +) +@Examples("map output", [ + ("map('key',[])", "{'key':[]}", Name("empty Array")), + ("map('key',[1,2,3])", "{'key':[1,2,3]}", Name("non-empty array of ints")), + ("map('key',['1','2','3'])", "{'key':['1','2','3']}", Name("non-empty array of strings")), + ("map('key',[map(1,2),map(2,3)])", "{'key':[{1:2},{2:3}]}", Name("non-empty array of maps")), + ("map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])])", "{'key':[{1:[{1:[1]}]},{2:[{2:[3]}]}]}", Name("non-empty array of maps of array of maps")), +]) +def select_map_with_value_array(self, map, output): + """Create a map using select that has value array type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") +) +@Examples("map output", [ + ("(map(1,127,2,0,3,-128))", '{1:127,2:0,3:-128}', Name("Int8")), + ("(map(1,0,2,255))", '{1:0,2:255}', Name("UInt8")), + ("(map(1,32767,2,0,3,-32768))", '{1:32767,2:0,3:-32768}', Name("Int16")), + ("(map(1,0,2,65535))", '{1:0,2:65535}', Name("UInt16")), + ("(map(1,2147483647,2,0,3,-2147483648))", '{1:2147483647,2:0,3:-2147483648}', Name("Int32")), + ("(map(1,0,2,4294967295))", '{1:0,2:4294967295}', Name("UInt32")), + ("(map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}', Name("Int64")), + ("(map(1,0,2,18446744073709551615))", '{1:0,2:18446744073709551615}', Name("UInt64")), + ("(map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{1:1.7014118346046923e38,2:0,3:-1.7014118346046923e38}', Name("Int128")), + ("(map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{1:5.78960446186581e76,2:0,3:-5.78960446186581e76}', Name("Int256")), + ("(map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{1:0,2:1.157920892373162e77}', Name("UInt256")), + ("(map(1,toNullable(1)))", '{1:1}', Name("toNullable")), + ("(map(1,toNullable(NULL)))", '{1:NULL}', Name("toNullable(NULL)")), +]) +def select_map_with_value_integer(self, map, output): + """Create a map using select that has value integer type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +) +@Examples("map output", [ + ("(map(127,1,0,1,-128,1))", '{127:1,0:1,-128:1}', Name("Int8")), + ("(map(0,1,255,1))", '{0:1,255:1}', Name("UInt8")), + ("(map(32767,1,0,1,-32768,1))", '{32767:1,0:1,-32768:1}', Name("Int16")), + ("(map(0,1,65535,1))", '{0:1,65535:1}', Name("UInt16")), + ("(map(2147483647,1,0,1,-2147483648,1))", '{2147483647:1,0:1,-2147483648:1}', Name("Int32")), + ("(map(0,1,4294967295,1))", '{0:1,4294967295:1}', Name("UInt32")), + ("(map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"9223372036854775807":1,"0":1,"-9223372036854775808":1}', Name("Int64")), + ("(map(0,1,18446744073709551615,1))", '{0:1,18446744073709551615:1}', Name("UInt64")), + ("(map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{1.7014118346046923e38:1,0:1,-1.7014118346046923e38:1}', Name("Int128")), + ("(map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{5.78960446186581e76:1,0:1,-5.78960446186581e76:1}', Name("Int256")), + ("(map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{0:1,1.157920892373162e77:1}', Name("UInt256")), + ("(map(toNullable(1),1))", '{1:1}', Name("toNullable")), + ("(map(toNullable(NULL),1))", '{NULL:1}', Name("toNullable(NULL)")), +]) +def select_map_with_key_integer(self, map, output): + """Create a map using select that has key integer type. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +) +@Examples("type data output", [ + ("Map(String, Int8)", "('2020-01-01', map('',1))", '{"d":"2020-01-01","m":{"":1}}', Name("empty string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1))", '{"d":"2020-01-01","m":{"hello":1}}', Name("non-empty string")), + ("Map(String, Int8)", "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", '{"d":"2020-01-01","m":{"Gãńdåłf_Thê_Gręât":1}}', Name("utf-8 string")), + ("Map(String, Int8)", "('2020-01-01', map('hello there',1))", '{"d":"2020-01-01","m":{"hello there":1}}', Name("multi word string")), + ("Map(String, Int8)", "('2020-01-01', map('hello',1,'there',2))", '{"d":"2020-01-01","m":{"hello":1,"there":2}}', Name("multiple keys")), + ("Map(String, Int8)", "('2020-01-01', map(toString(1),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("toString")), + ("Map(FixedString(1), Int8)", "('2020-01-01', map(toFixedString('1',1),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("FixedString")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("Nullable")), + ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("Nullable(NULL)")), + ("Map(LowCardinality(String), Int8)", "('2020-01-01', map(toLowCardinality('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(String)")), + ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(FixedString)")) +]) +def table_map_with_key_string(self, type, data, output): + """Check what values we can insert into map type column with key string. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +) +@Examples("type data output", [ + ("Map(String, String)", "('2020-01-01', map('key',''))", '{"d":"2020-01-01","m":{"key":""}}', Name("empty string")), + ("Map(String, String)", "('2020-01-01', map('key','hello'))", '{"d":"2020-01-01","m":{"key":"hello"}}', Name("non-empty string")), + ("Map(String, String)", "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", '{"d":"2020-01-01","m":{"key":"Gãńdåłf_Thê_Gręât"}}', Name("utf-8 string")), + ("Map(String, String)", "('2020-01-01', map('key', 'hello there'))", '{"d":"2020-01-01","m":{"key":"hello there"}}', Name("multi word string")), + ("Map(String, String)", "('2020-01-01', map('key','hello','key2','there'))", '{"d":"2020-01-01","m":{"key":"hello","key2":"there"}}', Name("multiple keys")), + ("Map(String, String)", "('2020-01-01', map('key', toString(1)))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("toString")), + ("Map(String, FixedString(1))", "('2020-01-01', map('key',toFixedString('1',1)))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("FixedString")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("Nullable")), + ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable(NULL)))", '{"d":"2020-01-01","m":{"key":null}}', Name("Nullable(NULL)")), + ("Map(String, LowCardinality(String))", "('2020-01-01', map('key',toLowCardinality('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(String)")), + ("Map(String, LowCardinality(FixedString(1)))", "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(FixedString)")) +]) +def table_map_with_value_string(self, type, data, output): + """Check what values we can insert into map type column with value string. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") +) +@Examples("type data output", [ + ("Map(Int8, Int8)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:127,2:0,3:-128}}', Name("Int8")), + ("Map(Int8, UInt8)", "('2020-01-01', map(1,0,2,255))", '{"d":"2020-01-01","m":{1:0,2:255}}', Name("UInt8")), + ("Map(Int8, Int16)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:32767,2:0,3:-32768}}', Name("Int16")), + ("Map(Int8, UInt16)", "('2020-01-01', map(1,0,2,65535))", '{"d":"2020-01-01","m":{1:0,2:65535}}', Name("UInt16")), + ("Map(Int8, Int32)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:2147483647,2:0,3:-2147483648}}', Name("Int32")), + ("Map(Int8, UInt32)", "('2020-01-01', map(1,0,2,4294967295))", '{"d":"2020-01-01","m":{1:0,2:4294967295}}', Name("UInt32")), + ("Map(Int8, Int64)", "('2020-01-01', map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{"d":"2020-01-01","m":{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}}', Name("Int64")), + ("Map(Int8, UInt64)", "('2020-01-01', map(1,0,2,18446744073709551615))", '{"d":"2020-01-01","m":{1:"0",2:"18446744073709551615"}}', Name("UInt64")), + ("Map(Int8, Int128)", "('2020-01-01', map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{"d":"2020-01-01","m":{1:"170141183460469231731687303715884105727",2:"0",3:"-170141183460469231731687303715884105728"}}', Name("Int128")), + ("Map(Int8, Int256)", "('2020-01-01', map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{"d":"2020-01-01","m":{1:"57896044618658097711785492504343953926634992332820282019728792003956564819967",2:"0",3:"-57896044618658097711785492504343953926634992332820282019728792003956564819968"}}', Name("Int256")), + ("Map(Int8, UInt256)", "('2020-01-01', map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{"d":"2020-01-01","m":{1:"0",2:"115792089237316195423570985008687907853269984665640564039457584007913129639935"}}', Name("UInt256")), + ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(1)))", '{"d":"2020-01-01","m":{1:1}}', Name("toNullable")), + ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(NULL)))", '{"d":"2020-01-01","m":{1:null}}', Name("toNullable(NULL)")), +]) +def table_map_with_value_integer(self, type, data, output): + """Check what values we can insert into map type column with value integer. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0") +) +@Examples("type data output", [ + ("Map(String, Array(Int8))", "('2020-01-01', map('key',[]))", '{"d":"2020-01-01","m":{"key":[]}}', Name("empty array")), + ("Map(String, Array(Int8))", "('2020-01-01', map('key',[1,2,3]))", '{"d":"2020-01-01","m":{"key":[1,2,3]}}', Name("non-empty array of ints")), + ("Map(String, Array(String))", "('2020-01-01', map('key',['1','2','3']))", '{"d":"2020-01-01","m":{"key":["1","2","3"]}}', Name("non-empty array of strings")), + ("Map(String, Array(Map(Int8, Int8)))", "('2020-01-01', map('key',[map(1,2),map(2,3)]))", '{"d":"2020-01-01","m":{"key":[{1:2},{2:3}]}}', Name("non-empty array of maps")), + ("Map(String, Array(Map(Int8, Array(Map(Int8, Array(Int8))))))", "('2020-01-01', map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])]))", '{"d":"2020-01-01","m":{"key":[{1:[{1:[1]}]},{2:[{2:[3]}]}]}}', Name("non-empty array of maps of array of maps")), +]) +def table_map_with_value_array(self, type, data, output): + """Check what values we can insert into map type column with value Array. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +) +@Examples("type data output", [ + ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{127:1,0:1,-128:1}}', Name("Int8")), + ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"d":"2020-01-01","m":{0:1,255:1}}', Name("UInt8")), + ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{32767:1,0:1,-32768:1}}', Name("Int16")), + ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"d":"2020-01-01","m":{0:1,65535:1}}', Name("UInt16")), + ("Map(Int32, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{2147483647:1,0:1,-2147483648:1}}', Name("Int32")), + ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"d":"2020-01-01","m":{0:1,4294967295:1}}', Name("UInt32")), + ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"d":"2020-01-01","m":{"9223372036854775807":1,"0":1,"-9223372036854775808":1}}', Name("Int64")), + ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"d":"2020-01-01","m":{"0":1,"18446744073709551615":1}}', Name("UInt64")), + ("Map(Int128, Int8)", "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{"d":"2020-01-01","m":{170141183460469231731687303715884105727:1,0:1,"-170141183460469231731687303715884105728":1}}', Name("Int128")), + ("Map(Int256, Int8)", "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{"d":"2020-01-01","m":{"57896044618658097711785492504343953926634992332820282019728792003956564819967":1,"0":1,"-57896044618658097711785492504343953926634992332820282019728792003956564819968":1}}', Name("Int256")), + ("Map(UInt256, Int8)", "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{"d":"2020-01-01","m":{"0":1,"115792089237316195423570985008687907853269984665640564039457584007913129639935":1}}', Name("UInt256")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(1),1))", '{"d":"2020-01-01","m":{1:1}}', Name("toNullable")), + ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("toNullable(NULL)")), +]) +def table_map_with_key_integer(self, type, data, output): + """Check what values we can insert into map type column with key integer. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps("1.0") +) +@Examples("type data output partition_by", [ + ("Array(Map(String, Int8))", + "('2020-01-01', [map('hello',1),map('hello',1,'there',2)])", + '{"d":"2020-01-01","m":[{"hello":1},{"hello":1,"there":2}]}', + "m", + Name("Array(Map(String, Int8))")), + ("Nested(x Map(String, Int8))", + "('2020-01-01', [map('hello',1)])", + '{"d":"2020-01-01","m.x":[{"hello":1}]}', + "m.x", + Name("Nested(x Map(String, Int8)")) +]) +def table_with_map_inside_another_type(self, type, data, output, partition_by): + """Check what values we can insert into a type that has map type. + """ + insert_into_table(type=type, data=data, output=output, partition_by=partition_by) + +@TestOutline +def insert_into_table(self, type, data, output, partition_by="m"): + """Check we can insert data into a table. + """ + uid = getuid() + node = self.context.node + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (d DATE, m " + type + ") ENGINE = MergeTree() PARTITION BY " + partition_by + " ORDER BY d" + + with Given(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data", description=data): + sql = f"INSERT INTO {table} VALUES {data}" + node.query(sql) + + with And("I select all rows from the table"): + r = node.query(f"SELECT * FROM {table} FORMAT JSONEachRow") + + with Then("I expect output to match", description=output): + assert r.output == output, error() + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes("1.0") +) +def select_map_with_invalid_mixed_key_and_value_types(self): + """Check that creating a map with mixed key types fails. + """ + node = self.context.node + exitcode = 130 + message = "DB::Exception: There is no supertype for types String, UInt8 because some of them are String/FixedString and some of them are not" + + with Check("attempt to create a map using SELECT with mixed key types then it fails"): + node.query("SELECT map('hello',1,2,3)", exitcode=exitcode, message=message) + + with Check("attempt to create a map using SELECT with mixed value types then it fails"): + node.query("SELECT map(1,'hello',2,2)", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments("1.0") +) +def select_map_with_invalid_number_of_arguments(self): + """Check that creating a map with invalid number of arguments fails. + """ + node = self.context.node + exitcode = 42 + message = "DB::Exception: Function map even number of arguments" + + with When("I create a map using SELECT with invalid number of arguments"): + node.query("SELECT map(1,2,3)", exitcode=exitcode, message=message) + +@TestScenario +def select_map_empty(self): + """Check that we can can create a empty map by not passing any arguments. + """ + node = self.context.node + + with When("I create a map using SELECT with no arguments"): + r = node.query("SELECT map()") + + with Then("it should create an empty map"): + assert r.output == "{}", error() + +@TestScenario +def insert_invalid_mixed_key_and_value_types(self): + """Check that inserting a map with mixed key or value types fails. + """ + uid = getuid() + node = self.context.node + exitcode = 130 + message = "DB::Exception: There is no supertype for types String, UInt8 because some of them are String/FixedString and some of them are not" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (d DATE, m Map(String, Int8)) ENGINE = MergeTree() PARTITION BY m ORDER BY d" + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert a map with mixed key types then it should fail"): + sql = f"INSERT INTO {table} VALUES ('2020-01-01', map('hello',1,2,3))" + node.query(sql, exitcode=exitcode, message=message) + + with When("I insert a map with mixed value types then it should fail"): + sql = f"INSERT INTO {table} VALUES ('2020-01-01', map(1,'hello',2,2))" + node.query(sql, exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0") +) +@Examples("type data output", [ + ("Map(String, String)", + "('2020-01-01', map('hello','there','hello','over there'))", + '{"d":"2020-01-01","m":{"hello":"there","hello":"over there"}}', + Name("Map(String, String))")), + ("Map(Int64, String)", + "('2020-01-01', map(12345,'there',12345,'over there'))", + '{"d":"2020-01-01","m":{"12345":"there","12345":"over there"}}', + Name("Map(Int64, String))")), +]) +def table_map_with_duplicated_keys(self, type, data, output): + """Check that map supports duplicated keys. + """ + insert_into_table(type=type, data=data, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0") +) +@Examples("map output", [ + ("map('hello','there','hello','over there')", "{'hello':'there','hello':'over there'}", Name("String")), + ("map(12345,'there',12345,'over there')", "{12345:'there',12345:'over there'}", Name("Integer")) +]) +def select_map_with_duplicated_keys(self, map, output): + """Check creating a map with duplicated keys. + """ + select_map(map=map, output=output) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0") +) +def select_map_key_not_found(self): + node = self.context.node + + with When("map is empty"): + node.query("SELECT map() AS m, m[1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("map has integer values"): + r = node.query("SELECT map(1,2) AS m, m[2] FORMAT Values") + with Then("zero should be returned for key that is not found"): + assert r.output == "({1:2},0)", error() + + with When("map has string values"): + r = node.query("SELECT map(1,'2') AS m, m[2] FORMAT Values") + with Then("empty string should be returned for key that is not found"): + assert r.output == "({1:'2'},'')", error() + + with When("map has array values"): + r = node.query("SELECT map(1,[2]) AS m, m[2] FORMAT Values") + with Then("empty array be returned for key that is not found"): + assert r.output == "({1:[2]},[])", error() + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0") +) +@Examples("type data select exitcode message", [ + ("Map(UInt8, UInt8), y Int8", "(y) VALUES (1)", "m[1] AS v", 0, '{"v":0}', Name("empty map")), + ("Map(UInt8, UInt8)", "VALUES (map(1,2))", "m[2] AS v", 0, '{"v":0}', Name("map has integer values")), + ("Map(UInt8, String)", "VALUES (map(1,'2'))", "m[2] AS v", 0, '{"v":""}', Name("map has string values")), + ("Map(UInt8, Array(Int8))", "VALUES (map(1,[2]))", "m[2] AS v", 0, '{"v":[]}', Name("map has array values")), +]) +def table_map_key_not_found(self, type, data, select, exitcode, message, order_by=None): + """Check values returned from a map column when key is not found. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0") +) +def invalid_key(self): + """Check when key is not valid. + """ + node = self.context.node + + with When("I try to use an integer key that is too large"): + node.query("SELECT map(1,2) AS m, m[256]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an integer key that is negative when key is unsigned"): + node.query("SELECT map(1,2) AS m, m[-1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use a string key when key is an integer"): + node.query("SELECT map(1,2) AS m, m['1']", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an integer key when key is a string"): + r = node.query("SELECT map('1',2) AS m, m[1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use an empty key when key is a string"): + r = node.query("SELECT map('1',2) AS m, m[]", exitcode=62, message="DB::Exception: Syntax error: failed at position") + + with When("I try to use wrong type conversion in key"): + r = node.query("SELECT map(1,2) AS m, m[toInt8('1')]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("in array of maps I try to use an integer key that is negative when key is unsigned"): + node.query("SELECT [map(1,2)] AS m, m[1][-1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + + with When("I try to use a NULL key when key is not nullable"): + r = node.query("SELECT map(1,2) AS m, m[NULL] FORMAT Values") + with Then("it should return NULL"): + assert r.output == "({1:2},NULL)", error() + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0") +) +@Examples("type data select exitcode message order_by", [ + ("Map(UInt8, UInt8)", "(map(1,2))", "m[256] AS v", 0, '{"v":0}', "m", Name("key too large)")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[-1] AS v", 0, '{"v":0}', "m", Name("key is negative")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("string when key is integer")), + ("Map(String, UInt8)", "(map('1',2))", "m[1] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("integer when key is string")), + ("Map(String, UInt8)", "(map('1',2))", "m[] AS v", 62, "DB::Exception: Syntax error: failed at position", "m", Name("empty when key is string")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[toInt8('1')] AS v", 0, '{"v":2}', "m", Name("wrong type conversion when key is integer")), + ("Map(String, UInt8)", "(map('1',2))", "m[toFixedString('1',1)] AS v", 0, '{"v":2}', "m", Name("wrong type conversion when key is string")), + ("Map(UInt8, UInt8)", "(map(1,2))", "m[NULL] AS v", 0, '{"v":null}', "m", Name("NULL key when key is not nullable")), + ("Array(Map(UInt8, UInt8))", "([map(1,2)])", "m[1]['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("string when key is integer in array of maps")), + ("Nested(x Map(UInt8, UInt8))", "([map(1,2)])", "m.x[1]['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m.x", Name("string when key is integer in nested map")), +]) +def table_map_invalid_key(self, type, data, select, exitcode, message, order_by="m"): + """Check selecting values from a map column using an invalid key. + """ + uid = getuid() + node = self.context.node + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} VALUES {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval("1.0") +) +@Examples("type data select filter exitcode message order_by", [ + ("Map(UInt8, UInt8)", "(map(1,1)),(map(1,2)),(map(2,3))", "m[1] AS v", "1=1 ORDER BY m[1]", 0, '{"v":0}\n{"v":1}\n{"v":2}', None, + Name("select the same key from all the rows")), + ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('e','f','c','d')", 0, '', None, + Name("filter rows by map having different pair order")), + ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('c','d','e','f')", 0, '{"m":{"c":"d","e":"f"}}', None, + Name("filter rows by map having the same pair order")), + ("Map(String, String)", "(map('a','b')),(map('e','f'))", "m", "m = map()", 43, 'DB::Exception: Illegal types of arguments ', None, + Name("filter rows by empty map")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1", 0, '{"m":{"a":1,"b":2}}', None, + Name("filter rows by map key value")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1 AND m['b'] = 2", 0, '{"m":{"a":1,"b":2}}', None, + Name("filter rows by map multiple key value combined with AND")), + ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1 OR m['b'] = 3", 0, '{"m":{"a":1,"b":2}}\n{"m":{"b":3}}', None, + Name("filter rows by map multiple key value combined with OR")), + ("Map(String, Array(Int8))", "(map('a',[])),(map('b',[1])),(map('c',[2]))", "m['b'] AS v", "m['b'] IN ([1],[2])", 0, '{"v":[1]}', None, + Name("filter rows by map array value using IN")), + ("Map(String, Nullable(String))", "(map('a',NULL)),(map('a',1))", "m", "isNull(m['a']) = 1", 0, '{"m":{"a":null}}', None, + Name("select map with nullable value")) +]) +def table_map_queries(self, type, data, select, filter, exitcode, message, order_by=None): + """Check retrieving map values and using maps in queries. + """ + uid = getuid() + node = self.context.node + + if order_by is None: + order_by = "m" + + with Given(f"table definition with {type}"): + sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + + with And(f"I create a table", description=sql): + table = create_table(name=uid, statement=sql) + + with When("I insert data into the map column"): + node.query(f"INSERT INTO {table} VALUES {data}") + + with And("I try to read from the table"): + node.query(f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing("1.0") +) +@Examples("type exitcode message", [ + ("Nullable(Map(String, String))", + 43, "DB::Exception: Nested type Map(String,String) cannot be inside Nullable type", + Name("nullable map")), + ("Map(Nothing, Nothing)", + 37, "DB::Exception: Column `m` with type Map(Nothing,Nothing) is not allowed in key expression, it's not comparable", + Name("map with nothing type for key and value")) +]) +def table_map_unsupported_types(self, type, exitcode, message): + """Check creating a table with unsupported map column types. + """ + uid = getuid() + node = self.context.node + + try: + with When(f"I create a table definition with {type}"): + sql = f"CREATE TABLE {uid} (m " + type + ") ENGINE = MergeTree() ORDER BY m" + node.query(sql, exitcode=exitcode, message=message) + finally: + with Finally("drop table if any"): + node.query(f"DROP TABLE IF EXISTS {uid}") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0") +) +@Examples("tuple type exitcode message", [ + ("([1, 2, 3], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 0, "{1:'Ready',2:'Steady',3:'Go'}", Name("int -> int")), + ("([1, 2, 3], ['Ready', 'Steady', 'Go'])", "Map(String, String)", + 0, "{'1':'Ready','2':'Steady','3':'Go'}", Name("int -> string")), + ("(['1', '2', '3'], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 0, "{1:'Ready',187:'Steady',143:'Go'}", Name("string -> int")), + ("([],[])", "Map(String, String)", + 0, "{}", Name("empty arrays to map str:str")), + ("([],[])", "Map(UInt8, Array(Int8))", + 0, "{}", Name("empty arrays to map uint8:array")), + ("([[1]],['hello'])", "Map(String, String)", + 0, "{'[1]':'hello'}", Name("array -> string")), + ("([(1,2),(3,4)])", "Map(UInt8, UInt8)", + 0, "{1:2,3:4}", Name("array of two tuples")), + ("([1, 2], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", + 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", + Name("unequal array sizes")), +]) +def cast_tuple_of_two_arrays_to_map(self, tuple, type, exitcode, message): + """Check casting Tuple(Array, Array) to a map type. + """ + node = self.context.node + + with When("I try to cast tuple", description=tuple): + node.query(f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0") +) +@Examples("tuple type exitcode message check_insert", [ + ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 0, '{"m":{1:"Ready",2:"Steady",3:"Go"}}', False, Name("int -> int")), + ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(String, String)", + 0, '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', False, Name("int -> string")), + ("((['1', '2', '3'], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 176, 'DB::Exception: Unknown codec family code', True, Name("string -> int")), + ("(([],[]))", "Map(String, String)", + 0, '{"m":{}}', False, Name("empty arrays to map str:str")), + ("(([],[]))", "Map(UInt8, Array(Int8))", + 0, '{"m":{}}', False, Name("empty arrays to map uint8:array")), + ("(([[1]],['hello']))", "Map(String, String)", + 53, 'DB::Exception: Type mismatch in IN or VALUES section', True, Name("array -> string")), + ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", + 0, '{"m":{1:2,3:4}}', False, Name("array of two tuples")), + ("(([1, 2], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", + 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", True, + Name("unequal array sizes")), +]) +def table_map_cast_tuple_of_arrays_to_map(self, tuple, type, exitcode, message, check_insert): + """Check converting Tuple(Array, Array) into map on insert into a map type column. + """ + table_map(type=type, data=tuple, select="*", filter="1=1", exitcode=exitcode, message=message, check_insert=check_insert) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") +) +@Examples("tuple type exitcode message", [ + ("([(1,2),(3,4)])", "Map(UInt8, UInt8)", 0, "{1:2,3:4}", + Name("array of two tuples")), + ("([(1,2),(3)])", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", + Name("not a tuple")), + ("([(1,2),(3,)])", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", + Name("invalid tuple")), +]) +def cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message): + """Check casting Array(Tuple(K,V)) to a map type. + """ + node = self.context.node + + with When("I try to cast tuple", description=tuple): + node.query(f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") +) +@Examples("tuple type exitcode message check_insert", [ + ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", 0, '{"m":{1:2,3:4}}', False, + Name("array of two tuples")), + ("(([(1,2),(3)]))", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", True, + Name("not a tuple")), + ("(([(1,2),(3,)]))", "Map(UInt8, UInt8)", 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", True, + Name("invalid tuple")), +]) +def table_map_cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message, check_insert): + """Check converting Array(Tuple(K,V),...) into map on insert into a map type column. + """ + table_map(type=type, data=tuple, select="*", filter="1=1", exitcode=exitcode, message=message, check_insert=check_insert) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap("1.0") +) +def subcolumns_keys_using_inline_defined_map(self): + node = self.context.node + exitcode = 47 + message = "DB::Exception: Missing columns: 'c.keys'" + + with When("I try to access keys sub-column using an inline defined map"): + node.query("SELECT map( 'aa', 4, '44' , 5) as c, c.keys", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap("1.0") +) +def subcolumns_values_using_inline_defined_map(self): + node = self.context.node + exitcode = 47 + message = "DB::Exception: Missing columns: 'c.values'" + + with When("I try to access values sub-column using an inline defined map"): + node.query("SELECT map( 'aa', 4, '44' , 5) as c, c.values", exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions("1.0") +) +@Examples("type data select filter exitcode message", [ + # keys + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.keys AS keys", "1=1", + 0, '{"keys":["a","c"]}\n{"keys":["e"]}', Name("select keys")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.keys AS keys", "has(m.keys, 'e')", + 0, '{"keys":["e"]}', Name("filter by using keys in an array function")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "has(m.keys, 'e') AS r", "1=1", + 0, '{"r":0}\n{"r":1}', Name("column that uses keys in an array function")), + # values + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.values AS values", "1=1", + 0, '{"values":["b","d"]}\n{"values":["f"]}', Name("select values")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.values AS values", "has(m.values, 'f')", + 0, '{"values":["f"]}', Name("filter by using values in an array function")), + ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "has(m.values, 'f') AS r", "1=1", + 0, '{"r":0}\n{"r":1}', Name("column that uses values in an array function")) +]) +def subcolumns(self, type, data, select, filter, exitcode, message, order_by=None): + """Check usage of sub-columns in queries. + """ + table_map(type=type, data=data, select=select, filter=filter, exitcode=exitcode, message=message, order_by=order_by) + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length("1.0") +) +def length(self): + """Check usage of length function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('a','b','c','d')),(map('e','f'))", + select="length(m) AS len, m", + filter="length(m) = 1", + exitcode=0, message='{"len":"1","m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty("1.0") +) +def empty(self): + """Check usage of empty function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('e','f'))", + select="empty(m) AS em, m", + filter="empty(m) <> 1", + exitcode=0, message='{"em":0,"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty("1.0") +) +def notempty(self): + """Check usage of notEmpty function with map data type. + """ + table_map(type="Map(String, String)", + data="(map('e','f'))", + select="notEmpty(m) AS em, m", + filter="notEmpty(m) = 1", + exitcode=0, message='{"em":1,"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd("1.0") +) +def cast_from_mapadd(self): + """Check converting the result of mapAdd function to a map data type. + """ + select_map(map="CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), 'Map(Int8, Int8)')", output="{1:2,2:2}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract("1.0") +) +def cast_from_mapsubstract(self): + """Check converting the result of mapSubstract function to a map data type. + """ + select_map(map="CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), 'Map(Int8, Int8)')", output="{1:-1,2:0}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries("1.0") +) +def cast_from_mappopulateseries(self): + """Check converting the result of mapPopulateSeries function to a map data type. + """ + select_map(map="CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), 'Map(Int8, Int8)')", output="{1:11,2:22,3:0,4:44,5:0}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains("1.0") +) +def mapcontains(self): + """Check usages of mapContains function with map data type. + """ + node = self.context.node + + with Example("key in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="mapContains(m, 'a')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("key not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT mapContains(m, 'a')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null key not in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="mapContains(m, NULL)", + exitcode=0, message='') + + with Example("null key in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="m", + filter="mapContains(m, NULL)", + exitcode=0, message='{null:"c"}') + + with Example("select nullable key"): + node.query("SELECT map(NULL, 1, 2, 3) AS m, mapContains(m, toNullable(toUInt8(2)))", exitcode=0, message="{2:3}") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys("1.0") +) +def mapkeys(self): + """Check usages of mapKeys function with map data type. + """ + with Example("key in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapKeys(m), 'a')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("key not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT has(mapKeys(m), 'a')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null key not in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapKeys(m), NULL)", + exitcode=0, message='') + + with Example("null key in map"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="m", + filter="has(mapKeys(m), NULL)", + exitcode=0, message='{"m":{null:"c"}}') + + with Example("select keys from column"): + table_map(type="Map(Nullable(String), String)", + data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", + select="mapKeys(m) AS keys", + filter="1 = 1", + exitcode=0, message='{"keys":["a"]}\n{"keys":["e"]}\n{"keys":[null]}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues("1.0") +) +def mapvalues(self): + """Check usages of mapValues function with map data type. + """ + with Example("value in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapValues(m), 'b')", + exitcode=0, message='{"m":{"a":"b"}}') + + with Example("value not in map"): + table_map(type="Map(String, String)", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="NOT has(mapValues(m), 'b')", + exitcode=0, message='{"m":{"e":"f"}}') + + with Example("null value not in map"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b'))", + select="m", + filter="has(mapValues(m), NULL)", + exitcode=0, message='') + + with Example("null value in map"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b')),(map('c',NULL))", + select="m", + filter="has(mapValues(m), NULL)", + exitcode=0, message='{"m":{"c":null}}') + + with Example("select values from column"): + table_map(type="Map(String, Nullable(String))", + data="(map('e','f')),(map('a','b')),(map('c',NULL))", + select="mapValues(m) AS values", + filter="1 = 1", + exitcode=0, message='{"values":["b"]}\n{"values":[null]}\n{"values":["f"]}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap("1.0") +) +def functions_with_inline_defined_map(self): + """Check that a map defined inline inside the select statement + can be used with functions that work with maps. + """ + with Example("mapKeys"): + select_map(map="map(1,2,3,4) as map, mapKeys(map) AS keys", output="{1:2,3:4}\t[1,3]") + + with Example("mapValyes"): + select_map(map="map(1,2,3,4) as map, mapValues(map) AS values", output="{1:2,3:4}\t[2,4]") + + with Example("mapContains"): + select_map(map="map(1,2,3,4) as map, mapContains(map, 1) AS contains", output="{1:2,3:4}\t1") + +@TestScenario +def empty_map(self): + """Check creating of an empty map `{}` using the map() function + when inserting data into a map type table column. + """ + table_map(type="Map(String, String)", + data="(map('e','f')),(map())", + select="m", + filter="1=1", + exitcode=0, message='{"m":{}}\n{"m":{"e":"f"}}') + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays("1.0") +) +def performance_vs_two_tuple_of_arrays(self, len=10, rows=6000000): + """Check performance of using map data type vs Tuple(Array, Array). + """ + uid = getuid() + node = self.context.node + + with Given(f"table with Tuple(Array(Int8),Array(Int8))"): + sql = "CREATE TABLE {name} (pairs Tuple(Array(Int8),Array(Int8))) ENGINE = MergeTree() ORDER BY pairs" + tuple_table = create_table(name=f"tuple_{uid}", statement=sql) + + with And(f"table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with tuples"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {tuple_table} SELECT ({keys},{values}) FROM numbers({rows})") + tuple_insert_time = time.time() - start_time + metric("tuple insert time", tuple_insert_time, "sec") + + with When("I insert data into table with a map"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with tuples"): + start_time = time.time() + node.query(f"SELECT sum(arrayFirst((v, k) -> k = {len-1}, tupleElement(pairs, 2), tupleElement(pairs, 1))) AS sum FROM {tuple_table}", + exitcode=0, message=f"{rows*(len-1)}") + tuple_select_time = time.time() - start_time + metric("tuple(array, array) select time", tuple_select_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + + metric("insert difference", (1 - map_insert_time/tuple_insert_time) * 100, "%") + metric("select difference", (1 - map_select_time/tuple_select_time) * 100, "%") + +@TestScenario +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples("1.0") +) +def performance_vs_array_of_tuples(self, len=10, rows=6000000): + """Check performance of using map data type vs Array(Tuple(K,V)). + """ + uid = getuid() + node = self.context.node + + with Given(f"table with Array(Tuple(K,V))"): + sql = "CREATE TABLE {name} (pairs Array(Tuple(Int8, Int8))) ENGINE = MergeTree() ORDER BY pairs" + array_table = create_table(name=f"tuple_{uid}", statement=sql) + + with And(f"table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with an array of tuples"): + pairs = list(zip(range(len),range(len))) + start_time = time.time() + node.query(f"INSERT INTO {array_table} SELECT ({pairs}) FROM numbers({rows})") + array_insert_time = time.time() - start_time + metric("array insert time", array_insert_time, "sec") + + with When("I insert data into table with a map"): + keys = range(len) + values = range(len) + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with an array of tuples"): + start_time = time.time() + node.query(f"SELECT sum(arrayFirst((v) -> v.1 = {len-1}, pairs).2) AS sum FROM {array_table}", + exitcode=0, message=f"{rows*(len-1)}") + array_select_time = time.time() - start_time + metric("array(tuple(k,v)) select time", array_select_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + + metric("insert difference", (1 - map_insert_time/array_insert_time) * 100, "%") + metric("select difference", (1 - map_select_time/array_select_time) * 100, "%") + +@TestScenario +def performance(self, len=10, rows=6000000): + """Check insert and select performance of using map data type. + """ + uid = getuid() + node = self.context.node + + with Given("table with Map(Int8,Int8)"): + sql = "CREATE TABLE {name} (pairs Map(Int8,Int8)) ENGINE = MergeTree() ORDER BY pairs" + map_table = create_table(name=f"map_{uid}", statement=sql) + + with When("I insert data into table with a map"): + values = [x for pair in zip(range(len),range(len)) for x in pair] + start_time = time.time() + node.query(f"INSERT INTO {map_table} SELECT (map({','.join([str(v) for v in values])})) FROM numbers({rows})") + map_insert_time = time.time() - start_time + metric("map insert time", map_insert_time, "sec") + + with And("I retrieve particular key value from table with map"): + start_time = time.time() + node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, message=f"{rows*(len-1)}") + map_select_time = time.time() - start_time + metric("map select time", map_select_time, "sec") + +# FIXME: add tests for different table engines + +@TestFeature +@Name("tests") +@Requirements( + RQ_SRS_018_ClickHouse_Map_DataType("1.0"), + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map("1.0") +) +def feature(self, node="clickhouse1"): + self.context.node = self.context.cluster.node(node) + + with Given("I allow experimental map type"): + allow_experimental_map_type() + + for scenario in loads(current_module(), Scenario): + scenario() From 4a085d7c2183794fc04dde6fac071f3826ce9de1 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 22 Feb 2021 15:06:45 -0500 Subject: [PATCH 002/512] Removing unused table types. --- tests/testflows/map_type/tests/common.py | 42 +----------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/tests/testflows/map_type/tests/common.py b/tests/testflows/map_type/tests/common.py index d4461062256..a3a0d0ef0b1 100644 --- a/tests/testflows/map_type/tests/common.py +++ b/tests/testflows/map_type/tests/common.py @@ -5,46 +5,6 @@ from testflows.core import * from testflows.core.name import basename, parentname from testflows._core.testtype import TestSubType -table_tuple = namedtuple("table_tuple", "create_statement cluster") - -table_types = { - "MergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, m Map(String, Int8)) ENGINE = MergeTree() PARTITION BY m ORDER BY d", None), - # "ReplacingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = ReplacingMergeTree() PARTITION BY y ORDER BY d", None), - # "SummingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) ENGINE = SummingMergeTree() PARTITION BY y ORDER BY d", None), - # "AggregatingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = AggregatingMergeTree() PARTITION BY y ORDER BY d", None), - # "CollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(sign) PARTITION BY y ORDER BY d", None), - # "VersionedCollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY y ORDER BY d", None), - # "GraphiteMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) ENGINE = GraphiteMergeTree('graphite_rollup_example') PARTITION BY y ORDER by d", None), - # "ReplicatedMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedReplacingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedReplacingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedSummingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ - # ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedSummingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ - # ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedAggregatingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedAggregatingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - # ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ - # ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ - # ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedVersionedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ - # ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedVersionedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ - # ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "one_shard_cluster"), - # "ReplicatedGraphiteMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ - # ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "sharded_cluster"), - # "ReplicatedGraphiteMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ - # ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "one_shard_cluster"), -} - def getuid(): if current().subtype == TestSubType.Example: testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" @@ -86,4 +46,4 @@ def create_table(self, name, statement, on_cluster=False): if on_cluster: node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster}") else: - node.query(f"DROP TABLE IF EXISTS {name}") \ No newline at end of file + node.query(f"DROP TABLE IF EXISTS {name}") From 8e770a850100ec1ae4a17a2883424461669723aa Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 23 Feb 2021 16:41:26 +0300 Subject: [PATCH 003/512] Update docker-compose.yml --- tests/testflows/map_type/docker-compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testflows/map_type/docker-compose/docker-compose.yml b/tests/testflows/map_type/docker-compose/docker-compose.yml index a3f5144c9ed..29f2ef52470 100755 --- a/tests/testflows/map_type/docker-compose/docker-compose.yml +++ b/tests/testflows/map_type/docker-compose/docker-compose.yml @@ -57,4 +57,4 @@ services: clickhouse3: condition: service_healthy zookeeper: - condition: service_healthy \ No newline at end of file + condition: service_healthy From 20da5e020ac9a751ff974d4077f41e6edbec9433 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Sun, 28 Feb 2021 22:27:22 +0300 Subject: [PATCH 004/512] DOCSUP-7086: Add s3 translation. --- .../engines/table-engines/integrations/s3.md | 14 +- .../engines/table-engines/integrations/s3.md | 158 ++++++++++++++++++ 2 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 docs/ru/engines/table-engines/integrations/s3.md diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 5858a0803e6..64c8698e370 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -26,19 +26,19 @@ ENGINE = S3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, **1.** Set up the `s3_engine_table` table: ``` sql -CREATE TABLE s3_engine_table (name String, value UInt32) ENGINE=S3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +CREATE TABLE s3_engine_table (name String, value UInt32) ENGINE=S3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip'); ``` **2.** Fill file: ``` sql -INSERT INTO s3_engine_table VALUES ('one', 1), ('two', 2), ('three', 3) +INSERT INTO s3_engine_table VALUES ('one', 1), ('two', 2), ('three', 3); ``` **3.** Query the data: ``` sql -SELECT * FROM s3_engine_table LIMIT 2 +SELECT * FROM s3_engine_table LIMIT 2; ``` ``` text @@ -83,19 +83,19 @@ Constructions with `{}` are similar to the [remote](../../../sql-reference/table ``` sql -CREATE TABLE table_with_range (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}', 'CSV') +CREATE TABLE table_with_range (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}', 'CSV'); ``` 3. Another way: ``` sql -CREATE TABLE table_with_question_mark (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_?', 'CSV') +CREATE TABLE table_with_question_mark (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_?', 'CSV'); ``` 4. Table consists of all the files in both directories (all files should satisfy format and schema described in query): ``` sql -CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV') +CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV'); ``` !!! warning "Warning" @@ -106,7 +106,7 @@ CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = S3('https: Create table with files named `file-000.csv`, `file-001.csv`, … , `file-999.csv`: ``` sql -CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV') +CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV'); ``` ## Virtual Columns {#virtual-columns} diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md new file mode 100644 index 00000000000..b0ef6b17f3a --- /dev/null +++ b/docs/ru/engines/table-engines/integrations/s3.md @@ -0,0 +1,158 @@ +--- +toc_priority: 4 +toc_title: S3 +--- + +# S3 {#table_engines-s3} + +Движок S3 обеспечивает интеграцию с экосистемой [Amazon S3](https://aws.amazon.com/s3/). Он похож на движок [HDFS](hdfs.md), но обладает специфическими для S3 функциями. + +## Использование движка {#ispolzovanie-dvizhka} + +``` sql +ENGINE = S3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compression]) +``` + +**Входные параметры** + +- `path` — URL бакета, включая путь к файлу. В режиме только для чтения поддерживаются следующие подстановочные знаки: `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, `’abc’`, `‘def’` — строки. +- `format` — [формат](../../../interfaces/formats.md#formats) файла. +- `structure` — структура таблицы. Указывается в формате `'column1_name column1_type, column2_name column2_type, ...'`. +- `compression` — опциональный параметр, необходимый, если требуется использовать определенный формат сжатия. Поддерживаемые значения: none, gzip/gz, brotli/br, xz/LZMA, zstd/zst. По умолчанию, it will autodetect compression by file extension. По умолчанию формат сжатия определяется автоматически исходя из расширения файла. + +**Примеры** + +**1.** Настройте таблицу `s3_engine_table`: + +``` sql +CREATE TABLE s3_engine_table (name String, value UInt32) ENGINE=S3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip'); +``` + +**2.** Заполните файл данными: + +``` sql +INSERT INTO s3_engine_table VALUES ('one', 1), ('two', 2), ('three', 3); +``` + +**3.** Прочитайте данные: + +Запрос: + +``` sql +SELECT * FROM s3_engine_table LIMIT 2; +``` + +Результат: + +``` text +┌─name─┬─value─┐ +│ one │ 1 │ +│ two │ 2 │ +└──────┴───────┘ +``` + +## Детали реализации {#detali-realizatsii} + +- поддерживается многопоточное чтение и запись. +- не поддерживается: + - использование операций `ALTER` и `SELECT...SAMPLE`; + - индексы; + - репликация. + +**Шаблоны в пути** + +Шаблоны могут содержаться в нескольких компонентах пути. Обрабатываются только существующие файлы, название которых целиком удовлетворяет шаблону (не только суффиксом или префиксом). Список файлов определяется во время выполнения запроса `SELECT` (а не во время `CREATE`). + +- `*` — заменяет любое количество любых символов кроме `/`, включая отсутствие символов. +- `?` — заменяет любой одиночный символ. +- `{some_string,another_string,yet_another_one}` — заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`. +- `{N..M}` — заменяет любое число в интервале от `N` до `M` включительно (может содержать ведущие нули, например: `000..078`). + +Конструкция с `{}` аналогична табличной функции [remote](../../../sql-reference/table-functions/remote.md). + +**Пример** + +1. Предположим, что у нас есть несколько файлов в формате TSV со следующими URL-адресами в HDFS: + +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_1.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_2.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_3.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_1.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_2.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_3.csv’ + +2. Существует несколько способов создать таблицу, объединяющую все шесть файлов: + +``` sql +CREATE TABLE table_with_range (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}', 'CSV'); +``` + +3. Еще один способ: + +``` sql +CREATE TABLE table_with_question_mark (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_?', 'CSV'); +``` + +4. Можно создать таблицу, содержащую все файлы из обоих директорий (все файлы должны соответствовать формату и схеме, описанным в запросе): + +``` sql +CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV'); +``` + +!!! warning "Предупреждение" + Если список файлов содержит диапазоны чисел с ведущими нулями, используйте конструкцию с фигурными скобками для каждой цифры отдельно или используйте `?`. + +**Пример** + +Создайте таблицу из файлов `file-000.csv`, `file-001.csv`, … , `file-999.csv`: + +``` sql +CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV'); +``` + +## Виртуальные столбцы {#virtualnye-stolbtsy} + +- `_path` — путь к файлу. +- `_file` — имя файла. + +**Смотрите также** + +- [Виртуальные столбцы](index.md#table_engines-virtual_columns) + +## Настройки, связанные с S3 {#settings} + +Следующие параметры могут быть указаны перед выполнением запроса или в конфигурационном файле: + +- `s3_max_single_part_upload_size` — значение по умолчанию `64Mb`. Максимальный размер объекта для одиночной загрузки в S3. +- `s3_min_upload_part_size` — значение по умолчанию `512Mb`. Минимальный размер части файла для [составной загрузки](https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). +- `s3_max_redirects` — значение по умолчанию `10`. Максимальное количество http-перенаправлений в запросе на хранилище S3. + +Соображения безопасности: чтобы избежать CSRF атак, когда злоумышленник может указать произвольные URL-адреса S3, следует установить `s3_max_redirects` значение `0`; или, обязательно указывать `remote_host_filter` в конфигурации сервера. + +## Настройки конечных точек {#endpointsettings} + +В файле конфигурации для определенной конечной точки (которая соответствует точному префиксу URL-адреса) можно указать следующие параметры: + +- `endpoint` — обязательный параметр, указывает префикс конечной точки. +- `access_key_id` and `secret_access_key` — опциональный параметр. Содержит учетные данные для использования с данной конечной точкой. +- `use_environment_credentials` — опциональный параметр, значение по умолчанию `false`. +Если установлено значение `true`, клиент S3 будет использовать учетные сведения из переменных сред и метаданных Amazon EC2, для данной конечной точки. +- `header` — опциональный параметр. Может быть указан несколько раз. Добавляет указанный HTTP-заголовок к запросу к заданной конечной точке. +- `server_side_encryption_customer_key_base64` — опциональный параметр. При его использовании, будут установлены необходимые заголовки для доступа к объектам S3 с помощью шифрования SSE-C. + +Пример: + +``` + + + https://storage.yandexcloud.net/my-test-bucket-768/ + + + + + + + +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/s3/) From bbdb66fb5828842e05ef2069ebd22ce4ac6cb836 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Sun, 28 Feb 2021 22:30:57 +0300 Subject: [PATCH 005/512] DOCSUP-7086: Add s3 translation. --- docs/ru/engines/table-engines/integrations/s3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md index b0ef6b17f3a..743d450f741 100644 --- a/docs/ru/engines/table-engines/integrations/s3.md +++ b/docs/ru/engines/table-engines/integrations/s3.md @@ -18,7 +18,7 @@ ENGINE = S3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, - `path` — URL бакета, включая путь к файлу. В режиме только для чтения поддерживаются следующие подстановочные знаки: `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, `’abc’`, `‘def’` — строки. - `format` — [формат](../../../interfaces/formats.md#formats) файла. - `structure` — структура таблицы. Указывается в формате `'column1_name column1_type, column2_name column2_type, ...'`. -- `compression` — опциональный параметр, необходимый, если требуется использовать определенный формат сжатия. Поддерживаемые значения: none, gzip/gz, brotli/br, xz/LZMA, zstd/zst. По умолчанию, it will autodetect compression by file extension. По умолчанию формат сжатия определяется автоматически исходя из расширения файла. +- `compression` — опциональный параметр, необходимый, если требуется использовать определенный формат сжатия. Поддерживаемые значения: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. По умолчанию формат сжатия определяется автоматически, исходя из расширения файла. **Примеры** From 34a8574920f77b7cb32887bcd8792de41222b808 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Sun, 28 Feb 2021 23:22:46 +0300 Subject: [PATCH 006/512] DOCSUP-7086: Add CH update translation. --- docs/en/operations/update.md | 7 +++---- .../ru/engines/table-engines/integrations/s3.md | 2 +- docs/ru/operations/update.md | 17 ++++++++++++++++- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/update.md b/docs/en/operations/update.md index 9fa9c44e130..df410e4c91e 100644 --- a/docs/en/operations/update.md +++ b/docs/en/operations/update.md @@ -15,7 +15,8 @@ $ sudo service clickhouse-server restart If you installed ClickHouse using something other than the recommended `deb` packages, use the appropriate update method. -ClickHouse does not support a distributed update. The operation should be performed consecutively on each separate server. Do not update all the servers on a cluster simultaneously, or the cluster will be unavailable for some time. +!!! warning "Warning" + ClickHouse does not support a distributed update. The operation should be performed consecutively on each separate server. Do not update all the servers on a cluster simultaneously, or the cluster will be unavailable for some time. The upgrade of older version of ClickHouse to specific version: @@ -29,6 +30,4 @@ $ sudo apt-get install clickhouse-server=xx.yy.a.b clickhouse-client=xx.yy.a.b c $ sudo service clickhouse-server restart ``` - - - +[Original article](https://clickhouse.tech/docs/en/operations/update/) diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md index 743d450f741..b4bd7f6b526 100644 --- a/docs/ru/engines/table-engines/integrations/s3.md +++ b/docs/ru/engines/table-engines/integrations/s3.md @@ -61,7 +61,7 @@ SELECT * FROM s3_engine_table LIMIT 2; **Шаблоны в пути** -Шаблоны могут содержаться в нескольких компонентах пути. Обрабатываются только существующие файлы, название которых целиком удовлетворяет шаблону (не только суффиксом или префиксом). Список файлов определяется во время выполнения запроса `SELECT` (а не во время `CREATE`). +Шаблоны могут содержаться в нескольких компонентах пути. Обрабатываются только существующие файлы, название которых целиком удовлетворяет шаблону (не только суффиксом или префиксом). Список файлов определяется во время выполнения запроса `SELECT` (не во время `CREATE`). - `*` — заменяет любое количество любых символов кроме `/`, включая отсутствие символов. - `?` — заменяет любой одиночный символ. diff --git a/docs/ru/operations/update.md b/docs/ru/operations/update.md index c74b28b3fd7..b0a8c7ef2f1 100644 --- a/docs/ru/operations/update.md +++ b/docs/ru/operations/update.md @@ -15,4 +15,19 @@ $ sudo service clickhouse-server restart Если ClickHouse установлен не из рекомендуемых deb-пакетов, используйте соответствующий метод обновления. -ClickHouse не поддерживает распределенное обновление. Операция должна выполняться последовательно на каждом отдельном сервере. Не обновляйте все серверы в кластере одновременно, иначе кластер становится недоступным в течение некоторого времени. +!!! warning "Предупреждение" + ClickHouse не поддерживает распределенное обновление. Операция должна выполняться последовательно на каждом отдельном сервере. Не обновляйте все серверы в кластере одновременно, иначе кластер становится недоступным в течение некоторого времени. + +Обновление старой версии ClickHouse до определенной версии: + +Пример: + +`xx.yy.a.b` - это текущая стабильная версия. Последнюю стабильную версию можно найти [здесь](https://github.com/ClickHouse/ClickHouse/releases) + +```bash +$ sudo apt-get update +$ sudo apt-get install clickhouse-server=xx.yy.a.b clickhouse-client=xx.yy.a.b clickhouse-common-static=xx.yy.a.b +$ sudo service clickhouse-server restart +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/update/) From 77a3b103ad9959db935175a318ddad63bb3d4dbb Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Sun, 28 Feb 2021 23:49:56 +0300 Subject: [PATCH 007/512] DOCSUP-7086: Add CH update translation. --- docs/en/engines/table-engines/integrations/s3.md | 2 +- docs/ru/engines/table-engines/integrations/s3.md | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 64c8698e370..bd15111e6b6 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -153,4 +153,4 @@ Example: ``` -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/s3/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/s3/) diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md index b4bd7f6b526..90b0ac43bb9 100644 --- a/docs/ru/engines/table-engines/integrations/s3.md +++ b/docs/ru/engines/table-engines/integrations/s3.md @@ -127,7 +127,7 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage. - `s3_min_upload_part_size` — значение по умолчанию `512Mb`. Минимальный размер части файла для [составной загрузки](https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). - `s3_max_redirects` — значение по умолчанию `10`. Максимальное количество http-перенаправлений в запросе на хранилище S3. -Соображения безопасности: чтобы избежать CSRF атак, когда злоумышленник может указать произвольные URL-адреса S3, следует установить `s3_max_redirects` значение `0`; или, обязательно указывать `remote_host_filter` в конфигурации сервера. +**Соображения безопасности**: чтобы избежать CSRF атак, когда злоумышленник может указать произвольные URL-адреса S3, следует установить `s3_max_redirects` значение `0`; или, обязательно указывать `remote_host_filter` в конфигурации сервера. ## Настройки конечных точек {#endpointsettings} @@ -135,8 +135,7 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage. - `endpoint` — обязательный параметр, указывает префикс конечной точки. - `access_key_id` and `secret_access_key` — опциональный параметр. Содержит учетные данные для использования с данной конечной точкой. -- `use_environment_credentials` — опциональный параметр, значение по умолчанию `false`. -Если установлено значение `true`, клиент S3 будет использовать учетные сведения из переменных сред и метаданных Amazon EC2, для данной конечной точки. +- `use_environment_credentials` — опциональный параметр, значение по умолчанию `false`. Если установлено значение `true`, клиент S3 будет использовать учетные сведения из переменных сред и метаданных Amazon EC2, для данной конечной точки. - `header` — опциональный параметр. Может быть указан несколько раз. Добавляет указанный HTTP-заголовок к запросу к заданной конечной точке. - `server_side_encryption_customer_key_base64` — опциональный параметр. При его использовании, будут установлены необходимые заголовки для доступа к объектам S3 с помощью шифрования SSE-C. @@ -155,4 +154,4 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage. ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/s3/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/s3/) From 16333e9f3a7cca295ac5156ec1eef318adbb6f05 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Mon, 1 Mar 2021 18:04:34 +0800 Subject: [PATCH 008/512] Suppport fuse aggregate functions:sum/avg/count to sumCount --- .../AggregateFunctionSumCount.cpp | 59 ++++ .../AggregateFunctionSumCount.h | 281 ++++++++++++++++++ .../registerAggregateFunctions.cpp | 2 + src/Core/Settings.h | 1 + src/Interpreters/GetAggregatesVisitor.h | 9 + src/Interpreters/TreeRewriter.cpp | 65 ++++ .../01744_fuse_sum_count_aggregate.reference | 6 + .../01744_fuse_sum_count_aggregate.sql | 8 + 8 files changed, 431 insertions(+) create mode 100644 src/AggregateFunctions/AggregateFunctionSumCount.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionSumCount.h create mode 100644 tests/queries/0_stateless/01744_fuse_sum_count_aggregate.reference create mode 100644 tests/queries/0_stateless/01744_fuse_sum_count_aggregate.sql diff --git a/src/AggregateFunctions/AggregateFunctionSumCount.cpp b/src/AggregateFunctions/AggregateFunctionSumCount.cpp new file mode 100644 index 00000000000..48770b629a8 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionSumCount.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include +#include "registerAggregateFunctions.h" + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +namespace +{ + +template +struct SumSimple +{ + /// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64 + using ResultType = std::conditional_t, + std::conditional_t, Decimal256, Decimal128>, + NearestFieldType>; + using AggregateDataType = AggregateFunctionSumCountData; + using Function = AggregateFunctionSumCount; +}; + + + +template using AggregateFunctionSumCountSimple = typename SumSimple::Function; + +template