thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into azat-jemalloc-MADV_DONTNEED-runtime-check

Browse Source
This commit is contained in:
Alexey Milovidov 2020-12-20 07:13:48 +03:00
commit d132f11e1a
47 changed files with 12467 additions and 2235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/engines/table-engines/mergetree-family/mergetree.md
View File

@ -98,6 +98,7 @@ For a description of parameters, see the [CREATE query description](../../../sql
- `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192.
- `storage_policy` — Storage policy. See [Using Multiple Block Devices for Data Storage](#table_engine-mergetree-multiple-volumes).
- `min_bytes_for_wide_part`, `min_rows_for_wide_part` — Minimum number of bytes/rows in a data part that can be stored in `Wide` format. You can set one, both or none of these settings. See [Data Storage](#mergetree-data-storage).
- `max_parts_in_total` — Maximum number of parts in all partitions.
**Example of Sections Setting**

10
docs/en/whats-new/roadmap.md
View File

@ -5,14 +5,6 @@ toc_title: Roadmap
# Roadmap {#roadmap}
## Q3 2020 {#q3-2020}
- High durability mode (`fsync` and WAL)
- Support spilling data to disk in `GLOBAL JOIN`
## Q4 2020 {#q4-2020}
- Improved efficiency of distributed queries
- Resource pools for more precise distribution of cluster capacity between users
The roadmap for year 2021 is published for open discussion [here](https://github.com/ClickHouse/ClickHouse/issues/17623).
{## [Original article](https://clickhouse.tech/docs/en/roadmap/) ##}

2176
docs/ru/whats-new/extended-roadmap.md
View File

File diff suppressed because it is too large Load Diff

1
docs/ru/whats-new/extended-roadmap.md Symbolic link
View File

@ -0,0 +1 @@
roadmap.md

2
tests/performance/encrypt_decrypt.xml
View File

@ -55,7 +55,7 @@
<substitution>
<name>table</name>
<values>
<value>numbers(200000)</value>
<value>numbers(2000000)</value>
</values>
</substitution>
<substitution>

49
tests/performance/encrypt_decrypt_slow.xml
View File

@ -1,49 +0,0 @@
<test>
<!-- "Strict" mode (key and iv length checks), non-empty plaintext.
Ciphers that are slow on every input are put here. -->
<substitutions>
<substitution>
<name>func</name>
<values>
<!-- decrypt + encrypt since it is really hard to compose decrypt-only case -->
<value>decrypt('aes-128-cfb1', encrypt('aes-128-cfb1', materialize(plaintext), key16, iv16), key16, iv16)</value>
<value>decrypt('aes-192-cfb1', encrypt('aes-192-cfb1', materialize(plaintext), key24, iv16), key24, iv16)</value>
<value>decrypt('aes-256-cfb1', encrypt('aes-256-cfb1', materialize(plaintext), key32, iv16), key32, iv16)</value>
</values>
</substitution>
<substitution>
<name>table_fast</name>
<values>
<value>numbers(50000)</value>
</values>
</substitution>
<substitution>
<name>table_slow</name>
<values>
<value>numbers(25000)</value>
</values>
</substitution>
<substitution>
<name>plaintext_fast</name>
<values>
<value>number</value>
<value>'paintext'</value>
</values>
</substitution>
<substitution>
<name>plaintext_slow</name>
<values>
<value>'\x12\x2B\xF9\x16\x93\xA4\xD6\x74\x22\xD9\x17\x5E\x38\xCD\x1D\x7B\xB0\x12\xEC\x43\x6B\xC7\x76\xFD\xA1\xA2\x4E\xFC\xBC\x19\x92\x3A\x12\x8B\xD4\xB3\x62\xA8\x9D\xBB\x3E\x0C\x08\x12\x67\x20\x7D\x02\x58\xCF\xE7\xD6\x06\xB8\xB0\x14\x0A\x70\xA1\x81\x94\x14\x24\x74'</value>
</values>
</substitution>
</substitutions>
<query>WITH {plaintext_fast} as plaintext, repeat('k', 32) as key32, substring(key32, 1, 24) as key24, substring(key32, 1, 16) as key16, repeat('iv', 8) as iv16, substring(iv16, 1, 12) as iv12 SELECT count() FROM {table_fast} WHERE NOT ignore({func})</query>
<query>WITH {plaintext_slow} as plaintext, repeat('k', 32) as key32, substring(key32, 1, 24) as key24, substring(key32, 1, 16) as key16, repeat('iv', 8) as iv16, substring(iv16, 1, 12) as iv12 SELECT count() FROM {table_slow} WHERE NOT ignore({func})</query>
</test>

6
tests/testflows/aes_encryption/configs/clickhouse/common.xml Normal file
View File

@ -0,0 +1,6 @@
<yandex>
<timezone>Europe/Moscow</timezone>
<listen_host replace="replace">0.0.0.0</listen_host>
<path>/var/lib/clickhouse/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
</yandex>

17
tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml Normal file
View File

@ -0,0 +1,17 @@
<yandex>
<shutdown_wait_unfinished>3</shutdown_wait_unfinished>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-server/log.log</log>
<errorlog>/var/log/clickhouse-server/log.err.log</errorlog>
<size>1000M</size>
<count>10</count>
<stderr>/var/log/clickhouse-server/stderr.log</stderr>
<stdout>/var/log/clickhouse-server/stdout.log</stdout>
</logger>
<part_log>
<database>system</database>
<table>part_log</table>
<flush_interval_milliseconds>500</flush_interval_milliseconds>
</part_log>
</yandex>

5
tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml Normal file
View File

@ -0,0 +1,5 @@
<?xml version="1.0"?>
<yandex>
<https_port>8443</https_port>
<tcp_port_secure>9440</tcp_port_secure>
</yandex>

107
tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml Normal file
View File

@ -0,0 +1,107 @@
<?xml version="1.0"?>
<yandex>
<remote_servers>
<replicated_cluster>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse1</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse2</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse3</host>
<port>9000</port>
</replica>
</shard>
</replicated_cluster>
<!--
<replicated_cluster_readonly>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse1</host>
<port>9000</port>
<user>readonly</user>
</replica>
<replica>
<host>clickhouse2</host>
<port>9000</port>
<user>readonly</user>
</replica>
<replica>
<host>clickhouse3</host>
<port>9000</port>
<user>readonly</user>
</replica>
</shard>
</replicated_cluster_readonly>
-->
<replicated_cluster_secure>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse1</host>
<port>9440</port>
<secure>1</secure>
</replica>
<replica>
<host>clickhouse2</host>
<port>9440</port>
<secure>1</secure>
</replica>
<replica>
<host>clickhouse3</host>
<port>9440</port>
<secure>1</secure>
</replica>
</shard>
</replicated_cluster_secure>
<sharded_cluster>
<shard>
<replica>
<host>clickhouse1</host>
<port>9000</port>
</replica>
</shard>
<shard>
<replica>
<host>clickhouse2</host>
<port>9000</port>
</replica>
</shard>
<shard>
<replica>
<host>clickhouse3</host>
<port>9000</port>
</replica>
</shard>
</sharded_cluster>
<sharded_cluster_secure>
<shard>
<replica>
<host>clickhouse1</host>
<port>9440</port>
<secure>1</secure>
</replica>
</shard>
<shard>
<replica>
<host>clickhouse2</host>
<port>9440</port>
<secure>1</secure>
</replica>
</shard>
<shard>
<replica>
<host>clickhouse3</host>
<port>9440</port>
<secure>1</secure>
</replica>
</shard>
</sharded_cluster_secure>
</remote_servers>
</yandex>

17
tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml Normal file
View File

@ -0,0 +1,17 @@
<yandex>
<openSSL>
<server>
<certificateFile>/etc/clickhouse-server/ssl/server.crt</certificateFile>
<privateKeyFile>/etc/clickhouse-server/ssl/server.key</privateKeyFile>
<verificationMode>none</verificationMode>
<cacheSessions>true</cacheSessions>
</server>
<client>
<cacheSessions>true</cacheSessions>
<verificationMode>none</verificationMode>
<invalidCertificateHandler>
<name>AcceptCertificateHandler</name>
</invalidCertificateHandler>
</client>
</openSSL>
</yandex>

20
tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml Normal file
View File

@ -0,0 +1,20 @@
<yandex>
<storage_configuration>
<disks>
<default>
<keep_free_space_bytes>1024</keep_free_space_bytes>
</default>
</disks>
<policies>
<default>
<volumes>
<default>
<disk>default</disk>
</default>
</volumes>
</default>
</policies>
</storage_configuration>
</yandex>

10
tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0"?>
<yandex>
<zookeeper>
<node index="1">
<host>zookeeper</host>
<port>2181</port>
</node>
<session_timeout_ms>15000</session_timeout_ms>
</zookeeper>
</yandex>

436
tests/testflows/aes_encryption/configs/clickhouse/config.xml Normal file
View File

@ -0,0 +1,436 @@
<?xml version="1.0"?>
<!--
NOTE: User and query level settings are set up in "users.xml" file.
-->
<yandex>
<logger>
<!-- Possible levels: https://github.com/pocoproject/poco/blob/develop/Foundation/include/Poco/Logger.h#L105 -->
<level>trace</level>
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
<size>1000M</size>
<count>10</count>
<!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) -->
</logger>
<!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<!-- For HTTPS and SSL over native protocol. -->
<!--
<https_port>8443</https_port>
<tcp_port_secure>9440</tcp_port_secure>
-->
<!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
<openSSL>
<server> <!-- Used for https server AND secure tcp port -->
<!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
<certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
<privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
<!-- openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 -->
<dhParamsFile>/etc/clickhouse-server/dhparam.pem</dhParamsFile>
<verificationMode>none</verificationMode>
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
</server>
<client> <!-- Used for connecting to https dictionary source -->
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
<!-- Use for self-signed: <verificationMode>none</verificationMode> -->
<invalidCertificateHandler>
<!-- Use for self-signed: <name>AcceptCertificateHandler</name> -->
<name>RejectCertificateHandler</name>
</invalidCertificateHandler>
</client>
</openSSL>
<!-- Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -->
<!--
<http_server_default_response><![CDATA[<html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>]]></http_server_default_response>
-->
<!-- Port for communication between replicas. Used for data exchange. -->
<interserver_http_port>9009</interserver_http_port>
<!-- Hostname that is used by other replicas to request this server.
If not specified, than it is determined analoguous to 'hostname -f' command.
This setting could be used to switch replication to another network interface.
-->
<!--
<interserver_http_host>example.yandex.ru</interserver_http_host>
-->
<!-- Listen specified host. use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. -->
<!-- <listen_host>::</listen_host> -->
<!-- Same for hosts with disabled ipv6: -->
<!-- <listen_host>0.0.0.0</listen_host> -->
<!-- Default values - try listen localhost on ipv4 and ipv6: -->
<!--
<listen_host>::1</listen_host>
<listen_host>127.0.0.1</listen_host>
-->
<!-- Don't exit if ipv6 or ipv4 unavailable, but listen_host with this protocol specified -->
<!-- <listen_try>0</listen_try> -->
<!-- Allow listen on same address:port -->
<!-- <listen_reuse_port>0</listen_reuse_port> -->
<!-- <listen_backlog>64</listen_backlog> -->
<max_connections>4096</max_connections>
<keep_alive_timeout>3</keep_alive_timeout>
<!-- Maximum number of concurrent queries. -->
<max_concurrent_queries>100</max_concurrent_queries>
<!-- Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve
correct maximum value. -->
<!-- <max_open_files>262144</max_open_files> -->
<!-- Size of cache of uncompressed blocks of data, used in tables of MergeTree family.
In bytes. Cache is single for server. Memory is allocated only on demand.
Cache is used when 'use_uncompressed_cache' user setting turned on (off by default).
Uncompressed cache is advantageous only for very short queries and in rare cases.
-->
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<!-- Approximate size of mark cache, used in tables of MergeTree family.
In bytes. Cache is single for server. Memory is allocated only on demand.
You should not lower this value.
-->
<mark_cache_size>5368709120</mark_cache_size>
<!-- Path to data directory, with trailing slash. -->
<path>/var/lib/clickhouse/</path>
<!-- Path to temporary data for processing hard queries. -->
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<!-- Directory with user provided files that are accessible by 'file' table function. -->
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<!-- Path to folder where users and roles created by SQL commands are stored. -->
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
<!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
<users_config>users.xml</users_config>
<!-- Default profile of settings. -->
<default_profile>default</default_profile>
<!-- System profile of settings. This settings are used by internal processes (Buffer storage, Distibuted DDL worker and so on). -->
<!-- <system_profile>default</system_profile> -->
<!-- Default database. -->
<default_database>default</default_database>
<!-- Server time zone could be set here.
Time zone is used when converting between String and DateTime types,
when printing DateTime in text formats and parsing DateTime from text,
it is used in date and time related functions, if specific time zone was not passed as an argument.
Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan.
If not specified, system time zone at server startup is used.
Please note, that server could display time zone alias instead of specified name.
Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC.
-->
<!-- <timezone>Europe/Moscow</timezone> -->
<!-- You can specify umask here (see "man umask"). Server will apply it on startup.
Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read).
-->
<!-- <umask>022</umask> -->
<!-- Perform mlockall after startup to lower first queries latency
and to prevent clickhouse executable from being paged out under high IO load.
Enabling this option is recommended but will lead to increased startup time for up to a few seconds.
-->
<mlock_executable>false</mlock_executable>
<!-- Configuration of clusters that could be used in Distributed tables.
https://clickhouse.yandex/docs/en/table_engines/distributed/
-->
<remote_servers incl="remote" >
<!-- Test only shard config for testing distributed storage -->
<test_shard_localhost>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
</test_shard_localhost>
<test_cluster_two_shards_localhost>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
</test_cluster_two_shards_localhost>
<test_shard_localhost_secure>
<shard>
<replica>
<host>localhost</host>
<port>9440</port>
<secure>1</secure>
</replica>
</shard>
</test_shard_localhost_secure>
<test_unavailable_shard>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
<shard>
<replica>
<host>localhost</host>
<port>1</port>
</replica>
</shard>
</test_unavailable_shard>
</remote_servers>
<!-- If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file.
By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element.
Values for substitutions are specified in /yandex/name_of_substitution elements in that file.
-->
<!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
Optional. If you don't use replicated tables, you could omit that.
See https://clickhouse.yandex/docs/en/table_engines/replication/
-->
<zookeeper incl="zookeeper" optional="true" />
<!-- Substitutions for parameters of replicated tables.
Optional. If you don't use replicated tables, you could omit that.
See https://clickhouse.yandex/docs/en/table_engines/replication/#creating-replicated-tables
-->
<macros incl="macros" optional="true" />
<!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. -->
<builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
<!-- Maximum session timeout, in seconds. Default: 3600. -->
<max_session_timeout>3600</max_session_timeout>
<!-- Default session timeout, in seconds. Default: 60. -->
<default_session_timeout>60</default_session_timeout>
<!-- Sending data to Graphite for monitoring. Several sections can be defined. -->
<!--
interval - send every X second
root_path - prefix for keys
hostname_in_path - append hostname to root_path (default = true)
metrics - send data from table system.metrics
events - send data from table system.events
asynchronous_metrics - send data from table system.asynchronous_metrics
-->
<!--
<graphite>
<host>localhost</host>
<port>42000</port>
<timeout>0.1</timeout>
<interval>60</interval>
<root_path>one_min</root_path>
<hostname_in_path>true</hostname_in_path>
<metrics>true</metrics>
<events>true</events>
<asynchronous_metrics>true</asynchronous_metrics>
</graphite>
<graphite>
<host>localhost</host>
<port>42000</port>
<timeout>0.1</timeout>
<interval>1</interval>
<root_path>one_sec</root_path>
<metrics>true</metrics>
<events>true</events>
<asynchronous_metrics>false</asynchronous_metrics>
</graphite>
-->
<!-- Query log. Used only for queries with setting log_queries = 1. -->
<query_log>
<!-- What table to insert data. If table is not exist, it will be created.
When query log structure is changed after system update,
then old table will be renamed and new table will be created automatically.
-->
<database>system</database>
<table>query_log</table>
<!--
PARTITION BY expr https://clickhouse.yandex/docs/en/table_engines/custom_partitioning_key/
Example:
event_date
toMonday(event_date)
toYYYYMM(event_date)
toStartOfHour(event_time)
-->
<partition_by>toYYYYMM(event_date)</partition_by>
<!-- Interval of flushing data. -->
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</query_log>
<!-- Trace log. Stores stack traces collected by query profilers.
See query_profiler_real_time_period_ns and query_profiler_cpu_time_period_ns settings. -->
<trace_log>
<database>system</database>
<table>trace_log</table>
<partition_by>toYYYYMM(event_date)</partition_by>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</trace_log>
<!-- Query thread log. Has information about all threads participated in query execution.
Used only for queries with setting log_query_threads = 1. -->
<query_thread_log>
<database>system</database>
<table>query_thread_log</table>
<partition_by>toYYYYMM(event_date)</partition_by>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</query_thread_log>
<!-- Uncomment if use part log.
Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads).
<part_log>
<database>system</database>
<table>part_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</part_log>
-->
<!-- Uncomment to write text log into table.
Text log contains all information from usual server log but stores it in structured and efficient way.
<text_log>
<database>system</database>
<table>text_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</text_log>
-->
<!-- Parameters for embedded dictionaries, used in Yandex.Metrica.
See https://clickhouse.yandex/docs/en/dicts/internal_dicts/
-->
<!-- Path to file with region hierarchy. -->
<!-- <path_to_regions_hierarchy_file>/opt/geo/regions_hierarchy.txt</path_to_regions_hierarchy_file> -->
<!-- Path to directory with files containing names of regions -->
<!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> -->
<!-- Configuration of external dictionaries. See:
https://clickhouse.yandex/docs/en/dicts/external_dicts/
-->
<dictionaries_config>*_dictionary.xml</dictionaries_config>
<!-- Uncomment if you want data to be compressed 30-100% better.
Don't do that if you just started using ClickHouse.
-->
<compression incl="compression">
<!--
<!- - Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. - ->
<case>
<!- - Conditions. All must be satisfied. Some conditions may be omitted. - ->
<min_part_size>10000000000</min_part_size> <!- - Min part size in bytes. - ->
<min_part_size_ratio>0.01</min_part_size_ratio> <!- - Min size of part relative to whole table size. - ->
<!- - What compression method to use. - ->
<method>zstd</method>
</case>
-->
</compression>
<!-- Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster.
Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -->
<distributed_ddl>
<!-- Path in ZooKeeper to queue with DDL queries -->
<path>/clickhouse/task_queue/ddl</path>
<!-- Settings from this profile will be used to execute DDL queries -->
<!-- <profile>default</profile> -->
</distributed_ddl>
<!-- Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h -->
<!--
<merge_tree>
<max_suspicious_broken_parts>5</max_suspicious_broken_parts>
</merge_tree>
-->
<!-- Protection from accidental DROP.
If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query.
If you want do delete one table and don't want to restart clickhouse-server, you could create special file <clickhouse-path>/flags/force_drop_table and make DROP once.
By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables.
The same for max_partition_size_to_drop.
Uncomment to disable protection.
-->
<!-- <max_table_size_to_drop>0</max_table_size_to_drop> -->
<!-- <max_partition_size_to_drop>0</max_partition_size_to_drop> -->
<!-- Example of parameters for GraphiteMergeTree table engine -->
<graphite_rollup_example>
<pattern>
<regexp>click_cost</regexp>
<function>any</function>
<retention>
<age>0</age>
<precision>3600</precision>
</retention>
<retention>
<age>86400</age>
<precision>60</precision>
</retention>
</pattern>
<default>
<function>max</function>
<retention>
<age>0</age>
<precision>60</precision>
</retention>
<retention>
<age>3600</age>
<precision>300</precision>
</retention>
<retention>
<age>86400</age>
<precision>3600</precision>
</retention>
</default>
</graphite_rollup_example>
<!-- Directory in <clickhouse-path> containing schema files for various input formats.
The directory will be created if it doesn't exist.
-->
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
<!-- Uncomment to disable ClickHouse internal DNS caching. -->
<!-- <disable_internal_dns_cache>1</disable_internal_dns_cache> -->
</yandex>

8
tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem Normal file
View File

@ -0,0 +1,8 @@
-----BEGIN DH PARAMETERS-----
MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR
XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5
Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb
9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf
slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb
dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg==
-----END DH PARAMETERS-----

19
tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt Normal file
View File

@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow
FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI
4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T
4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU
7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj
sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg
pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC
kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw
DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j
4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr
85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C
L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD
+UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L
P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp
0g==
-----END CERTIFICATE-----

28
tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key Normal file
View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn
DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L
OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus
TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7
JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F
KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6
ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP
TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f
7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN
MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5
212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu
plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr
/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu
Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa
CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g
1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5
bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq
cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez
on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K
GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54
nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU
2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh
vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa
dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr
vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1
KCBtmIrQLqzMpnBpLNuSY+Q=
-----END PRIVATE KEY-----

133
tests/testflows/aes_encryption/configs/clickhouse/users.xml Normal file
View File

@ -0,0 +1,133 @@
<?xml version="1.0"?>
<yandex>
<!-- Profiles of settings. -->
<profiles>
<!-- Default settings. -->
<default>
<!-- Maximum memory usage for processing single query, in bytes. -->
<max_memory_usage>10000000000</max_memory_usage>
<!-- Use cache of uncompressed blocks of data. Meaningful only for processing many of very short queries. -->
<use_uncompressed_cache>0</use_uncompressed_cache>
<!-- How to choose between replicas during distributed query processing.
random - choose random replica from set of replicas with minimum number of errors
nearest_hostname - from set of replicas with minimum number of errors, choose replica
with minimum number of different symbols between replica's hostname and local hostname
(Hamming distance).
in_order - first live replica is chosen in specified order.
first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
-->
<load_balancing>random</load_balancing>
</default>
<!-- Profile that allows only read queries. -->
<readonly>
<readonly>1</readonly>
</readonly>
</profiles>
<!-- Users and ACL. -->
<users>
<!-- If user name was not specified, 'default' user is used. -->
<default>
<!-- Password could be specified in plaintext or in SHA256 (in hex format).
If you want to specify password in plaintext (not recommended), place it in 'password' element.
Example: <password>qwerty</password>.
Password could be empty.
If you want to specify SHA256, place it in 'password_sha256_hex' element.
Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019).
If you want to specify double SHA1, place it in 'password_double_sha1_hex' element.
Example: <password_double_sha1_hex>e395796d6546b1b65db9d665cd43f0e858dd4303</password_double_sha1_hex>
How to generate decent password:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
In first line will be password and in second - corresponding SHA256.
How to generate double SHA1:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | openssl dgst -sha1 -binary | openssl dgst -sha1
In first line will be password and in second - corresponding double SHA1.
-->
<password></password>
<!-- List of networks with open access.
To open access from everywhere, specify:
<ip>::/0</ip>
To open access only from localhost, specify:
<ip>::1</ip>
<ip>127.0.0.1</ip>
Each element of list has one of the following forms:
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
<host> Hostname. Example: server01.yandex.ru.
To check access, DNS query is performed, and all received addresses compared to peer address.
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$
To check access, DNS PTR query is performed for peer address and then regexp is applied.
Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
Strongly recommended that regexp is ends with $
All results of DNS requests are cached till server restart.
-->
<networks incl="networks" replace="replace">
<ip>::/0</ip>
</networks>
<!-- Settings profile for user. -->
<profile>default</profile>
<!-- Quota for user. -->
<quota>default</quota>
<!-- Allow access management -->
<access_management>1</access_management>
<!-- Example of row level security policy. -->
<!-- <databases>
<test>
<filtered_table1>
<filter>a = 1</filter>
</filtered_table1>
<filtered_table2>
<filter>a + b &lt; 1 or c - d &gt; 5</filter>
</filtered_table2>
</test>
</databases> -->
</default>
<!-- Example of user with readonly access. -->
<!-- <readonly>
<password></password>
<networks incl="networks" replace="replace">
<ip>::1</ip>
<ip>127.0.0.1</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</readonly> -->
</users>
<!-- Quotas. -->
<quotas>
<!-- Name of quota. -->
<default>
<!-- Limits for time interval. You could specify many intervals with different limits. -->
<interval>
<!-- Length of interval. -->
<duration>3600</duration>
<!-- No limits. Just calculate resource usage for time interval. -->
<queries>0</queries>
<errors>0</errors>
<result_rows>0</result_rows>
<read_rows>0</read_rows>
<execution_time>0</execution_time>
</interval>
</default>
</quotas>
</yandex>

8
tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0"?>
<yandex>
<macros>
<replica>clickhouse1</replica>
<shard>01</shard>
<shard2>01</shard2>
</macros>
</yandex>

8
tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0"?>
<yandex>
<macros>
<replica>clickhouse2</replica>
<shard>01</shard>
<shard2>02</shard2>
</macros>
</yandex>

8
tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0"?>
<yandex>
<macros>
<replica>clickhouse3</replica>
<shard>01</shard>
<shard2>03</shard2>
</macros>
</yandex>

28
tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml Normal file
View File

@ -0,0 +1,28 @@
version: '2.3'
services:
clickhouse:
image: yandex/clickhouse-integration-test
expose:
- "9000"
- "9009"
- "8123"
volumes:
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml"
- "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse"
- "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge"
entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log"
healthcheck:
test: clickhouse client --query='select 1'
interval: 3s
timeout: 2s
retries: 40
start_period: 2s
cap_add:
- SYS_PTRACE
security_opt:
- label:disable

73
tests/testflows/aes_encryption/docker-compose/docker-compose.yml Normal file
View File

@ -0,0 +1,73 @@
version: '2.3'
services:
zookeeper:
extends:
file: zookeeper-service.yml
service: zookeeper
mysql1:
extends:
file: mysql-service.yml
service: mysql
hostname: mysql1
volumes:
- "${CLICKHOUSE_TESTS_DIR}/_instances/mysql1/database:/var/lib/mysql"
clickhouse1:
extends:
file: clickhouse-service.yml
service: clickhouse
hostname: clickhouse1
volumes:
- "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/"
- "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d"
depends_on:
zookeeper:
condition: service_healthy
clickhouse2:
extends:
file: clickhouse-service.yml
service: clickhouse
hostname: clickhouse2
volumes:
- "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/"
- "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d"
depends_on:
zookeeper:
condition: service_healthy
clickhouse3:
extends:
file: clickhouse-service.yml
service: clickhouse
hostname: clickhouse3
volumes:
- "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/"
- "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d"
- "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d"
depends_on:
zookeeper:
condition: service_healthy
# dummy service which does nothing, but allows to postpone
# 'docker-compose up -d' till all dependecies will go healthy
all_services_ready:
image: hello-world
depends_on:
mysql1:
condition: service_healthy
clickhouse1:
condition: service_healthy
clickhouse2:
condition: service_healthy
clickhouse3:
condition: service_healthy
zookeeper:
condition: service_healthy

19
tests/testflows/aes_encryption/docker-compose/mysql-service.yml Normal file
View File

@ -0,0 +1,19 @@
version: '2.3'
services:
mysql:
image: mysql:5.7.30
restart: always
environment:
MYSQL_DATABASE: 'db'
MYSQL_USER: 'user'
MYSQL_PASSWORD: 'password'
MYSQL_ROOT_PASSWORD: 'password'
expose:
- '3306'
healthcheck:
test: mysql -D db -u user --password=password -e "select 1;"
interval: 3s
timeout: 2s
retries: 40
start_period: 2s

18
tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml Normal file
View File

@ -0,0 +1,18 @@
version: '2.3'
services:
zookeeper:
image: zookeeper:3.4.12
expose:
- "2181"
environment:
ZOO_TICK_TIME: 500
ZOO_MY_ID: 1
healthcheck:
test: echo stat | nc localhost 2181
interval: 3s
timeout: 2s
retries: 5
start_period: 2s
security_opt:
- label:disable

76
tests/testflows/aes_encryption/regression.py Executable file
View File

@ -0,0 +1,76 @@
#!/usr/bin/env python3
import sys
from testflows.core import *
append_path(sys.path, "..")
from helpers.cluster import Cluster
from helpers.argparser import argparser
from aes_encryption.requirements import *
issue_18249 = "https://github.com/ClickHouse/ClickHouse/issues/18249"
issue_18250 = "https://github.com/ClickHouse/ClickHouse/issues/18250"
issue_18251 = "https://github.com/ClickHouse/ClickHouse/issues/18251"
xfails = {
# encrypt
"encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too short":
[(Fail, "known issue")],
"encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too long":
[(Fail, "known issue")],
# encrypt_mysql
"encrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None":
[(Fail, issue_18251)],
"encrypt_mysql/invalid parameters/iv not valid for mode":
[(Fail, issue_18251)],
# decrypt_mysql
"decrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None:":
[(Fail, issue_18251)],
# compatibility
"compatibility/insert/encrypt using materialized view/:":
[(Fail, issue_18249)],
"compatibility/insert/decrypt using materialized view/:":
[(Error, issue_18249)],
"compatibility/insert/aes encrypt mysql using materialized view/:":
[(Fail, issue_18249)],
"compatibility/insert/aes decrypt mysql using materialized view/:":
[(Error, issue_18249)],
"compatibility/select/decrypt unique":
[(Fail, issue_18249)],
"compatibility/mysql/:engine/decrypt/mysql_datatype='TEXT'/:":
[(Fail, issue_18250)],
"compatibility/mysql/:engine/decrypt/mysql_datatype='VARCHAR(100)'/:":
[(Fail, issue_18250)],
"compatibility/mysql/:engine/encrypt/mysql_datatype='TEXT'/:":
[(Fail, issue_18250)],
"compatibility/mysql/:engine/encrypt/mysql_datatype='VARCHAR(100)'/:":
[(Fail, issue_18250)]
}
@TestFeature
@Name("aes encryption")
@ArgumentParser(argparser)
@Specifications(SRS_008_ClickHouse_AES_Encryption_Functions)
@Requirements(
RQ_SRS008_AES_Functions("1.0"),
RQ_SRS008_AES_Functions_DifferentModes("1.0")
)
@XFails(xfails)
def regression(self, local, clickhouse_binary_path, stress=None, parallel=None):
"""ClickHouse AES encryption functions regression module.
"""
nodes = {
"clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"),
}
with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster:
self.context.cluster = cluster
Feature(run=load("aes_encryption.tests.encrypt", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.decrypt", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.encrypt_mysql", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.decrypt_mysql", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.compatibility.feature", "feature"), flags=TE)
if main():
regression()

1
tests/testflows/aes_encryption/requirements/__init__.py Normal file
View File

@ -0,0 +1 @@
from .requirements import *

934
tests/testflows/aes_encryption/requirements/requirements.md Normal file
View File

@ -0,0 +1,934 @@
# SRS-008 ClickHouse AES Encryption Functions
# Software Requirements Specification
## Table of Contents
* 1 [Revision History](#revision-history)
* 2 [Introduction](#introduction)
* 3 [Terminology](#terminology)
* 4 [Requirements](#requirements)
* 4.1 [Generic](#generic)
* 4.1.1 [RQ.SRS008.AES.Functions](#rqsrs008aesfunctions)
* 4.1.2 [RQ.SRS008.AES.Functions.Compatability.MySQL](#rqsrs008aesfunctionscompatabilitymysql)
* 4.1.3 [RQ.SRS008.AES.Functions.Compatability.Dictionaries](#rqsrs008aesfunctionscompatabilitydictionaries)
* 4.1.4 [RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL](#rqsrs008aesfunctionscompatabilityenginedatabasemysql)
* 4.1.5 [RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL](#rqsrs008aesfunctionscompatabilityenginetablemysql)
* 4.1.6 [RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL](#rqsrs008aesfunctionscompatabilitytablefunctionmysql)
* 4.1.7 [RQ.SRS008.AES.Functions.DifferentModes](#rqsrs008aesfunctionsdifferentmodes)
* 4.1.8 [RQ.SRS008.AES.Functions.DataFromMultipleSources](#rqsrs008aesfunctionsdatafrommultiplesources)
* 4.1.9 [RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues](#rqsrs008aesfunctionssuppressoutputofsensitivevalues)
* 4.1.10 [RQ.SRS008.AES.Functions.InvalidParameters](#rqsrs008aesfunctionsinvalidparameters)
* 4.1.11 [RQ.SRS008.AES.Functions.Mismatched.Key](#rqsrs008aesfunctionsmismatchedkey)
* 4.1.12 [RQ.SRS008.AES.Functions.Mismatched.IV](#rqsrs008aesfunctionsmismatchediv)
* 4.1.13 [RQ.SRS008.AES.Functions.Mismatched.AAD](#rqsrs008aesfunctionsmismatchedaad)
* 4.1.14 [RQ.SRS008.AES.Functions.Mismatched.Mode](#rqsrs008aesfunctionsmismatchedmode)
* 4.1.15 [RQ.SRS008.AES.Functions.Check.Performance](#rqsrs008aesfunctionscheckperformance)
* 4.1.16 [RQ.SRS008.AES.Function.Check.Performance.BestCase](#rqsrs008aesfunctioncheckperformancebestcase)
* 4.1.17 [RQ.SRS008.AES.Function.Check.Performance.WorstCase](#rqsrs008aesfunctioncheckperformanceworstcase)
* 4.1.18 [RQ.SRS008.AES.Functions.Check.Compression](#rqsrs008aesfunctionscheckcompression)
* 4.1.19 [RQ.SRS008.AES.Functions.Check.Compression.LowCardinality](#rqsrs008aesfunctionscheckcompressionlowcardinality)
* 4.2 [Specific](#specific)
* 4.2.1 [RQ.SRS008.AES.Encrypt.Function](#rqsrs008aesencryptfunction)
* 4.2.2 [RQ.SRS008.AES.Encrypt.Function.Syntax](#rqsrs008aesencryptfunctionsyntax)
* 4.2.3 [RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors](#rqsrs008aesencryptfunctionnisttestvectors)
* 4.2.4 [RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText](#rqsrs008aesencryptfunctionparametersplaintext)
* 4.2.5 [RQ.SRS008.AES.Encrypt.Function.Parameters.Key](#rqsrs008aesencryptfunctionparameterskey)
* 4.2.6 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode](#rqsrs008aesencryptfunctionparametersmode)
* 4.2.7 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesencryptfunctionparametersmodevaluesformat)
* 4.2.8 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesencryptfunctionparametersmodevalueinvalid)
* 4.2.9 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Values](#rqsrs008aesencryptfunctionparametersmodevalues)
* 4.2.10 [RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesencryptfunctionparametersinitializationvector)
* 4.2.11 [RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesencryptfunctionparametersadditionalauthenticateddata)
* 4.2.12 [RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesencryptfunctionparametersreturnvalue)
* 4.2.13 [RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesencryptfunctionkeylengthinvalidlengtherror)
* 4.2.14 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesencryptfunctioninitializationvectorlengthinvalidlengtherror)
* 4.2.15 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesencryptfunctioninitializationvectornotvalidformode)
* 4.2.16 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesencryptfunctionadditionalauthenticationdatanotvalidformode)
* 4.2.17 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesencryptfunctionadditionalauthenticationdatalength)
* 4.2.18 [RQ.SRS008.AES.Encrypt.Function.NonGCMMode.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionnongcmmodekeyandinitializationvectorlength)
* 4.2.19 [RQ.SRS008.AES.Encrypt.Function.GCMMode.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctiongcmmodekeyandinitializationvectorlength)
* 4.2.20 [RQ.SRS008.AES.Decrypt.Function](#rqsrs008aesdecryptfunction)
* 4.2.21 [RQ.SRS008.AES.Decrypt.Function.Syntax](#rqsrs008aesdecryptfunctionsyntax)
* 4.2.22 [RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText](#rqsrs008aesdecryptfunctionparametersciphertext)
* 4.2.23 [RQ.SRS008.AES.Decrypt.Function.Parameters.Key](#rqsrs008aesdecryptfunctionparameterskey)
* 4.2.24 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode](#rqsrs008aesdecryptfunctionparametersmode)
* 4.2.25 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesdecryptfunctionparametersmodevaluesformat)
* 4.2.26 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesdecryptfunctionparametersmodevalueinvalid)
* 4.2.27 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Values](#rqsrs008aesdecryptfunctionparametersmodevalues)
* 4.2.28 [RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesdecryptfunctionparametersinitializationvector)
* 4.2.29 [RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesdecryptfunctionparametersadditionalauthenticateddata)
* 4.2.30 [RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesdecryptfunctionparametersreturnvalue)
* 4.2.31 [RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesdecryptfunctionkeylengthinvalidlengtherror)
* 4.2.32 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesdecryptfunctioninitializationvectorlengthinvalidlengtherror)
* 4.2.33 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesdecryptfunctioninitializationvectornotvalidformode)
* 4.2.34 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesdecryptfunctionadditionalauthenticationdatanotvalidformode)
* 4.2.35 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesdecryptfunctionadditionalauthenticationdatalength)
* 4.2.36 [RQ.SRS008.AES.Decrypt.Function.NonGCMMode.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionnongcmmodekeyandinitializationvectorlength)
* 4.2.37 [RQ.SRS008.AES.Decrypt.Function.GCMMode.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctiongcmmodekeyandinitializationvectorlength)
* 4.3 [MySQL Specific Functions](#mysql-specific-functions)
* 4.3.1 [RQ.SRS008.AES.MySQL.Encrypt.Function](#rqsrs008aesmysqlencryptfunction)
* 4.3.2 [RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax](#rqsrs008aesmysqlencryptfunctionsyntax)
* 4.3.3 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText](#rqsrs008aesmysqlencryptfunctionparametersplaintext)
* 4.3.4 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key](#rqsrs008aesmysqlencryptfunctionparameterskey)
* 4.3.5 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode](#rqsrs008aesmysqlencryptfunctionparametersmode)
* 4.3.6 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqlencryptfunctionparametersmodevaluesformat)
* 4.3.7 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqlencryptfunctionparametersmodevalueinvalid)
* 4.3.8 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values](#rqsrs008aesmysqlencryptfunctionparametersmodevalues)
* 4.3.9 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevaluesgcmerror)
* 4.3.10 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevaluesctrerror)
* 4.3.11 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqlencryptfunctionparametersinitializationvector)
* 4.3.12 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqlencryptfunctionparametersreturnvalue)
* 4.3.13 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqlencryptfunctionkeylengthtooshorterror)
* 4.3.14 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqlencryptfunctionkeylengthtoolong)
* 4.3.15 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtooshorterror)
* 4.3.16 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtoolong)
* 4.3.17 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqlencryptfunctioninitializationvectornotvalidformode)
* 4.3.18 [RQ.SRS008.AES.MySQL.Encrypt.Function.Mode.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionmodekeyandinitializationvectorlength)
* 4.3.19 [RQ.SRS008.AES.MySQL.Decrypt.Function](#rqsrs008aesmysqldecryptfunction)
* 4.3.20 [RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax](#rqsrs008aesmysqldecryptfunctionsyntax)
* 4.3.21 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText](#rqsrs008aesmysqldecryptfunctionparametersciphertext)
* 4.3.22 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key](#rqsrs008aesmysqldecryptfunctionparameterskey)
* 4.3.23 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode](#rqsrs008aesmysqldecryptfunctionparametersmode)
* 4.3.24 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqldecryptfunctionparametersmodevaluesformat)
* 4.3.25 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqldecryptfunctionparametersmodevalueinvalid)
* 4.3.26 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values](#rqsrs008aesmysqldecryptfunctionparametersmodevalues)
* 4.3.27 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevaluesgcmerror)
* 4.3.28 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevaluesctrerror)
* 4.3.29 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqldecryptfunctionparametersinitializationvector)
* 4.3.30 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqldecryptfunctionparametersreturnvalue)
* 4.3.31 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqldecryptfunctionkeylengthtooshorterror)
* 4.3.32 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqldecryptfunctionkeylengthtoolong)
* 4.3.33 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtooshorterror)
* 4.3.34 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtoolong)
* 4.3.35 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqldecryptfunctioninitializationvectornotvalidformode)
* 4.3.36 [RQ.SRS008.AES.MySQL.Decrypt.Function.Mode.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionmodekeyandinitializationvectorlength)
* 5 [References](#references)
## Revision History
This document is stored in an electronic form using [Git] source control management software
hosted in a [GitHub Repository].
All the updates are tracked using the [Revision History].
## Introduction
Users need an ability to encrypt and decrypt column data with tenant specific keys.
Use cases include protection of sensitive column values and [GDPR] right to forget policies.
The implementation will support capabilities of the [MySQL aes_encrypt] and [MySQL aes_decrypt]
functions which encrypt and decrypt values using the [AES] (Advanced Encryption Standard)
algorithm. This functionality will enable encryption and decryption of data
accessed on remote [MySQL] servers via [MySQL Dictionary] or [MySQL Database Engine],
[MySQL Table Engine], or [MySQL Table Function].
## Terminology
* **AES** -
Advanced Encryption Standard ([AES])
## Requirements
### Generic
#### RQ.SRS008.AES.Functions
version: 1.0
[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data.
#### RQ.SRS008.AES.Functions.Compatability.MySQL
version: 1.0
[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7].
#### RQ.SRS008.AES.Functions.Compatability.Dictionaries
version: 1.0
[ClickHouse] SHALL support encryption and decryption of data accessed on remote
[MySQL] servers using [MySQL Dictionary].
#### RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL
version: 1.0
[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine],
#### RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL
version: 1.0
[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine].
#### RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL
version: 1.0
[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function].
#### RQ.SRS008.AES.Functions.DifferentModes
version: 1.0
[ClickHouse] SHALL allow different modes to be supported in a single SQL statement
using explicit function parameters.
#### RQ.SRS008.AES.Functions.DataFromMultipleSources
version: 1.0
[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources
in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary],
[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function]
with possibly different encryption schemes.
#### RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues
version: 1.0
[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log,
error log, and `query_log` table to prevent leakage of sensitive values.
#### RQ.SRS008.AES.Functions.InvalidParameters
version: 1.0
[ClickHouse] SHALL return an error when parameters are invalid.
#### RQ.SRS008.AES.Functions.Mismatched.Key
version: 1.0
[ClickHouse] SHALL return garbage for mismatched keys.
#### RQ.SRS008.AES.Functions.Mismatched.IV
version: 1.0
[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it.
#### RQ.SRS008.AES.Functions.Mismatched.AAD
version: 1.0
[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it.
#### RQ.SRS008.AES.Functions.Mismatched.Mode
version: 1.0
[ClickHouse] SHALL return an error or garbage for mismatched mode.
#### RQ.SRS008.AES.Functions.Check.Performance
version: 1.0
Performance of [AES] encryption functions SHALL be measured.
#### RQ.SRS008.AES.Function.Check.Performance.BestCase
version: 1.0
Performance of [AES] encryption functions SHALL be checked for the best case
scenario where there is one key, one initialization vector, and one large stream of data.
#### RQ.SRS008.AES.Function.Check.Performance.WorstCase
version: 1.0
Performance of [AES] encryption functions SHALL be checked for the worst case
where there are `N` keys, `N` initialization vectors and `N` very small streams of data.
#### RQ.SRS008.AES.Functions.Check.Compression
version: 1.0
Effect of [AES] encryption on column compression SHALL be measured.
#### RQ.SRS008.AES.Functions.Check.Compression.LowCardinality
version: 1.0
Effect of [AES] encryption on the compression of a column with [LowCardinality] data type
SHALL be measured.
### Specific
#### RQ.SRS008.AES.Encrypt.Function
version: 1.0
[ClickHouse] SHALL support `encrypt` function to encrypt data using [AES].
#### RQ.SRS008.AES.Encrypt.Function.Syntax
version: 1.0
[ClickHouse] SHALL support the following syntax for the `encrypt` function
```sql
encrypt(mode, plaintext, key, [iv, aad])
```
#### RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors
version: 1.0
[ClickHouse] `encrypt` function output SHALL produce output that matches [NIST test vectors].
#### RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText
version: 1.0
[ClickHouse] SHALL support `plaintext` accepting any data type as
the first parameter to the `encrypt` function that SHALL specify the data to be encrypted.
#### RQ.SRS008.AES.Encrypt.Function.Parameters.Key
version: 1.0
[ClickHouse] SHALL support `key` with `String` or `FixedString` data types
as the second parameter to the `encrypt` function that SHALL specify the encryption key.
#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode
version: 1.0
[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter
to the `encrypt` function that SHALL specify encryption key length and block encryption mode.
#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat
version: 1.0
[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter
of the `encrypt` function where
the `key_length` SHALL specifies the length of the key and SHALL accept
`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption
mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB] as well as
[CTR] and [GCM] as the values. For example, `aes-256-ofb`.
#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid
version: 1.0
[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `encrypt`
function is not valid with the exception where such a mode is supported by the underlying
[OpenSSL] implementation.
#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Values
version: 1.0
[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter
of the `encrypt` function:
* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key
* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key
* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key
* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key
* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key
* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key
* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key
* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key
* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key
* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key
* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key
* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key
* `aes-128-gcm` that SHALL use [GCM] block mode encryption with 128 bit key
and `AEAD` 16-byte tag is appended to the resulting ciphertext according to
the [RFC5116]
* `aes-192-gcm` that SHALL use [GCM] block mode encryption with 192 bit key
and `AEAD` 16-byte tag is appended to the resulting ciphertext according to
the [RFC5116]
* `aes-256-gcm` that SHALL use [GCM] block mode encryption with 256 bit key
and `AEAD` 16-byte tag is appended to the resulting ciphertext according to
the [RFC5116]
* `aes-128-ctr` that SHALL use [CTR] block mode encryption with 128 bit key
* `aes-192-ctr` that SHALL use [CTR] block mode encryption with 192 bit key
* `aes-256-ctr` that SHALL use [CTR] block mode encryption with 256 bit key
#### RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector
version: 1.0
[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth
parameter to the `encrypt` function that SHALL specify the initialization vector for block modes that require
it.
#### RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData
version: 1.0
[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth
parameter to the `encrypt` function that SHALL specify the additional authenticated data
for block modes that require it.
#### RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue
version: 1.0
[ClickHouse] SHALL return the encrypted value of the data
using `String` data type as the result of `encrypt` function.
#### RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError
version: 1.0
[ClickHouse] SHALL return an error if the `key` length is not exact for the `encrypt` function for a given block mode.
#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError
version: 1.0
[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `encrypt` function for a given block mode.
#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode
version: 1.0
[ClickHouse] SHALL return an error if the `iv` is specified for the `encrypt` function for a mode that does not need it.
#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode
version: 1.0
[ClickHouse] SHALL return an error if the `aad` is specified for the `encrypt` function for a mode that does not need it.
#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length
version: 1.0
[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `encrypt` function.
#### RQ.SRS008.AES.Encrypt.Function.NonGCMMode.KeyAndInitializationVector.Length
version: 1.0
[ClickHouse] SHALL return an error when the `encrypt` function is called with the following parameter values
when using non-GCM modes
* `aes-128-ecb` mode and `key` is not 16 bytes or `iv` or `aad` is specified
* `aes-192-ecb` mode and `key` is not 24 bytes or `iv` or `aad` is specified
* `aes-256-ecb` mode and `key` is not 32 bytes or `iv` or `aad` is specified
* `aes-128-cbc` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-192-cbc` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-cbc` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-cfb1` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-192-cfb1` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-cfb1` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-cfb8` mode and `key` is not 16 bytes and if specified `iv` is not 16 bytes
* `aes-192-cfb8` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-cfb8` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-cfb128` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-192-cfb128` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-cfb128` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-ofb` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-192-ofb` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-ofb` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-ctr` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes
* `aes-192-ctr` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes
* `aes-256-ctr` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes
#### RQ.SRS008.AES.Encrypt.Function.GCMMode.KeyAndInitializationVector.Length
version: 1.0
[ClickHouse] SHALL return an error when the `encrypt` function is called with the following parameter values
when using GCM modes
* `aes-128-gcm` mode and `key` is not 16 bytes or `iv` is not specified
* `aes-192-gcm` mode and `key` is not 24 bytes or `iv` is not specified
* `aes-256-gcm` mode and `key` is not 32 bytes or `iv` is not specified
#### RQ.SRS008.AES.Decrypt.Function
version: 1.0
[ClickHouse] SHALL support `decrypt` function to decrypt data using [AES].
#### RQ.SRS008.AES.Decrypt.Function.Syntax
version: 1.0
[ClickHouse] SHALL support the following syntax for the `decrypt` function
```sql
decrypt(mode, ciphertext, key, [iv, aad])
```
#### RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText
version: 1.0
[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as
the first parameter to the `decrypt` function that SHALL specify the data to be decrypted.
#### RQ.SRS008.AES.Decrypt.Function.Parameters.Key
version: 1.0
[ClickHouse] SHALL support `key` with `String` or `FixedString` data types
as the second parameter to the `decrypt` function that SHALL specify the encryption key.
#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode
version: 1.0
[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter
to the `decrypt` function that SHALL specify encryption key length and block encryption mode.
#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat
version: 1.0
[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter
of the `decrypt` function where
the `key_length` SHALL specifies the length of the key and SHALL accept
`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption
mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB] as well as
[CTR] and [GCM] as the values. For example, `aes-256-ofb`.
#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid
version: 1.0
[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `decrypt`
function is not valid with the exception where such a mode is supported by the underlying
[OpenSSL] implementation.
#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Values
version: 1.0
[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter
of the `decrypt` function:
* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key
* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key
* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key
* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key
* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key
* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key
* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key
* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key
* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key
* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key
* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key
* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key
* `aes-128-gcm` that SHALL use [GCM] block mode encryption with 128 bit key
and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to
the [RFC5116]
* `aes-192-gcm` that SHALL use [GCM] block mode encryption with 192 bit key
and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to
the [RFC5116]
* `aes-256-gcm` that SHALL use [GCM] block mode encryption with 256 bit key
and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to
the [RFC5116]
* `aes-128-ctr` that SHALL use [CTR] block mode encryption with 128 bit key
* `aes-192-ctr` that SHALL use [CTR] block mode encryption with 192 bit key
* `aes-256-ctr` that SHALL use [CTR] block mode encryption with 256 bit key
#### RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector
version: 1.0
[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth
parameter to the `decrypt` function that SHALL specify the initialization vector for block modes that require
it.
#### RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData
version: 1.0
[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth
parameter to the `decrypt` function that SHALL specify the additional authenticated data
for block modes that require it.
#### RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue
version: 1.0
[ClickHouse] SHALL return the decrypted value of the data
using `String` data type as the result of `decrypt` function.
#### RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError
version: 1.0
[ClickHouse] SHALL return an error if the `key` length is not exact for the `decrypt` function for a given block mode.
#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError
version: 1.0
[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `decrypt` function for a given block mode.
#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode
version: 1.0
[ClickHouse] SHALL return an error if the `iv` is specified for the `decrypt` function
for a mode that does not need it.
#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode
version: 1.0
[ClickHouse] SHALL return an error if the `aad` is specified for the `decrypt` function
for a mode that does not need it.
#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length
version: 1.0
[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `decrypt` function.
#### RQ.SRS008.AES.Decrypt.Function.NonGCMMode.KeyAndInitializationVector.Length
version: 1.0
[ClickHouse] SHALL return an error when the `decrypt` function is called with the following parameter values
when using non-GCM modes
* `aes-128-ecb` mode and `key` is not 16 bytes or `iv` or `aad` is specified
* `aes-192-ecb` mode and `key` is not 24 bytes or `iv` or `aad` is specified
* `aes-256-ecb` mode and `key` is not 32 bytes or `iv` or `aad` is specified
* `aes-128-cbc` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-192-cbc` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-cbc` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-cfb1` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-192-cfb1` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-cfb1` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-cfb8` mode and `key` is not 16 bytes and if specified `iv` is not 16 bytes
* `aes-192-cfb8` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-cfb8` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-cfb128` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-192-cfb128` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-cfb128` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-ofb` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-192-ofb` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-256-ofb` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified
* `aes-128-ctr` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes
* `aes-192-ctr` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes
* `aes-256-ctr` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes
#### RQ.SRS008.AES.Decrypt.Function.GCMMode.KeyAndInitializationVector.Length
version: 1.0
[ClickHouse] SHALL return an error when the `decrypt` function is called with the following parameter values
when using GCM modes
* `aes-128-gcm` mode and `key` is not 16 bytes or `iv` is not specified
* `aes-192-gcm` mode and `key` is not 24 bytes or `iv` is not specified
* `aes-256-gcm` mode and `key` is not 32 bytes or `iv` is not specified
### MySQL Specific Functions
#### RQ.SRS008.AES.MySQL.Encrypt.Function
version: 1.0
[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES].
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax
version: 1.0
[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function
```sql
aes_encrypt_mysql(mode, plaintext, key, [iv])
```
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText
version: 1.0
[ClickHouse] SHALL support `plaintext` accepting any data type as
the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key
version: 1.0
[ClickHouse] SHALL support `key` with `String` or `FixedString` data types
as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode
version: 1.0
[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter
to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat
version: 1.0
[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter
of the `aes_encrypt_mysql` function where
the `key_length` SHALL specifies the length of the key and SHALL accept
`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption
mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB]. For example, `aes-256-ofb`.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid
version: 1.0
[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql`
function is not valid with the exception where such a mode is supported by the underlying
[OpenSSL] implementation.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values
version: 1.0
[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter
of the `aes_encrypt_mysql` function:
* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key
* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key
* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key
* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key
* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key
* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key
* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key
* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key
* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key
* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key
* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key
* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.GCM.Error
version: 1.0
[ClickHouse] SHALL return an error if any of the following [GCM] modes are specified as the value
for the `mode` parameter of the `aes_encrypt_mysql` function
* `aes-128-gcm`
* `aes-192-gcm`
* `aes-256-gcm`
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.CTR.Error
version: 1.0
[ClickHouse] SHALL return an error if any of the following [CTR] modes are specified as the value
for the `mode` parameter of the `aes_encrypt_mysql` function
* `aes-128-ctr`
* `aes-192-ctr`
* `aes-256-ctr`
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector
version: 1.0
[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth
parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require
it.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue
version: 1.0
[ClickHouse] SHALL return the encrypted value of the data
using `String` data type as the result of `aes_encrypt_mysql` function.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError
version: 1.0
[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql`
function for a given block mode.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong
version: 1.0
[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required
for the `aes_encrypt_mysql` function for a given block mode.
```python
def fold_key(key, cipher_key_size):
key = list(key) if not isinstance(key, (list, tuple)) else key
folded_key = key[:cipher_key_size]
for i in range(cipher_key_size, len(key)):
print(i % cipher_key_size, i)
folded_key[i % cipher_key_size] ^= key[i]
return folded_key
```
#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError
version: 1.0
[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum
that is required for the `aes_encrypt_mysql` function for a given block mode.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong
version: 1.0
[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and
its length is longer than required for the `aes_encrypt_mysql` function for a given block mode.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode
version: 1.0
[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql`
function for a mode that does not need it.
#### RQ.SRS008.AES.MySQL.Encrypt.Function.Mode.KeyAndInitializationVector.Length
version: 1.0
[ClickHouse] SHALL return an error when the `aes_encrypt_mysql` function is called with the following parameter values
* `aes-128-ecb` mode and `key` is less than 16 bytes or `iv` is specified
* `aes-192-ecb` mode and `key` is less than 24 bytes or `iv` is specified
* `aes-256-ecb` mode and `key` is less than 32 bytes or `iv` is specified
* `aes-128-cbc` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes
* `aes-192-cbc` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-cbc` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
* `aes-128-cfb1` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes
* `aes-192-cfb1` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-cfb1` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
* `aes-128-cfb8` mode and `key` is less than 16 bytes and if specified `iv` is less than 16 bytes
* `aes-192-cfb8` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-cfb8` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
* `aes-128-cfb128` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes
* `aes-192-cfb128` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-cfb128` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
* `aes-128-ofb` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes
* `aes-192-ofb` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-ofb` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
#### RQ.SRS008.AES.MySQL.Decrypt.Function
version: 1.0
[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES].
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax
version: 1.0
[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function
```sql
aes_decrypt_mysql(mode, ciphertext, key, [iv])
```
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText
version: 1.0
[ClickHouse] SHALL support `ciphertext` accepting any data type as
the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key
version: 1.0
[ClickHouse] SHALL support `key` with `String` or `FixedString` data types
as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode
version: 1.0
[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter
to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat
version: 1.0
[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter
of the `aes_decrypt_mysql` function where
the `key_length` SHALL specifies the length of the key and SHALL accept
`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption
mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB]. For example, `aes-256-ofb`.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid
version: 1.0
[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql`
function is not valid with the exception where such a mode is supported by the underlying
[OpenSSL] implementation.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values
version: 1.0
[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter
of the `aes_decrypt_mysql` function:
* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key
* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key
* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key
* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key
* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key
* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key
* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key
* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key
* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key
* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key
* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key
* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.GCM.Error
version: 1.0
[ClickHouse] SHALL return an error if any of the following [GCM] modes are specified as the value
for the `mode` parameter of the `aes_decrypt_mysql` function
* `aes-128-gcm`
* `aes-192-gcm`
* `aes-256-gcm`
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.CTR.Error
version: 1.0
[ClickHouse] SHALL return an error if any of the following [CTR] modes are specified as the value
for the `mode` parameter of the `aes_decrypt_mysql` function
* `aes-128-ctr`
* `aes-192-ctr`
* `aes-256-ctr`
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector
version: 1.0
[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth
parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require
it.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue
version: 1.0
[ClickHouse] SHALL return the decrypted value of the data
using `String` data type as the result of `aes_decrypt_mysql` function.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError
version: 1.0
[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql`
function for a given block mode.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong
version: 1.0
[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required
for the `aes_decrypt_mysql` function for a given block mode.
```python
def fold_key(key, cipher_key_size):
key = list(key) if not isinstance(key, (list, tuple)) else key
folded_key = key[:cipher_key_size]
for i in range(cipher_key_size, len(key)):
print(i % cipher_key_size, i)
folded_key[i % cipher_key_size] ^= key[i]
return folded_key
```
#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError
version: 1.0
[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum
that is required for the `aes_decrypt_mysql` function for a given block mode.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong
version: 1.0
[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and
its length is longer than required for the `aes_decrypt_mysql` function for a given block mode.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode
version: 1.0
[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql`
function for a mode that does not need it.
#### RQ.SRS008.AES.MySQL.Decrypt.Function.Mode.KeyAndInitializationVector.Length
version: 1.0
[ClickHouse] SHALL return an error when the `aes_decrypt_mysql` function is called with the following parameter values
* `aes-128-ecb` mode and `key` is less than 16 bytes or `iv` is specified
* `aes-192-ecb` mode and `key` is less than 24 bytes or `iv` is specified
* `aes-256-ecb` mode and `key` is less than 32 bytes or `iv` is specified
* `aes-128-cbc` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes
* `aes-192-cbc` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-cbc` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
* `aes-128-cfb1` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes
* `aes-192-cfb1` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-cfb1` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
* `aes-128-cfb8` mode and `key` is less than 16 bytes and if specified `iv` is less than 16 bytes
* `aes-192-cfb8` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-cfb8` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
* `aes-128-cfb128` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes
* `aes-192-cfb128` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-cfb128` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
* `aes-128-ofb` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes
* `aes-192-ofb` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes
* `aes-256-ofb` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes
## References
* **GDPR:** https://en.wikipedia.org/wiki/General_Data_Protection_Regulation
* **MySQL:** https://www.mysql.com/
* **AES:** https://en.wikipedia.org/wiki/Advanced_Encryption_Standard
* **ClickHouse:** https://clickhouse.tech
* **Git:** https://git-scm.com/
[OpenSSL]: https://www.openssl.org/
[LowCardinality]: https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/
[MergeTree]: https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/
[MySQL Database Engine]: https://clickhouse.tech/docs/en/engines/database-engines/mysql/
[MySQL Table Engine]: https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/
[MySQL Table Function]: https://clickhouse.tech/docs/en/sql-reference/table-functions/mysql/
[MySQL Dictionary]: https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources/#dicts-external_dicts_dict_sources-mysql
[GCM]: https://en.wikipedia.org/wiki/Galois/Counter_Mode
[CTR]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR)
[CBC]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_block_chaining_(CBC)
[ECB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_codebook_(ECB)
[CFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB)
[CFB128]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB)
[OFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Output_feedback_(OFB)
[GDPR]: https://en.wikipedia.org/wiki/General_Data_Protection_Regulation
[RFC5116]: https://tools.ietf.org/html/rfc5116#section-5.1
[MySQL]: https://www.mysql.com/
[MySQL 5.7]: https://dev.mysql.com/doc/refman/5.7/en/
[MySQL aes_encrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-encrypt
[MySQL aes_decrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-decrypt
[AES]: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard
[ClickHouse]: https://clickhouse.tech
[GitHub repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/aes_encryption/requirements/requirements.md
[Revision history]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/aes_encryption/requirements/requirements.md
[Git]: https://git-scm.com/
[NIST test vectors]: https://csrc.nist.gov/Projects/Cryptographic-Algorithm-Validation-Program

2843
tests/testflows/aes_encryption/requirements/requirements.py Normal file
View File

File diff suppressed because it is too large Load Diff

128
tests/testflows/aes_encryption/tests/common.py Normal file
View File

@ -0,0 +1,128 @@
# -*- coding: utf-8 -*-
modes = [
# mode, key_len, iv_len, aad
("'aes-128-ecb'", 16, None, None),
("'aes-192-ecb'", 24, None, None),
("'aes-256-ecb'", 32, None, None),
# cbc
("'aes-128-cbc'", 16, None, None),
("'aes-192-cbc'", 24, None, None),
("'aes-256-cbc'", 32, None, None),
("'aes-128-cbc'", 16, 16, None),
("'aes-192-cbc'", 24, 16, None),
("'aes-256-cbc'", 32, 16, None),
# cfb128
("'aes-128-cfb128'", 16, None, None),
("'aes-192-cfb128'", 24, None, None),
("'aes-256-cfb128'", 32, None, None),
("'aes-128-cfb128'", 16, 16, None),
("'aes-192-cfb128'", 24, 16, None),
("'aes-256-cfb128'", 32, 16, None),
# ofb
("'aes-128-ofb'", 16, None, None),
("'aes-192-ofb'", 24, None, None),
("'aes-256-ofb'", 32, None, None),
("'aes-128-ofb'", 16, 16, None),
("'aes-192-ofb'", 24, 16, None),
("'aes-256-ofb'", 32, 16, None),
# gcm
("'aes-128-gcm'", 16, 12, None),
("'aes-192-gcm'", 24, 12, None),
("'aes-256-gcm'", 32, 12, None),
("'aes-128-gcm'", 16, 12, True),
("'aes-192-gcm'", 24, 12, True),
("'aes-256-gcm'", 32, 12, True),
# ctr
("'aes-128-ctr'", 16, None, None),
("'aes-192-ctr'", 24, None, None),
("'aes-256-ctr'", 32, None, None),
("'aes-128-ctr'", 16, 16, None),
("'aes-192-ctr'", 24, 16, None),
("'aes-256-ctr'", 32, 16, None),
]
mysql_modes = [
# mode, key_len, iv_len
("'aes-128-ecb'", 16, None),
("'aes-128-ecb'", 24, None),
("'aes-192-ecb'", 24, None),
("'aes-192-ecb'", 32, None),
("'aes-256-ecb'", 32, None),
("'aes-256-ecb'", 64, None),
# cbc
("'aes-128-cbc'", 16, None),
("'aes-192-cbc'", 24, None),
("'aes-256-cbc'", 32, None),
("'aes-128-cbc'", 16, 16),
("'aes-128-cbc'", 24, 24),
("'aes-192-cbc'", 24, 16),
("'aes-192-cbc'", 32, 32),
("'aes-256-cbc'", 32, 16),
("'aes-256-cbc'", 64, 64),
# cfb128
("'aes-128-cfb128'", 16, None),
("'aes-192-cfb128'", 24, None),
("'aes-256-cfb128'", 32, None),
("'aes-128-cfb128'", 16, 16),
("'aes-128-cfb128'", 24, 24),
("'aes-192-cfb128'", 24, 16),
("'aes-192-cfb128'", 32, 32),
("'aes-256-cfb128'", 32, 16),
("'aes-256-cfb128'", 64, 64),
# ofb
("'aes-128-ofb'", 16, None),
("'aes-192-ofb'", 24, None),
("'aes-256-ofb'", 32, None),
("'aes-128-ofb'", 16, 16),
("'aes-128-ofb'", 24, 24),
("'aes-192-ofb'", 24, 16),
("'aes-192-ofb'", 32, 32),
("'aes-256-ofb'", 32, 16),
("'aes-256-ofb'", 64, 64),
]
plaintexts = [
("bytes", "unhex('0')"),
("emptystring", "''"),
("utf8string", "'Gãńdåłf_Thê_Gręât'"),
("utf8fixedstring", "toFixedString('Gãńdåłf_Thê_Gręât', 24)"),
("String", "'1'"),
("FixedString", "toFixedString('1', 1)"),
("UInt8", "toUInt8('1')"),
("UInt16", "toUInt16('1')"),
("UInt32", "toUInt32('1')"),
("UInt64", "toUInt64('1')"),
("Int8", "toInt8('1')"),
("Int16", "toInt16('1')"),
("Int32", "toInt32('1')"),
("Int64", "toInt64('1')"),
("Float32", "toFloat32('1')"),
("Float64", "toFloat64('1')"),
("Decimal32", "toDecimal32(2, 4)"),
("Decimal64", "toDecimal64(2, 4)"),
("Decimal128", "toDecimal128(2, 4)"),
("UUID", "toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"),
("Date", "toDate('2020-01-01')"),
("DateTime", "toDateTime('2020-01-01 20:01:02')"),
("DateTime64", "toDateTime64('2020-01-01 20:01:02.123', 3)"),
("LowCardinality", "toLowCardinality('1')"),
("Array", "[1,2]"),
#("Tuple", "(1,'a')") - not supported
#("Nullable, "Nullable(X)") - not supported
("NULL", "toDateOrNull('foo')"),
("IPv4", "toIPv4('171.225.130.45')"),
("IPv6", "toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')"),
("Enum8", r"CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')"),
("Enum16", r"CAST('a', 'Enum16(\'a\' = 1, \'b\' = 2)')"),
]
_hex = hex
def hex(s):
"""Convert string to hex.
"""
if isinstance(s, str):
return "".join(['%X' % ord(c) for c in s])
if isinstance(s, bytes):
return "".join(['%X' % c for c in s])
return _hex(s)

0
tests/testflows/aes_encryption/tests/compatibility/__init__.py Normal file
View File

17
tests/testflows/aes_encryption/tests/compatibility/feature.py Normal file
View File

@ -0,0 +1,17 @@
from testflows.core import *
from aes_encryption.requirements import *
@TestFeature
@Name("compatibility")
@Requirements(
RQ_SRS008_AES_Functions_DataFromMultipleSources("1.0")
)
def feature(self, node="clickhouse1"):
"""Check encryption functions usage compatibility.
"""
self.context.node = self.context.cluster.node(node)
Feature(run=load("aes_encryption.tests.compatibility.insert", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.compatibility.select", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.compatibility.mysql.feature", "feature"), flags=TE)

414
tests/testflows/aes_encryption/tests/compatibility/insert.py Normal file
View File

@ -0,0 +1,414 @@
import os
import textwrap
from contextlib import contextmanager
from importlib.machinery import SourceFileLoader
from testflows.core import *
from testflows.core.name import basename
from testflows.asserts.helpers import varname
from testflows.asserts import values, error, snapshot
from aes_encryption.tests.common import modes, mysql_modes
@contextmanager
def table(name):
node = current().context.node
try:
with Given("table"):
sql = f"""
CREATE TABLE {name}
(
date Nullable(Date),
name Nullable(String),
secret Nullable(String)
)
ENGINE = Memory()
"""
with By("dropping table if exists"):
node.query(f"DROP TABLE IF EXISTS {name}")
with And("creating a table"):
node.query(textwrap.dedent(sql))
yield
finally:
with Finally("I drop the table", flags=TE):
node.query(f"DROP TABLE IF EXISTS {name}")
@contextmanager
def mv_transform(table, transform):
node = current().context.node
try:
with Given("tables for input transformation"):
with By("creating Null input table"):
sql = f"""
CREATE TABLE {table}_input
(
date Nullable(Date),
name Nullable(String),
secret Nullable(String),
mode String,
key String,
iv String,
aad String
)
ENGINE=Null()
"""
node.query(textwrap.dedent(sql))
with And("creating materialized view table"):
sql = f"""
CREATE MATERIALIZED VIEW {table}_input_mv TO {table} AS
SELECT date, name, {transform}
FROM {table}_input
"""
node.query(textwrap.dedent(sql))
yield
finally:
with Finally("I drop tables for input transformation", flags=TE):
with By("dropping materialized view table", flags=TE):
node.query(f"DROP TABLE IF EXISTS {table}_input_mv")
with And("dropping Null input table", flags=TE):
node.query(f"DROP TABLE IF EXISTS {table}_input")
@TestScenario
def encrypt_using_materialized_view(self):
"""Check that we can use `encrypt` function when inserting
data into a table using a materialized view for input
data transformation.
"""
node = self.context.node
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
example_transform = f"encrypt(mode, secret, key{', iv' if example_iv else ''}{', aad' if example_aad else ''})"
with table("user_data"):
with mv_transform("user_data", example_transform):
with When("I insert encrypted data"):
node.query(f"""
INSERT INTO user_data_input
(date, name, secret, mode, key)
VALUES
('2020-01-01', 'user0', 'user0_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}),
('2020-01-02', 'user1', 'user1_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}),
('2020-01-03', 'user2', 'user2_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""})
""")
with And("I read inserted data back"):
node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date")
with Then("output must match the snapshot"):
with values() as that:
assert that(snapshot(r.output.strip(), "insert", name=f"encrypt_mv_example_{varname(basename(self.name))}")), error()
@TestScenario
def aes_encrypt_mysql_using_materialized_view(self):
"""Check that we can use `aes_encrypt_mysql` function when inserting
data into a table using a materialized view for input
data transformation.
"""
node = self.context.node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
aad = "some random aad"
for mode, key_len, iv_len in mysql_modes:
with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_transform = f"aes_encrypt_mysql(mode, secret, key{', iv' if example_iv else ''})"
with table("user_data"):
with mv_transform("user_data", example_transform):
with When("I insert encrypted data"):
node.query(f"""
INSERT INTO user_data_input
(date, name, secret, mode, key)
VALUES
('2020-01-01', 'user0', 'user0_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}),
('2020-01-02', 'user1', 'user1_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}),
('2020-01-03', 'user2', 'user2_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""})
""")
with And("I read inserted data back"):
node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date")
with Then("output must match the snapshot"):
with values() as that:
assert that(snapshot(r.output.strip(), "insert", name=f"aes_encrypt_mysql_mv_example_{varname(basename(self.name))}")), error()
@TestScenario
def encrypt_using_input_table_function(self):
"""Check that we can use `encrypt` function when inserting
data into a table using insert select and `input()` table
function.
"""
node = self.context.node
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
example_transform = f"encrypt({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''}{(', ' + example_aad) if example_aad else ''})"
with table("user_data"):
with When("I insert encrypted data"):
node.query(f"""
INSERT INTO
user_data
SELECT
date, name, {example_transform}
FROM
input('date Date, name String, secret String')
FORMAT Values ('2020-01-01', 'user0', 'user0_secret'), ('2020-01-02', 'user1', 'user1_secret'), ('2020-01-03', 'user2', 'user2_secret')
""")
with And("I read inserted data back"):
r = node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date")
with Then("output must match the snapshot"):
with values() as that:
assert that(snapshot(r.output.strip(), "insert", name=f"encrypt_input_example_{varname(basename(example.name))}")), error()
@TestScenario
def aes_encrypt_mysql_using_input_table_function(self):
"""Check that we can use `aes_encrypt_mysql` function when inserting
data into a table using insert select and `input()` table
function.
"""
node = self.context.node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
aad = "some random aad"
for mode, key_len, iv_len in mysql_modes:
with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_transform = f"aes_encrypt_mysql({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})"
with table("user_data"):
with When("I insert encrypted data"):
node.query(f"""
INSERT INTO
user_data
SELECT
date, name, {example_transform}
FROM
input('date Date, name String, secret String')
FORMAT Values ('2020-01-01', 'user0', 'user0_secret'), ('2020-01-02', 'user1', 'user1_secret'), ('2020-01-03', 'user2', 'user2_secret')
""")
with And("I read inserted data back"):
r = node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date")
with Then("output must match the snapshot"):
with values() as that:
assert that(snapshot(r.output.strip(), "insert", name=f"aes_encrypt_mysql_input_example_{varname(basename(example.name))}")), error()
@TestScenario
def decrypt_using_materialized_view(self):
"""Check that we can use `decrypt` function when inserting
data into a table using a materialized view for input
data transformation.
"""
node = self.context.node
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
example_transform = f"decrypt(mode, secret, key{', iv' if example_iv else ''}{', aad' if example_aad else ''})"
with Given("I have ciphertexts"):
example_name = basename(example.name)
ciphertexts = getattr(snapshot_module, varname(f"encrypt_mv_example_{example_name}"))
example_ciphertexts = ["'{}'".format(l.split("\t")[-1].strup("'")) for l in ciphertexts.split("\n")]
with table("user_data"):
with mv_transform("user_data", example_transform):
with When("I insert encrypted data"):
node.query(f"""
INSERT INTO user_data_input
(date, name, secret, mode, key)
VALUES
('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}),
('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}),
('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""})
""")
with And("I read inserted data back"):
r = node.query("SELECT date, name, secret FROM user_data ORDER BY date")
with Then("output must match the expected"):
expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'"""
assert r.output == expected, error()
@TestScenario
def aes_decrypt_mysql_using_materialized_view(self):
"""Check that we can use `aes_decrypt_mysql` function when inserting
data into a table using a materialized view for input
data transformation.
"""
node = self.context.node
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
example_transform = f"aes_decrypt_mysql(mode, secret, key{', iv' if example_iv else ''})"
with Given("I have ciphertexts"):
example_name = basename(example.name)
ciphertexts = getattr(snapshot_module, varname(f"aes_encrypt_mysql_mv_example_{example_name}"))
example_ciphertexts = ["'{}'".format(l.split("\t")[-1].strup("'")) for l in ciphertexts.split("\n")]
with table("user_data"):
with mv_transform("user_data", example_transform):
with When("I insert encrypted data"):
node.query(f"""
INSERT INTO user_data_input
(date, name, secret, mode, key)
VALUES
('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}),
('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}),
('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""})
""")
with And("I read inserted data back"):
r = node.query("SELECT date, name, secret FROM user_data ORDER BY date")
with Then("output must match the expected"):
expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'"""
assert r.output == expected, error()
@TestScenario
def decrypt_using_input_table_function(self):
"""Check that we can use `decrypt` function when inserting
data into a table using insert select and `input()` table
function.
"""
node = self.context.node
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
example_transform = f"decrypt({mode}, unhex(secret), {example_key}{(', ' + example_iv) if example_iv else ''}{(', ' + example_aad) if example_aad else ''})"
with Given("I have ciphertexts"):
example_name = basename(example.name)
ciphertexts = getattr(snapshot_module, varname(f"encrypt_input_example_{example_name}"))
example_ciphertexts = [l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n")]
with table("user_data"):
with When("I insert decrypted data"):
node.query(textwrap.dedent(f"""
INSERT INTO
user_data
SELECT
date, name, {example_transform}
FROM
input('date Date, name String, secret String')
FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}')
"""))
with And("I read inserted data back"):
r = node.query("SELECT date, name, secret FROM user_data ORDER BY date")
expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret"""
with Then("output must match the expected", description=expected):
assert r.output == expected, error()
@TestScenario
def aes_decrypt_mysql_using_input_table_function(self):
"""Check that we can use `aes_decrypt_mysql` function when inserting
data into a table using insert select and `input()` table
function.
"""
node = self.context.node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
aad = "some random aad"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module()
for mode, key_len, iv_len in mysql_modes:
with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example:
example_key = f"'{key[:key_len]}'"
example_mode = mode
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_transform = f"aes_decrypt_mysql({mode}, unhex(secret), {example_key}{(', ' + example_iv) if example_iv else ''})"
with Given("I have ciphertexts"):
example_name = basename(example.name)
ciphertexts = getattr(snapshot_module, varname(f"aes_encrypt_mysql_input_example_{example_name}"))
example_ciphertexts = [l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n")]
with table("user_data"):
with When("I insert decrypted data"):
node.query(textwrap.dedent(f"""
INSERT INTO
user_data
SELECT
date, name, {example_transform}
FROM
input('date Date, name String, secret String')
FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}')
"""))
with And("I read inserted data back"):
r = node.query("SELECT date, name, secret FROM user_data ORDER BY date")
expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret"""
with Then("output must match the expected", description=expected):
assert r.output == expected, error()
@TestFeature
@Name("insert")
def feature(self, node="clickhouse1"):
"""Check encryption functions when used during data insertion into a table.
"""
self.context.node = self.context.cluster.node(node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

0
tests/testflows/aes_encryption/tests/compatibility/mysql/__init__.py Normal file
View File

196
tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py Normal file
View File

@ -0,0 +1,196 @@
import textwrap
from contextlib import contextmanager
from testflows.core import *
from testflows.asserts import error
from aes_encryption.requirements import *
from aes_encryption.tests.common import mysql_modes, hex
@contextmanager
def table(name, node, mysql_node, secret_type):
"""Create a table that can be accessed using MySQL database engine.
"""
try:
with Given("table in MySQL"):
sql = f"""
CREATE TABLE {name}(
id INT NOT NULL AUTO_INCREMENT,
date DATE,
name VARCHAR(100),
secret {secret_type},
PRIMARY KEY ( id )
);
"""
with When("I drop the table if exists"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
with And("I create a table"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I create a database using MySQL database engine"):
sql = f"""
CREATE DATABASE mysql_db
ENGINE = MySQL('{mysql_node.name}:3306', 'db', 'user', 'password')
"""
with When("I drop database if exists"):
node.query(f"DROP DATABASE IF EXISTS mysql_db")
with And("I create database"):
node.query(textwrap.dedent(sql))
yield
finally:
with And("I drop the database that is using MySQL database engine", flags=TE):
node.query(f"DROP DATABASE IF EXISTS mysql_db")
with And("I drop a table in MySQL", flags=TE):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
@TestOutline(Scenario)
@Examples("mysql_datatype", [
("VARBINARY(100)",),
#("VARCHAR(100)",),
("BLOB", ),
#("TEXT",)
])
def decrypt(self, mysql_datatype):
"""Check that when using a table provided by MySQL database engine that
contains a column encrypted in MySQL stored using specified data type
I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql`
functions in the select query.
"""
node = self.context.node
mysql_node = self.context.mysql_node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
for func in ["decrypt", "aes_decrypt_mysql"]:
for mode, key_len, iv_len in mysql_modes:
exact_key_size = int(mode.split("-")[1])//8
if "ecb" not in mode and not iv_len:
continue
if func == "decrypt":
if iv_len and iv_len != 16:
continue
if key_len != exact_key_size:
continue
with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""):
with table("user_data", node, mysql_node, mysql_datatype):
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
with When("I insert encrypted data in MySQL"):
sql = f"""
SET block_encryption_mode = {example_mode};
INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"}));
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I read encrypted data in MySQL to make sure it is valid"):
sql = f"""
SET block_encryption_mode = {example_mode};
SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data;
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I read raw encrypted data in MySQL"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0)
with And("I read raw data using MySQL database engine"):
output = node.query("SELECT id, date, name, hex(secret) AS secret FROM mysql_db.user_data")
with And("I read decrypted data using MySQL database engine"):
output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""").output.strip()
with Then("output should match the original plain text"):
assert output == hex("secret"), error()
@TestOutline(Scenario)
@Examples("mysql_datatype", [
("VARBINARY(100)",),
#("VARCHAR(100)",),
("BLOB", ),
#("TEXT",)
])
def encrypt(self, mysql_datatype):
"""Check that when using a table provided by MySQL database engine that
we can encrypt data during insert using the `aes_encrypt_mysql` function
and decrypt it in MySQL.
"""
node = self.context.node
mysql_node = self.context.mysql_node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
for func in ["encrypt", "aes_encrypt_mysql"]:
for mode, key_len, iv_len in mysql_modes:
exact_key_size = int(mode.split("-")[1])//8
if "ecb" not in mode and not iv_len:
continue
if func == "encrypt":
if iv_len and iv_len != 16:
continue
if key_len != exact_key_size:
continue
with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""):
with table("user_data", node, mysql_node, mysql_datatype):
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})"
with When("I insert encrypted data into a table provided by MySQL database engine"):
node.query(textwrap.dedent(f"""
INSERT INTO
mysql_db.user_data
SELECT
id, date, name, {example_transform}
FROM
input('id Int32, date Date, name String, secret String')
FORMAT Values (1, '2020-01-01', 'user0', 'secret')
"""))
with And("I read decrypted data using MySQL database engine"):
output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""").output.strip()
with Then("decrypted data from MySQL database engine should should match the original plain text"):
assert output == hex("secret"), error()
with And("I read raw data using MySQL database engine to get expected raw data"):
expected_raw_data = node.query("SELECT hex(secret) AS secret FROM mysql_db.user_data").output.strip()
with And("I read raw encrypted data in MySQL"):
output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip()
with Then("check that raw encryted data in MySQL matches the expected"):
assert expected_raw_data in output, error()
with And("I decrypt data in MySQL to make sure it is valid"):
sql = f"""
SET block_encryption_mode = {example_mode};
SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data;
"""
output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip()
with Then("decryted data in MySQL should match the original plain text"):
assert hex("secret") in output, error()
@TestFeature
@Name("database engine")
@Requirements(
RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL("1.0")
)
def feature(self, node="clickhouse1", mysql_node="mysql1"):
"""Check usage of encryption functions with [MySQL database engine].
[MySQL database engine]: https://clickhouse.tech/docs/en/engines/database-engines/mysql/
"""
self.context.node = self.context.cluster.node(node)
self.context.mysql_node = self.context.cluster.node(mysql_node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

251
tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py Normal file
View File

@ -0,0 +1,251 @@
import textwrap
from contextlib import contextmanager
from testflows.core import *
from testflows.asserts import error
from aes_encryption.requirements import *
from aes_encryption.tests.common import mysql_modes, hex
@contextmanager
def dictionary(name, node, mysql_node, secret_type):
"""Create a table in MySQL and use it a source for a dictionary.
"""
try:
with Given("table in MySQL"):
sql = f"""
CREATE TABLE {name}(
id INT NOT NULL AUTO_INCREMENT,
date DATE,
name VARCHAR(100),
secret {secret_type},
PRIMARY KEY ( id )
);
"""
with When("I drop the table if exists"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
with And("I create a table"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("dictionary that uses MySQL table as the external source"):
with When("I drop the dictionary if exists"):
node.query(f"DROP DICTIONARY IF EXISTS dict_{name}")
with And("I create the dictionary"):
sql = f"""
CREATE DICTIONARY dict_{name}
(
id Int32,
date Date,
name String,
secret String
)
PRIMARY KEY id
SOURCE(MYSQL(
USER 'user'
PASSWORD 'password'
DB 'db'
TABLE '{name}'
REPLICA(PRIORITY 1 HOST '{mysql_node.name}' PORT 3306)
))
LAYOUT(HASHED())
LIFETIME(0)
"""
node.query(textwrap.dedent(sql))
yield f"dict_{name}"
finally:
with Finally("I drop the dictionary", flags=TE):
node.query(f"DROP DICTIONARY IF EXISTS dict_{name}")
with And("I drop a table in MySQL", flags=TE):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
@contextmanager
def parameters_dictionary(name, node, mysql_node):
"""Create a table in MySQL and use it a source for a dictionary
that stores parameters for the encryption functions.
"""
try:
with Given("table in MySQL"):
sql = f"""
CREATE TABLE {name}(
`id` INT NOT NULL AUTO_INCREMENT,
`name` VARCHAR(100),
`mode` VARCHAR(100),
`key` BLOB,
`iv` BLOB,
`text` BLOB,
PRIMARY KEY ( id )
);
"""
with When("I drop the table if exists"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
with And("I create a table"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("dictionary that uses MySQL table as the external source"):
with When("I drop the dictionary if exists"):
node.query(f"DROP DICTIONARY IF EXISTS dict_{name}")
with And("I create the dictionary"):
sql = f"""
CREATE DICTIONARY dict_{name}
(
id Int32,
name String,
mode String,
key String,
iv String,
text String
)
PRIMARY KEY id
SOURCE(MYSQL(
USER 'user'
PASSWORD 'password'
DB 'db'
TABLE '{name}'
REPLICA(PRIORITY 1 HOST '{mysql_node.name}' PORT 3306)
))
LAYOUT(HASHED())
LIFETIME(0)
"""
node.query(textwrap.dedent(sql))
yield f"dict_{name}"
finally:
with Finally("I drop the dictionary", flags=TE):
node.query(f"DROP DICTIONARY IF EXISTS dict_{name}")
with And("I drop a table in MySQL", flags=TE):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
@TestScenario
def parameter_values(self):
"""Check that we can use a dictionary that uses MySQL table as a source
can be used as a parameters store for the `encrypt`, `decrypt`, and
`aes_encrypt_mysql`, `aes_decrypt_mysql` functions.
"""
node = self.context.node
mysql_node = self.context.mysql_node
mode = "'aes-128-cbc'"
key = f"'{'1' * 16}'"
iv = f"'{'2' * 16}'"
plaintext = "'secret'"
for encrypt, decrypt in [
("encrypt", "decrypt"),
("aes_encrypt_mysql", "aes_decrypt_mysql")
]:
with Example(f"{encrypt} and {decrypt}", description=f"Check using dictionary for parameters of {encrypt} and {decrypt} functions."):
with parameters_dictionary("parameters_data", node, mysql_node) as dict_name:
with When("I insert parameters values in MySQL"):
sql = f"""
INSERT INTO parameters_data VALUES (1, 'user0', {mode}, {key}, {iv}, {plaintext});
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I use dictionary values as parameters"):
sql = f"""
SELECT {decrypt}(
dictGet('default.{dict_name}', 'mode', toUInt64(1)),
{encrypt}(
dictGet('default.{dict_name}', 'mode', toUInt64(1)),
dictGet('default.{dict_name}', 'text', toUInt64(1)),
dictGet('default.{dict_name}', 'key', toUInt64(1)),
dictGet('default.{dict_name}', 'iv', toUInt64(1))
),
dictGet('default.{dict_name}', 'key', toUInt64(1)),
dictGet('default.{dict_name}', 'iv', toUInt64(1))
)
"""
output = node.query(textwrap.dedent(sql)).output.strip()
with Then("output should match the plain text"):
assert f"'{output}'" == plaintext, error()
@TestOutline(Scenario)
@Examples("mysql_datatype", [
("VARBINARY(100)",),
#("VARCHAR(100)",),
("BLOB", ),
#("TEXT",)
])
def decrypt(self, mysql_datatype):
"""Check that when using a dictionary that uses MySQL table as a source and
contains a data encrypted in MySQL and stored using specified data type
that we can decrypt data from the dictionary using
the `aes_decrypt_mysql` or `decrypt` functions in the select query.
"""
node = self.context.node
mysql_node = self.context.mysql_node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
for func in ["decrypt", "aes_decrypt_mysql"]:
for mode, key_len, iv_len in mysql_modes:
exact_key_size = int(mode.split("-")[1])//8
if "ecb" not in mode and not iv_len:
continue
if func == "decrypt":
if iv_len and iv_len != 16:
continue
if key_len != exact_key_size:
continue
with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""):
with dictionary("user_data", node, mysql_node, mysql_datatype) as dict_name:
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
with When("I insert encrypted data in MySQL"):
sql = f"""
SET block_encryption_mode = {example_mode};
INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"}));
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I read encrypted data in MySQL to make sure it is valid"):
sql = f"""
SET block_encryption_mode = {example_mode};
SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data;
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I read raw encrypted data in MySQL"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0)
with And("I read raw data using MySQL dictionary"):
output = node.query(f"SELECT hex(dictGet('default.{dict_name}', 'secret', toUInt64(1))) AS secret")
with And("I read decrypted data using MySQL dictionary"):
output = node.query(textwrap.dedent(f"""
SELECT hex(
{func}(
{example_mode},
dictGet('default.{dict_name}', 'secret', toUInt64(1)),
{example_key}{(", " + example_iv) if example_iv else ""}
)
)
""")).output.strip()
with Then("output should match the original plain text"):
assert output == hex("secret"), error()
@TestFeature
@Name("dictionary")
@Requirements(
RQ_SRS008_AES_Functions_Compatability_Dictionaries("1.0")
)
def feature(self, node="clickhouse1", mysql_node="mysql1"):
"""Check usage of encryption functions with [MySQL dictionary].
[MySQL dictionary]: https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources/#dicts-external_dicts_dict_sources-mysql
"""
self.context.node = self.context.cluster.node(node)
self.context.mysql_node = self.context.cluster.node(mysql_node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

18
tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py Normal file
View File

@ -0,0 +1,18 @@
from testflows.core import *
from aes_encryption.requirements import *
@TestFeature
@Name("mysql")
@Requirements(
RQ_SRS008_AES_Functions_Compatability_MySQL("1.0")
)
def feature(self, node="clickhouse1"):
"""Check encryption functions usage compatibility with MySQL.
"""
self.context.node = self.context.cluster.node(node)
Feature(run=load("aes_encryption.tests.compatibility.mysql.table_engine", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.compatibility.mysql.database_engine", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.compatibility.mysql.table_function", "feature"), flags=TE)
Feature(run=load("aes_encryption.tests.compatibility.mysql.dictionary", "feature"), flags=TE)

202
tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py Normal file
View File

@ -0,0 +1,202 @@
import textwrap
from contextlib import contextmanager
from testflows.core import *
from testflows.asserts import error
from aes_encryption.requirements import *
from aes_encryption.tests.common import mysql_modes, hex
@contextmanager
def table(name, node, mysql_node, secret_type):
"""Create a table that can be accessed using MySQL table engine.
"""
try:
with Given("table in MySQL"):
sql = f"""
CREATE TABLE {name}(
id INT NOT NULL AUTO_INCREMENT,
date DATE,
name VARCHAR(100),
secret {secret_type},
PRIMARY KEY ( id )
);
"""
with When("I drop the table if exists"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
with And("I create a table"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I create a table using MySQL table engine"):
sql = f"""
CREATE TABLE mysql_{name}
(
id Nullable(Int32),
date Nullable(Date),
name Nullable(String),
secret Nullable(String)
)
ENGINE = MySQL('{mysql_node.name}:3306', 'db', '{name}', 'user', 'password')
"""
with When("I drop table if exists"):
node.query(f"DROP TABLE IF EXISTS mysql_{name}")
with And("I create table"):
node.query(textwrap.dedent(sql))
yield
finally:
with And("I drop a table using MySQL table engine", flags=TE):
node.query(f"DROP TABLE IF EXISTS mysql_{name}")
with And("I drop a table in MySQL", flags=TE):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
@TestOutline(Scenario)
@Examples("mysql_datatype", [
("VARBINARY(100)",),
#("VARCHAR(100)",),
("BLOB", ),
#("TEXT",)
])
def decrypt(self, mysql_datatype):
"""Check that when using a table with MySQL table engine that
contains a column encrypted in MySQL stored using specified data type
I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql`
functions in the select query.
"""
node = self.context.node
mysql_node = self.context.mysql_node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
for func in ["decrypt", "aes_decrypt_mysql"]:
for mode, key_len, iv_len in mysql_modes:
exact_key_size = int(mode.split("-")[1])//8
if "ecb" not in mode and not iv_len:
continue
if func == "decrypt":
if iv_len and iv_len != 16:
continue
if key_len != exact_key_size:
continue
with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""):
with table("user_data", node, mysql_node, mysql_datatype):
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
with When("I insert encrypted data in MySQL"):
sql = f"""
SET block_encryption_mode = {example_mode};
INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"}));
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I read encrypted data in MySQL to make sure it is valid"):
sql = f"""
SET block_encryption_mode = {example_mode};
SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data;
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I read raw encrypted data in MySQL"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0)
with And("I read raw data using MySQL table engine"):
output = node.query("SELECT id, date, name, hex(secret) AS secret FROM mysql_user_data")
with And("I read decrypted data via MySQL table engine"):
output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""").output.strip()
with Then("the output should match the original plain text"):
assert output == hex("secret"), error()
@TestOutline(Scenario)
@Examples("mysql_datatype", [
("VARBINARY(100)",),
#("VARCHAR(100)",),
("BLOB", ),
#("TEXT",)
])
def encrypt(self, mysql_datatype):
"""Check that when using a table with MySQL table engine that
we can encrypt data during insert using the `encrypt` and `aes_encrypt_mysql`
functions and decrypt it in MySQL.
"""
node = self.context.node
mysql_node = self.context.mysql_node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
for func in ["encrypt", "aes_encrypt_mysql"]:
for mode, key_len, iv_len in mysql_modes:
exact_key_size = int(mode.split("-")[1])//8
if "ecb" not in mode and not iv_len:
continue
if func == "encrypt":
if iv_len and iv_len != 16:
continue
if key_len != exact_key_size:
continue
with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}"""):
with table("user_data", node, mysql_node, mysql_datatype):
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})"
with When("I insert encrypted data into MySQL table engine"):
node.query(textwrap.dedent(f"""
INSERT INTO
mysql_user_data
SELECT
id, date, name, {example_transform}
FROM
input('id Nullable(Int32), date Date, name String, secret String')
FORMAT Values (null, '2020-01-01', 'user0', 'secret')
"""))
with And("I read decrypted data via MySQL table engine"):
output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""").output.strip()
with Then("decrypted data from MySQL table engine should should match the original plain text"):
assert output == hex("secret"), error()
with And("I read raw data using MySQL table engine to get expected raw data"):
expected_raw_data = node.query("SELECT hex(secret) AS secret FROM mysql_user_data").output.strip()
with And("I read raw encrypted data in MySQL"):
output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip()
with Then("check that raw encryted data in MySQL matches the expected"):
assert expected_raw_data in output, error()
with And("I decrypt data in MySQL to make sure it is valid"):
sql = f"""
SET block_encryption_mode = {example_mode};
SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data;
"""
output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip()
with Then("decryted data in MySQL should match the original plain text"):
assert hex("secret") in output, error()
@TestFeature
@Name("table engine")
@Requirements(
RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL("1.0")
)
def feature(self, node="clickhouse1", mysql_node="mysql1"):
"""Check usage of encryption functions with [MySQL table engine].
[MySQL table engine]: https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/
"""
self.context.node = self.context.cluster.node(node)
self.context.mysql_node = self.context.cluster.node(mysql_node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

183
tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py Normal file
View File

@ -0,0 +1,183 @@
import textwrap
from contextlib import contextmanager
from testflows.core import *
from testflows.asserts import error
from aes_encryption.requirements import *
from aes_encryption.tests.common import mysql_modes, hex
@contextmanager
def table(name, node, mysql_node, secret_type):
"""Create a table that can be accessed using MySQL table function.
"""
try:
with Given("table in MySQL"):
sql = f"""
CREATE TABLE {name}(
id INT NOT NULL AUTO_INCREMENT,
date DATE,
name VARCHAR(100),
secret {secret_type},
PRIMARY KEY ( id )
);
"""
with When("I drop the table if exists"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
with And("I create a table"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
yield f"mysql('{mysql_node.name}:3306', 'db', 'user_data', 'user', 'password')"
finally:
with And("I drop a table in MySQL", flags=TE):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0)
@TestOutline(Scenario)
@Examples("mysql_datatype", [
("VARBINARY(100)",),
#("VARCHAR(100)",),
("BLOB", ),
#("TEXT",)
])
def decrypt(self, mysql_datatype):
"""Check that when using a table accessed through MySQL table function that
contains a column encrypted in MySQL stored using specified data type
I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql`
functions in the select query.
"""
node = self.context.node
mysql_node = self.context.mysql_node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
for func in ["decrypt", "aes_decrypt_mysql"]:
for mode, key_len, iv_len in mysql_modes:
exact_key_size = int(mode.split("-")[1])//8
if "ecb" not in mode and not iv_len:
continue
if func == "decrypt":
if iv_len and iv_len != 16:
continue
if key_len != exact_key_size:
continue
with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""):
with table("user_data", node, mysql_node, mysql_datatype) as table_function:
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
with When("I insert encrypted data in MySQL"):
sql = f"""
SET block_encryption_mode = {example_mode};
INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"}));
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I read encrypted data in MySQL to make sure it is valid"):
sql = f"""
SET block_encryption_mode = {example_mode};
SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data;
"""
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0)
with And("I read raw encrypted data in MySQL"):
mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0)
with And("I read raw data using MySQL table function"):
output = node.query(f"SELECT id, date, name, hex(secret) AS secret FROM {table_function}")
with And("I read decrypted data using MySQL table function"):
output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""").output.strip()
with Then("output should match the original plain text"):
assert output == hex("secret"), error()
@TestOutline(Scenario)
@Examples("mysql_datatype", [
("VARBINARY(100)",),
#("VARCHAR(100)",),
("BLOB", ),
#("TEXT",)
])
def encrypt(self, mysql_datatype):
"""Check that when using a table accessed through MySQL table function that
we can encrypt data during insert using the `aes_encrypt_mysql` function
and decrypt it in MySQL.
"""
node = self.context.node
mysql_node = self.context.mysql_node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
for func in ["encrypt", "aes_encrypt_mysql"]:
for mode, key_len, iv_len in mysql_modes:
exact_key_size = int(mode.split("-")[1])//8
if "ecb" not in mode and not iv_len:
continue
if func == "encrypt":
if iv_len and iv_len != 16:
continue
if key_len != exact_key_size:
continue
with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""):
with table("user_data", node, mysql_node, mysql_datatype) as table_function:
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})"
with When("I insert encrypted data into a table provided by MySQL database engine"):
node.query(textwrap.dedent(f"""
INSERT INTO TABLE FUNCTION
{table_function}
SELECT
id, date, name, {example_transform}
FROM
input('id Int32, date Date, name String, secret String')
FORMAT Values (1, '2020-01-01', 'user0', 'secret')
"""))
with And("I read decrypted data using MySQL database engine"):
output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""").output.strip()
with Then("decrypted data from MySQL database engine should should match the original plain text"):
assert output == hex("secret"), error()
with And("I read raw data using MySQL database engine to get expected raw data"):
expected_raw_data = node.query(f"SELECT hex(secret) AS secret FROM {table_function}").output.strip()
with And("I read raw encrypted data in MySQL"):
output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip()
with Then("check that raw encryted data in MySQL matches the expected"):
assert expected_raw_data in output, error()
with And("I decrypt data in MySQL to make sure it is valid"):
sql = f"""
SET block_encryption_mode = {example_mode};
SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data;
"""
output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip()
with Then("decryted data in MySQL should match the original plain text"):
assert hex("secret") in output, error()
@TestFeature
@Name("table function")
@Requirements(
RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL("1.0")
)
def feature(self, node="clickhouse1", mysql_node="mysql1"):
"""Check usage of encryption functions with [MySQL table function].
[MySQL table function]: https://clickhouse.tech/docs/en/sql-reference/table-functions/mysql/
"""
self.context.node = self.context.cluster.node(node)
self.context.mysql_node = self.context.cluster.node(mysql_node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

186
tests/testflows/aes_encryption/tests/compatibility/select.py Normal file
View File

@ -0,0 +1,186 @@
import textwrap
from contextlib import contextmanager
from testflows.core import *
from testflows.asserts.helpers import varname
from testflows.asserts import values, error, snapshot
from aes_encryption.tests.common import modes, mysql_modes
@contextmanager
def table(name, sql):
node = current().context.node
try:
with Given("table"):
with By("dropping table if exists"):
node.query(f"DROP TABLE IF EXISTS {name}")
with And("creating a table"):
node.query(textwrap.dedent(sql.format(name=name)))
yield
finally:
with Finally("I drop the table", flags=TE):
node.query(f"DROP TABLE IF EXISTS {name}")
@TestScenario
def decrypt(self):
"""Check decrypting column when reading data from a table.
"""
node = self.context.node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
aad = "some random aad"
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""") as example:
with table("user_table", """
CREATE TABLE {name}
(
date Nullable(Date),
name Nullable(String),
secret Nullable(String)
)
ENGINE = Memory()
"""):
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
with When("I insert encrypted data"):
encrypted_secret = node.query(f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""").output.strip()
node.query(textwrap.dedent(f"""
INSERT INTO user_table
(date, name, secret)
VALUES
('2020-01-01', 'user0', unhex('{encrypted_secret}'))
"""))
with And("I decrypt data during query"):
output = node.query(f"""SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table FORMAT JSONEachRow""").output.strip()
with Then("I should get back the original plain text"):
assert output == '{"name":"user0","secret":"secret"}', error()
@TestScenario
def decrypt_multiple(self, count=1000):
"""Check decrypting column when reading multiple entries
encrypted with the same parameters for the same user
from a table.
"""
node = self.context.node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
aad = "some random aad"
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""") as example:
with table("user_table", """
CREATE TABLE {name}
(
date Nullable(Date),
name Nullable(String),
secret Nullable(String)
)
ENGINE = Memory()
"""):
example_mode = mode
example_key = f"'{key[:key_len]}'"
example_iv = None if not iv_len else f"'{iv[:iv_len]}'"
example_aad = None if not aad_len else f"'{aad}'"
with When("I insert encrypted data"):
encrypted_secret = node.query(f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""").output.strip()
values = [f"('2020-01-01', 'user0', unhex('{encrypted_secret}'))"] * count
node.query(
"INSERT INTO user_table\n"
" (date, name, secret)\n"
f"VALUES {', '.join(values)}")
with And("I decrypt data", description="using a subquery and get the number of entries that match the plaintext"):
output = node.query(f"""SELECT count() AS count FROM (SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table) WHERE secret = 'secret' FORMAT JSONEachRow""").output.strip()
with Then("I should get back the expected result", description=f"{count}"):
assert output == f'{{"count":"{count}"}}', error()
@TestScenario
def decrypt_unique(self):
"""Check decrypting column when reading multiple entries
encrypted with the different parameters for each user
from a table.
"""
node = self.context.node
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
aad = "some random aad"
with table("user_table", """
CREATE TABLE {name}
(
id UInt64,
date Nullable(Date),
name Nullable(String),
secret Nullable(String)
)
ENGINE = Memory()
"""):
user_modes = []
user_keys = []
with When("I get encrypted data"):
user_id = 0
values = []
for mode, key_len, iv_len, aad_len in modes:
if "gcm" in mode:
continue
user_modes.append(mode)
user_keys.append(f"'{key[:key_len]}'")
with When(f"I get encrypted data for user {user_id}"):
encrypted_secret = node.query(
f"""SELECT hex(encrypt({user_modes[-1]}, 'secret', {user_keys[-1]}))"""
).output.strip()
values.append(f"({user_id}, '2020-01-01', 'user{user_id}', unhex('{encrypted_secret}'))")
user_id += 1
with And("I insert encrypted data for all users"):
node.query(
"INSERT INTO user_table\n"
" (id, date, name, secret)\n"
f"VALUES {', '.join(values)}")
with And("I read decrypted data for all users"):
output = node.query(textwrap.dedent(f"""
SELECT
count() AS count
FROM
(
SELECT
[{",".join(user_modes)}] AS modes,
[{",".join(user_keys)}] AS keys,
name,
decrypt(modes[id], secret, keys[id]) AS secret
FROM user_table
)
WHERE
secret = 'secret'
FORMAT JSONEachRow
""")).output.strip()
with Then("I should get back the expected result", description=f"{count}"):
assert output == f'{{"count":"{count}"}}', error()
@TestFeature
@Name("select")
def feature(self, node="clickhouse1"):
"""Check encryption functions when used during table querying.
"""
self.context.node = self.context.cluster.node(node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

192
tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot Normal file
View File

@ -0,0 +1,192 @@
aes_encrypt_mysql_input_example_mode_aes_128_ecb_key_16_iv_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'"""
aes_encrypt_mysql_input_example_mode_aes_128_ecb_key_24_iv_None = r"""'2020-01-01\tuser0\tB418FF12BCBF9E42FA7C19D6EE26BF0B\n2020-01-02\tuser1\t3147A3FEE47DF418D1D75CBC1BC14DE6\n2020-01-03\tuser2\tAECEFD40C6632A0FC033D040E44CCBCC'"""
aes_encrypt_mysql_input_example_mode_aes_192_ecb_key_24_iv_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'"""
aes_encrypt_mysql_input_example_mode_aes_192_ecb_key_32_iv_None = r"""'2020-01-01\tuser0\t044E715357AF77234FD95359666CAFF3\n2020-01-02\tuser1\tB633EF852CE85B4C97827401FD9B606B\n2020-01-03\tuser2\t2AFF7052C748E4BC3BDA8460AFD5A21D'"""
aes_encrypt_mysql_input_example_mode_aes_256_ecb_key_32_iv_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'"""
aes_encrypt_mysql_input_example_mode_aes_256_ecb_key_64_iv_None = r"""'2020-01-01\tuser0\tBFFEC9DF7285A3EC799C941E1450839C\n2020-01-02\tuser1\t3EA0ECBD06326D227A7B9519B1A2955D\n2020-01-03\tuser2\t1478C57DD49523ABDB83A0917F0EDA60'"""
aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_16_iv_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'"""
aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_24_iv_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'"""
aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_32_iv_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'"""
aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_16_iv_16 = r"""'2020-01-01\tuser0\tFC93C1D5E5E3B054C1F3A5692AAC0A61\n2020-01-02\tuser1\tD6DBC76ABCB14B7C6D93F1A5FCA66B9C\n2020-01-03\tuser2\tD4F4158A650D01EB505CC72EFE455486'"""
aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_24_iv_24 = r"""'2020-01-01\tuser0\t26CEE6B6EBDDE1BF887FDEB75F28FB52\n2020-01-02\tuser1\tF9EC1A75BEEFF70B4DEB39AAD075CEFF\n2020-01-03\tuser2\t3FF84AB3BD40FAEEF70F06BCF6AF9C42'"""
aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_24_iv_16 = r"""'2020-01-01\tuser0\t0E3BAF7F4E0BFCFFAE2589B67F71E277\n2020-01-02\tuser1\t2581CCEE9ABE5770480901D65B3D9222\n2020-01-03\tuser2\tED9F3BD8DB12FDF9F2462FFA572361E7'"""
aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_32_iv_32 = r"""'2020-01-01\tuser0\t07371B5DE2E378EE08A3A8B6B9FEAD13\n2020-01-02\tuser1\t3C0BF5D187421ECFFD3E00474A154452\n2020-01-03\tuser2\t05B253FA783D78D864AF7C4D5E6A492D'"""
aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_32_iv_16 = r"""'2020-01-01\tuser0\t72AC7BA6F283EA94A3C33C4D3E51C7D3\n2020-01-02\tuser1\tDACBBE79062F1C721A01CEEE3E85524F\n2020-01-03\tuser2\tFF5A09D19E5EB2ADD94581308588E44A'"""
aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_64_iv_64 = r"""'2020-01-01\tuser0\t573924F0BB4AA1780D45DB6451F123D6\n2020-01-02\tuser1\t007A54AA7ADE8EF844D28936486D75BC\n2020-01-03\tuser2\tAA7249B514398FE1EE827C44402BCE57'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_16_iv_None = r"""'2020-01-01\tuser0\t750BE8662F57A095EC0E610C\n2020-01-02\tuser1\t750BE8662E444A6284C0FC72\n2020-01-03\tuser2\t750BE8662C000B61CDCF1C94'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_24_iv_None = r"""'2020-01-01\tuser0\t5DCC67A043EB776D8B7F5B70\n2020-01-02\tuser1\t5DCC67A042B46DFCC10EFD66\n2020-01-03\tuser2\t5DCC67A040243A8C1346D2DD'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_32_iv_None = r"""'2020-01-01\tuser0\tFAAC1A7D2CE844F8DEB4C44E\n2020-01-02\tuser1\tFAAC1A7D2DF85A43828C0FF8\n2020-01-03\tuser2\tFAAC1A7D2FC7582CCEFCF330'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_16_iv_16 = r"""'2020-01-01\tuser0\t7670A865D13B1B65AD46F8ED\n2020-01-02\tuser1\t7670A865D046007A1E218286\n2020-01-03\tuser2\t7670A865D2E5B091492ECCFB'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_24_iv_24 = r"""'2020-01-01\tuser0\t51EADDE82195C31118D0C171\n2020-01-02\tuser1\t51EADDE82009A46518270271\n2020-01-03\tuser2\t51EADDE8235CB38F95766481'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_24_iv_16 = r"""'2020-01-01\tuser0\t7F38C051539074C0A635C937\n2020-01-02\tuser1\t7F38C051520A30DFACBE9564\n2020-01-03\tuser2\t7F38C051500DA29FF0E7B799'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_32_iv_32 = r"""'2020-01-01\tuser0\t2067186DB91666DE730D0708\n2020-01-02\tuser1\t2067186DB83E2E8B0019F839\n2020-01-03\tuser2\t2067186DBB540332BFC84955'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_32_iv_16 = r"""'2020-01-01\tuser0\t0A216A58A5C0A33215E8E722\n2020-01-02\tuser1\t0A216A58A4E94067ABF030B6\n2020-01-03\tuser2\t0A216A58A6822CAB0318C632'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_64_iv_64 = r"""'2020-01-01\tuser0\t81BD636E1BF4CA02399943E3\n2020-01-02\tuser1\t81BD636E1A93D5D6DD9DCD8D\n2020-01-03\tuser2\t81BD636E18F15168D19C8117'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_16_iv_None = r"""'2020-01-01\tuser0\t650D96B9698D20DB12E2E437\n2020-01-02\tuser1\t650D96B968F00D16ABF2852E\n2020-01-03\tuser2\t650D96B96B8141F425E60D6B'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_24_iv_None = r"""'2020-01-01\tuser0\t72C4724B2F528724A12041C0\n2020-01-02\tuser1\t72C4724B2EF3C6A6FF9E09A9\n2020-01-03\tuser2\t72C4724B2D6EAB1D47709E15'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FD6C94961765ED204F2BCA\n2020-01-02\tuser1\tC5FD6C9497AB1C1AF1DE671C\n2020-01-03\tuser2\tC5FD6C949491F4A3EA5069B3'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_16_iv_16 = r"""'2020-01-01\tuser0\t471D217E9CA3593FFEC955C8\n2020-01-02\tuser1\t471D217E9D7F484D85F81F19\n2020-01-03\tuser2\t471D217E9EBBFD2EA9841008'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_24_iv_24 = r"""'2020-01-01\tuser0\t2EE6147B830481BE36CBE350\n2020-01-02\tuser1\t2EE6147B82DE8F3197AF17A6\n2020-01-03\tuser2\t2EE6147B81FF826E798A0355'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_24_iv_16 = r"""'2020-01-01\tuser0\t1D98EFFAEB9907457BD3FCB2\n2020-01-02\tuser1\t1D98EFFAEA2D930825C6AE22\n2020-01-03\tuser2\t1D98EFFAE92C1D018438B98B'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_32_iv_32 = r"""'2020-01-01\tuser0\t4410165F7DCFDDBB1B15573F\n2020-01-02\tuser1\t4410165F7CFE6A0D2FD5CA9C\n2020-01-03\tuser2\t4410165F7FE8E0C081B3FB7B'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C07B443BB7D7D60E9999C1D\n2020-01-02\tuser1\t1C07B443BA9674A3F68FF3FE\n2020-01-03\tuser2\t1C07B443B95F4B68161A616F'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_64_iv_64 = r"""'2020-01-01\tuser0\tA6D2368A5F177157D73FBD9D\n2020-01-02\tuser1\tA6D2368A5E695ADF99475359\n2020-01-03\tuser2\tA6D2368A5DB96AFD43311124'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_16_iv_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_24_iv_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_16_iv_16 = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'"""
aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_24_iv_24 = r"""'2020-01-01\tuser0\t2E046787D9EFFED25D69C908\n2020-01-02\tuser1\t2E046787D8EFFED25D69C908\n2020-01-03\tuser2\t2E046787DBEFFED25D69C908'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_24_iv_16 = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'"""
aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_32_iv_32 = r"""'2020-01-01\tuser0\t44D3EB069FF443A121590842\n2020-01-02\tuser1\t44D3EB069EF443A121590842\n2020-01-03\tuser2\t44D3EB069DF443A121590842'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'"""
aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_64_iv_64 = r"""'2020-01-01\tuser0\tA69DAA2E8B265618D25D5FE4\n2020-01-02\tuser1\tA69DAA2E8A265618D25D5FE4\n2020-01-03\tuser2\tA69DAA2E89265618D25D5FE4'"""
aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_16_iv_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'"""
aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_24_iv_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'"""
aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'"""
aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_16_iv_16 = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'"""
aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_24_iv_24 = r"""'2020-01-01\tuser0\t2E046787D9EFFED25D69C908\n2020-01-02\tuser1\t2E046787D8EFFED25D69C908\n2020-01-03\tuser2\t2E046787DBEFFED25D69C908'"""
aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_24_iv_16 = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'"""
aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_32_iv_32 = r"""'2020-01-01\tuser0\t44D3EB069FF443A121590842\n2020-01-02\tuser1\t44D3EB069EF443A121590842\n2020-01-03\tuser2\t44D3EB069DF443A121590842'"""
aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'"""
aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_64_iv_64 = r"""'2020-01-01\tuser0\tA69DAA2E8B265618D25D5FE4\n2020-01-02\tuser1\tA69DAA2E8A265618D25D5FE4\n2020-01-03\tuser2\tA69DAA2E89265618D25D5FE4'"""
encrypt_input_example_mode_aes_128_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'"""
encrypt_input_example_mode_aes_192_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'"""
encrypt_input_example_mode_aes_256_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'"""
encrypt_input_example_mode_aes_128_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'"""
encrypt_input_example_mode_aes_192_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'"""
encrypt_input_example_mode_aes_256_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'"""
encrypt_input_example_mode_aes_128_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\tFC93C1D5E5E3B054C1F3A5692AAC0A61\n2020-01-02\tuser1\tD6DBC76ABCB14B7C6D93F1A5FCA66B9C\n2020-01-03\tuser2\tD4F4158A650D01EB505CC72EFE455486'"""
encrypt_input_example_mode_aes_192_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\t0E3BAF7F4E0BFCFFAE2589B67F71E277\n2020-01-02\tuser1\t2581CCEE9ABE5770480901D65B3D9222\n2020-01-03\tuser2\tED9F3BD8DB12FDF9F2462FFA572361E7'"""
encrypt_input_example_mode_aes_256_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\t72AC7BA6F283EA94A3C33C4D3E51C7D3\n2020-01-02\tuser1\tDACBBE79062F1C721A01CEEE3E85524F\n2020-01-03\tuser2\tFF5A09D19E5EB2ADD94581308588E44A'"""
encrypt_input_example_mode_aes_128_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\t750BE8662F57A095EC0E610C\n2020-01-02\tuser1\t750BE8662E444A6284C0FC72\n2020-01-03\tuser2\t750BE8662C000B61CDCF1C94'"""
encrypt_input_example_mode_aes_192_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\t5DCC67A043EB776D8B7F5B70\n2020-01-02\tuser1\t5DCC67A042B46DFCC10EFD66\n2020-01-03\tuser2\t5DCC67A040243A8C1346D2DD'"""
encrypt_input_example_mode_aes_256_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\tFAAC1A7D2CE844F8DEB4C44E\n2020-01-02\tuser1\tFAAC1A7D2DF85A43828C0FF8\n2020-01-03\tuser2\tFAAC1A7D2FC7582CCEFCF330'"""
encrypt_input_example_mode_aes_128_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t7670A865D13B1B65AD46F8ED\n2020-01-02\tuser1\t7670A865D046007A1E218286\n2020-01-03\tuser2\t7670A865D2E5B091492ECCFB'"""
encrypt_input_example_mode_aes_192_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t7F38C051539074C0A635C937\n2020-01-02\tuser1\t7F38C051520A30DFACBE9564\n2020-01-03\tuser2\t7F38C051500DA29FF0E7B799'"""
encrypt_input_example_mode_aes_256_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t0A216A58A5C0A33215E8E722\n2020-01-02\tuser1\t0A216A58A4E94067ABF030B6\n2020-01-03\tuser2\t0A216A58A6822CAB0318C632'"""
encrypt_input_example_mode_aes_128_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\t650D96B9698D20DB12E2E437\n2020-01-02\tuser1\t650D96B968F00D16ABF2852E\n2020-01-03\tuser2\t650D96B96B8141F425E60D6B'"""
encrypt_input_example_mode_aes_192_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C4724B2F528724A12041C0\n2020-01-02\tuser1\t72C4724B2EF3C6A6FF9E09A9\n2020-01-03\tuser2\t72C4724B2D6EAB1D47709E15'"""
encrypt_input_example_mode_aes_256_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FD6C94961765ED204F2BCA\n2020-01-02\tuser1\tC5FD6C9497AB1C1AF1DE671C\n2020-01-03\tuser2\tC5FD6C949491F4A3EA5069B3'"""
encrypt_input_example_mode_aes_128_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t471D217E9CA3593FFEC955C8\n2020-01-02\tuser1\t471D217E9D7F484D85F81F19\n2020-01-03\tuser2\t471D217E9EBBFD2EA9841008'"""
encrypt_input_example_mode_aes_192_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t1D98EFFAEB9907457BD3FCB2\n2020-01-02\tuser1\t1D98EFFAEA2D930825C6AE22\n2020-01-03\tuser2\t1D98EFFAE92C1D018438B98B'"""
encrypt_input_example_mode_aes_256_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C07B443BB7D7D60E9999C1D\n2020-01-02\tuser1\t1C07B443BA9674A3F68FF3FE\n2020-01-03\tuser2\t1C07B443B95F4B68161A616F'"""
encrypt_input_example_mode_aes_128_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'"""
encrypt_input_example_mode_aes_192_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'"""
encrypt_input_example_mode_aes_256_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'"""
encrypt_input_example_mode_aes_128_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'"""
encrypt_input_example_mode_aes_192_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'"""
encrypt_input_example_mode_aes_256_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'"""
encrypt_input_example_mode_aes_128_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'"""
encrypt_input_example_mode_aes_192_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'"""
encrypt_input_example_mode_aes_256_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'"""
encrypt_input_example_mode_aes_128_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'"""
encrypt_input_example_mode_aes_192_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'"""
encrypt_input_example_mode_aes_256_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'"""
encrypt_input_example_mode_aes_128_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t98E5A430C4A01C4429B0F37A4B3CDBC2BDB491651A36D7F904E231E0\n2020-01-02\tuser1\t98E5A430C5A01C4429B0F37A6E108322C2863C1ABF9BC7098CD369DB\n2020-01-03\tuser2\t98E5A430C6A01C4429B0F37A01646A0243D1CB9A516CF61814808196'"""
encrypt_input_example_mode_aes_192_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t3F89C3B657596C86202B59F4350807B364DA1E94682EAB679617575D\n2020-01-02\tuser1\t3F89C3B656596C86202B59F4FA03602ED37788B312FDE2AFDBB7F097\n2020-01-03\tuser2\t3F89C3B655596C86202B59F4691EC8880B8132DA9D8838F70D5618C8'"""
encrypt_input_example_mode_aes_256_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t23B80948CCDB54DC6D0B62F215132A07B30BA6F15593B4F946726B11\n2020-01-02\tuser1\t23B80948CDDB54DC6D0B62F2A01C1BAE07B8D6B26F60116040CDDB55\n2020-01-03\tuser2\t23B80948CEDB54DC6D0B62F2BD0D4954DA6D46772074FFCB4B0D0B98'"""
encrypt_input_example_mode_aes_128_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t98E5A430C4A01C4429B0F37AF9758E0EA4B44A50A7F964C8E51A913C\n2020-01-02\tuser1\t98E5A430C5A01C4429B0F37ADC59D6EEDB86E72F025474386D2BC907\n2020-01-03\tuser2\t98E5A430C6A01C4429B0F37AB32D3FCE5AD110AFECA34529F578214A'"""
encrypt_input_example_mode_aes_192_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t3F89C3B657596C86202B59F4B6C662DFF6347EF3B46C170A2F80E946\n2020-01-02\tuser1\t3F89C3B656596C86202B59F479CD05424199E8D4CEBF5EC262204E8C\n2020-01-03\tuser2\t3F89C3B655596C86202B59F4EAD0ADE4996F52BD41CA849AB4C1A6D3'"""
encrypt_input_example_mode_aes_256_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t23B80948CCDB54DC6D0B62F28787710BBF3F9A594C387B9F7CA2372B\n2020-01-02\tuser1\t23B80948CDDB54DC6D0B62F2328840A20B8CEA1A76CBDE067A1D876F\n2020-01-03\tuser2\t23B80948CEDB54DC6D0B62F22F991258D6597ADF39DF30AD71DD57A2'"""
encrypt_input_example_mode_aes_128_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'"""
encrypt_input_example_mode_aes_192_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'"""
encrypt_input_example_mode_aes_256_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'"""
encrypt_input_example_mode_aes_128_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'"""
encrypt_input_example_mode_aes_192_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'"""
encrypt_input_example_mode_aes_256_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'"""

609
tests/testflows/aes_encryption/tests/decrypt.py Normal file
View File

@ -0,0 +1,609 @@
# -*- coding: utf-8 -*-
import os
from importlib.machinery import SourceFileLoader
from testflows.core import *
from testflows.core.name import basename
from testflows.asserts.helpers import varname
from testflows.asserts import error
from aes_encryption.requirements.requirements import *
from aes_encryption.tests.common import *
@TestOutline
def decrypt(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, step=When, cast=None, endcast=None, compare=None, no_checks=False):
"""Execute `decrypt` function with the specified parameters.
"""
params = []
if mode is not None:
params.append(mode)
if ciphertext is not None:
params.append(ciphertext)
if key is not None:
params.append(key)
if iv is not None:
params.append(iv)
if aad is not None:
params.append(aad)
sql = f"decrypt(" + ", ".join(params) + ")"
if cast:
sql = f"{cast}({sql}){endcast or ''}"
if compare:
sql = f"{compare} = {sql}"
sql = f"SELECT {sql}"
return current().context.node.query(sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks)
@TestScenario
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText("1.0"),
)
def invalid_ciphertext(self):
"""Check that `decrypt` function does not crash when invalid
`ciphertext` is passed as the first parameter.
"""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
invalid_ciphertexts = plaintexts
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}"""):
d_iv = None if not iv_len else f"'{iv[:iv_len]}'"
d_aad = None if not aad_len else f"'{aad}'"
for datatype, ciphertext in invalid_ciphertexts:
if datatype in ["NULL"]:
continue
with When(f"invalid ciphertext={ciphertext}"):
if "cfb" in mode or "ofb" in mode or "ctr" in mode:
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, aad=d_aad, cast="hex")
else:
with When("I execute decrypt function"):
r = decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, aad=d_aad, no_checks=True, step=By)
with Then("exitcode is not zero"):
assert r.exitcode in [198, 36]
with And("exception is present in the output"):
assert "DB::Exception:" in r.output
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_InvalidParameters("1.0")
)
def invalid_parameters(self):
"""Check that `decrypt` function returns an error when
we call it with invalid parameters.
"""
ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')"
with Example("no parameters"):
decrypt(exitcode=42, message="DB::Exception: Incorrect number of arguments for function decrypt provided 0, expected 3 to 5")
with Example("missing key and mode"):
decrypt(ciphertext=ciphertext, exitcode=42,
message="DB::Exception: Incorrect number of arguments for function decrypt provided 1")
with Example("missing mode"):
decrypt(ciphertext=ciphertext, key="'123'", exitcode=42,
message="DB::Exception: Incorrect number of arguments for function decrypt provided 2")
with Example("bad key type - UInt8"):
decrypt(ciphertext=ciphertext, key="123", mode="'aes-128-ecb'", exitcode=43,
message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3")
with Example("bad mode type - forgot quotes"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="aes-128-ecb", exitcode=47,
message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query")
with Example("bad mode type - UInt8"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="128", exitcode=43,
message="DB::Exception: Illegal type of argument #1 'mode'")
with Example("bad iv type - UInt8"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43,
message="DB::Exception: Illegal type of argument")
with Example("bad aad type - UInt8"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=43,
message="DB::Exception: Illegal type of argument")
with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36,
message="DB::Exception: aes-128-ecb does not support IV")
with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]):
decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=36,
message="DB::Exception: aes-128-ecb does not support IV")
with Example("aad not valid for mode", requirements=[RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0")]):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=36,
message="DB::Exception: AAD can be only set for GCM-mode")
with Example("invalid mode value", requirements=[RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0")]):
with When("using unsupported cfb1 mode"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cfb1'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128-cfb1")
with When("using unsupported cfb8 mode"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cfb8'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128-cfb8")
with When("typo in the block algorithm"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128-eeb")
with When("typo in the key size"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-127-ecb")
with When("typo in the aes prefix"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aee-128-ecb")
with When("missing last dash"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128ecb")
with When("missing first dash"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes128-ecb")
with When("all capitals"):
decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36,
message="DB::Exception: Invalid mode: AES-128-ECB")
@TestOutline(Scenario)
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError("1.0"),
RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"),
RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0"),
RQ_SRS008_AES_Decrypt_Function_NonGCMMode_KeyAndInitializationVector_Length("1.0")
)
@Examples("mode key_len iv_len aad", [
# ECB
("'aes-128-ecb'", 16, None, None),
("'aes-192-ecb'", 24, None, None),
("'aes-256-ecb'", 32, None, None),
# CBC
("'aes-128-cbc'", 16, 16, None),
("'aes-192-cbc'", 24, 16, None),
("'aes-256-cbc'", 32, 16, None),
# CFB128
("'aes-128-cfb128'", 16, 16, None),
("'aes-192-cfb128'", 24, 16, None),
("'aes-256-cfb128'", 32, 16, None),
# OFB
("'aes-128-ofb'", 16, 16, None),
("'aes-192-ofb'", 24, 16, None),
("'aes-256-ofb'", 32, 16, None),
# CTR
("'aes-128-ctr'", 16, 16, None),
("'aes-192-ctr'", 24, 16, None),
("'aes-256-ctr'", 32, 16, None)
], "%-16s %-10s %-10s %-10s")
def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad):
"""Check that an error is returned when key or iv length does not match
the expected value for the mode.
"""
ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')"
key = "0123456789" * 4
iv = "0123456789" * 4
with When("key is too short"):
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
with When("key is too long"):
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
if iv_len is not None:
with When("iv is too short"):
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size")
with When("iv is too long"):
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size")
if aad is None:
with When("aad is specified but not needed"):
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", aad="'AAD'", mode=mode, exitcode=36, message="DB::Exception: AAD can be only set for GCM-mode")
else:
with When("iv is specified but not needed"):
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: {} does not support IV".format(mode.strip("'")))
@TestOutline(Scenario)
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError("1.0"),
RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"),
RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0"),
RQ_SRS008_AES_Decrypt_Function_GCMMode_KeyAndInitializationVector_Length("1.0")
)
@Examples("mode key_len iv_len aad", [
# GCM
("'aes-128-gcm'", 16, 8, "'hello there aad'"),
("'aes-128-gcm'", 16, None, "'hello there aad'"),
("'aes-192-gcm'", 24, 8, "''"),
("'aes-192-gcm'", 24, None, "''"),
("'aes-256-gcm'", 32, 8, "'a'"),
("'aes-256-gcm'", 32, None, "'a'")
], "%-16s %-10s %-10s %-10s")
def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad):
"""Check that an error is returned when key or iv length does not match
the expected value for the GCM mode.
"""
ciphertext = "'hello there'"
plaintext = "hello there"
key = "0123456789" * 4
iv = "0123456789" * 4
with When("key is too short"):
ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')"
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
with When("key is too long"):
ciphertext = "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')"
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
if iv_len is not None:
with When(f"iv is too short"):
ciphertext = "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')"
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=198, message="DB::Exception:")
else:
with When("iv is not specified"):
ciphertext = "unhex('1CD4EC93A4B0C687926E8F8C2AA3B4CE1943D006DAE3A774CB1AE5')"
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size 0 != expected size 12")
@TestScenario
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData("1.0"),
RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length("1.0")
)
def aad_parameter_types_and_length(self):
"""Check that `decrypt` function accepts `aad` parameter as the fifth argument
of either `String` or `FixedString` types and that the length is not limited.
"""
plaintext = "hello there"
iv = "'012345678912'"
mode = "'aes-128-gcm'"
key = "'0123456789123456'"
with When("aad is specified using String type"):
ciphertext = "unhex('19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526')"
decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'aad'", message=plaintext)
with When("aad is specified using String with UTF8 characters"):
ciphertext = "unhex('19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39')"
decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'Gãńdåłf_Thê_Gręât'", message=plaintext)
with When("aad is specified using FixedString type"):
ciphertext = "unhex('19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526')"
decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="toFixedString('aad', 3)", message=plaintext)
with When("aad is specified using FixedString with UTF8 characters"):
ciphertext = "unhex('19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39')"
decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", message=plaintext)
with When("aad is 0 bytes"):
ciphertext = "unhex('19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9')"
decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="''", message=plaintext)
with When("aad is 1 byte"):
ciphertext = "unhex('19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035')"
decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'1'", message=plaintext)
with When("aad is 256 bytes"):
ciphertext = "unhex('19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7')"
decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad=f"'{'1' * 256}'", message=plaintext)
@TestScenario
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector("1.0")
)
def iv_parameter_types(self):
"""Check that `decrypt` function accepts `iv` parameter as the fourth argument
of either `String` or `FixedString` types.
"""
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("iv is specified using String type"):
decrypt(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=iv, message="hello there")
with When("iv is specified using String with UTF8 characters"):
decrypt(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="hello there")
with When("iv is specified using FixedString type"):
decrypt(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="hello there")
with When("iv is specified using FixedString with UTF8 characters"):
decrypt(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv=f"toFixedString('Gãńdåłf_Thê', 16)", message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Parameters_Key("1.0")
)
def key_parameter_types(self):
"""Check that `decrypt` function accepts `key` parameter as the second argument
of either `String` or `FixedString` types.
"""
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("key is specified using String type"):
decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there")
with When("key is specified using String with UTF8 characters"):
decrypt(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key="'Gãńdåłf_Thê'", mode=mode, message="hello there")
with When("key is specified using FixedString type"):
decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=f"toFixedString({key}, 16)", mode=mode, message="hello there")
with When("key is specified using FixedString with UTF8 characters"):
decrypt(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key=f"toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Parameters_Mode("1.0"),
)
def mode_parameter_types(self):
"""Check that `decrypt` function accepts `mode` parameter as the third argument
of either `String` or `FixedString` types.
"""
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("mode is specified using String type"):
decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there")
with When("mode is specified using FixedString type"):
decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=f"toFixedString({mode}, 12)", message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue("1.0")
)
def return_value(self):
"""Check that `decrypt` functions returns String data type.
"""
ciphertext = "unhex('F024F9372FA0D8B974894D29FFB8A7F7')"
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("I get type of the return value"):
sql = "SELECT toTypeName(decrypt(" + mode + "," + ciphertext + "," + key + "," + iv + "))"
r = self.context.node.query(sql)
with Then("type should be String"):
assert r.output.strip() == "String", error()
with When("I get the return value"):
decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Syntax("1.0"),
)
def syntax(self):
"""Check that `decrypt` function supports syntax
```sql
decrypt(ciphertext, key, mode, [iv, aad])
```
"""
ciphertext = "19A1183335B374C626B242A6F6E8712E2B64DCDC6A468B2F654614"
sql = f"SELECT decrypt('aes-128-gcm', unhex('{ciphertext}'), '0123456789123456', '012345678912', 'AAD')"
self.context.node.query(sql, step=When, message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText("1.0"),
RQ_SRS008_AES_Decrypt_Function_Parameters_Mode("1.0"),
RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat("1.0"),
RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Values("1.0")
)
def decryption(self):
"""Check that `decrypt` functions accepts `ciphertext` as the second parameter
and `mode` as the first parameter and we can convert the decrypted value into the original
value with the original data type.
"""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(),
"snapshots", "encrypt.py.encrypt.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
for datatype, plaintext in plaintexts:
with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
cast = None
endcast = None
ciphertext = f"unhex({ciphertext})"
compare = plaintext
if datatype == "IPv4":
cast = "toIPv4(IPv4NumToString(reinterpretAsUInt32"
endcast = "))"
elif datatype in ["DateTime64", "UUID", "IPv6", "LowCardinality", "Enum8", "Enum16", "Decimal32", "Decimal64", "Decimal128", "Array"]:
xfail(reason="no conversion")
elif datatype == "NULL":
ciphertext = "NULL"
cast = "isNull"
compare = None
elif datatype in ["Float32", "Float64", "Date", "DateTime"] or "Int" in datatype:
cast = f"reinterpretAs{datatype}"
decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"),
aad=(None if not aad_len else f"'{aad}'"),
cast=cast, endcast=endcast, compare=compare, message="1")
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_Mismatched_Key("1.0")
)
def mismatched_key(self):
"""Check that `decrypt` function returns garbage or an error when key parameter does not match.
"""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
datatype = "String"
plaintext = "'1'"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(),
"snapshots", "encrypt.py.encrypt.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
with When("I decrypt using a mismatched key"):
r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'a{key[:key_len-1]}'", mode=mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"),
aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex")
with Then("exitcode shoud be 0 or 198"):
assert r.exitcode in [0, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
assert "Exception: Failed to decrypt" in output or output != "31", error()
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_Mismatched_IV("1.0")
)
def mismatched_iv(self):
"""Check that `decrypt` function returns garbage or an error when iv parameter does not match.
"""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
datatype = "String"
plaintext = "'1'"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
if not iv_len:
continue
with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
with When("I decrypt using a mismatched iv"):
r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode,
iv=f"'a{iv[:iv_len-1]}'",
aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex")
with Then("exitcode shoud be 0 or 198"):
assert r.exitcode in [0, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
assert "Exception: Failed to decrypt" in output or output != "31", error()
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_Mismatched_AAD("1.0")
)
def mismatched_aad(self):
"""Check that `decrypt` function returns garbage or an error when aad parameter does not match.
"""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
datatype = "String"
plaintext = "'1'"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
if not aad_len:
continue
with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
with When("I decrypt using a mismatched aad"):
r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"),
aad=(None if not aad_len else f"'a{aad}'"), no_checks=True, cast="hex")
with Then("exitcode shoud be 0 or 198"):
assert r.exitcode in [0, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
assert "Exception: Failed to decrypt" in output or output != "31", error()
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_Mismatched_Mode("1.0")
)
def mismatched_mode(self):
"""Check that `decrypt` function returns garbage or an error when mode parameter does not match.
"""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
plaintext = hex('Gãńdåłf_Thê_Gręât'.encode("utf-8"))
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module()
for mode, key_len, iv_len, aad_len in modes:
with Example(f"""mode={mode.strip("'")} datatype=utf8string iv={iv_len} aad={aad_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
for mismatched_mode, _, _, _ in modes:
if mismatched_mode == mode:
continue
with When(f"I decrypt using mismatched mode {mismatched_mode}"):
r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mismatched_mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"),
aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex")
with Then("exitcode shoud be 0 or 36 or 198"):
assert r.exitcode in [0, 36, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
condition = "Exception: Failed to decrypt" in output \
or 'Exception: Invalid key size' in output \
or output != plaintext
assert condition, error()
@TestFeature
@Name("decrypt")
@Requirements(
RQ_SRS008_AES_Decrypt_Function("1.0")
)
def feature(self, node="clickhouse1"):
"""Check the behavior of the `decrypt` function.
"""
self.context.node = self.context.cluster.node(node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

500
tests/testflows/aes_encryption/tests/decrypt_mysql.py Normal file
View File

@ -0,0 +1,500 @@
# -*- coding: utf-8 -*-
import os
from importlib.machinery import SourceFileLoader
from testflows.core import *
from testflows.core.name import basename
from testflows.asserts.helpers import varname
from testflows.asserts import error
from aes_encryption.requirements.requirements import *
from aes_encryption.tests.common import *
@TestOutline
def aes_decrypt_mysql(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None,
step=When, cast=None, endcast=None, compare=None, no_checks=False):
"""Execute `aes_decrypt_mysql` function with the specified parameters.
"""
params = []
if mode is not None:
params.append(mode)
if ciphertext is not None:
params.append(ciphertext)
if key is not None:
params.append(key)
if iv is not None:
params.append(iv)
if aad is not None:
params.append(aad)
sql = f"aes_decrypt_mysql(" + ", ".join(params) + ")"
if cast:
sql = f"{cast}({sql}){endcast or ''}"
if compare:
sql = f"{compare} = {sql}"
sql = f"SELECT {sql}"
return current().context.node.query(sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks)
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText("1.0"),
)
def invalid_ciphertext(self):
"""Check that `aes_decrypt_mysql` function does not crash when invalid
`ciphertext` is passed as the first parameter.
"""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
invalid_ciphertexts = plaintexts
for mode, key_len, iv_len in mysql_modes:
with Example(f"""mode={mode.strip("'")} iv={iv_len}"""):
d_iv = None if not iv_len else f"'{iv[:iv_len]}'"
for datatype, ciphertext in invalid_ciphertexts:
if datatype in ["NULL"]:
continue
with When(f"invalid ciphertext={ciphertext}"):
if "cfb" in mode or "ofb" in mode or "ctr" in mode:
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, cast="hex")
else:
with When("I execute aes_decrypt_mysql function"):
r = aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, no_checks=True, step=By)
with Then("exitcode is not zero"):
assert r.exitcode in [198, 36]
with And("exception is present in the output"):
assert "DB::Exception:" in r.output
@TestOutline(Scenario)
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values_GCM_Error("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values_CTR_Error("1.0")
)
@Examples("mode", [
("'aes-128-gcm'",),
("'aes-192-gcm'",),
("'aes-256-gcm'",),
("'aes-128-ctr'",),
("'aes-192-ctr'",),
("'aes-256-ctr'",)
])
def unsupported_modes(self, mode):
"""Check that `aes_decrypt_mysql` function returns an error when unsupported modes are specified.
"""
ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')"
aes_decrypt_mysql(ciphertext=ciphertext, mode=mode, key=f"'{'1'* 32}'", exitcode=36, message="DB::Exception: Unsupported cipher mode")
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_InvalidParameters("1.0")
)
def invalid_parameters(self):
"""Check that `aes_decrypt_mysql` function returns an error when
we call it with invalid parameters.
"""
ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')"
with Example("no parameters"):
aes_decrypt_mysql(exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 0, expected 3 to 4")
with Example("missing key and mode"):
aes_decrypt_mysql(ciphertext=ciphertext, exitcode=42,
message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 1")
with Example("missing mode"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'123'", exitcode=42,
message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 2")
with Example("bad key type - UInt8"):
aes_decrypt_mysql(ciphertext=ciphertext, key="123", mode="'aes-128-ecb'", exitcode=43,
message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3")
with Example("bad mode type - forgot quotes"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="aes-128-ecb", exitcode=47,
message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query")
with Example("bad mode type - UInt8"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="128", exitcode=43,
message="DB::Exception: Illegal type of argument #1 'mode'")
with Example("bad iv type - UInt8"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43,
message="DB::Exception: Illegal type of argument")
with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]):
aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=0,
message=None)
with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]):
aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=0,
message=None)
with Example("aad passed by mistake"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=42,
message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5")
with Example("aad passed by mistake type - UInt8"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=42,
message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5")
with Example("invalid mode value", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0")]):
with When("typo in the block algorithm"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128-eeb")
with When("typo in the key size"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-127-ecb")
with When("typo in the aes prefix"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aee-128-ecb")
with When("missing last dash"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128ecb")
with When("missing first dash"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes128-ecb")
with When("all capitals"):
aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36,
message="DB::Exception: Invalid mode: AES-128-ECB")
@TestOutline(Scenario)
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_Mode_KeyAndInitializationVector_Length("1.0")
)
@Examples("mode key_len iv_len", [
# ECB
("'aes-128-ecb'", 16, None),
("'aes-192-ecb'", 24, None),
("'aes-256-ecb'", 32, None),
# CBC
("'aes-128-cbc'", 16, 16),
("'aes-192-cbc'", 24, 16),
("'aes-256-cbc'", 32, 16),
# CFB128
("'aes-128-cfb128'", 16, 16),
("'aes-192-cfb128'", 24, 16),
("'aes-256-cfb128'", 32, 16),
# OFB
("'aes-128-ofb'", 16, 16),
("'aes-192-ofb'", 24, 16),
("'aes-256-ofb'", 32, 16)
], "%-16s %-10s %-10s")
def key_or_iv_length_for_mode(self, mode, key_len, iv_len):
"""Check that key or iv length for mode.
"""
ciphertext = "unhex('31F4C847CAB873AB34584368E3E85E3A')"
if mode == "'aes-128-ecb'":
ciphertext = "unhex('31F4C847CAB873AB34584368E3E85E3B')"
elif mode == "'aes-192-ecb'":
ciphertext = "unhex('073868ECDECA94133A61A0FFA282E877')"
elif mode == "'aes-256-ecb'":
ciphertext = "unhex('1729E5354D6EC44D89900ABDB09DC297')"
key = "0123456789" * 4
iv = "0123456789" * 4
with When("key is too short"):
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
with When("key is too long"):
if "ecb" in mode or "cbc" in mode:
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt")
else:
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, cast="hex")
if iv_len is not None:
with When("iv is too short"):
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size")
with When("iv is too long"):
if "ecb" in mode or "cbc" in mode:
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt")
else:
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, cast="hex")
else:
with When("iv is specified but not needed"):
if "ecb" in mode or "cbc" in mode:
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt")
else:
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode)
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector("1.0")
)
def iv_parameter_types(self):
"""Check that `aes_decrypt_mysql` function accepts `iv` parameter as the fourth argument
of either `String` or `FixedString` types.
"""
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("iv is specified using String type"):
aes_decrypt_mysql(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=iv, message="hello there")
with When("iv is specified using String with UTF8 characters"):
aes_decrypt_mysql(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="hello there")
with When("iv is specified using FixedString type"):
aes_decrypt_mysql(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="hello there")
with When("iv is specified using FixedString with UTF8 characters"):
aes_decrypt_mysql(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv=f"toFixedString('Gãńdåłf_Thê', 16)", message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key("1.0")
)
def key_parameter_types(self):
"""Check that `aes_decrypt` function accepts `key` parameter as the second argument
of either `String` or `FixedString` types.
"""
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("key is specified using String type"):
aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there")
with When("key is specified using String with UTF8 characters"):
aes_decrypt_mysql(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key="'Gãńdåłf_Thê'", mode=mode, message="hello there")
with When("key is specified using FixedString type"):
aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=f"toFixedString({key}, 16)", mode=mode, message="hello there")
with When("key is specified using FixedString with UTF8 characters"):
aes_decrypt_mysql(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key=f"toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode("1.0"),
)
def mode_parameter_types(self):
"""Check that `aes_decrypt_mysql` function accepts `mode` parameter as the third argument
of either `String` or `FixedString` types.
"""
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("mode is specified using String type"):
aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there")
with When("mode is specified using FixedString type"):
aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=f"toFixedString({mode}, 12)", message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue("1.0")
)
def return_value(self):
"""Check that `aes_decrypt_mysql` functions returns String data type.
"""
ciphertext = "unhex('F024F9372FA0D8B974894D29FFB8A7F7')"
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("I get type of the return value"):
sql = "SELECT toTypeName(aes_decrypt_mysql(" + mode + "," + ciphertext + "," + key + "," + iv + "))"
r = self.context.node.query(sql)
with Then("type should be String"):
assert r.output.strip() == "String", error()
with When("I get the return value"):
aes_decrypt_mysql(ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax("1.0"),
)
def syntax(self):
"""Check that `aes_decrypt_mysql` function supports syntax
```sql
aes_decrypt_mysql(ciphertext, key, mode, [iv])
```
"""
ciphertext = "70FE78410D6EE237C2DE4A"
sql = f"SELECT aes_decrypt_mysql('aes-128-ofb', unhex('{ciphertext}'), '0123456789123456', '0123456789123456')"
self.context.node.query(sql, step=When, message="hello there")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat("1.0"),
RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values("1.0")
)
def decryption(self):
"""Check that `aes_decrypt_mysql` functions accepts `mode` as the first parameter
and `ciphertext` as the second parameter and we can convert the decrypted value into the original
value with the original data type.
"""
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(),
"snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module()
for mode, key_len, iv_len in mysql_modes:
for datatype, plaintext in plaintexts:
with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
cast = None
endcast = None
ciphertext = f"unhex({ciphertext})"
compare = plaintext
if datatype == "IPv4":
cast = "toIPv4(IPv4NumToString(reinterpretAsUInt32"
endcast = "))"
elif datatype in ["DateTime64", "UUID", "IPv6", "LowCardinality", "Enum8", "Enum16", "Decimal32", "Decimal64", "Decimal128", "Array"]:
xfail(reason="no conversion")
elif datatype == "NULL":
ciphertext = "NULL"
cast = "isNull"
compare = None
elif datatype in ["Float32", "Float64", "Date", "DateTime"] or "Int" in datatype:
cast = f"reinterpretAs{datatype}"
aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"),
cast=cast, endcast=endcast, compare=compare, message="1")
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_Mismatched_Key("1.0")
)
def mismatched_key(self):
"""Check that `aes_decrypt_mysql` function returns garbage or an error when key parameter does not match.
"""
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(),
"snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module()
for mode, key_len, iv_len in mysql_modes:
with Example(f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
with When("I decrypt using a mismatched key"):
r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'a{key[:key_len-1]}'", mode=mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"),
cast="hex", no_checks=True)
with Then("exitcode shoud be 0 or 198"):
assert r.exitcode in [0, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
assert "Exception: Failed to decrypt" in output or output != "31", error()
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_Mismatched_IV("1.0")
)
def mismatched_iv(self):
"""Check that `aes_decrypt_mysql` function returns garbage or an error when iv parameter does not match.
"""
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(),
"snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module()
for mode, key_len, iv_len in mysql_modes:
if not iv_len:
continue
with Example(f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
with When("I decrypt using a mismatched key"):
r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode,
iv=f"'a{iv[:iv_len-1]}'",
cast="hex", no_checks=True)
with Then("exitcode shoud be 0 or 198"):
assert r.exitcode in [0, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
assert "Exception: Failed to decrypt" in output or output != "31", error()
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_Mismatched_Mode("1.0")
)
def mismatched_mode(self):
"""Check that `aes_decrypt_mysql` function returns garbage or an error when mode parameter does not match.
"""
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
plaintext = hex('Gãńdåłf_Thê_Gręât'.encode("utf-8"))
with Given("I load encrypt snapshots"):
snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(),
"snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module()
for mode, key_len, iv_len in mysql_modes:
if not iv_len:
continue
with Example(f"""mode={mode.strip("'")} datatype=utf8string key={key_len} iv={iv_len}""") as example:
with Given("I have ciphertext"):
example_name = basename(example.name)
ciphertext = getattr(snapshot_module, varname(f"example_{example_name}"))
for mismatched_mode, _, _ in mysql_modes:
if mismatched_mode == mode:
continue
with When(f"I decrypt using a mismatched mode {mismatched_mode}"):
r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mismatched_mode,
iv=f"'{iv[:iv_len]}'",
cast="hex", no_checks=True)
with Then("exitcode shoud be 0 or 36 or 198"):
assert r.exitcode in [0, 36, 198], error()
with And("output should be garbage or an error"):
output = r.output.strip()
assert "Exception: Failed to decrypt" in output or output != plaintext, error()
@TestFeature
@Name("decrypt_mysql")
@Requirements(
RQ_SRS008_AES_MySQL_Decrypt_Function("1.0")
)
def feature(self, node="clickhouse1"):
"""Check the behavior of the `aes_decrypt_mysql` function.
"""
self.context.node = self.context.cluster.node(node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

384
tests/testflows/aes_encryption/tests/encrypt.py Normal file
View File

@ -0,0 +1,384 @@
# -*- coding: utf-8 -*-
from testflows.core import *
from testflows.core.name import basename
from testflows.asserts import values, error, snapshot
from aes_encryption.requirements.requirements import *
from aes_encryption.tests.common import *
@TestOutline
def encrypt(self, plaintext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, step=When):
"""Execute `encrypt` function with the specified parameters.
"""
params = []
if mode is not None:
params.append(mode)
if plaintext is not None:
params.append(plaintext)
if key is not None:
params.append(key)
if iv is not None:
params.append(iv)
if aad is not None:
params.append(aad)
sql = "SELECT hex(encrypt(" + ", ".join(params) + "))"
return current().context.node.query(sql, step=step, exitcode=exitcode, message=message)
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_InvalidParameters("1.0")
)
def invalid_parameters(self):
"""Check that `encrypt` function returns an error when
we call it with invalid parameters.
"""
with Example("no parameters"):
encrypt(exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 0, expected 3 to 5")
with Example("missing key and mode"):
encrypt(plaintext="'hello there'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 1")
with Example("missing mode"):
encrypt(plaintext="'hello there'", key="'123'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 2")
with Example("bad key type - UInt8"):
encrypt(plaintext="'hello there'", key="123", mode="'aes-128-ecb'", exitcode=43,
message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3")
with Example("bad mode type - forgot quotes"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="aes-128-ecb", exitcode=47,
message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query")
with Example("bad mode type - UInt8"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="128", exitcode=43,
message="DB::Exception: Illegal type of argument #1 'mode'")
with Example("bad iv type - UInt8"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43,
message="DB::Exception: Illegal type of argument")
with Example("bad aad type - UInt8"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=43,
message="DB::Exception: Illegal type of argument")
with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36,
message="DB::Exception: aes-128-ecb does not support IV")
with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=36,
message="DB::Exception: aes-128-ecb does not support IV")
with Example("aad not valid for mode", requirements=[RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0")]):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=36,
message="DB::Exception: AAD can be only set for GCM-mode")
with Example("invalid mode value", requirements=[RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0")]):
with When("using unsupported cfb1 mode"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cfb1'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128-cfb1")
with When("using unsupported cfb8 mode"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cfb8'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128-cfb8")
with When("typo in the block algorithm"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128-eeb")
with When("typo in the key size"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-127-ecb")
with When("typo in the aes prefix"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aee-128-ecb")
with When("missing last dash"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128ecb")
with When("missing first dash"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes128-ecb")
with When("all capitals"):
encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36,
message="DB::Exception: Invalid mode: AES-128-ECB")
@TestOutline(Scenario)
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError("1.0"),
RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"),
RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0"),
RQ_SRS008_AES_Encrypt_Function_NonGCMMode_KeyAndInitializationVector_Length("1.0")
)
@Examples("mode key_len iv_len aad", [
# ECB
("'aes-128-ecb'", 16, None, None),
("'aes-192-ecb'", 24, None, None),
("'aes-256-ecb'", 32, None, None),
# CBC
("'aes-128-cbc'", 16, 16, None),
("'aes-192-cbc'", 24, 16, None),
("'aes-256-cbc'", 32, 16, None),
# CFB128
("'aes-128-cfb128'", 16, 16, None),
("'aes-192-cfb128'", 24, 16, None),
("'aes-256-cfb128'", 32, 16, None),
# OFB
("'aes-128-ofb'", 16, 16, None),
("'aes-192-ofb'", 24, 16, None),
("'aes-256-ofb'", 32, 16, None),
# CTR
("'aes-128-ctr'", 16, 16, None),
("'aes-192-ctr'", 24, 16, None),
("'aes-256-ctr'", 32, 16, None),
], "%-16s %-10s %-10s %-10s")
def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad):
"""Check that an error is returned when key or iv length does not match
the expected value for the mode.
"""
plaintext = "'hello there'"
key = "0123456789" * 4
iv = "0123456789" * 4
with When("key is too short"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
with When("key is too long"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
if iv_len is not None:
with When("iv is too short"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size")
with When("iv is too long"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size")
if aad is None:
with When("aad is specified but not needed"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", aad="'AAD'", mode=mode, exitcode=36, message="DB::Exception: AAD can be only set for GCM-mode")
else:
with When("iv is specified but not needed"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: {} does not support IV".format(mode.strip("'")))
@TestOutline(Scenario)
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError("1.0"),
RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"),
RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0"),
RQ_SRS008_AES_Encrypt_Function_GCMMode_KeyAndInitializationVector_Length("1.0")
)
@Examples("mode key_len iv_len aad", [
# GCM
("'aes-128-gcm'", 16, 8, "'hello there aad'"),
("'aes-192-gcm'", 24, 8, "''"),
("'aes-256-gcm'", 32, 8, "'a'"),
], "%-16s %-10s %-10s %-10s")
def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad):
"""Check that an error is returned when key or iv length does not match
the expected value for the GCM mode.
"""
plaintext = "'hello there'"
key = "0123456789" * 4
iv = "0123456789" * 4
with When("key is too short"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len-1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
with When("key is too long"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len+1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
if iv_len is not None:
with When(f"iv is too short"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=0)
else:
with When("iv is not specified"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size")
if aad is not None:
with When(f"aad is {aad}"):
encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len]}'", aad=f"{aad}", mode=mode)
@TestScenario
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData("1.0"),
RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length("1.0")
)
def aad_parameter_types_and_length(self):
"""Check that `encrypt` function accepts `aad` parameter as the fifth argument
of either `String` or `FixedString` types and that the length is not limited.
"""
plaintext = "'hello there'"
iv = "'012345678912'"
mode = "'aes-128-gcm'"
key = "'0123456789123456'"
with When("aad is specified using String type"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'aad'", message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526")
with When("aad is specified using String with UTF8 characters"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'Gãńdåłf_Thê_Gręât'", message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39")
with When("aad is specified using FixedString type"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="toFixedString('aad', 3)", message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526")
with When("aad is specified using FixedString with UTF8 characters"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39")
with When("aad is 0 bytes"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="''", message="19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9")
with When("aad is 1 byte"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'1'", message="19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035")
with When("aad is 256 bytes"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad=f"'{'1' * 256}'", message="19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7")
@TestScenario
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector("1.0")
)
def iv_parameter_types(self):
"""Check that `encrypt` function accepts `iv` parameter as the fourth argument
of either `String` or `FixedString` types.
"""
plaintext = "'hello there'"
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("iv is specified using String type"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7")
with When("iv is specified using String with UTF8 characters"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="7A4EC0FF3796F46BED281F4778ACE1DC")
with When("iv is specified using FixedString type"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="F024F9372FA0D8B974894D29FFB8A7F7")
with When("iv is specified using FixedString with UTF8 characters"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv="toFixedString('Gãńdåłf_Thê', 16)", message="7A4EC0FF3796F46BED281F4778ACE1DC")
@TestScenario
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Parameters_Key("1.0")
)
def key_parameter_types(self):
"""Check that `encrypt` function accepts `key` parameter as the second argument
of either `String` or `FixedString` types.
"""
plaintext = "'hello there'"
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("key is specified using String type"):
encrypt(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576")
with When("key is specified using String with UTF8 characters"):
encrypt(plaintext=plaintext, key="'Gãńdåłf_Thê'", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D")
with When("key is specified using FixedString type"):
encrypt(plaintext=plaintext, key=f"toFixedString({key}, 16)", mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576")
with When("key is specified using FixedString with UTF8 characters"):
encrypt(plaintext=plaintext, key="toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D")
@TestScenario
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Parameters_Mode("1.0"),
)
def mode_parameter_types(self):
"""Check that `encrypt` function accepts `mode` parameter as the third argument
of either `String` or `FixedString` types.
"""
plaintext = "'hello there'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("mode is specified using String type"):
encrypt(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576")
with When("mode is specified using FixedString type"):
encrypt(plaintext=plaintext, key=key, mode=f"toFixedString({mode}, 12)", message="49C9ADB81BA9B58C485E7ADB90E70576")
@TestScenario
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText("1.0"),
RQ_SRS008_AES_Encrypt_Function_Parameters_Mode("1.0"),
RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat("1.0"),
RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Values("1.0")
)
def encryption(self):
"""Check that `encrypt` functions accepts `plaintext` as the second parameter
with any data type and `mode` as the first parameter.
"""
key = f"{'1' * 36}"
iv = f"{'2' * 16}"
aad = "some random aad"
for mode, key_len, iv_len, aad_len in modes:
for datatype, plaintext in plaintexts:
with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example:
r = encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"), aad=(None if not aad_len else f"'{aad}'"))
with Then("I check output against snapshot"):
with values() as that:
example_name = basename(example.name)
assert that(snapshot(r.output.strip(), "encrypt", name=f"example_{example_name.replace(' ', '_')}")), error()
@TestScenario
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue("1.0")
)
def return_value(self):
"""Check that `encrypt` functions returns String data type.
"""
plaintext = "'hello there'"
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("I get type of the return value"):
sql = "SELECT toTypeName(encrypt(" + mode + "," + plaintext + "," + key + "," + iv + "))"
r = self.context.node.query(sql)
with Then("type should be String"):
assert r.output.strip() == "String", error()
with When("I get return ciphertext as hex"):
encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7")
@TestScenario
@Requirements(
RQ_SRS008_AES_Encrypt_Function_Syntax("1.0"),
)
def syntax(self):
"""Check that `encrypt` function supports syntax
```sql
encrypt(plaintext, key, mode, [iv, aad])
```
"""
sql = "SELECT hex(encrypt('aes-128-gcm', 'hello there', '0123456789123456', '012345678912', 'AAD'))"
self.context.node.query(sql, step=When, message="19A1183335B374C626B242A6F6E8712E2B64DCDC6A468B2F654614")
@TestFeature
@Name("encrypt")
@Requirements(
RQ_SRS008_AES_Encrypt_Function("1.0")
)
def feature(self, node="clickhouse1"):
"""Check the behavior of the `encrypt` function.
"""
self.context.node = self.context.cluster.node(node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

304
tests/testflows/aes_encryption/tests/encrypt_mysql.py Normal file
View File

@ -0,0 +1,304 @@
from testflows.core import *
from testflows.core.name import basename
from testflows.asserts import values, error, snapshot
from aes_encryption.requirements.requirements import *
from aes_encryption.tests.common import *
@TestOutline
def aes_encrypt_mysql(self, plaintext=None, key=None, mode=None, iv=None, exitcode=0, message=None, step=When):
"""Execute `aes_encrypt_mysql` function with the specified parameters.
"""
params = []
if mode is not None:
params.append(mode)
if plaintext is not None:
params.append(plaintext)
if key is not None:
params.append(key)
if iv is not None:
params.append(iv)
sql = "SELECT hex(aes_encrypt_mysql(" + ", ".join(params) + "))"
return current().context.node.query(sql, step=step, exitcode=exitcode, message=message)
@TestOutline(Scenario)
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values_GCM_Error("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values_CTR_Error("1.0")
)
@Examples("mode", [
("'aes-128-gcm'",),
("'aes-192-gcm'",),
("'aes-256-gcm'",),
("'aes-128-ctr'",),
("'aes-192-ctr'",),
("'aes-256-ctr'",),
])
def unsupported_modes(self, mode):
"""Check that `aes_encrypt_mysql` function returns an error when unsupported modes are specified.
"""
aes_encrypt_mysql(plaintext="'hello there'", mode=mode, key=f"'{'1'* 32}'", exitcode=36, message="DB::Exception: Unsupported cipher mode")
@TestScenario
@Requirements(
RQ_SRS008_AES_Functions_InvalidParameters("1.0")
)
def invalid_parameters(self):
"""Check that `aes_encrypt_mysql` function returns an error when
we call it with invalid parameters.
"""
with Example("no parameters"):
aes_encrypt_mysql(exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 0, expected 3 to 4")
with Example("missing key and mode"):
aes_encrypt_mysql(plaintext="'hello there'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 1")
with Example("missing mode"):
aes_encrypt_mysql(plaintext="'hello there'", key="'123'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 2")
with Example("bad key type - UInt8"):
aes_encrypt_mysql(plaintext="'hello there'", key="123", mode="'aes-128-ecb'", exitcode=43,
message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3")
with Example("bad mode type - forgot quotes"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="aes-128-ecb", exitcode=47,
message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query")
with Example("bad mode type - UInt8"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="128", exitcode=43,
message="DB::Exception: Illegal type of argument #1 'mode'")
with Example("bad iv type - UInt8"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43,
message="DB::Exception: Illegal type of argument")
with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36,
message="DB::Exception: aes-128-ecb does not support IV")
with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=0,
message=None)
with Example("invalid mode value", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0")]):
with When("typo in the block algorithm"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128-eeb")
with When("typo in the key size"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-127-ecb")
with When("typo in the aes prefix"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aee-128-ecb")
with When("missing last dash"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes-128ecb")
with When("missing first dash"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36,
message="DB::Exception: Invalid mode: aes128-ecb")
with When("all capitals"):
aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36,
message="DB::Exception: Invalid mode: AES-128-ECB")
@TestOutline(Scenario)
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_Mode_KeyAndInitializationVector_Length("1.0")
)
@Examples("mode key_len iv_len", [
# ECB
("'aes-128-ecb'", 16, None),
("'aes-192-ecb'", 24, None),
("'aes-256-ecb'", 32, None),
# CBC
("'aes-128-cbc'", 16, 16),
("'aes-192-cbc'", 24, 16),
("'aes-256-cbc'", 32, 16),
# CFB128
("'aes-128-cfb128'", 16, 16),
("'aes-192-cfb128'", 24, 16),
("'aes-256-cfb128'", 32, 16),
# OFB
("'aes-128-ofb'", 16, 16),
("'aes-192-ofb'", 24, 16),
("'aes-256-ofb'", 32, 16)
], "%-16s %-10s %-10s")
def key_or_iv_length_for_mode(self, mode, key_len, iv_len):
"""Check that key or iv length for mode.
"""
plaintext = "'hello there'"
key = "0123456789" * 4
iv = "0123456789" * 4
with When("key is too short"):
aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size")
with When("key is too long"):
aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len+1]}'", mode=mode)
if iv_len is not None:
with When("iv is too short"):
aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size")
with When("iv is too long"):
aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode)
else:
with When("iv is specified but not needed"):
aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector("1.0")
)
def iv_parameter_types(self):
"""Check that `aes_encrypt_mysql` function accepts `iv` parameter as the fourth argument
of either `String` or `FixedString` types.
"""
plaintext = "'hello there'"
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("iv is specified using String type"):
aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7")
with When("iv is specified using String with UTF8 characters"):
aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="7A4EC0FF3796F46BED281F4778ACE1DC")
with When("iv is specified using FixedString type"):
aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="F024F9372FA0D8B974894D29FFB8A7F7")
with When("iv is specified using FixedString with UTF8 characters"):
aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv="toFixedString('Gãńdåłf_Thê', 16)", message="7A4EC0FF3796F46BED281F4778ACE1DC")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key("1.0")
)
def key_parameter_types(self):
"""Check that `aes_encrypt_mysql` function accepts `key` parameter as the second argument
of either `String` or `FixedString` types.
"""
plaintext = "'hello there'"
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("key is specified using String type"):
aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576")
with When("key is specified using String with UTF8 characters"):
aes_encrypt_mysql(plaintext=plaintext, key="'Gãńdåłf_Thê'", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D")
with When("key is specified using FixedString type"):
aes_encrypt_mysql(plaintext=plaintext, key=f"toFixedString({key}, 16)", mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576")
with When("key is specified using FixedString with UTF8 characters"):
aes_encrypt_mysql(plaintext=plaintext, key="toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode("1.0"),
)
def mode_parameter_types(self):
"""Check that `aes_encrypt_mysql` function accepts `mode` parameter as the third argument
of either `String` or `FixedString` types.
"""
plaintext = "'hello there'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("mode is specified using String type"):
aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576")
with When("mode is specified using FixedString type"):
aes_encrypt_mysql(plaintext=plaintext, key=key, mode=f"toFixedString({mode}, 12)", message="49C9ADB81BA9B58C485E7ADB90E70576")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue("1.0")
)
def return_value(self):
"""Check that `aes_encrypt_mysql` functions returns String data type.
"""
plaintext = "'hello there'"
iv = "'0123456789123456'"
mode = "'aes-128-cbc'"
key = "'0123456789123456'"
with When("I get type of the return value"):
sql = "SELECT toTypeName(aes_encrypt_mysql("+ mode + "," + plaintext + "," + key + "," + iv + "))"
r = self.context.node.query(sql)
with Then("type should be String"):
assert r.output.strip() == "String", error()
with When("I get return ciphertext as hex"):
aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax("1.0"),
)
def syntax(self):
"""Check that `aes_encrypt_mysql` function supports syntax
```sql
aes_encrypt_mysql(plaintext, key, mode, [iv])
```
"""
sql = "SELECT hex(aes_encrypt_mysql('aes-128-ofb', 'hello there', '0123456789123456', '0123456789123456'))"
self.context.node.query(sql, step=When, message="70FE78410D6EE237C2DE4A")
@TestScenario
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat("1.0"),
RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values("1.0")
)
def encryption(self):
"""Check that `aes_encrypt_mysql` functions accepts `plaintext` as the second parameter
with any data type and `mode` as the first parameter.
"""
key = f"{'1' * 64}"
iv = f"{'2' * 64}"
for mode, key_len, iv_len in mysql_modes:
for datatype, plaintext in plaintexts:
with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""") as example:
r = aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode,
iv=(None if not iv_len else f"'{iv[:iv_len]}'"))
with Then("I check output against snapshot"):
with values() as that:
example_name = basename(example.name)
assert that(snapshot(r.output.strip(), "encrypt_mysql", name=f"example_{example_name.replace(' ', '_')}")), error()
@TestFeature
@Name("encrypt_mysql")
@Requirements(
RQ_SRS008_AES_MySQL_Encrypt_Function("1.0")
)
def feature(self, node="clickhouse1"):
"""Check the behavior of the `aes_encrypt_mysql` function.
"""
self.context.node = self.context.cluster.node(node)
for scenario in loads(current_module(), Scenario):
Scenario(run=scenario, flags=TE)

2076
tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot Normal file
View File

File diff suppressed because it is too large Load Diff

1980
tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot Normal file
View File

File diff suppressed because it is too large Load Diff

1
tests/testflows/regression.py
View File

@ -17,6 +17,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None):
Feature(test=load("example.regression", "regression"))(**args)
Feature(test=load("ldap.regression", "regression"))(**args)
Feature(test=load("rbac.regression", "regression"))(**args)
Feature(test=load("aes_encryption.regression", "regression"))(**args)
if main():
regression()