Test various combinations of deleted and policy-filtered rows

tests/queries/0_stateless/02473_multistep_prewhere.python

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+import requests
+import os
+
+
+class Tester:
+    '''
+    - Creates test table
+    - Deletes the specified range of rows
+    - Masks another range using row-level policy
+    - Runs some read queries     
+    '''
+    def __init__(self, session, url, index_granularity, total_rows):
+        self.session = session
+        self.url = url
+        self.index_granularity = index_granularity
+        self.total_rows = total_rows
+        self.reported_errors = set()
+        self.repro_queries = []
+
+    def query(self, query_text, include_in_repro_steps = True):
+        self.repro_queries.append(query_text)
+        resp = self.session.post(self.url, data=query_text)
+        if resp.status_code != 200:
+            # Group similar errors 
+            error = resp.text[0:40]
+            if error not in self.reported_errors:
+                print('Code:', resp.status_code)
+                print('Result:', resp.text)
+                print('Repro steps:', '\n\n\t'.join(self.repro_queries))
+                self.reported_errors.add(error)
+                exit(1)
+
+        if not include_in_repro_steps:
+            self.repro_queries.pop()
+
+        return resp.text
+
+    
+    def check_data(self, delete_range_start, delete_range_end, row_level_policy_range_start, row_level_policy_range_end):
+        result = self.query('SELECT count() FROM tab_02473;', False)
+
+        delta = 10
+        for query_range_start in [0, delta]:
+            for query_range_end in [self.total_rows - delta]: #, self.total_rows]:
+                result = self.query('SELECT count() from tab_02473 PREWHERE b > {} AND b <= {} WHERE a == 0;'.format(
+                    query_range_start, query_range_end), False)
+                result = self.query('SELECT count() from tab_02473 PREWHERE c > {} AND c <= {} WHERE a == 0;'.format(
+                    query_range_start, query_range_end), False)
+                result = self.query('SELECT count() from tab_02473 PREWHERE c <= {} OR c > {} WHERE a == 0;'.format(
+                    query_range_start, query_range_end), False)
+
+
+    def run_test(self, delete_range_start, delete_range_end, row_level_policy_range_start, row_level_policy_range_end):
+        self.repro_queries = []
+
+        self.query('''
+            CREATE TABLE tab_02473 (a Int8, b Int32, c Int32, PRIMARY KEY (a))
+            ENGINE = MergeTree() ORDER BY (a, b)
+            SETTINGS min_bytes_for_wide_part = 0, index_granularity = {};'''.format(self.index_granularity))
+
+        self.query('INSERT INTO tab_02473 select 0, number+1, number+1 FROM numbers({});'.format(self.total_rows))
+
+        self.query('OPTIMIZE TABLE tab_02473 FINAL SETTINGS mutations_sync=2;')
+
+        self.check_data(-100, -100, -100, -100)
+
+        self.query('DELETE FROM tab_02473 WHERE a = 0 AND b > {} AND b <= {};'.format(
+            delete_range_start, delete_range_end))
+
+        self.check_data(delete_range_start, delete_range_end, -100, -100)
+
+        self.query('CREATE ROW POLICY policy_tab_02473 ON tab_02473 FOR SELECT USING b <= {} OR b > {} TO default;'.format(
+            row_level_policy_range_start, row_level_policy_range_end))
+
+        self.check_data(delete_range_start, delete_range_end, row_level_policy_range_start, row_level_policy_range_end)
+
+        self.query('DROP POLICY policy_tab_02473 ON tab_02473;')
+
+        self.query('DROP TABLE tab_02473;')
+
+
+
+def main():
+    # Set mutations to synchromous mode and enable lightweight DELETE's
+    url = os.environ['CLICKHOUSE_URL'] + '&mutations_sync=2&allow_experimental_lightweight_delete=1&max_threads=1'
+
+    default_index_granularity = 10;
+    total_rows = 5 * default_index_granularity
+    step = default_index_granularity
+    session = requests.Session()
+    for index_granularity in [default_index_granularity-1, default_index_granularity]: # [default_index_granularity-1, default_index_granularity+1, default_index_granularity]:
+        tester = Tester(session, url, index_granularity, total_rows)
+        # Test combinations of ranges of various size masked by lightweight DELETES
+        # along with ranges of various size masked by row-level policies
+        for delete_range_start in range(0, total_rows, step):
+            for delete_range_end in range(delete_range_start, total_rows, step):
+                for row_level_policy_range_start in range(0, total_rows, step):
+                    for row_level_policy_range_end in range(row_level_policy_range_start, total_rows, step):
+                        tester.run_test(delete_range_start, delete_range_end, row_level_policy_range_start, row_level_policy_range_end)
+
+
+if __name__ == "__main__":
+    main()
+

tests/queries/0_stateless/02473_multistep_prewhere.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# We should have correct env vars from shell_config.sh to run this test
+
+python3 "$CURDIR"/02473_multistep_prewhere.python
+