Allow to specify min and max for random settings in the test

docs/en/development/tests.md

@@ -91,6 +91,28 @@ SELECT 1
 In addition to the above settings, you can use `USE_*` flags from `system.build_options` to define usage of particular ClickHouse features.
 For example, if your test uses a MySQL table, you should add a tag `use-mysql`.
 
+### Specifying limits for random settings
+
+A test can specify minimum and maximum allowed values for settings that can be randomized during test run.
+
+For `.sh` tests limits are written as a comment on the line next to tags or on the second line if no tags are specified:
+
+```bash
+#!/usr/bin/env bash
+# Tags: no-fasttest
+# Random settings limits: max_block_size=(1000, 10000), index_granularity=(100, None)
+```
+
+For `.sql` tests tags are placed as a SQL comment in the line next to tags or in the first line:
+
+```sql
+-- Tags: no-fasttest
+-- Random settings limits: max_block_size=(1000, 10000), index_granularity=(100, None)
+SELECT 1
+```
+
+If you need to specify only one limit, you can use `None` for another one.
+
 ### Choosing the Test Name
 
 The name of the test starts with a five-digit prefix followed by a descriptive name, such as `00422_hash_function_constexpr.sql`. To choose the prefix, find the largest prefix already present in the directory, and increment it by one. In the meantime, some other tests might be added with the same numeric prefix, but this is OK and does not lead to any problems, you don't have to change it later.

tests/clickhouse-test

@@ -39,6 +39,7 @@ from errno import ESRCH
 from subprocess import PIPE, Popen
 from time import sleep, time
 from typing import Dict, List, Optional, Set, Tuple, Union
+from ast import literal_eval as make_tuple
 
 try:
     import termcolor  # type: ignore
@@ -1068,9 +1069,25 @@ class TestCase:
 
         return description + "\n"
 
+    def apply_random_settings_limits(self, random_settings):
+        print("Random settings limits:", self.random_settings_limits)
+        for setting in random_settings:
+            if setting in self.random_settings_limits:
+                min = self.random_settings_limits[setting][0]
+                if min and random_settings[setting] < min:
+                    random_settings[setting] = min
+                max = self.random_settings_limits[setting][1]
+                if max and random_settings[setting] > max:
+                    random_settings[setting] = max
+
     def __init__(self, suite, case: str, args, is_concurrent: bool):
         self.case: str = case  # case file name
         self.tags: Set[str] = suite.all_tags[case] if case in suite.all_tags else set()
+        self.random_settings_limits = (
+            suite.all_random_settings_limits[case]
+            if case in suite.all_random_settings_limits
+            else dict()
+        )
 
         for tag in os.getenv("GLOBAL_TAGS", "").split(","):
             self.tags.add(tag.strip())
@@ -1112,11 +1129,13 @@ class TestCase:
 
         if self.randomize_settings:
             self.random_settings = SettingsRandomizer.get_random_settings(args)
+            self.apply_random_settings_limits(self.random_settings)
 
         if self.randomize_merge_tree_settings:
             self.merge_tree_random_settings = (
                 MergeTreeSettingsRandomizer.get_random_settings(args)
             )
+            self.apply_random_settings_limits(self.merge_tree_random_settings)
 
         self.base_url_params = (
             os.environ["CLICKHOUSE_URL_PARAMS"]
@@ -1900,7 +1919,9 @@ class TestSuite:
         return test_name
 
     @staticmethod
-    def read_test_tags(suite_dir: str, all_tests: List[str]) -> Dict[str, Set[str]]:
+    def read_test_tags_and_random_settings_limits(
+        suite_dir: str, all_tests: List[str]
+    ) -> (Dict[str, Set[str]], Dict[str, Dict[str, Tuple[int, int]]]):
         def get_comment_sign(filename):
             if filename.endswith(".sql") or filename.endswith(".sql.j2"):
                 return "--"
@@ -1925,22 +1946,48 @@ class TestSuite:
             tags = {tag.strip() for tag in tags}
             return tags
 
+        def parse_random_settings_limits_from_line(
+            line, comment_sign
+        ) -> Dict[str, Tuple[int, int]]:
+            if not line.startswith(comment_sign):
+                return {}
+            random_settings_limits_str = line[
+                len(comment_sign) :
+            ].lstrip()  # noqa: ignore E203
+            random_settings_limits_prefix = "Random settings limits:"
+            if not random_settings_limits_str.startswith(random_settings_limits_prefix):
+                return {}
+            random_settings_limits_str = random_settings_limits_str[
+                len(random_settings_limits_prefix) :
+            ]  # noqa: ignore E203
+            # limits are specified in a form 'setting1=(min, max), setting2=(min,max), ...'
+            random_settings_limits = re.findall(
+                "([^=, ]+) *= *(\([^=]+\))", random_settings_limits_str
+            )
+            random_settings_limits = {
+                pair[0]: make_tuple(pair[1]) for pair in random_settings_limits
+            }
+            return random_settings_limits
+
         def is_shebang(line: str) -> bool:
             return line.startswith("#!")
 
         def find_tag_line(file):
-            for line in file:
+            line = file.readline()
+            while line != "":
                 line = line.strip()
                 if line and not is_shebang(line):
                     return line
+                line = file.readline()
             return ""
 
-        def load_tags_from_file(filepath):
+        def load_tags_and_random_settings_limits_from_file(filepath):
             comment_sign = get_comment_sign(filepath)
             need_query_params = False
             with open(filepath, "r", encoding="utf-8") as file:
                 try:
                     tag_line = find_tag_line(file)
+                    next_line = file.readline()
                 except UnicodeDecodeError:
                     return []
                 try:
@@ -1950,21 +1997,35 @@ class TestSuite:
                                 need_query_params = True
                 except UnicodeDecodeError:
                     pass
-            parsed_tags = parse_tags_from_line(tag_line, comment_sign)
-            if need_query_params:
-                parsed_tags.add("need-query-parameters")
-            return parsed_tags
+                parsed_tags = parse_tags_from_line(tag_line, comment_sign)
+                if need_query_params:
+                    parsed_tags.add("need-query-parameters")
+                random_settings_limits_line = next_line if parsed_tags else tag_line
+                random_settings_limits = parse_random_settings_limits_from_line(
+                    random_settings_limits_line, comment_sign
+                )
+            return parsed_tags, random_settings_limits
 
         all_tags = {}
+        all_random_settings_limits = {}
         start_time = datetime.now()
         for test_name in all_tests:
-            tags = load_tags_from_file(os.path.join(suite_dir, test_name))
+            (
+                tags,
+                random_settings_limits,
+            ) = load_tags_and_random_settings_limits_from_file(
+                os.path.join(suite_dir, test_name)
+            )
             if tags:
                 all_tags[test_name] = tags
+            if random_settings_limits:
+                all_random_settings_limits[test_name] = random_settings_limits
         elapsed = (datetime.now() - start_time).total_seconds()
         if elapsed > 1:
-            print(f"Tags for suite {suite_dir} read in {elapsed:.2f} seconds")
-        return all_tags
+            print(
+                f"Tags and random settings limits for suite {suite_dir} read in {elapsed:.2f} seconds"
+            )
+        return all_tags, all_random_settings_limits
 
     def __init__(self, args, suite_path: str, suite_tmp_path: str, suite: str):
         self.args = args
@@ -1994,9 +2055,16 @@ class TestSuite:
         self.all_tests: List[str] = self.get_tests_list(
             self.tests_in_suite_key_func, filter_func
         )
-        self.all_tags: Dict[str, Set[str]] = self.read_test_tags(
-            self.suite_path, self.all_tests
+
+        all_tags_and_random_settings_limits = (
+            self.read_test_tags_and_random_settings_limits(
+                self.suite_path, self.all_tests
+            )
         )
+        self.all_tags: Dict[str, Set[str]] = all_tags_and_random_settings_limits[0]
+        self.all_random_settings_limits: Dict[
+            str, Dict[str, (int, int)]
+        ] = all_tags_and_random_settings_limits[1]
 
         self.sequential_tests = []
         self.parallel_tests = []

tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql

@@ -1,4 +1,5 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None)
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql

@@ -1,4 +1,5 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None)
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql

@@ -1,4 +1,6 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
+
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -31,3 +33,5 @@ optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
 drop table test;
+
+select 1;

tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql

@@ -1,4 +1,6 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
+
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;

tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql

@@ -1,4 +1,6 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
+
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;

tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql

@@ -1,4 +1,6 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
+
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;

tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql

@@ -1,4 +1,5 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
 
 set allow_experimental_dynamic_type = 1;
 

tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql

@@ -1,4 +1,5 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
 
 set allow_experimental_dynamic_type = 1;
 

tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql

@@ -1,4 +1,5 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
 
 set allow_experimental_dynamic_type = 1;
 

tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql

@@ -1,4 +1,5 @@
 -- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
 
 set allow_experimental_dynamic_type = 1;
 

tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql

@@ -1,4 +1,5 @@
 -- Tags: long
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql

@@ -1,4 +1,5 @@
 -- Tags: long
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql

@@ -1,4 +1,5 @@
 -- Tags: long
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql

@@ -1,4 +1,5 @@
 -- Tags: long
+-- Random settings limits: index_granularity=(100, None), merge_max_block_size=(100, None)
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;