Support jijna templates for sql files in clickhouse-test

docker/test/fasttest/Dockerfile

@@ -65,7 +65,7 @@ RUN apt-get update \
         unixodbc \
        --yes --no-install-recommends
 
-RUN pip3 install numpy scipy pandas
+RUN pip3 install numpy scipy pandas Jinja2
 
 # This symlink required by gcc to find lld compiler
 RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld

docker/test/stateless/Dockerfile

@@ -32,7 +32,7 @@ RUN apt-get update -y \
             postgresql-client \
             sqlite3
 
-RUN pip3 install numpy scipy pandas
+RUN pip3 install numpy scipy pandas Jinja2
 
 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
    && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \

tests/.gitignore

@@ -3,3 +3,6 @@
 *.error
 *.dump
 test_data
+
+/queries/0_stateless/*.gen.sql
+/queries/0_stateless/*.gen.reference

tests/clickhouse-test

@@ -29,6 +29,13 @@ import string
 import multiprocessing
 from contextlib import closing
 
+USE_JINJA = True
+try:
+    import jinja2
+except ImportError:
+    USE_JINJA = False
+    print('WARNING: jinja2 not installed! Template tests will be skipped.')
+
 DISTRIBUTED_DDL_TIMEOUT_MSG = "is executing longer than distributed_ddl_task_timeout"
 
 MESSAGES_TO_RETRY = [
@@ -47,6 +54,8 @@ MESSAGES_TO_RETRY = [
 
 MAX_RETRIES = 3
 
+TEST_FILE_EXTENSIONS = ['.sql', '.sql.j2', '.sh', '.py', '.expect']
+
 class Terminated(KeyboardInterrupt):
     pass
 
@@ -458,7 +467,7 @@ def run_tests_array(all_tests_with_params):
                             break
 
                     file_suffix = ('.' + str(os.getpid())) if is_concurrent and args.test_runs > 1 else ''
-                    reference_file = os.path.join(suite_dir, name) + '.reference'
+                    reference_file = get_reference_file(suite_dir, name)
                     stdout_file = os.path.join(suite_tmp_dir, name) + file_suffix + '.stdout'
                     stderr_file = os.path.join(suite_tmp_dir, name) + file_suffix + '.stderr'
 
@@ -535,7 +544,7 @@ def run_tests_array(all_tests_with_params):
                             status += " - having exception:\n{}\n".format(
                                 '\n'.join(stdout.split('\n')[:100]))
                             status += 'Database: ' + testcase_args.testcase_database
-                        elif not os.path.isfile(reference_file):
+                        elif reference_file is None:
                             status += MSG_UNKNOWN
                             status += print_test_time(total_time)
                             status += " - no reference file\n"
@@ -760,6 +769,99 @@ def do_run_tests(jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_test
         return num_tests
 
 
+def is_test_from_dir(suite_dir, case):
+    case_file = os.path.join(suite_dir, case)
+    # We could also test for executable files (os.access(case_file, os.X_OK),
+    # but it interferes with 01610_client_spawn_editor.editor, which is invoked
+    # as a query editor in the test, and must be marked as executable.
+    return os.path.isfile(case_file) and any(case_file.endswith(suppotred_ext) for suppotred_ext in TEST_FILE_EXTENSIONS)
+
+
+def removesuffix(str, *suffixes):
+    """
+    Added in python 3.9
+    https://www.python.org/dev/peps/pep-0616/
+
+    This version can work with severtal possible suffixes
+    """
+    for suffix in suffixes:
+        if suffix and str.endswith(suffix):
+            return str[:-len(suffix)]
+    return str
+
+
+def render_test_template(j2env, suite_dir, test_name):
+    """
+    Render template for test and reference file if needed
+    """
+
+    if j2env is None:
+        return test_name
+
+    test_base_name = removesuffix(test_name, ".sql.j2", ".sql")
+
+    reference_file_name = test_base_name  + ".reference.j2"
+    reference_file_path = os.path.join(suite_dir, reference_file_name)
+    if os.path.isfile(reference_file_path):
+        tpl = j2env.get_template(reference_file_name)
+        tpl.stream().dump(os.path.join(suite_dir, test_base_name) + ".gen.reference")
+
+    if test_name.endswith(".sql.j2"):
+        tpl = j2env.get_template(test_name)
+        generated_test_name = test_base_name + ".gen.sql"
+        tpl.stream().dump(os.path.join(suite_dir, generated_test_name))
+        return generated_test_name
+
+    return test_name
+
+
+def get_selected_tests(suite_dir, patterns):
+    """
+    Find all files with tests, filter, render templates
+    """
+
+    j2env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(suite_dir),
+        keep_trailing_newline=True,
+    ) if USE_JINJA else None
+
+    for test_name in os.listdir(suite_dir):
+        if not is_test_from_dir(suite_dir, test_name):
+            continue
+        if patterns and not any(re.search(pattern, test_name) for pattern in patterns):
+            continue
+        if USE_JINJA and test_name.endswith(".gen.sql"):
+            continue
+        if not USE_JINJA and test_name.endswith(".j2"):
+            continue
+        test_name = render_test_template(j2env, suite_dir, test_name)
+        yield test_name
+
+
+def get_tests_list(suite_dir, patterns, test_runs, sort_key):
+    """
+    Return list of tests file names to run
+    """
+
+    all_tests = list(get_selected_tests(suite_dir, patterns))
+    all_tests = all_tests * test_runs
+    all_tests.sort(key=sort_key)
+    return all_tests
+
+
+def get_reference_file(suite_dir, name):
+    """
+    Returns reference file name for specified test
+    """
+
+    name = removesuffix(name, ".gen")
+    for ext in ['.reference', '.gen.reference']:
+        reference_file = os.path.join(suite_dir, name) + ext
+        if os.path.isfile(reference_file):
+            return reference_file
+    return None
+
+
 def main(args):
     global server_died
     global stop_time
@@ -844,14 +946,6 @@ def main(args):
         create_common_database(args, args.database)
     create_common_database(args, "test")
 
-    def is_test_from_dir(suite_dir, case):
-        case_file = os.path.join(suite_dir, case)
-        (_, ext) = os.path.splitext(case)
-        # We could also test for executable files (os.access(case_file, os.X_OK),
-        # but it interferes with 01610_client_spawn_editor.editor, which is invoked
-        # as a query editor in the test, and must be marked as executable.
-        return os.path.isfile(case_file) and (ext in ['.sql', '.sh', '.py', '.expect'])
-
     def sute_key_func(item):
         if args.order == 'random':
             return random.random()
@@ -911,12 +1005,7 @@ def main(args):
                 except ValueError:
                     return 99997
 
-            all_tests = os.listdir(suite_dir)
+            all_tests = get_tests_list(suite_dir, args.test, args.test_runs, key_func)
-            all_tests = [case for case in all_tests if is_test_from_dir(suite_dir, case)]
-            if args.test:
-                all_tests = [t for t in all_tests if any(re.search(r, t) for r in args.test)]
-            all_tests = all_tests * args.test_runs
-            all_tests.sort(key=key_func)
 
             jobs = args.jobs
             parallel_tests = []