Merge pull request #28079 from vdimir/fuzz-j2

Use jinja template tests in fuzzer

docker/test/fuzzer/Dockerfile

@@ -16,6 +16,8 @@ RUN apt-get update \
             p7zip-full \
             parallel \
             psmisc \
+            python3 \
+            python3-pip \
             rsync \
             tree \
             tzdata \
@@ -25,6 +27,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+RUN pip3 install Jinja2
+
 COPY * /
 
 SHELL ["/bin/bash", "-c"]

docker/test/fuzzer/generate-test-j2.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+import os
+import jinja2
+
+
+def removesuffix(text, suffix):
+    """
+    Added in python 3.9
+    https://www.python.org/dev/peps/pep-0616/
+    """
+    if suffix and text.endswith(suffix):
+        return text[:-len(suffix)]
+    else:
+        return text[:]
+
+
+def render_test_template(j2env, suite_dir, test_name):
+    """
+    Render template for test and reference file if needed
+    """
+
+    test_base_name = removesuffix(test_name, ".sql.j2")
+
+    reference_file_name = test_base_name + ".reference.j2"
+    reference_file_path = os.path.join(suite_dir, reference_file_name)
+    if os.path.isfile(reference_file_path):
+        tpl = j2env.get_template(reference_file_name)
+        tpl.stream().dump(os.path.join(suite_dir, test_base_name) + ".gen.reference")
+
+    if test_name.endswith(".sql.j2"):
+        tpl = j2env.get_template(test_name)
+        generated_test_name = test_base_name + ".gen.sql"
+        tpl.stream().dump(os.path.join(suite_dir, generated_test_name))
+        return generated_test_name
+
+    return test_name
+
+
+def main(args):
+    suite_dir = args.path
+
+    print(f"Scanning {suite_dir} directory...")
+
+    j2env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(suite_dir),
+        keep_trailing_newline=True,
+    )
+
+    test_names = os.listdir(suite_dir)
+    for test_name in test_names:
+        if not test_name.endswith(".sql.j2"):
+            continue
+        new_name = render_test_template(j2env, suite_dir, test_name)
+        print(f"File {new_name} generated")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description="Jinja2 test generator")
+    parser.add_argument("-p", "--path", help="Path to test dir", required=True)
+    main(parser.parse_args())

docker/test/fuzzer/run-fuzzer.sh

@@ -71,12 +71,12 @@ function watchdog
     kill -9 -- $fuzzer_pid ||:
 }
 
-function filter_exists
+function filter_exists_and_template
 {
     local path
     for path in "$@"; do
         if [ -e "$path" ]; then
-            echo "$path"
+            echo "$path" | sed -n 's/\.sql\.j2$/.gen.sql/'
         else
             echo "'$path' does not exists" >&2
         fi
@@ -85,11 +85,13 @@ function filter_exists
 
 function fuzz
 {
+    /generate-test-j2.py --path ch/tests/queries/0_stateless
+
     # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests.
     # Don't overwrite the NEW_TESTS_OPT so that it can be set from the environment.
-    NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\)$!ch/\1!p' ci-changed-files.txt | sort -R)"
+    NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\(\.j2\)\?\)$!ch/\1!p' ci-changed-files.txt | sort -R)"
     # ci-changed-files.txt contains also files that has been deleted/renamed, filter them out.
-    NEW_TESTS="$(filter_exists $NEW_TESTS)"
+    NEW_TESTS="$(filter_exists_and_template $NEW_TESTS)"
     if [[ -n "$NEW_TESTS" ]]
     then
         NEW_TESTS_OPT="${NEW_TESTS_OPT:---interleave-queries-file ${NEW_TESTS}}"

tests/queries/0_stateless/01720_join_implicit_cast.sql.j2

@@ -42,7 +42,6 @@ SELECT sum(a) + sum(t_ab2.a) - 1, sum(b) + sum(t_ab2.b) - 1 FROM t_ab1 RIGHT JOI
 SELECT sum(a) + sum(t_ab2.a) - 1, sum(b) + sum(t_ab2.b) - 1 FROM t_ab1 INNER JOIN t_ab2 ON (t_ab1.a == t_ab2.a AND t_ab1.b == t_ab2.b);
 
 SELECT '= types =';
-
 SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(b)) == 'Nullable(Int64)' FROM t_ab1 FULL JOIN t_ab2 USING (a, b);
 SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(b)) == 'Nullable(Int64)' FROM t_ab1 LEFT JOIN t_ab2 USING (a, b);
 SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(b)) == 'Nullable(Int64)' FROM t_ab1 RIGHT JOIN t_ab2 USING (a, b);