From 0e37203dcd4e560de184cb072cfbaaa63b2fdc5f Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@yandex-team.ru>
Date: Tue, 24 Aug 2021 15:27:50 +0300
Subject: [PATCH 1/4] Use jinja template tests in fuzzer

---
 docker/test/fuzzer/Dockerfile          |  4 ++
 docker/test/fuzzer/generate-test-j2.py | 62 ++++++++++++++++++++++++++
 docker/test/fuzzer/run-fuzzer.sh       | 10 +++--
 3 files changed, 72 insertions(+), 4 deletions(-)
 create mode 100644 docker/test/fuzzer/generate-test-j2.py

diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile
index 18684145636..9a96ac1dfa7 100644
--- a/docker/test/fuzzer/Dockerfile
+++ b/docker/test/fuzzer/Dockerfile
@@ -16,6 +16,8 @@ RUN apt-get update \
             p7zip-full \
             parallel \
             psmisc \
+            python3 \
+            python3-pip \
             rsync \
             tree \
             tzdata \
@@ -25,6 +27,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+RUN pip3 install Jinja2
+
 COPY * /
 
 SHELL ["/bin/bash", "-c"]
diff --git a/docker/test/fuzzer/generate-test-j2.py b/docker/test/fuzzer/generate-test-j2.py
new file mode 100644
index 00000000000..bcc1bf6bc84
--- /dev/null
+++ b/docker/test/fuzzer/generate-test-j2.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+import os
+import jinja2
+
+
+def removesuffix(text, suffix):
+    """
+    Added in python 3.9
+    https://www.python.org/dev/peps/pep-0616/
+    """
+    if suffix and text.endswith(suffix):
+        return text[:-len(suffix)]
+    else:
+        return text[:]
+
+
+def render_test_template(j2env, suite_dir, test_name):
+    """
+    Render template for test and reference file if needed
+    """
+
+    test_base_name = removesuffix(test_name, ".sql.j2")
+
+    reference_file_name = test_base_name + ".reference.j2"
+    reference_file_path = os.path.join(suite_dir, reference_file_name)
+    if os.path.isfile(reference_file_path):
+        tpl = j2env.get_template(reference_file_name)
+        tpl.stream().dump(os.path.join(suite_dir, test_base_name) + ".gen.reference")
+
+    if test_name.endswith(".sql.j2"):
+        tpl = j2env.get_template(test_name)
+        generated_test_name = test_base_name + ".gen.sql"
+        tpl.stream().dump(os.path.join(suite_dir, generated_test_name))
+        return generated_test_name
+
+    return test_name
+
+
+def main(args):
+    suite_dir = args.path
+
+    print(f"Scanning {suite_dir} directory...")
+
+    j2env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(suite_dir),
+        keep_trailing_newline=True,
+    )
+
+    test_names = os.listdir(suite_dir)
+    for test_name in test_names:
+        if not test_name.endswith(".sql.j2"):
+            continue
+        new_name = render_test_template(j2env, suite_dir, test_name)
+        print(f"File {new_name} generated")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description="Jinja2 test generator")
+    parser.add_argument("-p", "--path", help="Path to test dir", required=True)
+    main(parser.parse_args())
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 44183a50ae5..8cf85a014a3 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -71,12 +71,12 @@ function watchdog
     kill -9 -- $fuzzer_pid ||:
 }
 
-function filter_exists
+function filter_exists_and_template
 {
     local path
     for path in "$@"; do
         if [ -e "$path" ]; then
-            echo "$path"
+            echo "$path" | sed -n 's/\.sql\.j2$/.gen.sql/'
         else
             echo "'$path' does not exists" >&2
         fi
@@ -87,9 +87,9 @@ function fuzz
 {
     # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests.
     # Don't overwrite the NEW_TESTS_OPT so that it can be set from the environment.
-    NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\)$!ch/\1!p' ci-changed-files.txt | sort -R)"
+    NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\(\.j2\)\?\)$!ch/\1!p' ci-changed-files.txt | sort -R)"
     # ci-changed-files.txt contains also files that has been deleted/renamed, filter them out.
-    NEW_TESTS="$(filter_exists $NEW_TESTS)"
+    NEW_TESTS="$(filter_exists_and_template $NEW_TESTS)"
     if [[ -n "$NEW_TESTS" ]]
     then
         NEW_TESTS_OPT="${NEW_TESTS_OPT:---interleave-queries-file ${NEW_TESTS}}"
@@ -97,6 +97,8 @@ function fuzz
         NEW_TESTS_OPT="${NEW_TESTS_OPT:-}"
     fi
 
+    /generate-test-j2.py --path ch/tests/queries/0_stateless
+
     export CLICKHOUSE_WATCHDOG_ENABLE=0 # interferes with gdb
     clickhouse-server --config-file db/config.xml -- --path db 2>&1 | tail -100000 > server.log &
     server_pid=$!

From 9ea6ee7fb2a64f723a5505357799af7bc92a1ca4 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@yandex-team.ru>
Date: Tue, 24 Aug 2021 16:15:58 +0300
Subject: [PATCH 2/4] Dummy change in 01720_join_implicit_cast.sql.j2

---
 tests/queries/0_stateless/01720_join_implicit_cast.sql.j2 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/queries/0_stateless/01720_join_implicit_cast.sql.j2 b/tests/queries/0_stateless/01720_join_implicit_cast.sql.j2
index f2b13e9824b..d1de6d06593 100644
--- a/tests/queries/0_stateless/01720_join_implicit_cast.sql.j2
+++ b/tests/queries/0_stateless/01720_join_implicit_cast.sql.j2
@@ -42,7 +42,6 @@ SELECT sum(a) + sum(t_ab2.a) - 1, sum(b) + sum(t_ab2.b) - 1 FROM t_ab1 RIGHT JOI
 SELECT sum(a) + sum(t_ab2.a) - 1, sum(b) + sum(t_ab2.b) - 1 FROM t_ab1 INNER JOIN t_ab2 ON (t_ab1.a == t_ab2.a AND t_ab1.b == t_ab2.b);
 
 SELECT '= types =';
-
 SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(b)) == 'Nullable(Int64)' FROM t_ab1 FULL JOIN t_ab2 USING (a, b);
 SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(b)) == 'Nullable(Int64)' FROM t_ab1 LEFT JOIN t_ab2 USING (a, b);
 SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(b)) == 'Nullable(Int64)' FROM t_ab1 RIGHT JOIN t_ab2 USING (a, b);

From b9b0b17ad09673c0ba3dad0e45fae6b743ce5b54 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@yandex-team.ru>
Date: Wed, 25 Aug 2021 11:20:42 +0300
Subject: [PATCH 3/4] chmod +x docker/test/fuzzer/generate-test-j2.py

---
 docker/test/fuzzer/generate-test-j2.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 docker/test/fuzzer/generate-test-j2.py

diff --git a/docker/test/fuzzer/generate-test-j2.py b/docker/test/fuzzer/generate-test-j2.py
old mode 100644
new mode 100755

From 92f073a4350213a66700625f6623c20284966c40 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@yandex-team.ru>
Date: Wed, 25 Aug 2021 16:05:16 +0300
Subject: [PATCH 4/4] Call generate-test-j2.py in run-fuzzer.sh before
 calculating NEW_TESTS_OPT

---
 docker/test/fuzzer/run-fuzzer.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 8cf85a014a3..9a389edc5b2 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -85,6 +85,8 @@ function filter_exists_and_template
 
 function fuzz
 {
+    /generate-test-j2.py --path ch/tests/queries/0_stateless
+
     # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests.
     # Don't overwrite the NEW_TESTS_OPT so that it can be set from the environment.
     NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\(\.j2\)\?\)$!ch/\1!p' ci-changed-files.txt | sort -R)"
@@ -97,8 +99,6 @@ function fuzz
         NEW_TESTS_OPT="${NEW_TESTS_OPT:-}"
     fi
 
-    /generate-test-j2.py --path ch/tests/queries/0_stateless
-
     export CLICKHOUSE_WATCHDOG_ENABLE=0 # interferes with gdb
     clickhouse-server --config-file db/config.xml -- --path db 2>&1 | tail -100000 > server.log &
     server_pid=$!