Merge pull request #52293 from ClickHouse/sqltest

Add SQLTest

.github/workflows/pull_request.yml

@@ -5227,3 +5227,39 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+##############################################################################################
+##################################### SQL TEST ###############################################
+##############################################################################################
+  SQLTest:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, fuzzer-unit-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/sqltest
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=SQLTest
+          REPO_COPY=${{runner.temp}}/sqltest/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: SQLTest
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 sqltest.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"

docker/README.md

@@ -2,4 +2,4 @@
 
 This directory contain Dockerfiles for `clickhouse-server`. They are updated in each release.
 
-Also there is bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`.
+Also, there is a bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`.

docker/images.json

@@ -125,6 +125,7 @@
             "docker/test/keeper-jepsen",
             "docker/test/server-jepsen",
             "docker/test/sqllogic",
+            "docker/test/sqltest",
             "docker/test/stateless"
          ]
     },
@@ -155,13 +156,16 @@
     },
     "docker/docs/builder": {
         "name": "clickhouse/docs-builder",
-        "dependent": [
-        ]
+        "dependent": []
     },
     "docker/test/sqllogic": {
         "name": "clickhouse/sqllogic-test",
         "dependent": []
     },
+    "docker/test/sqltest": {
+        "name": "clickhouse/sqltest",
+        "dependent": []
+    },
     "docker/test/integration/nginx_dav": {
         "name": "clickhouse/nginx-dav",
         "dependent": []

docker/test/sqllogic/run.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+
 set -exu
 trap "exit" INT TERM
 

docker/test/sqltest/Dockerfile

@@ -0,0 +1,30 @@
+# docker build -t clickhouse/sqltest .
+ARG FROM_TAG=latest
+FROM clickhouse/test-base:$FROM_TAG
+
+RUN apt-get update --yes \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            wget \
+            git \
+            python3 \
+            python3-dev \
+            python3-pip \
+            sudo \
+    && apt-get clean
+
+RUN pip3 install \
+    pyyaml \
+    clickhouse-driver
+
+ARG sqltest_repo="https://github.com/elliotchance/sqltest/"
+
+RUN git clone ${sqltest_repo}
+
+ENV TZ=UTC
+ENV MAX_RUN_TIME=900
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+COPY run.sh /
+COPY test.py /
+CMD ["/bin/bash", "/run.sh"]

docker/test/sqltest/run.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# shellcheck disable=SC2015
+
+set -x
+set -e
+set -u
+set -o pipefail
+
+BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-16_debug_none_unsplitted_disable_False_binary"}
+BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
+
+function wget_with_retry
+{
+    for _ in 1 2 3 4; do
+        if wget -nv -nd -c "$1";then
+            return 0
+        else
+            sleep 0.5
+        fi
+    done
+    return 1
+}
+
+wget_with_retry "$BINARY_URL_TO_DOWNLOAD"
+chmod +x clickhouse
+./clickhouse install --noninteractive
+
+echo "
+users:
+  default:
+    access_management: 1" > /etc/clickhouse-server/users.d/access_management.yaml
+
+clickhouse start
+
+# Wait for start
+for _ in {1..100}
+do
+    clickhouse-client --query "SELECT 1" && break ||:
+    sleep 1
+done
+
+# Run the test
+pushd sqltest/standards/2016/
+/test.py
+mv report.html test.log /workspace
+popd
+
+zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.log
+zstd --threads=0 /var/log/clickhouse-server/clickhouse-server.err.log
+
+mv /var/log/clickhouse-server/clickhouse-server.log.zst /var/log/clickhouse-server/clickhouse-server.err.log.zst /workspace

docker/test/sqltest/test.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+import html
+import random
+import string
+from clickhouse_driver import Client
+
+
+client = Client(host="localhost", port=9000)
+settings = {
+    "default_table_engine": "Memory",
+    "union_default_mode": "DISTINCT",
+    "calculate_text_stack_trace": 0,
+}
+
+database_name = "sqltest_" + "".join(
+    random.choice(string.ascii_lowercase) for _ in range(10)
+)
+
+client.execute(f"DROP DATABASE IF EXISTS {database_name}", settings=settings)
+client.execute(f"CREATE DATABASE {database_name}", settings=settings)
+
+client = Client(host="localhost", port=9000, database=database_name)
+
+summary = {"success": 0, "total": 0, "results": {}}
+
+log_file = open("test.log", "w")
+report_html_file = open("report.html", "w")
+
+with open("features.yml", "r") as file:
+    yaml_content = yaml.safe_load(file)
+
+    for category in yaml_content:
+        log_file.write(category.capitalize() + " features:\n")
+        summary["results"][category] = {"success": 0, "total": 0, "results": {}}
+
+        for test in yaml_content[category]:
+            log_file.write(test + ": " + yaml_content[category][test] + "\n")
+            summary["results"][category]["results"][test] = {
+                "success": 0,
+                "total": 0,
+                "description": yaml_content[category][test],
+            }
+
+            test_path = test[0] + "/" + test + ".tests.yml"
+            if os.path.exists(test_path):
+                with open(test_path, "r") as test_file:
+                    test_yaml_content = yaml.load_all(test_file, Loader=yaml.FullLoader)
+
+                    for test_case in test_yaml_content:
+                        queries = test_case["sql"]
+                        if not isinstance(queries, list):
+                            queries = [queries]
+
+                        for query in queries:
+                            # Example: E011-01
+                            test_group = ""
+                            if "-" in test:
+                                test_group = test.split("-", 1)[0]
+                                summary["results"][category]["results"][test_group][
+                                    "total"
+                                ] += 1
+                            summary["results"][category]["results"][test]["total"] += 1
+                            summary["results"][category]["total"] += 1
+                            summary["total"] += 1
+
+                            log_file.write(query + "\n")
+
+                            try:
+                                result = client.execute(query, settings=settings)
+                                log_file.write(str(result) + "\n")
+
+                                if test_group:
+                                    summary["results"][category]["results"][test_group][
+                                        "success"
+                                    ] += 1
+                                summary["results"][category]["results"][test][
+                                    "success"
+                                ] += 1
+                                summary["results"][category]["success"] += 1
+                                summary["success"] += 1
+
+                            except Exception as e:
+                                log_file.write(f"Error occurred: {str(e)}\n")
+
+client.execute(f"DROP DATABASE {database_name}", settings=settings)
+
+
+def enable_color(ratio):
+    if ratio == 0:
+        return "<b style='color: red;'>"
+    elif ratio < 0.5:
+        return "<b style='color: orange;'>"
+    elif ratio < 1:
+        return "<b style='color: gray;'>"
+    else:
+        return "<b style='color: green;'>"
+
+
+reset_color = "</b>"
+
+
+def print_ratio(indent, name, success, total, description):
+    report_html_file.write(
+        "{}{}: {}{} / {} ({:.1%}){}{}\n".format(
+            " " * indent,
+            name.capitalize(),
+            enable_color(success / total),
+            success,
+            total,
+            success / total,
+            reset_color,
+            f" - " + html.escape(description) if description else "",
+        )
+    )
+
+
+report_html_file.write(
+    "<html><body><pre style='font-size: 16pt; padding: 1em; line-height: 1.25;'>\n"
+)
+
+print_ratio(0, "Total", summary["success"], summary["total"], "")
+
+for category in summary["results"]:
+    cat_summary = summary["results"][category]
+
+    if cat_summary["total"] == 0:
+        continue
+
+    print_ratio(2, category, cat_summary["success"], cat_summary["total"], "")
+
+    for test in summary["results"][category]["results"]:
+        test_summary = summary["results"][category]["results"][test]
+
+        if test_summary["total"] == 0:
+            continue
+
+        print_ratio(
+            6 if "-" in test else 4,
+            test,
+            test_summary["success"],
+            test_summary["total"],
+            test_summary["description"],
+        )
+
+report_html_file.write("</pre></body></html>\n")

tests/ci/ast_fuzzer_check.py

@@ -79,7 +79,7 @@ def main():
             build_url = url
             break
     else:
-        raise Exception("Cannot binary clickhouse among build results")
+        raise Exception("Cannot find the clickhouse binary among build results")
 
     logging.info("Got build url %s", build_url)
 

tests/ci/ci_config.py

@@ -276,6 +276,7 @@ CI_CONFIG = CiConfig(
         "SQLancer (release)": TestConfig("package_release"),
         "SQLancer (debug)": TestConfig("package_debug"),
         "Sqllogic test (release)": TestConfig("package_release"),
+        "SQLTest": TestConfig("package_release"),
     },
 )
 CI_CONFIG.validate()

tests/ci/docker_test.py

@@ -40,6 +40,12 @@ class TestDockerImageCheck(unittest.TestCase):
             [
                 di.DockerImage("docker/test/base", "clickhouse/test-base", False),
                 di.DockerImage("docker/docs/builder", "clickhouse/docs-builder", True),
+                di.DockerImage(
+                    "docker/test/sqltest",
+                    "clickhouse/sqltest",
+                    False,
+                    "clickhouse/test-base",  # type: ignore
+                ),
                 di.DockerImage(
                     "docker/test/stateless",
                     "clickhouse/stateless-test",

tests/ci/sqltest.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+
+import logging
+import subprocess
+import os
+import sys
+
+from github import Github
+
+from build_download_helper import get_build_name_for_check, read_build_urls
+from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
+from commit_status_helper import (
+    RerunHelper,
+    get_commit,
+    post_commit_status,
+)
+from docker_pull_helper import get_image_with_version
+from env_helper import (
+    GITHUB_RUN_URL,
+    REPORTS_PATH,
+    TEMP_PATH,
+)
+from get_robot_token import get_best_robot_token
+from pr_info import PRInfo
+from report import TestResult
+from s3_helper import S3Helper
+from stopwatch import Stopwatch
+
+IMAGE_NAME = "clickhouse/sqltest"
+
+
+def get_run_command(pr_number, sha, download_url, workspace_path, image):
+    return (
+        f"docker run "
+        # For sysctl
+        "--privileged "
+        "--network=host "
+        f"--volume={workspace_path}:/workspace "
+        "--cap-add syslog --cap-add sys_admin --cap-add=SYS_PTRACE "
+        f'-e PR_TO_TEST={pr_number} -e SHA_TO_TEST={sha} -e BINARY_URL_TO_DOWNLOAD="{download_url}" '
+        f"{image}"
+    )
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+
+    stopwatch = Stopwatch()
+
+    temp_path = TEMP_PATH
+    reports_path = REPORTS_PATH
+
+    check_name = sys.argv[1]
+
+    if not os.path.exists(temp_path):
+        os.makedirs(temp_path)
+
+    pr_info = PRInfo()
+
+    gh = Github(get_best_robot_token(), per_page=100)
+    commit = get_commit(gh, pr_info.sha)
+
+    rerun_helper = RerunHelper(commit, check_name)
+    if rerun_helper.is_already_finished_by_status():
+        logging.info("Check is already finished according to github status, exiting")
+        sys.exit(0)
+
+    docker_image = get_image_with_version(reports_path, IMAGE_NAME)
+
+    build_name = get_build_name_for_check(check_name)
+    print(build_name)
+    urls = read_build_urls(build_name, reports_path)
+    if not urls:
+        raise Exception("No build URLs found")
+
+    for url in urls:
+        if url.endswith("/clickhouse"):
+            build_url = url
+            break
+    else:
+        raise Exception("Cannot find the clickhouse binary among build results")
+
+    logging.info("Got build url %s", build_url)
+
+    workspace_path = os.path.join(temp_path, "workspace")
+    if not os.path.exists(workspace_path):
+        os.makedirs(workspace_path)
+
+    run_command = get_run_command(
+        pr_info.number, pr_info.sha, build_url, workspace_path, docker_image
+    )
+    logging.info("Going to run %s", run_command)
+
+    run_log_path = os.path.join(temp_path, "run.log")
+    with open(run_log_path, "w", encoding="utf-8") as log:
+        with subprocess.Popen(
+            run_command, shell=True, stderr=log, stdout=log
+        ) as process:
+            retcode = process.wait()
+            if retcode == 0:
+                logging.info("Run successfully")
+            else:
+                logging.info("Run failed")
+
+    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
+
+    check_name_lower = (
+        check_name.lower().replace("(", "").replace(")", "").replace(" ", "")
+    )
+    s3_prefix = f"{pr_info.number}/{pr_info.sha}/sqltest_{check_name_lower}/"
+    paths = {
+        "run.log": run_log_path,
+        "server.log.zst": os.path.join(workspace_path, "server.log.zst"),
+        "server.err.log.zst": os.path.join(workspace_path, "server.err.log.zst"),
+        "report.html": os.path.join(workspace_path, "report.html"),
+        "test.log": os.path.join(workspace_path, "test.log"),
+    }
+
+    s3_helper = S3Helper()
+    for f in paths:
+        try:
+            paths[f] = s3_helper.upload_test_report_to_s3(paths[f], s3_prefix + f)
+        except Exception as ex:
+            logging.info("Exception uploading file %s text %s", f, ex)
+            paths[f] = ""
+
+    report_url = GITHUB_RUN_URL
+    if paths["report.html"]:
+        report_url = paths["report.html"]
+
+    status = "success"
+    description = "See the report"
+    test_result = TestResult(description, "OK")
+
+    ch_helper = ClickHouseHelper()
+
+    prepared_events = prepare_tests_results_for_clickhouse(
+        pr_info,
+        [test_result],
+        status,
+        stopwatch.duration_seconds,
+        stopwatch.start_time_str,
+        report_url,
+        check_name,
+    )
+
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
+
+    logging.info("Result: '%s', '%s', '%s'", status, description, report_url)
+    print(f"::notice ::Report url: {report_url}")
+    post_commit_status(commit, status, report_url, description, check_name, pr_info)
+
+
+if __name__ == "__main__":
+    main()

tests/ci/tests/docker_images_for_tests.json

@@ -119,7 +119,8 @@
             "docker/test/stateless",
             "docker/test/integration/base",
             "docker/test/fuzzer",
-            "docker/test/keeper-jepsen"
+            "docker/test/keeper-jepsen",
+            "docker/test/sqltest"
          ]
     },
     "docker/test/integration/kerberized_hadoop": {
@@ -153,5 +154,9 @@
     "docker/test/sqllogic": {
         "name": "clickhouse/sqllogic-test",
         "dependent": []
+    },
+    "docker/test/sqltest": {
+        "name": "clickhouse/sqltest",
+        "dependent": []
     }
 }