Merge pull request #42397 from qoega/revive-sqlancer

Revive SQLancer

.github/workflows/master.yml

@@ -2994,6 +2994,77 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+##############################################################################################
+###################################### SQLANCER FUZZERS ######################################
+##############################################################################################
+  SQLancerTestRelease:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, fuzzer-unit-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/sqlancer_release
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=SQLancer (release)
+          REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: SQLancer
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 sqlancer_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  SQLancerTestDebug:
+    needs: [BuilderDebDebug]
+    runs-on: [self-hosted, fuzzer-unit-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/sqlancer_debug
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=SQLancer (debug)
+          REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: SQLancer
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 sqlancer_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
   FinishCheck:
     needs:
       - DockerHubPush
@@ -3053,6 +3124,8 @@ jobs:
       - UnitTestsUBsan
       - UnitTestsReleaseClang
       - SharedBuildSmokeTest
+      - SQLancerTestRelease
+      - SQLancerTestDebug
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Clear repository

.github/workflows/pull_request.yml

@@ -3491,6 +3491,77 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
+##############################################################################################
+###################################### SQLANCER FUZZERS ######################################
+##############################################################################################
+  SQLancerTestRelease:
+    needs: [BuilderDebRelease]
+    runs-on: [self-hosted, fuzzer-unit-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/sqlancer_release
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=SQLancer (release)
+          REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: SQLancer
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 sqlancer_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  SQLancerTestDebug:
+    needs: [BuilderDebDebug]
+    runs-on: [self-hosted, fuzzer-unit-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/sqlancer_debug
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=SQLancer (debug)
+          REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: SQLancer
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 sqlancer_check.py "$CHECK_NAME"
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
 #############################################################################################
 ###################################### JEPSEN TESTS #########################################
 #############################################################################################
@@ -3501,7 +3572,6 @@ jobs:
     if: contains(github.event.pull_request.labels.*.name, 'jepsen-test')
     needs: [BuilderBinRelease]
     uses: ./.github/workflows/jepsen.yml
-
   FinishCheck:
     needs:
       - StyleCheck
@@ -3576,6 +3646,8 @@ jobs:
       - SharedBuildSmokeTest
       - CompatibilityCheck
       - IntegrationTestsFlakyCheck
+      - SQLancerTestRelease
+      - SQLancerTestDebug
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Clear repository

.gitignore

@@ -154,3 +154,6 @@ website/package-lock.json
 /programs/server/metadata
 /programs/server/store
 
+# temporary test files
+tests/queries/0_stateless/test_*
+tests/queries/0_stateless/*.binary

docker/test/sqlancer/Dockerfile

@@ -1,5 +1,5 @@
 # docker build -t clickhouse/sqlancer-test .
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"

docker/test/sqlancer/process_sqlancer_result.py

@@ -11,13 +11,15 @@ def process_result(result_folder):
     summary = []
     paths = []
     tests = [
-        "TLPWhere",
+        "TLPAggregate",
+        "TLPDistinct",
         "TLPGroupBy",
         "TLPHaving",
+        "TLPWhere",
         "TLPWhereGroupBy",
-        "TLPDistinct",
-        "TLPAggregate",
+        "NoREC",
     ]
+    failed_tests = []
 
     for test in tests:
         err_path = "{}/{}.err".format(result_folder, test)
@@ -33,15 +35,11 @@ def process_result(result_folder):
             with open(err_path, "r") as f:
                 if "AssertionError" in f.read():
                     summary.append((test, "FAIL"))
+                    failed_tests.append(test)
                     status = "failure"
                 else:
                     summary.append((test, "OK"))
 
-    logs_path = "{}/logs.tar.gz".format(result_folder)
-    if not os.path.exists(logs_path):
-        logging.info("No logs tar on path %s", logs_path)
-    else:
-        paths.append(logs_path)
     stdout_path = "{}/stdout.log".format(result_folder)
     if not os.path.exists(stdout_path):
         logging.info("No stdout log on path %s", stdout_path)
@@ -53,18 +51,23 @@ def process_result(result_folder):
     else:
         paths.append(stderr_path)
 
-    description = "SQLancer test run. See report"
+    description = "SQLancer run successfully"
+    if status == "failure":
+        description = f"Failed oracles: {failed_tests}"
 
     return status, description, summary, paths
 
 
-def write_results(results_file, status_file, results, status):
+def write_results(
+    results_file, status_file, description_file, results, status, description
+):
     with open(results_file, "w") as f:
         out = csv.writer(f, delimiter="\t")
         out.writerows(results)
     with open(status_file, "w") as f:
-        out = csv.writer(f, delimiter="\t")
-        out.writerow(status)
+        f.write(status + "\n")
+    with open(description_file, "w") as f:
+        f.write(description + "\n")
 
 
 if __name__ == "__main__":
@@ -72,13 +75,20 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="ClickHouse script for parsing results of sqlancer test"
     )
-    parser.add_argument("--in-results-dir", default="/test_output/")
-    parser.add_argument("--out-results-file", default="/test_output/test_results.tsv")
-    parser.add_argument("--out-status-file", default="/test_output/check_status.tsv")
+    parser.add_argument("--in-results-dir", default="/workspace/")
+    parser.add_argument("--out-results-file", default="/workspace/summary.tsv")
+    parser.add_argument("--out-description-file", default="/workspace/description.txt")
+    parser.add_argument("--out-status-file", default="/workspace/status.txt")
     args = parser.parse_args()
 
-    state, description, test_results, logs = process_result(args.in_results_dir)
+    status, description, summary, logs = process_result(args.in_results_dir)
     logging.info("Result parsed")
-    status = (state, description)
-    write_results(args.out_results_file, args.out_status_file, test_results, status)
+    write_results(
+        args.out_results_file,
+        args.out_status_file,
+        args.out_description_file,
+        summary,
+        status,
+        description,
+    )
     logging.info("Result written")

docker/test/sqlancer/run.sh

@@ -1,33 +1,62 @@
 #!/bin/bash
+set -exu
+trap "exit" INT TERM
 
-set -e -x
+function wget_with_retry
+{
+    for _ in 1 2 3 4; do
+        if wget -nv -nd -c "$1";then
+            return 0
+        else
+            sleep 0.5
+        fi
+    done
+    return 1
+}
 
-dpkg -i package_folder/clickhouse-common-static_*.deb
-dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
-dpkg -i package_folder/clickhouse-server_*.deb
-dpkg -i package_folder/clickhouse-client_*.deb
+if [ -z ${BINARY_URL_TO_DOWNLOAD+x} ]
+then
+    echo "No BINARY_URL_TO_DOWNLOAD provided."
+else
+    wget_with_retry "$BINARY_URL_TO_DOWNLOAD"
+    chmod +x /clickhouse
+fi
 
-service clickhouse-server start && sleep 5
+if [[ -f "/clickhouse" ]]; then
+    echo "/clickhouse exists"
+else
+    exit 1
+fi
+
+cd /workspace
+/clickhouse server -P /workspace/clickhouse-server.pid -L /workspace/clickhouse-server.log -E /workspace/clickhouse-server.log.err --daemon
+
+for _ in $(seq 1 60); do if [[ $(wget -q 'localhost:8123' -O-) == 'Ok.' ]]; then break ; else sleep 1; fi ; done
 
 cd /sqlancer/sqlancer-master
 
-export TIMEOUT=300
-export NUM_QUERIES=1000
+TIMEOUT=300
+NUM_QUERIES=1000
+NUM_THREADS=10
+TESTS=( "TLPGroupBy" "TLPHaving" "TLPWhere" "TLPDistinct" "TLPAggregate" "NoREC" )
+echo "${TESTS[@]}"
 
-( java -jar target/sqlancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES  --username default --password "" clickhouse --oracle TLPWhere | tee /test_output/TLPWhere.out )  3>&1 1>&2 2>&3 | tee /test_output/TLPWhere.err
-( java -jar target/sqlancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES  --username default --password "" clickhouse --oracle TLPGroupBy | tee /test_output/TLPGroupBy.out )  3>&1 1>&2 2>&3 | tee /test_output/TLPGroupBy.err
-( java -jar target/sqlancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES  --username default --password "" clickhouse --oracle TLPHaving | tee /test_output/TLPHaving.out )  3>&1 1>&2 2>&3 | tee /test_output/TLPHaving.err
-( java -jar target/sqlancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES  --username default --password "" clickhouse --oracle TLPWhere --oracle TLPGroupBy | tee /test_output/TLPWhereGroupBy.out )  3>&1 1>&2 2>&3 | tee /test_output/TLPWhereGroupBy.err
-( java -jar target/sqlancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES  --username default --password "" clickhouse --oracle TLPDistinct | tee /test_output/TLPDistinct.out )  3>&1 1>&2 2>&3 | tee /test_output/TLPDistinct.err
-( java -jar target/sqlancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES  --username default --password "" clickhouse --oracle TLPAggregate | tee /test_output/TLPAggregate.out )  3>&1 1>&2 2>&3 | tee /test_output/TLPAggregate.err
+for TEST in "${TESTS[@]}"; do
+    echo "$TEST"
+    if [[ $(wget -q 'localhost:8123' -O-) == 'Ok.' ]]
+    then
+        echo "Server is OK"
+        ( java -jar target/sqlancer-*.jar --log-each-select true --print-failed false --num-threads "$NUM_THREADS" --timeout-seconds "$TIMEOUT" --num-queries "$NUM_QUERIES"  --username default --password "" clickhouse --oracle "$TEST" | tee "/workspace/$TEST.out" )  3>&1 1>&2 2>&3 | tee "/workspace/$TEST.err"
+    else
+        touch "/workspace/$TEST.err" "/workspace/$TEST.out"
+        echo "Server is not responding" | tee /workspace/server_crashed.log
+    fi
+done
 
-service clickhouse stop
+ls /workspace
+pkill -F /workspace/clickhouse-server.pid || true
 
-ls /var/log/clickhouse-server/
-tar czf /test_output/logs.tar.gz -C /var/log/clickhouse-server/ .
-tail -n 1000 /var/log/clickhouse-server/stderr.log > /test_output/stderr.log
-tail -n 1000 /var/log/clickhouse-server/stdout.log > /test_output/stdout.log
-tail -n 1000 /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log
+for _ in $(seq 1 60); do if [[ $(wget -q 'localhost:8123' -O-) == 'Ok.' ]]; then sleep 1 ; else break; fi ; done
 
-/process_sqlancer_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
-ls /test_output
+/process_sqlancer_result.py || echo -e "failure\tCannot parse results" > /workspace/check_status.tsv
+ls /workspace

tests/ci/ast_fuzzer_check.py

@@ -69,7 +69,7 @@ if __name__ == "__main__":
         logging.info("Check is already finished according to github status, exiting")
         sys.exit(0)
 
-    docker_image = get_image_with_version(temp_path, IMAGE_NAME)
+    docker_image = get_image_with_version(reports_path, IMAGE_NAME)
 
     build_name = get_build_name_for_check(check_name)
     print(build_name)

tests/ci/ci_config.py

@@ -355,6 +355,12 @@ CI_CONFIG = {
             "required_build": "package_aarch64",
             "test_grep_exclude_filter": "",
         },
+        "SQLancer (release)": {
+            "required_build": "package_release",
+        },
+        "SQLancer (debug)": {
+            "required_build": "package_debug",
+        },
     },
 }  # type: dict
 

tests/ci/sqlancer_check.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+
+import logging
+import subprocess
+import os
+import sys
+
+from github import Github
+
+from env_helper import (
+    GITHUB_REPOSITORY,
+    GITHUB_RUN_URL,
+    REPORTS_PATH,
+    REPO_COPY,
+    TEMP_PATH,
+)
+from s3_helper import S3Helper
+from get_robot_token import get_best_robot_token
+from pr_info import PRInfo
+from build_download_helper import get_build_name_for_check, read_build_urls
+from docker_pull_helper import get_image_with_version
+from commit_status_helper import post_commit_status
+from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
+from upload_result_helper import upload_results
+from stopwatch import Stopwatch
+from rerun_helper import RerunHelper
+
+IMAGE_NAME = "clickhouse/sqlancer-test"
+
+
+def get_run_command(download_url, workspace_path, image):
+    return (
+        f"docker run "
+        # For sysctl
+        "--privileged "
+        "--network=host "
+        f"--volume={workspace_path}:/workspace "
+        "--cap-add syslog --cap-add sys_admin --cap-add=SYS_PTRACE "
+        f'-e BINARY_URL_TO_DOWNLOAD="{download_url}" '
+        f"{image}"
+    )
+
+
+def get_commit(gh, commit_sha):
+    repo = gh.get_repo(GITHUB_REPOSITORY)
+    commit = repo.get_commit(commit_sha)
+    return commit
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    stopwatch = Stopwatch()
+
+    temp_path = TEMP_PATH
+    repo_path = REPO_COPY
+    reports_path = REPORTS_PATH
+
+    check_name = sys.argv[1]
+
+    if not os.path.exists(temp_path):
+        os.makedirs(temp_path)
+
+    pr_info = PRInfo()
+
+    gh = Github(get_best_robot_token(), per_page=100)
+
+    rerun_helper = RerunHelper(gh, pr_info, check_name)
+    if rerun_helper.is_already_finished_by_status():
+        logging.info("Check is already finished according to github status, exiting")
+        sys.exit(0)
+
+    docker_image = get_image_with_version(reports_path, IMAGE_NAME)
+
+    build_name = get_build_name_for_check(check_name)
+    print(build_name)
+    urls = read_build_urls(build_name, reports_path)
+    if not urls:
+        raise Exception("No build URLs found")
+
+    for url in urls:
+        if url.endswith("/clickhouse"):
+            build_url = url
+            break
+    else:
+        raise Exception("Cannot find binary clickhouse among build results")
+
+    logging.info("Got build url %s", build_url)
+
+    workspace_path = os.path.join(temp_path, "workspace")
+    if not os.path.exists(workspace_path):
+        os.makedirs(workspace_path)
+
+    run_command = get_run_command(build_url, workspace_path, docker_image)
+    logging.info("Going to run %s", run_command)
+
+    run_log_path = os.path.join(workspace_path, "runlog.log")
+    with open(run_log_path, "w", encoding="utf-8") as log:
+        with subprocess.Popen(
+            run_command, shell=True, stderr=log, stdout=log
+        ) as process:
+            retcode = process.wait()
+            if retcode == 0:
+                logging.info("Run successfully")
+            else:
+                logging.info("Run failed")
+
+    subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
+
+    check_name_lower = (
+        check_name.lower().replace("(", "").replace(")", "").replace(" ", "")
+    )
+    s3_prefix = f"{pr_info.number}/{pr_info.sha}/{check_name_lower}/"
+
+    tests = [
+        "TLPGroupBy",
+        "TLPHaving",
+        "TLPWhere",
+        "TLPDistinct",
+        "TLPAggregate",
+        "NoREC",
+    ]
+
+    paths = [
+        run_log_path,
+        os.path.join(workspace_path, "clickhouse-server.log"),
+        os.path.join(workspace_path, "stderr.log"),
+        os.path.join(workspace_path, "stdout.log"),
+    ]
+    for t in tests:
+        err_name = f"{t}.err"
+        log_name = f"{t}.out"
+        paths.append(os.path.join(workspace_path, err_name))
+        paths.append(os.path.join(workspace_path, log_name))
+
+    s3_helper = S3Helper()
+    report_url = GITHUB_RUN_URL
+
+    status = "success"
+    test_results = []
+    # Try to get status message saved by the SQLancer
+    try:
+        # with open(
+        #     os.path.join(workspace_path, "status.txt"), "r", encoding="utf-8"
+        # ) as status_f:
+        #     status = status_f.readline().rstrip("\n")
+        if os.path.exists(os.path.join(workspace_path, "server_crashed.log")):
+            test_results.append("Server crashed", "FAIL")
+        with open(
+            os.path.join(workspace_path, "summary.tsv"), "r", encoding="utf-8"
+        ) as summary_f:
+            for line in summary_f:
+                l = line.split("\t")
+                test_results.append((l[0], l[1]))
+
+        with open(
+            os.path.join(workspace_path, "description.txt"), "r", encoding="utf-8"
+        ) as desc_f:
+            description = desc_f.readline().rstrip("\n")[:140]
+    except:
+        # status = "failure"
+        description = "Task failed: $?=" + str(retcode)
+
+    report_url = upload_results(
+        s3_helper,
+        pr_info.number,
+        pr_info.sha,
+        test_results,
+        paths,
+        check_name,
+        False,
+    )
+
+    post_commit_status(gh, pr_info.sha, check_name, description, status, report_url)
+
+    print(f"::notice:: {check_name} Report url: {report_url}")
+
+    ch_helper = ClickHouseHelper()
+
+    prepared_events = prepare_tests_results_for_clickhouse(
+        pr_info,
+        test_results,
+        status,
+        stopwatch.duration_seconds,
+        stopwatch.start_time_str,
+        report_url,
+        check_name,
+    )
+
+    ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
+
+    print(f"::notice Result: '{status}', '{description}', '{report_url}'")
+    post_commit_status(gh, pr_info.sha, check_name, description, status, report_url)

tests/ci/upload_result_helper.py

@@ -14,6 +14,8 @@ from report import ReportColorTheme, create_test_html_report
 def process_logs(
     s3_client, additional_logs, s3_path_prefix, test_results, with_raw_logs
 ):
+    logging.info("Upload files to s3 %s", additional_logs)
+
     processed_logs = {}
     # Firstly convert paths of logs from test_results to urls to s3.
     for test_result in test_results: