Better stylecheck

.github/workflows/hello-world.yml

@@ -1,4 +1,5 @@
 name: GitHub Actions Hello self hosted
+desction: Trying GithubActions
 on:
   push:
     branches:
@@ -18,7 +19,11 @@ jobs:
         uses: actions/checkout@v2
       - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
       - run: echo "🖥️ The workflow is now ready to test your code on the runner."
-      - run: cd $GITHUB_WORKSPACE/tests/ci && python3 style_check.py
+      - name: Style Check
+        env:
+          YANDEX_S3_ACCESS_KEY_ID: ${{ secrets.YANDEX_S3_ACCESS_KEY_ID }}
+          YANDEX_S3_ACCESS_SECRET_KEY: ${{ secrets.YANDEX_S3_ACCESS_SECRET_KEY }}
+        run: cd $GITHUB_WORKSPACE/tests/ci && python3 style_check.py
       - name: List files in the repository
         run: |
           ls ${{ github.workspace }}

tests/ci/compress_files.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import subprocess
+import logging
+import os
+
+def compress_file_fast(path, archive_path):
+    if os.path.exists('/usr/bin/pigz'):
+        subprocess.check_call("pigz < {} > {}".format(path, archive_path), shell=True)
+    else:
+        subprocess.check_call("gzip < {} > {}".format(path, archive_path), shell=True)
+
+
+def compress_fast(path, archive_path, exclude=None):
+    pigz_part = ''
+    if os.path.exists('/usr/bin/pigz'):
+        logging.info("pigz found, will compress and decompress faster")
+        pigz_part = "--use-compress-program='pigz'"
+    else:
+        pigz_part = '-z'
+        logging.info("no pigz, compressing with default tar")
+
+    if exclude is None:
+        exclude_part = ""
+    elif isinstance(exclude, list):
+        exclude_part = " ".join(["--exclude {}".format(x) for x in exclude])
+    else:
+        exclude_part = "--exclude {}".format(str(exclude))
+
+    fname = os.path.basename(path)
+    if os.path.isfile(path):
+        path = os.path.dirname(path)
+    else:
+        path += "/.."
+    cmd = "tar {} {} -cf {} -C {} {}".format(pigz_part, exclude_part, archive_path, path, fname)
+    logging.debug("compress_fast cmd:{}".format(cmd))
+    subprocess.check_call(cmd, shell=True)
+
+
+def decompress_fast(archive_path, result_path=None):
+    pigz_part = ''
+    if os.path.exists('/usr/bin/pigz'):
+        logging.info("pigz found, will compress and decompress faster ('{}' -> '{}')".format(archive_path, result_path))
+        pigz_part = "--use-compress-program='pigz'"
+    else:
+        pigz_part = '-z'
+        logging.info("no pigz, decompressing with default tar ('{}' -> '{}')".format(archive_path, result_path))
+
+    if result_path is None:
+        subprocess.check_call("tar {} -xf {}".format(pigz_part, archive_path), shell=True)
+    else:
+        subprocess.check_call("tar {} -xf {} -C {}".format(pigz_part, archive_path, result_path), shell=True)

tests/ci/s3_helper.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+import hashlib
+import logging
+import os
+import boto3
+from botocore.exceptions import ClientError, BotoCoreError
+from multiprocessing.dummy import Pool
+from compress_files import compress_file_fast
+
+def _md5(fname):
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    logging.debug("MD5 for {} is {}".format(fname, hash_md5.hexdigest()))
+    return hash_md5.hexdigest()
+
+
+def _flatten_list(lst):
+    result = []
+    for elem in lst:
+        if isinstance(elem, list):
+            result += _flatten_list(elem)
+        else:
+            result.append(elem)
+    return result
+
+
+class S3Helper(object):
+    def __init__(self, host, aws_access_key_id, aws_secret_access_key):
+        self.session = boto3.session.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
+        self.client = self.session.client('s3', endpoint_url=host)
+
+    def _upload_file_to_s3(self, bucket_name, file_path, s3_path):
+        logging.debug("Start uploading {} to bucket={} path={}".format(file_path, bucket_name, s3_path))
+        metadata = {}
+        if os.path.getsize(file_path) < 64 * 1024 * 1024:
+            if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"):
+                metadata['ContentType'] = "text/plain; charset=utf-8"
+                logging.info("Content type %s for file path %s", "text/plain; charset=utf-8", file_path)
+            elif s3_path.endswith("html"):
+                metadata['ContentType'] = "text/html; charset=utf-8"
+                logging.info("Content type %s for file path %s", "text/html; charset=utf-8", file_path)
+            else:
+                logging.info("No content type provied for %s", file_path)
+        else:
+            if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"):
+                logging.info("Going to compress file log file %s to %s", file_path, file_path + ".gz")
+                compress_file_fast(file_path, file_path + ".gz")
+                file_path += ".gz"
+                s3_path += ".gz"
+            else:
+                logging.info("Processing file without compression")
+            logging.info("File is too large, do not provide content type")
+
+        self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata)
+        logging.info("Upload {} to {}. Meta: {}".format(file_path, s3_path, metadata))
+        return "https://storage.yandexcloud.net/{bucket}/{path}".format(bucket=bucket_name, path=s3_path)
+
+    def upload_test_report_to_s3(self, file_path, s3_path):
+        return self._upload_file_to_s3('clickhouse-test-reports', file_path, s3_path)
+
+    def upload_build_file_to_s3(self, file_path, s3_path):
+        return self._upload_file_to_s3('clickhouse-builds', file_path, s3_path)
+
+    def _upload_folder_to_s3(self, folder_path, s3_folder_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks):
+        logging.info("Upload folder '{}' to bucket={} of s3 folder '{}'".format(folder_path, bucket_name, s3_folder_path))
+        if not os.path.exists(folder_path):
+            return []
+        files = os.listdir(folder_path)
+        if not files:
+            return []
+
+        p = Pool(min(len(files), 30))
+
+        def task(file_name):
+            full_fs_path = os.path.join(folder_path, file_name)
+            if keep_dirs_in_s3_path:
+                full_s3_path = s3_folder_path + "/" + os.path.basename(folder_path)
+            else:
+                full_s3_path = s3_folder_path
+
+            if os.path.isdir(full_fs_path):
+                return self._upload_folder_to_s3(full_fs_path, full_s3_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks)
+
+            if os.path.islink(full_fs_path):
+                if upload_symlinks:
+                    return self._upload_file_to_s3(bucket_name, full_fs_path, full_s3_path + "/" + file_name)
+                return []
+
+            return self._upload_file_to_s3(bucket_name, full_fs_path, full_s3_path + "/" + file_name)
+
+        return sorted(_flatten_list(list(p.map(task, files))))
+
+    def upload_build_folder_to_s3(self, folder_path, s3_folder_path, keep_dirs_in_s3_path=True, upload_symlinks=True):
+        return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-builds', keep_dirs_in_s3_path, upload_symlinks)
+
+    def upload_test_folder_to_s3(self, folder_path, s3_folder_path):
+        return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-test-reports', True, True)

tests/ci/style_check.py

@@ -4,6 +4,19 @@ import logging
 import subprocess
 import os
 import csv
+from s3_helper import S3Helper
+
+
+def process_logs(s3_client, additional_logs, s3_path_prefix):
+    additional_urls = []
+    for log_path in additional_logs:
+        if log_path:
+            additional_urls.append(
+                s3_client.upload_test_report_to_s3(
+                    log_path,
+                    s3_path_prefix + "/" + os.path.basename(log_path)))
+
+    return additional_urls
 
 
 def process_result(result_folder):
@@ -34,6 +47,31 @@ def process_result(result_folder):
             state, description = "error", "Failed to read test_results.tsv"
         return state, description, test_results, additional_files
 
+def upload_results(s3_client, pr_number, commit_sha, state, description, test_results, additional_files):
+    s3_path_prefix = f"{pr_number}/{commit_sha}/style_check"
+    additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
+
+     # Add link to help. Anchors in the docs must be adjusted accordingly.
+    branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
+    branch_name = "master"
+    if pr_number != 0:
+        branch_name = "PR #{}".format(pr_number)
+        branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
+    commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
+
+    task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{run_id}"
+
+    raw_log_url = additional_urls[0]
+    additional_urls.pop(0)
+
+    html_report = create_test_html_report("Style Check (actions)", test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
+    with open('report.html', 'w') as f:
+        f.write(html_report)
+
+    url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
+    logging.info("Search result in url %s", url)
+
+
 def get_pr_url_from_ref(ref):
     try:
         return ref.split("/")[2]
@@ -41,24 +79,25 @@ def get_pr_url_from_ref(ref):
         return "master"
 
 if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
     repo_path = os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../"))
     temp_path = os.getenv("RUNNER_TEMP", os.path.abspath("./temp"))
     run_id = os.getenv("GITHUB_RUN_ID", 0)
     commit_sha = os.getenv("GITHUB_SHA", 0)
     ref = os.getenv("GITHUB_REF", "")
+    aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
+    aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
+
     docker_image_version = os.getenv("DOCKER_IMAGE_VERSION", "latest")
+    if not aws_secret_key_id  or not aws_secret_key:
+        logging.info("No secrets, will not upload anything to S3")
+
+    s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
 
     if not os.path.exists(temp_path):
         os.makedirs(temp_path)
 
     subprocess.check_output(f"docker run --cap-add=SYS_PTRACE --volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output clickhouse/style-test:{docker_image_version}", shell=True)
-    state, description, test_results, additional_files = process_result(temp_path)
-    task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{run_id}"
-    branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(get_pr_url_from_ref(ref))
-    branch_name = "PR #" + str(get_pr_url_from_ref(ref))
-    commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
-    raw_log_url = "noop"
 
-    html_report = create_test_html_report("Style Check (actions)", test_results, raw_log_url, task_url, branch_url, branch_name, commit_url)
-    with open(os.path.join(temp_path, 'report.html'), 'w') as f:
-        f.write(html_report)
+    state, description, test_results, additional_files = process_result(temp_path)
+    upload_results(s3_helper, get_pr_url_from_ref(ref), commit_sha, state, description, test_results, additional_files)