Merge pull request #31227 from ClickHouse/download_ccache_in_builds

Download ccache if not exists in builds

.github/workflows/main.yml

@@ -160,8 +160,47 @@ jobs:
           docker kill $(docker ps -q) ||:
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr $TEMP_PATH
+  BuilderDebRelease:
+    needs: DockerHubPush
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'pr-documentation') && !contains(github.event.pull_request.labels.*.name, 'pr-doc-fix') }}
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/images_path
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        env:
+          TEMP_PATH: ${{runner.temp}}/build_check
+          IMAGES_PATH: ${{runner.temp}}/images_path
+          REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH: ${{runner.temp}}/../ccaches
+          CHECK_NAME: 'ClickHouse build check (actions)'
+          BUILD_NUMBER: 0
+        run: |
+          sudo rm -fr $TEMP_PATH
+          mkdir -p $TEMP_PATH
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
+      - name: Upload build URLs to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_NAME }}
+          path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH
   BuilderReport:
-    needs: [BuilderDebDebug, BuilderDebAsan]
+    needs: [BuilderDebDebug, BuilderDebAsan, BuilderDebRelease]
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Download json reports

tests/ci/build_check.py

@@ -11,6 +11,7 @@ from s3_helper import S3Helper
 from pr_info import PRInfo
 from get_robot_token import get_best_robot_token
 from version_helper import get_version_from_repo, update_version_local
+from ccache_utils import get_ccache_if_not_exists, upload_ccache
 
 
 def get_build_config(build_check_name, build_number, repo_path):
@@ -41,7 +42,7 @@ def _can_export_binaries(build_config):
     return False
 
 
-def get_packager_cmd(build_config, packager_path, output_path, build_version, image_version, ccache_path):
+def get_packager_cmd(build_config, packager_path, output_path, build_version, image_version, ccache_path, pr_info):
     package_type = build_config['package-type']
     comp = build_config['compiler']
     cmd = f"cd {packager_path} && ./packager --output-dir={output_path} --package-type={package_type} --compiler={comp}"
@@ -61,7 +62,8 @@ def get_packager_cmd(build_config, packager_path, output_path, build_version, im
     cmd += ' --ccache_dir={}'.format(ccache_path)
 
     if 'alien_pkgs' in build_config and build_config['alien_pkgs']:
-        cmd += ' --alien-pkgs'
+        if pr_info == 0 or 'release' in pr_info.labels:
+            cmd += ' --alien-pkgs rpm tgz'
 
     cmd += ' --docker-image-version={}'.format(image_version)
     cmd += ' --version={}'.format(build_version)
@@ -163,10 +165,16 @@ if __name__ == "__main__":
         os.makedirs(build_output_path)
 
     ccache_path = os.path.join(caches_path, build_name + '_ccache')
+    s3_helper = S3Helper('https://s3.amazonaws.com')
+
+    logging.info("Will try to fetch cache for our build")
+    get_ccache_if_not_exists(ccache_path, s3_helper, pr_info.number, temp_path)
+
     if not os.path.exists(ccache_path):
+        logging.info("cache was not fetched, will create empty dir")
         os.makedirs(ccache_path)
 
-    packager_cmd = get_packager_cmd(build_config, os.path.join(repo_path, "docker/packager"), build_output_path, version.get_version_string(), image_version, ccache_path)
+    packager_cmd = get_packager_cmd(build_config, os.path.join(repo_path, "docker/packager"), build_output_path, version.get_version_string(), image_version, ccache_path, pr_info)
     logging.info("Going to run packager with %s", packager_cmd)
 
     build_clickhouse_log = os.path.join(temp_path, "build_log")
@@ -180,7 +188,10 @@ if __name__ == "__main__":
     subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {ccache_path}", shell=True)
     logging.info("Build finished with %s, log path %s", success, log_path)
 
-    s3_helper = S3Helper('https://s3.amazonaws.com')
+
+    logging.info("Will upload cache")
+    upload_ccache(ccache_path, s3_helper, pr_info.number, temp_path)
+
     s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + build_check_name.lower().replace(' ', '_') + "/" + build_name
     if os.path.exists(log_path):
         log_url = s3_helper.upload_build_file_to_s3(log_path, s3_path_prefix + "/" + os.path.basename(log_path))

tests/ci/ccache_utils.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+import logging
+import time
+import sys
+import os
+import shutil
+from pathlib import Path
+
+import requests
+
+from compress_files import decompress_fast, compress_fast
+
+DOWNLOAD_RETRIES_COUNT = 5
+
+def dowload_file_with_progress(url, path):
+    logging.info("Downloading from %s to temp path %s", url, path)
+    for i in range(DOWNLOAD_RETRIES_COUNT):
+        try:
+            with open(path, 'wb') as f:
+                response = requests.get(url, stream=True)
+                response.raise_for_status()
+                total_length = response.headers.get('content-length')
+                if total_length is None or int(total_length) == 0:
+                    logging.info("No content-length, will download file without progress")
+                    f.write(response.content)
+                else:
+                    dl = 0
+                    total_length = int(total_length)
+                    logging.info("Content length is %ld bytes", total_length)
+                    for data in response.iter_content(chunk_size=4096):
+                        dl += len(data)
+                        f.write(data)
+                        if sys.stdout.isatty():
+                            done = int(50 * dl / total_length)
+                            percent = int(100 * float(dl) / total_length)
+                            eq_str = '=' * done
+                            space_str = ' ' * (50 - done)
+                            sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
+                            sys.stdout.flush()
+            break
+        except Exception as ex:
+            sys.stdout.write("\n")
+            time.sleep(3)
+            logging.info("Exception while downloading %s, retry %s", ex, i + 1)
+            if os.path.exists(path):
+                os.remove(path)
+    else:
+        raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
+
+    sys.stdout.write("\n")
+    logging.info("Downloading finished")
+
+
+def get_ccache_if_not_exists(path_to_ccache_dir, s3_helper, current_pr_number, temp_path):
+    if os.path.exists(path_to_ccache_dir):
+        shutil.rmtree(path_to_ccache_dir)
+        logging.info("Ccache already exists, removing it")
+
+    ccache_name = os.path.basename(path_to_ccache_dir)
+    cache_found = False
+    prs_to_check = [current_pr_number]
+    if current_pr_number != 0:
+        prs_to_check.append(0)
+    for pr_number in prs_to_check:
+        logging.info("Searching cache for pr %s", pr_number)
+        s3_path_prefix = str(pr_number) + "/ccaches"
+        objects = s3_helper.list_prefix(s3_path_prefix)
+        logging.info("Found %s objects for pr", len(objects))
+        for obj in objects:
+            if ccache_name in obj:
+                logging.info("Found ccache on path %s", obj)
+                url = "https://s3.amazonaws.com/clickhouse-builds/" + obj
+                compressed_cache = os.path.join(temp_path, os.path.basename(obj))
+                dowload_file_with_progress(url, compressed_cache)
+
+                path_to_decompress = str(Path(path_to_ccache_dir).parent)
+                if not os.path.exists(path_to_decompress):
+                    os.makedirs(path_to_decompress)
+
+                logging.info("Decompressing cache to path %s", path_to_decompress)
+                decompress_fast(compressed_cache, path_to_decompress)
+                logging.info("Files on path %s", os.listdir(path_to_decompress))
+                cache_found = True
+                break
+        if cache_found:
+            break
+
+    if not cache_found:
+        logging.info("ccache not found anywhere, cannot download anything :(")
+    else:
+        logging.info("ccache downloaded")
+
+def upload_ccache(path_to_ccache_dir, s3_helper, current_pr_number, temp_path):
+    logging.info("Uploading cache %s for pr %s", path_to_ccache_dir, current_pr_number)
+    ccache_name = os.path.basename(path_to_ccache_dir)
+    compressed_cache_path = os.path.join(temp_path, ccache_name + ".tar.gz")
+    compress_fast(path_to_ccache_dir, compressed_cache_path)
+
+    s3_path = str(current_pr_number) + "/ccaches/" + os.path.basename(compressed_cache_path)
+    logging.info("Will upload %s to path %s", compressed_cache_path, s3_path)
+    s3_helper.upload_build_file_to_s3(compressed_cache_path, s3_path)
+    logging.info("Upload finished")

tests/ci/s3_helper.py

@@ -103,3 +103,12 @@ class S3Helper():
 
     def upload_test_folder_to_s3(self, folder_path, s3_folder_path):
         return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-test-reports', True, True)
+
+    def list_prefix(self, s3_prefix_path, bucket='clickhouse-builds'):
+        objects = self.client.list_objects_v2(Bucket=bucket, Prefix=s3_prefix_path)
+        result = []
+        if 'Contents' in objects:
+            for obj in objects['Contents']:
+                result.append(obj['Key'])
+
+        return result