Fixes

.github/workflows/main.yml

@@ -64,9 +64,9 @@ jobs:
         env:
           TEMP_PATH: ${{runner.temp}}/build_check
           REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
-          CACHES_PATH: ${{runner.temp}}/..
+          CACHES_PATH: ${{runner.temp}}/ccaches/..
           CHECK_NAME: 'ClickHouse build check (actions)'
-        run: mkdir -p $TEMP_PATH && cp -r $GITHUB_WORKSPACE $TEMP_PATH && cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" ${{ matrix.build_number }} '21.10.1.1'
+        run: mkdir -p $TEMP_PATH && cp -r $GITHUB_WORKSPACE $TEMP_PATH && cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" ${{ matrix.build_number }}
       - name: Upload build URLs to artifacts
         uses: actions/upload-artifact@v2
         with:

tests/ci/build_check.py

@@ -13,6 +13,7 @@ from get_robot_token import get_best_robot_token, get_parameter_from_ssm
 import os
 import sys
 import time
+from version_helper import get_version_from_repo, update_version_local
 
 
 def get_build_config(build_check_name, build_number, repo_path):
@@ -113,7 +114,6 @@ if __name__ == "__main__":
 
     build_check_name = sys.argv[1]
     build_number = int(sys.argv[2])
-    build_version = sys.argv[3]
 
     build_config = get_build_config(build_check_name, build_number, repo_path)
 
@@ -151,6 +151,9 @@ if __name__ == "__main__":
     else:
         raise Exception(f"Cannot pull dockerhub for image docker pull {image_name}:{image_version}")
 
+    version = get_version_from_repo(repo_path)
+    version.tweak_update()
+    update_version_local(repo_path, pr_info.sha, version)
 
     build_name = build_config_to_string(build_config)
     logging.info(f"Build short name {build_name}")
@@ -164,7 +167,7 @@ if __name__ == "__main__":
     if not os.path.exists(ccache_path):
         os.makedirs(ccache_path)
 
-    packager_cmd = get_packager_cmd(build_config, os.path.join(repo_path, "docker/packager"), build_output_path, build_version, image_version, ccache_path)
+    packager_cmd = get_packager_cmd(build_config, os.path.join(repo_path, "docker/packager"), build_output_path, version.get_version_string(), image_version, ccache_path)
     logging.info("Going to run packager with %s", packager_cmd)
 
     build_clickhouse_log = os.path.join(temp_path, "build_log")

tests/ci/s3_helper.py

@@ -56,7 +56,8 @@ class S3Helper(object):
 
         self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata)
         logging.info("Upload {} to {}. Meta: {}".format(file_path, s3_path, metadata))
-        return "https://s3.amazonaws.com/{bucket}/{path}".format(bucket=bucket_name, path=s3_path)
+        # last two replacements are specifics of AWS urls: https://jamesd3142.wordpress.com/2018/02/28/amazon-s3-and-the-plus-symbol/
+        return "https://s3.amazonaws.com/{bucket}/{path}".format(bucket=bucket_name, path=s3_path).replace('+', '%2B').replace(' ', '%20')
 
     def upload_test_report_to_s3(self, file_path, s3_path):
         return self._upload_file_to_s3('clickhouse-test-reports', file_path, s3_path)

tests/ci/version_helper.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import datetime
+
+FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt"
+CHANGELOG_IN_PATH = "debian/changelog.in"
+CHANGELOG_PATH = "debian/changelog"
+CONTRIBUTORS_SCRIPT_DIR = "src/Storages/System/"
+
+
+class ClickHouseVersion(object):
+    def __init__(self, major, minor, patch, tweak, revision):
+        self.major = major
+        self.minor = minor
+        self.patch = patch
+        self.tweak = tweak
+        self.revision = revision
+
+    def minor_update(self):
+        return ClickHouseVersion(
+            self.major,
+            self.minor + 1,
+            1,
+            1,
+            self.revision + 1)
+
+    def patch_update(self):
+        return ClickHouseVersion(
+            self.major,
+            self.minor,
+            self.patch + 1,
+            1,
+            self.revision)
+
+    def tweak_update(self):
+        return ClickHouseVersion(
+            self.major,
+            self.minor,
+            self.patch,
+            self.tweak + 1,
+            self.revision)
+
+    def get_version_string(self):
+        return '.'.join([
+            str(self.major),
+            str(self.minor),
+            str(self.patch),
+            str(self.tweak)
+        ])
+
+    def as_tuple(self):
+        return (self.major, self.minor, self.patch, self.tweak)
+
+
+class VersionType(object):
+    STABLE = "stable"
+    TESTING = "testing"
+
+
+def build_version_description(version, version_type):
+    return "v" + version.get_version_string() + "-" + version_type
+
+
+def _get_version_from_line(line):
+    _, ver_with_bracket = line.strip().split(' ')
+    return ver_with_bracket[:-1]
+
+
+def get_version_from_repo(repo_path):
+    path_to_file = os.path.join(repo_path, FILE_WITH_VERSION_PATH)
+    major = 0
+    minor = 0
+    patch = 0
+    tweak = 0
+    version_revision = 0
+    with open(path_to_file, 'r') as ver_file:
+        for line in ver_file:
+            if "VERSION_MAJOR" in line and "math" not in line and "SET" in line:
+                major = _get_version_from_line(line)
+            elif "VERSION_MINOR" in line and "math" not in line and "SET" in line:
+                minor = _get_version_from_line(line)
+            elif "VERSION_PATCH" in line and "math" not in line and "SET" in line:
+                patch = _get_version_from_line(line)
+            elif "VERSION_REVISION" in line and "math" not in line:
+                version_revision = _get_version_from_line(line)
+    return ClickHouseVersion(major, minor, patch, tweak, version_revision)
+
+
+def _update_cmake_version(repo_path, version, sha, version_type):
+    cmd = """sed -i --follow-symlinks -e "s/SET(VERSION_REVISION [^) ]*/SET(VERSION_REVISION {revision}/g;" \
+            -e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE {version_desc}/g;" \
+            -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH {sha}/g;" \
+            -e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR {major}/g;" \
+            -e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR {minor}/g;" \
+            -e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH {patch}/g;" \
+            -e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING {version_string}/g;" \
+            {path}""".format(
+        revision=version.revision,
+        version_desc=build_version_description(version, version_type),
+        sha=sha,
+        major=version.major,
+        minor=version.minor,
+        patch=version.patch,
+        version_string=version.get_version_string(),
+        path=os.path.join(repo_path, FILE_WITH_VERSION_PATH),
+    )
+    subprocess.check_call(cmd, shell=True)
+
+
+def _update_changelog(repo_path, version):
+    cmd = """sed \
+        -e "s/[@]VERSION_STRING[@]/{version_str}/g" \
+        -e "s/[@]DATE[@]/{date}/g" \
+        -e "s/[@]AUTHOR[@]/clickhouse-release/g" \
+        -e "s/[@]EMAIL[@]/clickhouse-release@yandex-team.ru/g" \
+        < {in_path} > {changelog_path}
+    """.format(
+        version_str=version.get_version_string(),
+        date=datetime.datetime.now().strftime("%a, %d %b %Y %H:%M:%S") + " +0300",
+        in_path=os.path.join(repo_path, CHANGELOG_IN_PATH),
+        changelog_path=os.path.join(repo_path, CHANGELOG_PATH)
+    )
+    subprocess.check_call(cmd, shell=True)
+
+def _update_contributors(repo_path):
+    cmd = "cd {} && ./StorageSystemContributors.sh".format(os.path.join(repo_path, CONTRIBUTORS_SCRIPT_DIR))
+    subprocess.check_call(cmd, shell=True)
+
+def _update_dockerfile(repo_path, version):
+    version_str_for_docker = '.'.join([str(version.major), str(version.minor), str(version.patch), '*'])
+    cmd = "ls -1 {path}/docker/*/Dockerfile | xargs sed -i -r -e 's/ARG version=.+$/ARG version='{ver}'/'".format(path=repo_path, ver=version_str_for_docker)
+    subprocess.check_call(cmd, shell=True)
+
+def update_version_local(repo_path, sha, version, version_type="testing"):
+    _update_contributors(repo_path)
+    _update_cmake_version(repo_path, version, sha, version_type)
+    _update_changelog(repo_path, version)
+    _update_dockerfile(repo_path, version)