thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-30 03:22:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #36091 from ClickHouse/thin-ckeckout

Browse Source 
Thin checkout
This commit is contained in:
alesapin 2022-04-22 11:22:46 +02:00 committed by GitHub
commit 32f5bc8d52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 232 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.github/workflows/backport_branches.yml vendored
View File

@ -142,11 +142,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -181,23 +180,26 @@ jobs:
uses: actions/download-artifact@v2 uses: actions/download-artifact@v2
with: with:
name: changed_images name: changed_images
path: ${{ runner.temp }}/images_path path: ${{ env.IMAGES_PATH }}
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME" cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
- name: Upload build URLs to artifacts - name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v2
with: with:
name: ${{ env.BUILD_URLS }} name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup - name: Cleanup
if: always() if: always()
run: | run: |
@ -228,11 +230,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -273,11 +274,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -318,11 +318,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

69
.github/workflows/master.yml vendored
View File

@ -207,10 +207,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -249,10 +250,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -293,10 +295,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true' fetch-depth: 0 # is needed for ancestor commit search
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -338,10 +341,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -382,11 +386,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -427,11 +430,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -472,11 +474,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -517,11 +518,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -562,11 +562,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -607,11 +606,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -655,11 +653,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -700,11 +697,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -746,10 +742,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -791,10 +788,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -836,10 +834,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -881,10 +880,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -926,10 +926,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

1
.github/workflows/nightly.yml vendored
View File

@ -100,7 +100,6 @@ jobs:
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true' submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"

95
.github/workflows/pull_request.yml vendored
View File

@ -265,11 +265,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -311,10 +310,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true' fetch-depth: 0 # is needed for ancestor commit search
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -355,11 +355,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -395,13 +394,15 @@ jobs:
with: with:
name: changed_images name: changed_images
path: ${{ runner.temp }}/images_path path: ${{ runner.temp }}/images_path
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -411,7 +412,7 @@ jobs:
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v2
with: with:
name: ${{ env.BUILD_URLS }} name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup - name: Cleanup
if: always() if: always()
run: | run: |
@ -437,13 +438,15 @@ jobs:
with: with:
name: changed_images name: changed_images
path: ${{ runner.temp }}/images_path path: ${{ runner.temp }}/images_path
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -453,7 +456,7 @@ jobs:
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v2
with: with:
name: ${{ env.BUILD_URLS }} name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup - name: Cleanup
if: always() if: always()
run: | run: |
@ -484,11 +487,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -529,11 +531,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -574,11 +575,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -619,11 +619,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -664,11 +663,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -712,11 +710,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -757,11 +754,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -802,11 +798,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -847,11 +842,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -892,11 +886,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -937,11 +930,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -982,11 +974,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

31
.github/workflows/release_branches.yml vendored
View File

@ -134,10 +134,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -176,10 +177,11 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -219,11 +221,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -264,11 +265,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -309,11 +309,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -354,11 +353,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -399,11 +397,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v2 uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build - name: Build
run: | run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH" sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH" mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

2
docker/test/fasttest/run.sh
View File

@ -179,7 +179,7 @@ function clone_submodules
) )
git submodule sync git submodule sync
git submodule update --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}" git submodule update --jobs=16 --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
git submodule foreach git reset --hard git submodule foreach git reset --hard
git submodule foreach git checkout @ -f git submodule foreach git checkout @ -f
git submodule foreach git clean -xfd git submodule foreach git clean -xfd

10
tests/ci/build_check.py
View File

@ -14,6 +14,7 @@ from s3_helper import S3Helper
from pr_info import PRInfo from pr_info import PRInfo
from version_helper import ( from version_helper import (
ClickHouseVersion, ClickHouseVersion,
Git,
get_version_from_repo, get_version_from_repo,
update_version_local, update_version_local,
) )
@ -209,7 +210,7 @@ def main():
s3_helper = S3Helper("https://s3.amazonaws.com") s3_helper = S3Helper("https://s3.amazonaws.com")
version = get_version_from_repo() version = get_version_from_repo(git=Git(True))
release_or_pr = get_release_or_pr(pr_info, build_config, version) release_or_pr = get_release_or_pr(pr_info, build_config, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name)) s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))
@ -284,9 +285,12 @@ def main():
if build_config["package_type"] == "performance" and pr_info.number != 0: if build_config["package_type"] == "performance" and pr_info.number != 0:
# because perf tests store some information about git commits # because perf tests store some information about git commits
subprocess.check_call( cmd = (
f"cd {REPO_COPY} && git fetch origin master:master", shell=True f"cd {REPO_COPY} && git fetch --depth=60 --no-recurse-submodules "
"--no-tags origin master:master"
) )
logging.info("Fetch master branch with a command: %s", cmd)
subprocess.check_call(cmd, shell=True)
packager_cmd = get_packager_cmd( packager_cmd = get_packager_cmd(
build_config, build_config,

71
tests/ci/git_helper.py
View File

@ -13,6 +13,9 @@ TAG_REGEXP = (
) )
SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z" SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z"
CWD = p.dirname(p.realpath(__file__))
TWEAK = 1
# Py 3.8 removeprefix and removesuffix # Py 3.8 removeprefix and removesuffix
def removeprefix(string: str, prefix: str): def removeprefix(string: str, prefix: str):
@ -46,8 +49,8 @@ def release_branch(name: str):
class Runner: class Runner:
"""lightweight check_output wrapper with stripping last NEW_LINE""" """lightweight check_output wrapper with stripping last NEW_LINE"""
def __init__(self, cwd: str = p.dirname(p.realpath(__file__))): def __init__(self, cwd: str = CWD):
self.cwd = cwd self._cwd = cwd
def run(self, cmd: str, cwd: Optional[str] = None) -> str: def run(self, cmd: str, cwd: Optional[str] = None) -> str:
if cwd is None: if cwd is None:
@ -56,22 +59,47 @@ class Runner:
cmd, shell=True, cwd=cwd, encoding="utf-8" cmd, shell=True, cwd=cwd, encoding="utf-8"
).strip() ).strip()
@property
def cwd(self) -> str:
return self._cwd
@cwd.setter
def cwd(self, value: str):
# Set _cwd only once, then set it to readonly
if self._cwd != CWD:
return
self._cwd = value
git_runner = Runner()
# Set cwd to abs path of git root
git_runner.cwd = p.relpath(
p.join(git_runner.cwd, git_runner.run("git rev-parse --show-cdup"))
)
def get_tags() -> List[str]:
if git_runner.run("git rev-parse --is-shallow-repository") == "true":
raise RuntimeError("attempt to run on a shallow repository")
return git_runner.run("git tag").split()
class Git: class Git:
"""A small wrapper around subprocess to invoke git commands""" """A small wrapper around subprocess to invoke git commands"""
def __init__(self): _tag_pattern = re.compile(TAG_REGEXP)
runner = Runner()
rel_root = runner.run("git rev-parse --show-cdup") def __init__(self, ignore_no_tags: bool = False):
self.root = p.realpath(p.join(runner.cwd, rel_root)) self.root = git_runner.cwd
self._tag_pattern = re.compile(TAG_REGEXP) self._ignore_no_tags = ignore_no_tags
runner.cwd = self.root self.run = git_runner.run
self.run = runner.run self.latest_tag = ""
self.new_tag = ""
self.new_branch = "" self.new_branch = ""
self.branch = "" self.branch = ""
self.sha = "" self.sha = ""
self.sha_short = "" self.sha_short = ""
self.description = "" self.description = "shallow-checkout"
self.commits_since_tag = 0 self.commits_since_tag = 0
self.update() self.update()
@ -82,6 +110,19 @@ class Git:
self.sha_short = self.sha[:11] self.sha_short = self.sha[:11]
# The following command shows the most recent tag in a graph # The following command shows the most recent tag in a graph
# Format should match TAG_REGEXP # Format should match TAG_REGEXP
if (
self._ignore_no_tags
and self.run("git rev-parse --is-shallow-repository") == "true"
):
try:
self._update_tags()
except subprocess.CalledProcessError:
pass
return
self._update_tags()
def _update_tags(self):
self.latest_tag = self.run("git describe --tags --abbrev=0") self.latest_tag = self.run("git describe --tags --abbrev=0")
# Format should be: {latest_tag}-{commits_since_tag}-g{sha_short} # Format should be: {latest_tag}-{commits_since_tag}-g{sha_short}
self.description = self.run("git describe --tags --long") self.description = self.run("git describe --tags --long")
@ -89,10 +130,11 @@ class Git:
self.run(f"git rev-list {self.latest_tag}..HEAD --count") self.run(f"git rev-list {self.latest_tag}..HEAD --count")
) )
def check_tag(self, value: str): @staticmethod
def check_tag(value: str):
if value == "": if value == "":
return return
if not self._tag_pattern.match(value): if not Git._tag_pattern.match(value):
raise ValueError(f"last tag {value} doesn't match the pattern") raise ValueError(f"last tag {value} doesn't match the pattern")
@property @property
@ -118,10 +160,7 @@ class Git:
if not self.latest_tag.endswith("-testing"): if not self.latest_tag.endswith("-testing"):
# When we are on the tag, we still need to have tweak=1 to not # When we are on the tag, we still need to have tweak=1 to not
# break cmake with versions like 12.13.14.0 # break cmake with versions like 12.13.14.0
return self.commits_since_tag or 1 return self.commits_since_tag or TWEAK
version = self.latest_tag.split("-", maxsplit=1)[0] version = self.latest_tag.split("-", maxsplit=1)[0]
return int(version.split(".")[-1]) + self.commits_since_tag return int(version.split(".")[-1]) + self.commits_since_tag
def get_tags(self) -> List[str]:
return self.run("git tag").split()

15
tests/ci/git_test.py
View File

@ -4,7 +4,7 @@ from unittest.mock import patch
import os.path as p import os.path as p
import unittest import unittest
from git_helper import Git, Runner from git_helper import Git, Runner, CWD
class TestRunner(unittest.TestCase): class TestRunner(unittest.TestCase):
@ -19,6 +19,18 @@ class TestRunner(unittest.TestCase):
output = runner.run("echo 1") output = runner.run("echo 1")
self.assertEqual(output, "1") self.assertEqual(output, "1")
def test_one_time_writeable_cwd(self):
runner = Runner()
self.assertEqual(runner.cwd, CWD)
runner.cwd = "/bin"
self.assertEqual(runner.cwd, "/bin")
runner.cwd = "/"
self.assertEqual(runner.cwd, "/bin")
runner = Runner("/")
self.assertEqual(runner.cwd, "/")
runner.cwd = "/bin"
self.assertEqual(runner.cwd, "/")
class TestGit(unittest.TestCase): class TestGit(unittest.TestCase):
def setUp(self): def setUp(self):
@ -31,6 +43,7 @@ class TestGit(unittest.TestCase):
self.addCleanup(update_patcher.stop) self.addCleanup(update_patcher.stop)
self.git = Git() self.git = Git()
update_mock.assert_called_once() update_mock.assert_called_once()
self.git.run("test")
self.run_mock.assert_called_once() self.run_mock.assert_called_once()
self.git.new_branch = "NEW_BRANCH_NAME" self.git.new_branch = "NEW_BRANCH_NAME"
self.git.new_tag = "v21.12.333.22222-stable" self.git.new_tag = "v21.12.333.22222-stable"

29
tests/ci/release.py
View File

@ -9,14 +9,20 @@ import logging
from git_helper import commit, release_branch from git_helper import commit, release_branch
from version_helper import ( from version_helper import (
FILE_WITH_VERSION_PATH, FILE_WITH_VERSION_PATH,
GENERATED_CONTRIBUTORS,
ClickHouseVersion, ClickHouseVersion,
Git,
VersionType, VersionType,
get_abs_path, get_abs_path,
get_version_from_repo, get_version_from_repo,
update_cmake_version, update_cmake_version,
update_contributors,
) )
git = Git()
class Repo: class Repo:
VALID = ("ssh", "https", "origin") VALID = ("ssh", "https", "origin")
@ -47,14 +53,16 @@ class Repo:
class Release: class Release:
BIG = ("major", "minor") BIG = ("major", "minor")
SMALL = ("patch",) SMALL = ("patch",)
CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH)
CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS)
def __init__(self, repo: Repo, release_commit: str, release_type: str): def __init__(self, repo: Repo, release_commit: str, release_type: str):
self.repo = repo self.repo = repo
self._release_commit = "" self._release_commit = ""
self.release_commit = release_commit self.release_commit = release_commit
self.release_type = release_type self.release_type = release_type
self._version = get_version_from_repo() self._git = git
self._git = self._version._git self._version = get_version_from_repo(git=self._git)
self._release_branch = "" self._release_branch = ""
self._rollback_stack = [] # type: List[str] self._rollback_stack = [] # type: List[str]
@ -75,7 +83,7 @@ class Release:
def read_version(self): def read_version(self):
self._git.update() self._git.update()
self.version = get_version_from_repo() self.version = get_version_from_repo(git=self._git)
def check_prerequisites(self): def check_prerequisites(self):
""" """
@ -179,14 +187,15 @@ class Release:
self.version = self.version.update(self.release_type) self.version = self.version.update(self.release_type)
self.version.with_description(version_type) self.version.with_description(version_type)
update_cmake_version(self.version) update_cmake_version(self.version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) update_contributors(raise_error=True)
# Checkouting the commit of the branch and not the branch itself, # Checkouting the commit of the branch and not the branch itself,
# then we are able to skip rollback # then we are able to skip rollback
with self._checkout(f"{self.release_branch}@{{0}}", False): with self._checkout(f"{self.release_branch}@{{0}}", False):
current_commit = self.run("git rev-parse HEAD") current_commit = self.run("git rev-parse HEAD")
self.run( self.run(
f"git commit -m " f"git commit -m "
f"'Update version to {self.version.string}' '{cmake_path}'" f"'Update version to {self.version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
) )
with self._push( with self._push(
"HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch "HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch
@ -243,9 +252,10 @@ class Release:
new_version = self.version.patch_update() new_version = self.version.patch_update()
new_version.with_description("prestable") new_version.with_description("prestable")
update_cmake_version(new_version) update_cmake_version(new_version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) update_contributors(raise_error=True)
self.run( self.run(
f"git commit -m 'Update version to {new_version.string}' '{cmake_path}'" f"git commit -m 'Update version to {new_version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
) )
with self._push(self.release_branch): with self._push(self.release_branch):
with self._create_gh_label( with self._create_gh_label(
@ -271,9 +281,10 @@ class Release:
self.version = self.version.update(self.release_type) self.version = self.version.update(self.release_type)
self.version.with_description("testing") self.version.with_description("testing")
update_cmake_version(self.version) update_cmake_version(self.version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) update_contributors(raise_error=True)
self.run( self.run(
f"git commit -m 'Update version to {self.version.string}' '{cmake_path}'" f"git commit -m 'Update version to {self.version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
) )
with self._push(helper_branch): with self._push(helper_branch):
body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md") body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")

44
tests/ci/version_helper.py
View File

@ -2,9 +2,9 @@
import logging import logging
import os.path as p import os.path as p
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError
from typing import Dict, List, Tuple, Union from typing import Dict, List, Optional, Tuple, Union
from git_helper import Git, removeprefix from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix
FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt" FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt"
CHANGELOG_IN_PATH = "debian/changelog.in" CHANGELOG_IN_PATH = "debian/changelog.in"
@ -34,8 +34,6 @@ SET(VERSION_STRING {string})
# end of autochange # end of autochange
""" """
git = Git()
class ClickHouseVersion: class ClickHouseVersion:
"""Immutable version class. On update returns a new instance""" """Immutable version class. On update returns a new instance"""
@ -46,7 +44,7 @@ class ClickHouseVersion:
minor: Union[int, str], minor: Union[int, str],
patch: Union[int, str], patch: Union[int, str],
revision: Union[int, str], revision: Union[int, str],
git: Git, git: Optional[Git],
tweak: str = None, tweak: str = None,
): ):
self._major = int(major) self._major = int(major)
@ -54,9 +52,11 @@ class ClickHouseVersion:
self._patch = int(patch) self._patch = int(patch)
self._revision = int(revision) self._revision = int(revision)
self._git = git self._git = git
self._tweak = None self._tweak = TWEAK
if tweak is not None: if tweak is not None:
self._tweak = int(tweak) self._tweak = int(tweak)
elif self._git is not None:
self._tweak = self._git.tweak
self._describe = "" self._describe = ""
def update(self, part: str) -> "ClickHouseVersion": def update(self, part: str) -> "ClickHouseVersion":
@ -91,7 +91,7 @@ class ClickHouseVersion:
@property @property
def tweak(self) -> int: def tweak(self) -> int:
return self._tweak or self._git.tweak return self._tweak
@property @property
def revision(self) -> int: def revision(self) -> int:
@ -99,7 +99,9 @@ class ClickHouseVersion:
@property @property
def githash(self) -> str: def githash(self) -> str:
return self._git.sha if self._git is not None:
return self._git.sha
return "0000000000000000000000000000000000000000"
@property @property
def describe(self): def describe(self):
@ -171,7 +173,7 @@ def validate_version(version: str):
def get_abs_path(path: str) -> str: def get_abs_path(path: str) -> str:
return p.abspath(p.join(git.root, path)) return p.abspath(p.join(git_runner.cwd, path))
def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS: def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
@ -197,6 +199,7 @@ def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
def get_version_from_repo( def get_version_from_repo(
versions_path: str = FILE_WITH_VERSION_PATH, versions_path: str = FILE_WITH_VERSION_PATH,
git: Optional[Git] = None,
) -> ClickHouseVersion: ) -> ClickHouseVersion:
versions = read_versions(versions_path) versions = read_versions(versions_path)
return ClickHouseVersion( return ClickHouseVersion(
@ -208,14 +211,16 @@ def get_version_from_repo(
) )
def get_version_from_string(version: str) -> ClickHouseVersion: def get_version_from_string(
version: str, git: Optional[Git] = None
) -> ClickHouseVersion:
validate_version(version) validate_version(version)
parts = version.split(".") parts = version.split(".")
return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3]) return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3])
def get_version_from_tag(tag: str) -> ClickHouseVersion: def get_version_from_tag(tag: str) -> ClickHouseVersion:
git.check_tag(tag) Git.check_tag(tag)
tag = tag[1:].split("-")[0] tag = tag[1:].split("-")[0]
return get_version_from_string(tag) return get_version_from_string(tag)
@ -236,7 +241,7 @@ def version_arg(version: str) -> ClickHouseVersion:
def get_tagged_versions() -> List[ClickHouseVersion]: def get_tagged_versions() -> List[ClickHouseVersion]:
versions = [] versions = []
for tag in git.get_tags(): for tag in get_tags():
try: try:
version = get_version_from_tag(tag) version = get_version_from_tag(tag)
versions.append(version) versions.append(version)
@ -255,21 +260,26 @@ def update_cmake_version(
def update_contributors( def update_contributors(
relative_contributors_path: str = GENERATED_CONTRIBUTORS, force: bool = False relative_contributors_path: str = GENERATED_CONTRIBUTORS,
force: bool = False,
raise_error: bool = False,
): ):
# Check if we have shallow checkout by comparing number of lines # Check if we have shallow checkout by comparing number of lines
# '--is-shallow-repository' is in git since 2.15, 2017-10-30 # '--is-shallow-repository' is in git since 2.15, 2017-10-30
if git.run("git rev-parse --is-shallow-repository") == "true" and not force: if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force:
logging.warning("The repository is shallow, refusing to update contributors") logging.warning("The repository is shallow, refusing to update contributors")
if raise_error:
raise RuntimeError("update_contributors executed on a shallow repository")
return return
contributors = git.run("git shortlog HEAD --summary") # format: " 1016 Alexey Arno"
shortlog = git_runner.run("git shortlog HEAD --summary")
contributors = sorted( contributors = sorted(
[c.split(maxsplit=1)[-1].replace('"', r"\"") for c in contributors.split("\n")], [c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")],
) )
contributors = [f' "{c}",' for c in contributors] contributors = [f' "{c}",' for c in contributors]
executer = p.relpath(p.realpath(__file__), git.root) executer = p.relpath(p.realpath(__file__), git_runner.cwd)
content = CONTRIBUTORS_TEMPLATE.format( content = CONTRIBUTORS_TEMPLATE.format(
executer=executer, contributors="\n".join(contributors) executer=executer, contributors="\n".join(contributors)
) )