thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-29 19:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #36091 from ClickHouse/thin-ckeckout

Browse Source 
Thin checkout
This commit is contained in:
alesapin 2022-04-22 11:22:46 +02:00 committed by GitHub
commit 32f5bc8d52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 232 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.github/workflows/backport_branches.yml vendored
View File

@ -142,11 +142,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -181,23 +180,26 @@ jobs:
uses: actions/download-artifact@v2
with:
name: changed_images
path: ${{ runner.temp }}/images_path
path: ${{ env.IMAGES_PATH }}
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -228,11 +230,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -273,11 +274,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -318,11 +318,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

69
.github/workflows/master.yml vendored
View File

@ -207,10 +207,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -249,10 +250,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -293,10 +295,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
fetch-depth: 0 # is needed for ancestor commit search
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -338,10 +341,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -382,11 +386,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -427,11 +430,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -472,11 +474,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -517,11 +518,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -562,11 +562,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -607,11 +606,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -655,11 +653,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -700,11 +697,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -746,10 +742,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -791,10 +788,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -836,10 +834,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -881,10 +880,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -926,10 +926,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

1
.github/workflows/nightly.yml vendored
View File

@ -100,7 +100,6 @@ jobs:
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"

95
.github/workflows/pull_request.yml vendored
View File

@ -265,11 +265,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -311,10 +310,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
fetch-depth: 0 # is needed for ancestor commit search
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -355,11 +355,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -395,13 +394,15 @@ jobs:
with:
name: changed_images
path: ${{ runner.temp }}/images_path
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -411,7 +412,7 @@ jobs:
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -437,13 +438,15 @@ jobs:
with:
name: changed_images
path: ${{ runner.temp }}/images_path
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -453,7 +456,7 @@ jobs:
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -484,11 +487,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -529,11 +531,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -574,11 +575,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -619,11 +619,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -664,11 +663,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -712,11 +710,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -757,11 +754,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -802,11 +798,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -847,11 +842,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -892,11 +886,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -937,11 +930,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -982,11 +974,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

31
.github/workflows/release_branches.yml vendored
View File

@ -134,10 +134,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -176,10 +177,11 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -219,11 +221,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -264,11 +265,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -309,11 +309,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -354,11 +353,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
@ -399,11 +397,10 @@ jobs:
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync --recursive
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"

2
docker/test/fasttest/run.sh
View File

@ -179,7 +179,7 @@ function clone_submodules
)
git submodule sync
git submodule update --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
git submodule update --jobs=16 --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
git submodule foreach git reset --hard
git submodule foreach git checkout @ -f
git submodule foreach git clean -xfd

10
tests/ci/build_check.py
View File

@ -14,6 +14,7 @@ from s3_helper import S3Helper
from pr_info import PRInfo
from version_helper import (
ClickHouseVersion,
Git,
get_version_from_repo,
update_version_local,
)
@ -209,7 +210,7 @@ def main():
s3_helper = S3Helper("https://s3.amazonaws.com")
version = get_version_from_repo()
version = get_version_from_repo(git=Git(True))
release_or_pr = get_release_or_pr(pr_info, build_config, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))
@ -284,9 +285,12 @@ def main():
if build_config["package_type"] == "performance" and pr_info.number != 0:
# because perf tests store some information about git commits
subprocess.check_call(
f"cd {REPO_COPY} && git fetch origin master:master", shell=True
cmd = (
f"cd {REPO_COPY} && git fetch --depth=60 --no-recurse-submodules "
"--no-tags origin master:master"
)
logging.info("Fetch master branch with a command: %s", cmd)
subprocess.check_call(cmd, shell=True)
packager_cmd = get_packager_cmd(
build_config,

71
tests/ci/git_helper.py
View File

@ -13,6 +13,9 @@ TAG_REGEXP = (
)
SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z"
CWD = p.dirname(p.realpath(__file__))
TWEAK = 1
# Py 3.8 removeprefix and removesuffix
def removeprefix(string: str, prefix: str):
@ -46,8 +49,8 @@ def release_branch(name: str):
class Runner:
"""lightweight check_output wrapper with stripping last NEW_LINE"""
def __init__(self, cwd: str = p.dirname(p.realpath(__file__))):
self.cwd = cwd
def __init__(self, cwd: str = CWD):
self._cwd = cwd
def run(self, cmd: str, cwd: Optional[str] = None) -> str:
if cwd is None:
@ -56,22 +59,47 @@ class Runner:
cmd, shell=True, cwd=cwd, encoding="utf-8"
).strip()
@property
def cwd(self) -> str:
return self._cwd
@cwd.setter
def cwd(self, value: str):
# Set _cwd only once, then set it to readonly
if self._cwd != CWD:
return
self._cwd = value
git_runner = Runner()
# Set cwd to abs path of git root
git_runner.cwd = p.relpath(
p.join(git_runner.cwd, git_runner.run("git rev-parse --show-cdup"))
)
def get_tags() -> List[str]:
if git_runner.run("git rev-parse --is-shallow-repository") == "true":
raise RuntimeError("attempt to run on a shallow repository")
return git_runner.run("git tag").split()
class Git:
"""A small wrapper around subprocess to invoke git commands"""
def __init__(self):
runner = Runner()
rel_root = runner.run("git rev-parse --show-cdup")
self.root = p.realpath(p.join(runner.cwd, rel_root))
self._tag_pattern = re.compile(TAG_REGEXP)
runner.cwd = self.root
self.run = runner.run
_tag_pattern = re.compile(TAG_REGEXP)
def __init__(self, ignore_no_tags: bool = False):
self.root = git_runner.cwd
self._ignore_no_tags = ignore_no_tags
self.run = git_runner.run
self.latest_tag = ""
self.new_tag = ""
self.new_branch = ""
self.branch = ""
self.sha = ""
self.sha_short = ""
self.description = ""
self.description = "shallow-checkout"
self.commits_since_tag = 0
self.update()
@ -82,6 +110,19 @@ class Git:
self.sha_short = self.sha[:11]
# The following command shows the most recent tag in a graph
# Format should match TAG_REGEXP
if (
self._ignore_no_tags
and self.run("git rev-parse --is-shallow-repository") == "true"
):
try:
self._update_tags()
except subprocess.CalledProcessError:
pass
return
self._update_tags()
def _update_tags(self):
self.latest_tag = self.run("git describe --tags --abbrev=0")
# Format should be: {latest_tag}-{commits_since_tag}-g{sha_short}
self.description = self.run("git describe --tags --long")
@ -89,10 +130,11 @@ class Git:
self.run(f"git rev-list {self.latest_tag}..HEAD --count")
)
def check_tag(self, value: str):
@staticmethod
def check_tag(value: str):
if value == "":
return
if not self._tag_pattern.match(value):
if not Git._tag_pattern.match(value):
raise ValueError(f"last tag {value} doesn't match the pattern")
@property
@ -118,10 +160,7 @@ class Git:
if not self.latest_tag.endswith("-testing"):
# When we are on the tag, we still need to have tweak=1 to not
# break cmake with versions like 12.13.14.0
return self.commits_since_tag or 1
return self.commits_since_tag or TWEAK
version = self.latest_tag.split("-", maxsplit=1)[0]
return int(version.split(".")[-1]) + self.commits_since_tag
def get_tags(self) -> List[str]:
return self.run("git tag").split()

15
tests/ci/git_test.py
View File

@ -4,7 +4,7 @@ from unittest.mock import patch
import os.path as p
import unittest
from git_helper import Git, Runner
from git_helper import Git, Runner, CWD
class TestRunner(unittest.TestCase):
@ -19,6 +19,18 @@ class TestRunner(unittest.TestCase):
output = runner.run("echo 1")
self.assertEqual(output, "1")
def test_one_time_writeable_cwd(self):
runner = Runner()
self.assertEqual(runner.cwd, CWD)
runner.cwd = "/bin"
self.assertEqual(runner.cwd, "/bin")
runner.cwd = "/"
self.assertEqual(runner.cwd, "/bin")
runner = Runner("/")
self.assertEqual(runner.cwd, "/")
runner.cwd = "/bin"
self.assertEqual(runner.cwd, "/")
class TestGit(unittest.TestCase):
def setUp(self):
@ -31,6 +43,7 @@ class TestGit(unittest.TestCase):
self.addCleanup(update_patcher.stop)
self.git = Git()
update_mock.assert_called_once()
self.git.run("test")
self.run_mock.assert_called_once()
self.git.new_branch = "NEW_BRANCH_NAME"
self.git.new_tag = "v21.12.333.22222-stable"

29
tests/ci/release.py
View File

@ -9,14 +9,20 @@ import logging
from git_helper import commit, release_branch
from version_helper import (
FILE_WITH_VERSION_PATH,
GENERATED_CONTRIBUTORS,
ClickHouseVersion,
Git,
VersionType,
get_abs_path,
get_version_from_repo,
update_cmake_version,
update_contributors,
)
git = Git()
class Repo:
VALID = ("ssh", "https", "origin")
@ -47,14 +53,16 @@ class Repo:
class Release:
BIG = ("major", "minor")
SMALL = ("patch",)
CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH)
CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS)
def __init__(self, repo: Repo, release_commit: str, release_type: str):
self.repo = repo
self._release_commit = ""
self.release_commit = release_commit
self.release_type = release_type
self._version = get_version_from_repo()
self._git = self._version._git
self._git = git
self._version = get_version_from_repo(git=self._git)
self._release_branch = ""
self._rollback_stack = [] # type: List[str]
@ -75,7 +83,7 @@ class Release:
def read_version(self):
self._git.update()
self.version = get_version_from_repo()
self.version = get_version_from_repo(git=self._git)
def check_prerequisites(self):
"""
@ -179,14 +187,15 @@ class Release:
self.version = self.version.update(self.release_type)
self.version.with_description(version_type)
update_cmake_version(self.version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH)
update_contributors(raise_error=True)
# Checkouting the commit of the branch and not the branch itself,
# then we are able to skip rollback
with self._checkout(f"{self.release_branch}@{{0}}", False):
current_commit = self.run("git rev-parse HEAD")
self.run(
f"git commit -m "
f"'Update version to {self.version.string}' '{cmake_path}'"
f"'Update version to {self.version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
)
with self._push(
"HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch
@ -243,9 +252,10 @@ class Release:
new_version = self.version.patch_update()
new_version.with_description("prestable")
update_cmake_version(new_version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH)
update_contributors(raise_error=True)
self.run(
f"git commit -m 'Update version to {new_version.string}' '{cmake_path}'"
f"git commit -m 'Update version to {new_version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
)
with self._push(self.release_branch):
with self._create_gh_label(
@ -271,9 +281,10 @@ class Release:
self.version = self.version.update(self.release_type)
self.version.with_description("testing")
update_cmake_version(self.version)
cmake_path = get_abs_path(FILE_WITH_VERSION_PATH)
update_contributors(raise_error=True)
self.run(
f"git commit -m 'Update version to {self.version.string}' '{cmake_path}'"
f"git commit -m 'Update version to {self.version.string}' "
f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"
)
with self._push(helper_branch):
body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")

44
tests/ci/version_helper.py
View File

@ -2,9 +2,9 @@
import logging
import os.path as p
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError
from typing import Dict, List, Tuple, Union
from typing import Dict, List, Optional, Tuple, Union
from git_helper import Git, removeprefix
from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix
FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt"
CHANGELOG_IN_PATH = "debian/changelog.in"
@ -34,8 +34,6 @@ SET(VERSION_STRING {string})
# end of autochange
"""
git = Git()
class ClickHouseVersion:
"""Immutable version class. On update returns a new instance"""
@ -46,7 +44,7 @@ class ClickHouseVersion:
minor: Union[int, str],
patch: Union[int, str],
revision: Union[int, str],
git: Git,
git: Optional[Git],
tweak: str = None,
):
self._major = int(major)
@ -54,9 +52,11 @@ class ClickHouseVersion:
self._patch = int(patch)
self._revision = int(revision)
self._git = git
self._tweak = None
self._tweak = TWEAK
if tweak is not None:
self._tweak = int(tweak)
elif self._git is not None:
self._tweak = self._git.tweak
self._describe = ""
def update(self, part: str) -> "ClickHouseVersion":
@ -91,7 +91,7 @@ class ClickHouseVersion:
@property
def tweak(self) -> int:
return self._tweak or self._git.tweak
return self._tweak
@property
def revision(self) -> int:
@ -99,7 +99,9 @@ class ClickHouseVersion:
@property
def githash(self) -> str:
return self._git.sha
if self._git is not None:
return self._git.sha
return "0000000000000000000000000000000000000000"
@property
def describe(self):
@ -171,7 +173,7 @@ def validate_version(version: str):
def get_abs_path(path: str) -> str:
return p.abspath(p.join(git.root, path))
return p.abspath(p.join(git_runner.cwd, path))
def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
@ -197,6 +199,7 @@ def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS:
def get_version_from_repo(
versions_path: str = FILE_WITH_VERSION_PATH,
git: Optional[Git] = None,
) -> ClickHouseVersion:
versions = read_versions(versions_path)
return ClickHouseVersion(
@ -208,14 +211,16 @@ def get_version_from_repo(
)
def get_version_from_string(version: str) -> ClickHouseVersion:
def get_version_from_string(
version: str, git: Optional[Git] = None
) -> ClickHouseVersion:
validate_version(version)
parts = version.split(".")
return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3])
def get_version_from_tag(tag: str) -> ClickHouseVersion:
git.check_tag(tag)
Git.check_tag(tag)
tag = tag[1:].split("-")[0]
return get_version_from_string(tag)
@ -236,7 +241,7 @@ def version_arg(version: str) -> ClickHouseVersion:
def get_tagged_versions() -> List[ClickHouseVersion]:
versions = []
for tag in git.get_tags():
for tag in get_tags():
try:
version = get_version_from_tag(tag)
versions.append(version)
@ -255,21 +260,26 @@ def update_cmake_version(
def update_contributors(
relative_contributors_path: str = GENERATED_CONTRIBUTORS, force: bool = False
relative_contributors_path: str = GENERATED_CONTRIBUTORS,
force: bool = False,
raise_error: bool = False,
):
# Check if we have shallow checkout by comparing number of lines
# '--is-shallow-repository' is in git since 2.15, 2017-10-30
if git.run("git rev-parse --is-shallow-repository") == "true" and not force:
if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force:
logging.warning("The repository is shallow, refusing to update contributors")
if raise_error:
raise RuntimeError("update_contributors executed on a shallow repository")
return
contributors = git.run("git shortlog HEAD --summary")
# format: " 1016 Alexey Arno"
shortlog = git_runner.run("git shortlog HEAD --summary")
contributors = sorted(
[c.split(maxsplit=1)[-1].replace('"', r"\"") for c in contributors.split("\n")],
[c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")],
)
contributors = [f' "{c}",' for c in contributors]
executer = p.relpath(p.realpath(__file__), git.root)
executer = p.relpath(p.realpath(__file__), git_runner.cwd)
content = CONTRIBUTORS_TEMPLATE.format(
executer=executer, contributors="\n".join(contributors)
)