thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 16:42:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #47945 from ClickHouse/artifacts-s3-prefix

Browse Source 
Artifacts s3 prefix
This commit is contained in:
robot-clickhouse 2023-03-26 00:15:50 +01:00 committed by GitHub
commit 6ac86ed3e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 24 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docker/test/performance-comparison/download.sh
View File

@ -3,7 +3,9 @@ set -ex
set -o pipefail
trap "exit" INT TERM
trap 'kill $(jobs -pr) ||:' EXIT
S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
BUILD_NAME=${BUILD_NAME:-package_release}
export S3_URL BUILD_NAME
mkdir db0 ||:
mkdir left ||:
@ -28,8 +30,9 @@ function download
# Historically there were various paths for the performance test package.
# Test all of them.
declare -a urls_to_try=(
"https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
"https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/$BUILD_NAME/performance.tgz"
"$S3_URL/PRs/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
"$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
"$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tgz"
)
for path in "${urls_to_try[@]}"

20
docker/test/performance-comparison/entrypoint.sh
View File

@ -6,11 +6,7 @@ export CHPC_CHECK_START_TIMESTAMP
S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
BUILD_NAME=${BUILD_NAME:-package_release}
COMMON_BUILD_PREFIX="/clickhouse_build_check"
if [[ $S3_URL == *"s3.amazonaws.com"* ]]; then
COMMON_BUILD_PREFIX=""
fi
export S3_URL BUILD_NAME
# Sometimes AWS responde with DNS error and it's impossible to retry it with
# current curl version options.
@ -66,8 +62,9 @@ function find_reference_sha
# test all of them.
unset found
declare -a urls_to_try=(
"https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
"https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/$BUILD_NAME/performance.tgz"
"$S3_URL/PRs/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
"$S3_URL/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
"$S3_URL/0/$REF_SHA/$BUILD_NAME/performance.tgz"
)
for path in "${urls_to_try[@]}"
do
@ -92,10 +89,15 @@ chmod 777 workspace output
cd workspace
# Download the package for the version we are going to test.
if curl_with_retry "$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/$BUILD_NAME/performance.tar.zst"
# A temporary solution for migrating into PRs directory
for prefix in "$S3_URL/PRs" "$S3_URL";
do
if curl_with_retry "$prefix/$PR_TO_TEST/$SHA_TO_TEST/$BUILD_NAME/performance.tar.zst"
then
right_path="$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/$BUILD_NAME/performance.tar.zst"
right_path="$prefix/$PR_TO_TEST/$SHA_TO_TEST/$BUILD_NAME/performance.tar.zst"
break
fi
done
mkdir right
wget -nv -nd -c "$right_path" -O- | tar -C right --no-same-owner --strip-components=1 --zstd --extract --verbose

8
tests/ci/build_check.py
View File

@ -196,19 +196,21 @@ def create_json_artifact(
def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str, str]:
"Return prefixes for S3 artifacts paths"
# FIXME performance
# performance builds are havily relies on a fixed path for artifacts, that's why
# we need to preserve 0 for anything but PR number
# It should be fixed in performance-comparison image eventually
performance_pr = "0"
# For performance tests we always set PRs prefix
performance_pr = "PRs/0"
if "release" in pr_info.labels or "release-lts" in pr_info.labels:
# for release pull requests we use branch names prefixes, not pr numbers
return pr_info.head_ref, performance_pr
elif pr_info.number == 0:
if pr_info.number == 0:
# for pushes to master - major version
return f"{version.major}.{version.minor}", performance_pr
# PR number for anything else
pr_number = str(pr_info.number)
pr_number = f"PRs/{pr_info.number}"
return pr_number, pr_number

2
tests/ci/build_report_check.py
View File

@ -247,7 +247,7 @@ def main():
str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path
)
url = s3_helper.upload_build_file_to_s3(
url = s3_helper.upload_test_report_to_s3(
report_path, s3_path_prefix + "/report.html"
)
logging.info("Report url %s", url)