thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-28 18:42:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #47945 from ClickHouse/artifacts-s3-prefix

Browse Source 
Artifacts s3 prefix
This commit is contained in:
robot-clickhouse 2023-03-26 00:15:50 +01:00 committed by GitHub
commit 6ac86ed3e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 24 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docker/test/performance-comparison/download.sh
View File

@ -3,7 +3,9 @@ set -ex
set -o pipefail set -o pipefail
trap "exit" INT TERM trap "exit" INT TERM
trap 'kill $(jobs -pr) ||:' EXIT trap 'kill $(jobs -pr) ||:' EXIT
S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
BUILD_NAME=${BUILD_NAME:-package_release} BUILD_NAME=${BUILD_NAME:-package_release}
export S3_URL BUILD_NAME
mkdir db0 ||: mkdir db0 ||:
mkdir left ||: mkdir left ||:
@ -28,8 +30,9 @@ function download
# Historically there were various paths for the performance test package. # Historically there were various paths for the performance test package.
# Test all of them. # Test all of them.
declare -a urls_to_try=( declare -a urls_to_try=(
"https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst" "$S3_URL/PRs/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
"https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/$BUILD_NAME/performance.tgz" "$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
"$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tgz"
) )
for path in "${urls_to_try[@]}" for path in "${urls_to_try[@]}"

20
docker/test/performance-comparison/entrypoint.sh
View File

@ -6,11 +6,7 @@ export CHPC_CHECK_START_TIMESTAMP
S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"} S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
BUILD_NAME=${BUILD_NAME:-package_release} BUILD_NAME=${BUILD_NAME:-package_release}
export S3_URL BUILD_NAME
COMMON_BUILD_PREFIX="/clickhouse_build_check"
if [[ $S3_URL == *"s3.amazonaws.com"* ]]; then
COMMON_BUILD_PREFIX=""
fi
# Sometimes AWS responde with DNS error and it's impossible to retry it with # Sometimes AWS responde with DNS error and it's impossible to retry it with
# current curl version options. # current curl version options.
@ -66,8 +62,9 @@ function find_reference_sha
# test all of them. # test all of them.
unset found unset found
declare -a urls_to_try=( declare -a urls_to_try=(
"https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/$BUILD_NAME/performance.tar.zst" "$S3_URL/PRs/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
"https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/$BUILD_NAME/performance.tgz" "$S3_URL/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
"$S3_URL/0/$REF_SHA/$BUILD_NAME/performance.tgz"
) )
for path in "${urls_to_try[@]}" for path in "${urls_to_try[@]}"
do do
@ -92,10 +89,15 @@ chmod 777 workspace output
cd workspace cd workspace
# Download the package for the version we are going to test. # Download the package for the version we are going to test.
if curl_with_retry "$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/$BUILD_NAME/performance.tar.zst" # A temporary solution for migrating into PRs directory
for prefix in "$S3_URL/PRs" "$S3_URL";
do
if curl_with_retry "$prefix/$PR_TO_TEST/$SHA_TO_TEST/$BUILD_NAME/performance.tar.zst"
then then
right_path="$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/$BUILD_NAME/performance.tar.zst" right_path="$prefix/$PR_TO_TEST/$SHA_TO_TEST/$BUILD_NAME/performance.tar.zst"
break
fi fi
done
mkdir right mkdir right
wget -nv -nd -c "$right_path" -O- | tar -C right --no-same-owner --strip-components=1 --zstd --extract --verbose wget -nv -nd -c "$right_path" -O- | tar -C right --no-same-owner --strip-components=1 --zstd --extract --verbose

8
tests/ci/build_check.py
View File

@ -196,19 +196,21 @@ def create_json_artifact(
def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str, str]: def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str, str]:
"Return prefixes for S3 artifacts paths"
# FIXME performance # FIXME performance
# performance builds are havily relies on a fixed path for artifacts, that's why # performance builds are havily relies on a fixed path for artifacts, that's why
# we need to preserve 0 for anything but PR number # we need to preserve 0 for anything but PR number
# It should be fixed in performance-comparison image eventually # It should be fixed in performance-comparison image eventually
performance_pr = "0" # For performance tests we always set PRs prefix
performance_pr = "PRs/0"
if "release" in pr_info.labels or "release-lts" in pr_info.labels: if "release" in pr_info.labels or "release-lts" in pr_info.labels:
# for release pull requests we use branch names prefixes, not pr numbers # for release pull requests we use branch names prefixes, not pr numbers
return pr_info.head_ref, performance_pr return pr_info.head_ref, performance_pr
elif pr_info.number == 0: if pr_info.number == 0:
# for pushes to master - major version # for pushes to master - major version
return f"{version.major}.{version.minor}", performance_pr return f"{version.major}.{version.minor}", performance_pr
# PR number for anything else # PR number for anything else
pr_number = str(pr_info.number) pr_number = f"PRs/{pr_info.number}"
return pr_number, pr_number return pr_number, pr_number

2
tests/ci/build_report_check.py
View File

@ -247,7 +247,7 @@ def main():
str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path
) )
url = s3_helper.upload_build_file_to_s3( url = s3_helper.upload_test_report_to_s3(
report_path, s3_path_prefix + "/report.html" report_path, s3_path_prefix + "/report.html"
) )
logging.info("Report url %s", url) logging.info("Report url %s", url)