thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34792 from ClickHouse/scripts-no-options

Browse Source 
Improve `--no-*` options code in ci scripts
This commit is contained in:
alesapin 2022-02-23 10:38:14 +03:00 committed by GitHub
commit c4077c8683
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 81 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
tests/ci/docker_images_check.py
View File

@ -349,14 +349,20 @@ def parse_args() -> argparse.Namespace:
help="list of image paths to build instead of using pr_info + diff URL, "
"e.g. 'docker/packager/binary'",
)
parser.add_argument("--reports", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-reports",
action="store_true",
action="store_false",
dest="reports",
default=argparse.SUPPRESS,
help="don't push reports to S3 and github",
)
parser.add_argument("--push", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-push-images",
action="store_true",
action="store_false",
dest="push",
default=argparse.SUPPRESS,
help="don't push images to docker hub",
)
@ -375,8 +381,7 @@ def main():
else:
changed_json = os.path.join(TEMP_PATH, "changed_images.json")
push = not args.no_push_images
if push:
if args.push:
subprocess.check_output( # pylint: disable=unexpected-keyword-arg
"docker login --username 'robotclickhouse' --password-stdin",
input=get_parameter_from_ssm("dockerhub_robot_password"),
@ -408,7 +413,7 @@ def main():
images_processing_result = []
for image in changed_images:
images_processing_result += process_image_with_parents(
image, image_versions, push
image, image_versions, args.push
)
result_images[image.repo] = result_version
@ -437,7 +442,7 @@ def main():
print(f"::notice ::Report url: {url}")
print(f'::set-output name=url_output::"{url}"')
if args.no_reports:
if not args.reports:
return
gh = Github(get_best_robot_token())

63
tests/ci/docker_manifests_merge.py
View File

@ -44,14 +44,20 @@ def parse_args() -> argparse.Namespace:
default=RUNNER_TEMP,
help="path to changed_images_*.json files",
)
parser.add_argument("--reports", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-reports",
action="store_true",
action="store_false",
dest="reports",
default=argparse.SUPPRESS,
help="don't push reports to S3 and github",
)
parser.add_argument("--push", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-push-images",
action="store_true",
action="store_false",
dest="push",
default=argparse.SUPPRESS,
help="don't push images to docker hub",
)
@ -63,7 +69,7 @@ def parse_args() -> argparse.Namespace:
def load_images(path: str, suffix: str) -> Images:
with open(os.path.join(path, CHANGED_IMAGES.format(suffix)), "r") as images:
with open(os.path.join(path, CHANGED_IMAGES.format(suffix)), "rb") as images:
return json.load(images)
@ -125,39 +131,37 @@ def merge_images(to_merge: Dict[str, Images]) -> Dict[str, List[List[str]]]:
def create_manifest(image: str, tags: List[str], push: bool) -> Tuple[str, str]:
tag = tags[0]
manifest = f"{image}:{tag}"
cmd = "docker manifest create --amend {}".format(
" ".join((f"{image}:{t}" for t in tags))
)
cmd = "docker manifest create --amend " + " ".join((f"{image}:{t}" for t in tags))
logging.info("running: %s", cmd)
popen = subprocess.Popen(
with subprocess.Popen(
cmd,
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
universal_newlines=True,
)
retcode = popen.wait()
if retcode != 0:
output = popen.stdout.read() # type: ignore
logging.error("failed to create manifest for %s:\n %s\n", manifest, output)
return manifest, "FAIL"
if not push:
return manifest, "OK"
) as popen:
retcode = popen.wait()
if retcode != 0:
output = popen.stdout.read() # type: ignore
logging.error("failed to create manifest for %s:\n %s\n", manifest, output)
return manifest, "FAIL"
if not push:
return manifest, "OK"
cmd = f"docker manifest push {manifest}"
logging.info("running: %s", cmd)
popen = subprocess.Popen(
with subprocess.Popen(
cmd,
shell=True,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
universal_newlines=True,
)
retcode = popen.wait()
if retcode != 0:
output = popen.stdout.read() # type: ignore
logging.error("failed to push %s:\n %s\n", manifest, output)
return manifest, "FAIL"
) as popen:
retcode = popen.wait()
if retcode != 0:
output = popen.stdout.read() # type: ignore
logging.error("failed to push %s:\n %s\n", manifest, output)
return manifest, "FAIL"
return manifest, "OK"
@ -167,8 +171,7 @@ def main():
stopwatch = Stopwatch()
args = parse_args()
push = not args.no_push_images
if push:
if args.push:
subprocess.check_output( # pylint: disable=unexpected-keyword-arg
"docker login --username 'robotclickhouse' --password-stdin",
input=get_parameter_from_ssm("dockerhub_robot_password"),
@ -189,12 +192,14 @@ def main():
test_results = [] # type: List[Tuple[str, str]]
for image, versions in merged.items():
for tags in versions:
manifest, test_result = create_manifest(image, tags, push)
manifest, test_result = create_manifest(image, tags, args.push)
test_results.append((manifest, test_result))
if test_result != "OK":
status = "failure"
with open(os.path.join(args.path, "changed_images.json"), "w") as ci:
with open(
os.path.join(args.path, "changed_images.json"), "w", encoding="utf-8"
) as ci:
json.dump(changed_images, ci)
pr_info = PRInfo()
@ -202,10 +207,10 @@ def main():
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME)
print("::notice ::Report url: {}".format(url))
print('::set-output name=url_output::"{}"'.format(url))
print(f"::notice ::Report url: {url}")
print(f'::set-output name=url_output::"{url}"')
if args.no_reports:
if not args.reports:
return
if changed_images:

14
tests/ci/push_to_artifactory.py
View File

@ -253,15 +253,21 @@ def parse_args() -> argparse.Namespace:
default="https://clickhousedb.jfrog.io/artifactory",
help="SaaS Artifactory url",
)
parser.add_argument("--artifactory", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"-n",
"--no-artifactory",
action="store_true",
action="store_false",
dest="artifactory",
default=argparse.SUPPRESS,
help="do not push packages to artifactory",
)
parser.add_argument("--force-download", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-force-download",
action="store_true",
action="store_false",
dest="force_download",
default=argparse.SUPPRESS,
help="do not download packages again if they exist already",
)
@ -303,10 +309,10 @@ def main():
args.commit,
args.check_name,
args.release.version,
not args.no_force_download,
args.force_download,
)
art_client = None
if not args.no_artifactory:
if args.artifactory:
art_client = Artifactory(args.artifactory_url, args.release.type)
if args.deb:

42
tests/ci/release.py
View File

@ -78,21 +78,21 @@ class Release:
self._git.update()
self.version = get_version_from_repo()
def do(self, no_check_dirty: bool, no_check_branch: bool, no_prestable: bool):
def do(self, check_dirty: bool, check_branch: bool, with_prestable: bool):
if not no_check_dirty:
if check_dirty:
logging.info("Checking if repo is clean")
self.run("git diff HEAD --exit-code")
self.set_release_branch()
if not no_check_branch:
if check_branch:
self.check_branch()
with self._checkout(self.release_commit, True):
if self.release_type in self.BIG:
# Checkout to the commit, it will provide the correct current version
if no_prestable:
if with_prestable:
logging.info("Skipping prestable stage")
else:
with self.prestable():
@ -405,28 +405,38 @@ def parse_args() -> argparse.Namespace:
dest="release_type",
help="a release type, new branch is created only for 'major' and 'minor'",
)
parser.add_argument(
"--no-prestable",
action="store_true",
help=f"for release types in {Release.BIG} skip creating prestable release and "
"release branch",
)
parser.add_argument(
"--commit",
default=git.sha,
type=commit,
help="commit create a release, default to HEAD",
)
parser.add_argument("--with-prestable", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-prestable",
dest="with_prestable",
action="store_false",
default=argparse.SUPPRESS,
help=f"if set, for release types in {Release.BIG} skip creating prestable "
"release and release branch",
)
parser.add_argument("--check-dirty", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-check-dirty",
action="store_true",
help="(dangerous) skip check repository for uncommited changes",
dest="check_dirty",
action="store_false",
default=argparse.SUPPRESS,
help="(dangerous) if set, skip check repository for uncommited changes",
)
parser.add_argument("--check-branch", default=True, help=argparse.SUPPRESS)
parser.add_argument(
"--no-check-branch",
action="store_true",
help="(debug or development only) by default, 'major' and 'minor' types work "
"only for master, and 'patch' works only for a release branches, that name "
dest="check_branch",
action="store_false",
default=argparse.SUPPRESS,
help="(debug or development only) if set, skip the branch check for a run. "
"By default, 'major' and 'minor' types workonly for master, and 'patch' works "
"only for a release branches, that name "
"should be the same as '$MAJOR.$MINOR' version, e.g. 22.2",
)
@ -439,7 +449,7 @@ def main():
repo = Repo(args.repo, args.remote_protocol)
release = Release(repo, args.commit, args.release_type)
release.do(args.no_check_dirty, args.no_check_branch, args.no_prestable)
release.do(args.check_dirty, args.check_branch, args.with_prestable)
if __name__ == "__main__":