From 0b17e6c85ae20ba179a2e479f4bc98c2d7857b55 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 16 Mar 2019 14:02:34 +0300 Subject: [PATCH] Add freebsd build to packager and vagrant file --- docker/packager/freebsd/Vagrantfile | 5 +++ docker/packager/packager | 62 +++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 docker/packager/freebsd/Vagrantfile diff --git a/docker/packager/freebsd/Vagrantfile b/docker/packager/freebsd/Vagrantfile new file mode 100644 index 00000000000..3667a9feca7 --- /dev/null +++ b/docker/packager/freebsd/Vagrantfile @@ -0,0 +1,5 @@ +Vagrant.configure("2") do |config| + config.vm.box = "robot-clickhouse/clickhouse-freebsd" + config.vm.synced_folder ".", "/vagrant", disabled: true +end + diff --git a/docker/packager/packager b/docker/packager/packager index feb5e3488b7..88fcc870cc9 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -6,11 +6,52 @@ import argparse import logging import sys +SCRIPT_PATH = os.path.realpath(__file__) + IMAGE_MAP = { "deb": "yandex/clickhouse-deb-builder", "binary": "yandex/clickhouse-binary-builder", + "freebsd": os.path.join(os.path.dirname(SCRIPT_PATH), "freebsd"), } +class Vagrant(object): + def __init__(self, path_to_vagrant_file): + self.prefix = "VAGRANT_CWD=" + path_to_vagrant_file + + def __enter__(self): + subprocess.check_call("{} vagrant up".format(self.prefix), shell=True) + self.ssh_path = "/tmp/vagrant-ssh" + subprocess.check_call("{} vagrant ssh-config > {}".format(self.prefix, self.ssh_path), shell=True) + return self + + def copy_to_image(self, local_path, remote_path): + cmd = "scp -F {ssh} -r {lpath} default:{rpath}".format(ssh=self.ssh_path, lpath=local_path, rpath=remote_path) + logging.info("Copying to image %s", cmd) + subprocess.check_call( + cmd, + shell=True + ) + + def copy_from_image(self, remote_path, local_path): + cmd = "scp -F {ssh} -r default:{rpath} {lpath}".format(ssh=self.ssh_path, rpath=remote_path, lpath=local_path) + logging.info("Copying from image %s", cmd) + subprocess.check_call( + cmd, + shell=True + ) + + def execute_cmd(self, cmd): + cmd = '{} vagrant ssh -c "{}"'.format(self.prefix, cmd) + logging.info("Executin cmd %s", cmd) + subprocess.check_call( + cmd, + shell=True + ) + + def __exit__(self, exc_type, exc_val, exc_tb): + logging.info("Destroying image") + subprocess.check_call("{} vagrant destroy --force".format(self.prefix), shell=True) + def check_image_exists_locally(image_name): try: @@ -30,7 +71,7 @@ def pull_image(image_name): def build_image(image_name, filepath): subprocess.check_call("docker build --network=host -t {} -f {} .".format(image_name, filepath), shell=True) -def run_image_with_env(image_name, output, env_variables, ch_root): +def run_docker_image_with_env(image_name, output, env_variables, ch_root): env_part = " -e ".join(env_variables) if env_part: env_part = " -e " + env_part @@ -52,6 +93,15 @@ def run_image_with_env(image_name, output, env_variables, ch_root): subprocess.check_call(cmd, shell=True) +def run_vagrant_box_with_env(image_path, output_dir, ch_root): + with Vagrant(image_path) as vagrant: + logging.info("Copying folder to vagrant machine") + vagrant.copy_to_image(ch_root, "~/ClickHouse") + logging.info("Running build") + vagrant.execute_cmd("cd ~/ClickHouse && cmake . && ninja") + logging.info("Copying binary back") + vagrant.copy_from_image("~/ClickHouse/dbms/programs/clickhouse", output_dir) + def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, distcc_hosts, unbundled, split_binary, version, author): result = [] if package_type == "deb": @@ -91,7 +141,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="ClickHouse building script via docker") + parser = argparse.ArgumentParser(description="ClickHouse building script via virtualization mechanisms") parser.add_argument("--package-type", choices=IMAGE_MAP.keys(), required=True) parser.add_argument("--clickhouse-repo-path", default="../../") parser.add_argument("--output-dir", required=True) @@ -118,12 +168,16 @@ if __name__ == "__main__": ch_root = args.clickhouse_repo_path dockerfile = os.path.join(ch_root, "docker/packager", args.package_type, "Dockerfile") - if not check_image_exists_locally(image_name) or args.force_build_image: + if args.package_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image: if not pull_image(image_name) or args.force_build_image: build_image(image_name, dockerfile) env_prepared = parse_env_variables( args.build_type, args.compiler, args.sanitizer, args.package_type, args.cache, args.distcc_hosts, args.unbundled, args.split_binary, args.version, args.author) - run_image_with_env(image_name, args.output_dir, env_prepared, ch_root) + if args.package_type != "freebsd": + run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root) + else: + logging.info("Running freebsd build, arguments will be ignored") + run_vagrant_box_with_env(image_name, args.output_dir, ch_root) logging.info("Output placed into {}".format(args.output_dir))