ClickHouse/docs/tools/website.py

import json
import logging
import os
import shutil
import bs4

import util


def handle_iframe(iframe, soup):
    allowed_domains = ["https://www.youtube.com/"]
    illegal_domain = True
    iframe_src = iframe.attrs["src"]
    for domain in allowed_domains:
        if iframe_src.startswith(domain):
            illegal_domain = False
            break
    if illegal_domain:
        raise RuntimeError(f"iframe from illegal domain: {iframe_src}")
    wrapper = soup.new_tag("div")
    wrapper.attrs["class"] = ["embed-responsive", "embed-responsive-16by9"]
    iframe.insert_before(wrapper)
    iframe.extract()
    wrapper.insert(0, iframe)
    if "width" in iframe.attrs:
        del iframe.attrs["width"]
    if "height" in iframe.attrs:
        del iframe.attrs["height"]
    iframe.attrs[
        "allow"
    ] = "accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture"
    iframe.attrs["class"] = "embed-responsive-item"
    iframe.attrs["frameborder"] = "0"
    iframe.attrs["allowfullscreen"] = "1"


def adjust_markdown_html(content):
    soup = bs4.BeautifulSoup(content, features="html.parser")

    for a in soup.find_all("a"):
        a_class = a.attrs.get("class")
        a_href = a.attrs.get("href")
        if a_class and "headerlink" in a_class:
            a.string = "\xa0"
        if a_href and a_href.startswith("http"):
            a.attrs["target"] = "_blank"

    for code in soup.find_all("code"):
        code_class = code.attrs.get("class")
        if code_class:
            code.attrs["class"] = code_class + ["syntax"]
        else:
            code.attrs["class"] = "syntax"

    for iframe in soup.find_all("iframe"):
        handle_iframe(iframe, soup)

    for img in soup.find_all("img"):
        if img.attrs.get("alt") == "iframe":
            img.name = "iframe"
            img.string = ""
            handle_iframe(img, soup)
            continue
        img_class = img.attrs.get("class")
        if img_class:
            img.attrs["class"] = img_class + ["img-fluid"]
        else:
            img.attrs["class"] = "img-fluid"

    for details in soup.find_all("details"):
        for summary in details.find_all("summary"):
            if summary.parent != details:
                summary.extract()
                details.insert(0, summary)

    for dd in soup.find_all("dd"):
        dd_class = dd.attrs.get("class")
        if dd_class:
            dd.attrs["class"] = dd_class + ["pl-3"]
        else:
            dd.attrs["class"] = "pl-3"

    for div in soup.find_all("div"):
        div_class = div.attrs.get("class")
        is_admonition = div_class and "admonition" in div.attrs.get("class")
        if is_admonition:
            for a in div.find_all("a"):
                a_class = a.attrs.get("class")
                if a_class:
                    a.attrs["class"] = a_class + ["alert-link"]
                else:
                    a.attrs["class"] = "alert-link"

        for p in div.find_all("p"):
            p_class = p.attrs.get("class")
            if is_admonition and p_class and ("admonition-title" in p_class):
                p.attrs["class"] = p_class + [
                    "alert-heading",
                    "display-4",
                    "text-reset",
                    "mb-2",
                ]

        if is_admonition:
            div.attrs["role"] = "alert"
            if ("info" in div_class) or ("note" in div_class):
                mode = "alert-primary"
            elif ("attention" in div_class) or ("warning" in div_class):
                mode = "alert-warning"
            elif "important" in div_class:
                mode = "alert-danger"
            elif "tip" in div_class:
                mode = "alert-info"
            else:
                mode = "alert-secondary"
            div.attrs["class"] = div_class + ["alert", "pb-0", "mb-4", mode]

    return str(soup)


def build_website(args):
    logging.info("Building website")
    env = util.init_jinja2_env(args)

    shutil.copytree(
        args.website_dir,
        args.output_dir,
        ignore=shutil.ignore_patterns(
            "*.md",
            "*.sh",
            "*.css",
            "*.json",
            "js/*.js",
            "build",
            "docs",
            "public",
            "node_modules",
            "src",
            "templates",
            "locale",
            ".gitkeep",
        ),
    )

    shutil.copytree(
        os.path.join(args.website_dir, "images"),
        os.path.join(args.output_dir, "docs", "images"),
    )

    # This file can be requested to check for available ClickHouse releases.
    shutil.copy2(
        os.path.join(args.src_dir, "utils", "list-versions", "version_date.tsv"),
        os.path.join(args.output_dir, "data", "version_date.tsv"),
    )

    # This file can be requested to install ClickHouse.
    shutil.copy2(
        os.path.join(args.src_dir, "docs", "_includes", "install", "universal.sh"),
        os.path.join(args.output_dir, "data", "install.sh"),
    )

    for root, _, filenames in os.walk(args.output_dir):
        for filename in filenames:
            if filename == "main.html":
                continue

            path = os.path.join(root, filename)
            if not filename.endswith(".html"):
                continue
            logging.info("Processing %s", path)
            with open(path, "rb") as f:
                content = f.read().decode("utf-8")

            template = env.from_string(content)
            content = template.render(args.__dict__)

            with open(path, "wb") as f:
                f.write(content.encode("utf-8"))


def process_benchmark_results(args):
    benchmark_root = os.path.join(args.website_dir, "benchmark")
    required_keys = {
        "dbms": ["result"],
        "hardware": ["result", "system", "system_full", "kind"],
        "versions": ["version", "system"],
    }
    for benchmark_kind in ["dbms", "hardware", "versions"]:
        results = []
        results_root = os.path.join(benchmark_root, benchmark_kind, "results")
        for result in sorted(os.listdir(results_root)):
            result_file = os.path.join(results_root, result)
            logging.debug(f"Reading benchmark result from {result_file}")
            with open(result_file, "r") as f:
                result = json.loads(f.read())
                for item in result:
                    for required_key in required_keys[benchmark_kind]:
                        assert (
                            required_key in item
                        ), f'No "{required_key}" in {result_file}'
                results += result
        results_js = os.path.join(
            args.output_dir, "benchmark", benchmark_kind, "results.js"
        )
        with open(results_js, "w") as f:
            data = json.dumps(results)
            f.write(f"var results = {data};")