mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-19 22:22:00 +00:00
268 lines
8.4 KiB
Python
Executable File
268 lines
8.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
|
||
import argparse
|
||
import datetime
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
import jinja2
|
||
import livereload
|
||
import markdown.util
|
||
|
||
import nav # monkey patches mkdocs
|
||
|
||
from mkdocs import config
|
||
from mkdocs import exceptions
|
||
import mkdocs.commands.build
|
||
|
||
import amp
|
||
import blog
|
||
import mdx_clickhouse
|
||
import redirects
|
||
import single_page
|
||
import test
|
||
import util
|
||
import website
|
||
|
||
from cmake_in_clickhouse_generator import generate_cmake_flags_files
|
||
|
||
|
||
class ClickHouseMarkdown(markdown.extensions.Extension):
|
||
class ClickHousePreprocessor(markdown.util.Processor):
|
||
def run(self, lines):
|
||
for line in lines:
|
||
if "<!--hide-->" not in line:
|
||
yield line
|
||
|
||
def extendMarkdown(self, md):
|
||
md.preprocessors.register(
|
||
self.ClickHousePreprocessor(), "clickhouse_preprocessor", 31
|
||
)
|
||
|
||
|
||
markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown
|
||
|
||
|
||
def build_for_lang(lang, args):
|
||
logging.info(f"Building {lang} docs")
|
||
os.environ["SINGLE_PAGE"] = "0"
|
||
|
||
try:
|
||
theme_cfg = {
|
||
"name": None,
|
||
"custom_dir": os.path.join(os.path.dirname(__file__), "..", args.theme_dir),
|
||
"language": lang,
|
||
"direction": "rtl" if lang == "fa" else "ltr",
|
||
"static_templates": ["404.html"],
|
||
"extra": {
|
||
"now": int(
|
||
time.mktime(datetime.datetime.now().timetuple())
|
||
) # TODO better way to avoid caching
|
||
},
|
||
}
|
||
|
||
# the following list of languages is sorted according to
|
||
# https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers
|
||
languages = {"en": "English", "zh": "中文", "ru": "Русский", "ja": "日本語"}
|
||
|
||
site_names = {
|
||
"en": "ClickHouse %s Documentation",
|
||
"zh": "ClickHouse文档 %s",
|
||
"ru": "Документация ClickHouse %s",
|
||
"ja": "ClickHouseドキュメント %s",
|
||
}
|
||
|
||
assert len(site_names) == len(languages)
|
||
|
||
site_dir = os.path.join(args.docs_output_dir, lang)
|
||
|
||
plugins = ["macros"]
|
||
if args.htmlproofer:
|
||
plugins.append("htmlproofer")
|
||
|
||
website_url = "https://clickhouse.com"
|
||
site_name = site_names.get(lang, site_names["en"]) % ""
|
||
site_name = site_name.replace(" ", " ")
|
||
|
||
raw_config = dict(
|
||
site_name=site_name,
|
||
site_url=f"{website_url}/docs/{lang}/",
|
||
docs_dir=os.path.join(args.docs_dir, lang),
|
||
site_dir=site_dir,
|
||
strict=True,
|
||
theme=theme_cfg,
|
||
copyright="©2016–2022 ClickHouse, Inc.",
|
||
use_directory_urls=True,
|
||
repo_name="ClickHouse/ClickHouse",
|
||
repo_url="https://github.com/ClickHouse/ClickHouse/",
|
||
edit_uri=f"edit/master/docs/{lang}",
|
||
markdown_extensions=mdx_clickhouse.MARKDOWN_EXTENSIONS,
|
||
plugins=plugins,
|
||
extra=dict(
|
||
now=datetime.datetime.now().isoformat(),
|
||
single_page=False,
|
||
rev=args.rev,
|
||
rev_short=args.rev_short,
|
||
rev_url=args.rev_url,
|
||
website_url=website_url,
|
||
events=args.events,
|
||
languages=languages,
|
||
includes_dir=os.path.join(os.path.dirname(__file__), "..", "_includes"),
|
||
is_amp=False,
|
||
is_blog=False,
|
||
),
|
||
)
|
||
|
||
# Clean to be safe if last build finished abnormally
|
||
single_page.remove_temporary_files(lang, args)
|
||
|
||
raw_config["nav"] = nav.build_docs_nav(lang, args)
|
||
|
||
cfg = config.load_config(**raw_config)
|
||
|
||
if not args.skip_multi_page:
|
||
mkdocs.commands.build.build(cfg)
|
||
|
||
if not args.skip_amp:
|
||
amp.build_amp(lang, args, cfg)
|
||
|
||
if not args.skip_single_page:
|
||
single_page.build_single_page_version(
|
||
lang, args, raw_config.get("nav"), cfg
|
||
)
|
||
|
||
mdx_clickhouse.PatchedMacrosPlugin.disabled = False
|
||
|
||
logging.info(f"Finished building {lang} docs")
|
||
|
||
except exceptions.ConfigurationError as e:
|
||
raise SystemExit("\n" + str(e))
|
||
|
||
|
||
def build_docs(args):
|
||
tasks = []
|
||
for lang in args.lang.split(","):
|
||
if lang:
|
||
tasks.append(
|
||
(
|
||
lang,
|
||
args,
|
||
)
|
||
)
|
||
util.run_function_in_parallel(build_for_lang, tasks, threads=False)
|
||
redirects.build_docs_redirects(args)
|
||
|
||
|
||
def build(args):
|
||
if os.path.exists(args.output_dir):
|
||
shutil.rmtree(args.output_dir)
|
||
|
||
if not args.skip_website:
|
||
website.build_website(args)
|
||
|
||
if not args.skip_docs:
|
||
generate_cmake_flags_files()
|
||
|
||
build_docs(args)
|
||
|
||
if not args.skip_blog:
|
||
blog.build_blog(args)
|
||
|
||
if not args.skip_website:
|
||
website.process_benchmark_results(args)
|
||
website.minify_website(args)
|
||
redirects.build_static_redirects(args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
os.chdir(os.path.join(os.path.dirname(__file__), ".."))
|
||
|
||
# A root path to ClickHouse source code.
|
||
src_dir = ".."
|
||
|
||
website_dir = os.path.join(src_dir, "website")
|
||
|
||
arg_parser = argparse.ArgumentParser()
|
||
arg_parser.add_argument("--lang", default="en,ru,zh,ja")
|
||
arg_parser.add_argument("--blog-lang", default="en")
|
||
arg_parser.add_argument("--docs-dir", default=".")
|
||
arg_parser.add_argument("--theme-dir", default=website_dir)
|
||
arg_parser.add_argument("--website-dir", default=website_dir)
|
||
arg_parser.add_argument("--src-dir", default=src_dir)
|
||
arg_parser.add_argument("--blog-dir", default=os.path.join(website_dir, "blog"))
|
||
arg_parser.add_argument("--output-dir", default="build")
|
||
arg_parser.add_argument("--nav-limit", type=int, default="0")
|
||
arg_parser.add_argument("--skip-multi-page", action="store_true")
|
||
arg_parser.add_argument("--skip-single-page", action="store_true")
|
||
arg_parser.add_argument("--skip-amp", action="store_true")
|
||
arg_parser.add_argument("--skip-website", action="store_true")
|
||
arg_parser.add_argument("--skip-blog", action="store_true")
|
||
arg_parser.add_argument("--skip-git-log", action="store_true")
|
||
arg_parser.add_argument("--skip-docs", action="store_true")
|
||
arg_parser.add_argument("--test-only", action="store_true")
|
||
arg_parser.add_argument("--minify", action="store_true")
|
||
arg_parser.add_argument("--htmlproofer", action="store_true")
|
||
arg_parser.add_argument("--no-docs-macros", action="store_true")
|
||
arg_parser.add_argument("--save-raw-single-page", type=str)
|
||
arg_parser.add_argument("--livereload", type=int, default="0")
|
||
arg_parser.add_argument("--verbose", action="store_true")
|
||
|
||
args = arg_parser.parse_args()
|
||
args.minify = False # TODO remove
|
||
|
||
logging.basicConfig(
|
||
level=logging.DEBUG if args.verbose else logging.INFO, stream=sys.stderr
|
||
)
|
||
|
||
logging.getLogger("MARKDOWN").setLevel(logging.INFO)
|
||
|
||
args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), "docs")
|
||
args.blog_output_dir = os.path.join(os.path.abspath(args.output_dir), "blog")
|
||
|
||
from github import get_events
|
||
|
||
args.rev = (
|
||
subprocess.check_output("git rev-parse HEAD", shell=True)
|
||
.decode("utf-8")
|
||
.strip()
|
||
)
|
||
args.rev_short = (
|
||
subprocess.check_output("git rev-parse --short HEAD", shell=True)
|
||
.decode("utf-8")
|
||
.strip()
|
||
)
|
||
args.rev_url = f"https://github.com/ClickHouse/ClickHouse/commit/{args.rev}"
|
||
args.events = get_events(args)
|
||
|
||
if args.test_only:
|
||
args.skip_multi_page = True
|
||
args.skip_blog = True
|
||
args.skip_website = True
|
||
args.skip_amp = True
|
||
|
||
if args.skip_git_log or args.skip_amp:
|
||
mdx_clickhouse.PatchedMacrosPlugin.skip_git_log = True
|
||
|
||
from build import build
|
||
|
||
build(args)
|
||
|
||
if args.livereload:
|
||
new_args = [arg for arg in sys.argv if not arg.startswith("--livereload")]
|
||
new_args = sys.executable + " " + " ".join(new_args)
|
||
|
||
server = livereload.Server()
|
||
server.watch(
|
||
args.docs_dir + "**/*", livereload.shell(new_args, cwd="tools", shell=True)
|
||
)
|
||
server.watch(
|
||
args.website_dir + "**/*",
|
||
livereload.shell(new_args, cwd="tools", shell=True),
|
||
)
|
||
server.serve(root=args.output_dir, host="0.0.0.0", port=args.livereload)
|
||
sys.exit(0)
|