ClickHouse/docs/tools/build.py
2020-05-12 17:09:36 +03:00

296 lines
9.9 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
import argparse
import datetime
import logging
import os
import shutil
import subprocess
import sys
import time
import jinja2
import livereload
import markdown.util
import nav # monkey patches mkdocs
from mkdocs import config
from mkdocs import exceptions
import mkdocs.commands.build
import amp
import mdx_clickhouse
import redirects
import single_page
import test
import util
import website
class ClickHouseMarkdown(markdown.extensions.Extension):
class ClickHousePreprocessor(markdown.util.Processor):
def run(self, lines):
for line in lines:
if '<!--hide-->' not in line:
yield line
def extendMarkdown(self, md):
md.preprocessors.register(self.ClickHousePreprocessor(), 'clickhouse_preprocessor', 31)
markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown
def build_for_lang(lang, args):
logging.info(f'Building {lang} docs')
os.environ['SINGLE_PAGE'] = '0'
config_path = os.path.join(args.docs_dir, f'toc_{lang}.yml')
if args.is_stable_release and not os.path.exists(config_path):
logging.warning(f'Skipping {lang} docs, because {config} does not exist')
return
try:
theme_cfg = {
'name': None,
'custom_dir': os.path.join(os.path.dirname(__file__), '..', args.theme_dir),
'language': lang,
'direction': 'rtl' if lang == 'fa' else 'ltr',
# TODO: cleanup
'feature': {
'tabs': False
},
'palette': {
'primary': 'white',
'accent': 'white'
},
'font': False,
'logo': 'images/logo.svg',
'favicon': 'assets/images/favicon.ico',
'static_templates': ['404.html'],
'extra': {
'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching
}
}
# the following list of languages is sorted according to
# https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers
languages = {
'en': 'English',
'zh': '中文',
'es': 'Español',
'fr': 'Français',
'ru': 'Русский',
'ja': '日本語',
'tr': 'Türkçe',
'fa': 'فارسی'
}
site_names = {
'en': 'ClickHouse %s Documentation',
'zh': 'ClickHouse文档 %s',
'es': 'Documentación de ClickHouse %s',
'fr': 'Documentation ClickHouse %s',
'ru': 'Документация ClickHouse %s',
'ja': 'ClickHouseドキュメント %s',
'tr': 'ClickHouse Belgeleri %s',
'fa': 'مستندات %sClickHouse'
}
assert len(site_names) == len(languages)
if args.version_prefix:
site_dir = os.path.join(args.docs_output_dir, args.version_prefix, lang)
else:
site_dir = os.path.join(args.docs_output_dir, lang)
markdown_extensions = [
'mdx_clickhouse',
'admonition',
'attr_list',
'codehilite',
'nl2br',
'sane_lists',
'pymdownx.details',
'pymdownx.magiclink',
'pymdownx.superfences',
'extra',
{
'toc': {
'permalink': True,
'slugify': mdx_clickhouse.slugify
}
}
]
plugins = ['macros']
if args.htmlproofer:
plugins.append('htmlproofer')
website_url = 'https://clickhouse.tech'
site_name = site_names.get(lang, site_names['en']) % args.version_prefix
site_name = site_name.replace(' ', ' ')
raw_config = dict(
site_name=site_name,
site_url=f'{website_url}/docs/{lang}/',
docs_dir=os.path.join(args.docs_dir, lang),
site_dir=site_dir,
strict=not args.version_prefix,
theme=theme_cfg,
copyright='©20162020 Yandex LLC',
use_directory_urls=True,
repo_name='ClickHouse/ClickHouse',
repo_url='https://github.com/ClickHouse/ClickHouse/',
edit_uri=f'edit/master/docs/{lang}',
markdown_extensions=markdown_extensions,
plugins=plugins,
extra=dict(
now=datetime.datetime.now().isoformat(),
stable_releases=args.stable_releases,
version_prefix=args.version_prefix,
single_page=False,
rev=args.rev,
rev_short=args.rev_short,
rev_url=args.rev_url,
website_url=website_url,
events=args.events,
languages=languages,
includes_dir=os.path.join(os.path.dirname(__file__), '..', '_includes'),
is_amp=False
)
)
if os.path.exists(config_path):
raw_config['config_file'] = config_path
else:
raw_config['nav'] = nav.build_nav(lang, args)
cfg = config.load_config(**raw_config)
if not args.skip_multi_page:
try:
mkdocs.commands.build.build(cfg)
except jinja2.exceptions.TemplateError:
if not args.version_prefix:
raise
mdx_clickhouse.PatchedMacrosPlugin.disabled = True
mkdocs.commands.build.build(cfg)
if not (args.skip_amp or args.version_prefix):
amp.build_amp(lang, args, cfg)
if not args.skip_single_page:
single_page.build_single_page_version(lang, args, raw_config.get('nav'), cfg)
mdx_clickhouse.PatchedMacrosPlugin.disabled = False
logging.info(f'Finished building {lang} docs')
except exceptions.ConfigurationError as e:
raise SystemExit('\n' + str(e))
def build_docs(args):
tasks = []
for lang in args.lang.split(','):
if lang:
tasks.append((lang, args,))
util.run_function_in_parallel(build_for_lang, tasks, threads=False)
redirects.build_redirects(args)
def build(args):
if os.path.exists(args.output_dir):
shutil.rmtree(args.output_dir)
if not args.skip_website:
website.build_website(args)
test.test_templates(args.website_dir)
build_docs(args)
from github import build_releases
build_releases(args, build_docs)
if not args.skip_website:
website.process_benchmark_results(args)
website.minify_website(args)
redirects.build_static_redirects(args)
if __name__ == '__main__':
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
website_dir = os.path.join('..', 'website')
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--lang', default='en,es,fr,ru,zh,ja,tr,fa')
arg_parser.add_argument('--docs-dir', default='.')
arg_parser.add_argument('--theme-dir', default=website_dir)
arg_parser.add_argument('--website-dir', default=website_dir)
arg_parser.add_argument('--output-dir', default='build')
arg_parser.add_argument('--enable-stable-releases', action='store_true')
arg_parser.add_argument('--stable-releases-limit', type=int, default='4')
arg_parser.add_argument('--lts-releases-limit', type=int, default='2')
arg_parser.add_argument('--nav-limit', type=int, default='0')
arg_parser.add_argument('--version-prefix', type=str, default='')
arg_parser.add_argument('--is-stable-release', action='store_true')
arg_parser.add_argument('--skip-multi-page', action='store_true')
arg_parser.add_argument('--skip-single-page', action='store_true')
arg_parser.add_argument('--skip-amp', action='store_true')
arg_parser.add_argument('--skip-pdf', action='store_true')
arg_parser.add_argument('--skip-website', action='store_true')
arg_parser.add_argument('--skip-git-log', action='store_true')
arg_parser.add_argument('--test-only', action='store_true')
arg_parser.add_argument('--minify', action='store_true')
arg_parser.add_argument('--htmlproofer', action='store_true')
arg_parser.add_argument('--no-docs-macros', action='store_true')
arg_parser.add_argument('--save-raw-single-page', type=str)
arg_parser.add_argument('--livereload', type=int, default='0')
arg_parser.add_argument('--verbose', action='store_true')
args = arg_parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
stream=sys.stderr
)
logging.getLogger('MARKDOWN').setLevel(logging.INFO)
args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), 'docs')
from github import choose_latest_releases, get_events
args.stable_releases = choose_latest_releases(args) if args.enable_stable_releases else []
args.rev = subprocess.check_output('git rev-parse HEAD', shell=True).decode('utf-8').strip()
args.rev_short = subprocess.check_output('git rev-parse --short HEAD', shell=True).decode('utf-8').strip()
args.rev_url = f'https://github.com/ClickHouse/ClickHouse/commit/{args.rev}'
args.events = get_events(args)
if args.test_only:
args.skip_multi_page = True
args.skip_website = True
args.skip_pdf = True
args.skip_amp = True
if args.skip_git_log or args.skip_amp:
mdx_clickhouse.PatchedMacrosPlugin.skip_git_log = True
from build import build
build(args)
if args.livereload:
new_args = [arg for arg in sys.argv if not arg.startswith('--livereload')]
new_args = sys.executable + ' ' + ' '.join(new_args)
server = livereload.Server()
server.watch(args.docs_dir + '**/*', livereload.shell(new_args, cwd='tools', shell=True))
server.watch(args.website_dir + '**/*', livereload.shell(new_args, cwd='tools', shell=True))
server.serve(
root=args.output_dir,
host='0.0.0.0',
port=args.livereload
)
sys.exit(0)