mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 15:42:02 +00:00
283 lines
10 KiB
Python
Executable File
283 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
|
||
import argparse
|
||
import datetime
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
import jinja2
|
||
import livereload
|
||
import markdown.util
|
||
|
||
import nav # monkey patches mkdocs
|
||
|
||
from mkdocs import config
|
||
from mkdocs import exceptions
|
||
import mkdocs.commands.build
|
||
|
||
import amp
|
||
import blog
|
||
import mdx_clickhouse
|
||
import redirects
|
||
import single_page
|
||
import test
|
||
import util
|
||
import website
|
||
|
||
from cmake_in_clickhouse_generator import generate_cmake_flags_files
|
||
|
||
class ClickHouseMarkdown(markdown.extensions.Extension):
|
||
class ClickHousePreprocessor(markdown.util.Processor):
|
||
def run(self, lines):
|
||
for line in lines:
|
||
if '<!--hide-->' not in line:
|
||
yield line
|
||
|
||
def extendMarkdown(self, md):
|
||
md.preprocessors.register(self.ClickHousePreprocessor(), 'clickhouse_preprocessor', 31)
|
||
|
||
|
||
markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown
|
||
|
||
|
||
def build_for_lang(lang, args):
|
||
logging.info(f'Building {lang} docs')
|
||
os.environ['SINGLE_PAGE'] = '0'
|
||
|
||
config_path = os.path.join(args.docs_dir, f'toc_{lang}.yml')
|
||
if args.is_stable_release and not os.path.exists(config_path):
|
||
logging.warning(f'Skipping {lang} docs, because {config} does not exist')
|
||
return
|
||
|
||
try:
|
||
theme_cfg = {
|
||
'name': None,
|
||
'custom_dir': os.path.join(os.path.dirname(__file__), '..', args.theme_dir),
|
||
'language': lang,
|
||
'direction': 'rtl' if lang == 'fa' else 'ltr',
|
||
'static_templates': ['404.html'],
|
||
'extra': {
|
||
'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching
|
||
}
|
||
}
|
||
|
||
# the following list of languages is sorted according to
|
||
# https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers
|
||
languages = {
|
||
'en': 'English',
|
||
'zh': '中文',
|
||
'es': 'Español',
|
||
'fr': 'Français',
|
||
'ru': 'Русский',
|
||
'ja': '日本語',
|
||
'tr': 'Türkçe',
|
||
'fa': 'فارسی'
|
||
}
|
||
|
||
site_names = {
|
||
'en': 'ClickHouse %s Documentation',
|
||
'zh': 'ClickHouse文档 %s',
|
||
'es': 'Documentación de ClickHouse %s',
|
||
'fr': 'Documentation ClickHouse %s',
|
||
'ru': 'Документация ClickHouse %s',
|
||
'ja': 'ClickHouseドキュメント %s',
|
||
'tr': 'ClickHouse Belgeleri %s',
|
||
'fa': 'مستندات %sClickHouse'
|
||
}
|
||
|
||
assert len(site_names) == len(languages)
|
||
|
||
if args.version_prefix:
|
||
site_dir = os.path.join(args.docs_output_dir, args.version_prefix, lang)
|
||
else:
|
||
site_dir = os.path.join(args.docs_output_dir, lang)
|
||
|
||
plugins = ['macros']
|
||
if args.htmlproofer:
|
||
plugins.append('htmlproofer')
|
||
|
||
website_url = 'https://clickhouse.tech'
|
||
site_name = site_names.get(lang, site_names['en']) % args.version_prefix
|
||
site_name = site_name.replace(' ', ' ')
|
||
raw_config = dict(
|
||
site_name=site_name,
|
||
site_url=f'{website_url}/docs/{lang}/',
|
||
docs_dir=os.path.join(args.docs_dir, lang),
|
||
site_dir=site_dir,
|
||
strict=not args.version_prefix,
|
||
theme=theme_cfg,
|
||
copyright='©2016–2020 Yandex LLC',
|
||
use_directory_urls=True,
|
||
repo_name='ClickHouse/ClickHouse',
|
||
repo_url='https://github.com/ClickHouse/ClickHouse/',
|
||
edit_uri=f'edit/master/docs/{lang}',
|
||
markdown_extensions=mdx_clickhouse.MARKDOWN_EXTENSIONS,
|
||
plugins=plugins,
|
||
extra=dict(
|
||
now=datetime.datetime.now().isoformat(),
|
||
stable_releases=args.stable_releases,
|
||
version_prefix=args.version_prefix,
|
||
single_page=False,
|
||
rev=args.rev,
|
||
rev_short=args.rev_short,
|
||
rev_url=args.rev_url,
|
||
website_url=website_url,
|
||
events=args.events,
|
||
languages=languages,
|
||
includes_dir=os.path.join(os.path.dirname(__file__), '..', '_includes'),
|
||
is_amp=False,
|
||
is_blog=False
|
||
)
|
||
)
|
||
|
||
if os.path.exists(config_path):
|
||
raw_config['config_file'] = config_path
|
||
else:
|
||
raw_config['nav'] = nav.build_docs_nav(lang, args)
|
||
|
||
cfg = config.load_config(**raw_config)
|
||
|
||
if not args.skip_multi_page:
|
||
try:
|
||
mkdocs.commands.build.build(cfg)
|
||
except jinja2.exceptions.TemplateError:
|
||
if not args.version_prefix:
|
||
raise
|
||
mdx_clickhouse.PatchedMacrosPlugin.disabled = True
|
||
mkdocs.commands.build.build(cfg)
|
||
|
||
if not (args.skip_amp or args.version_prefix):
|
||
amp.build_amp(lang, args, cfg)
|
||
|
||
if not args.skip_single_page:
|
||
single_page.build_single_page_version(lang, args, raw_config.get('nav'), cfg)
|
||
|
||
mdx_clickhouse.PatchedMacrosPlugin.disabled = False
|
||
|
||
logging.info(f'Finished building {lang} docs')
|
||
|
||
except exceptions.ConfigurationError as e:
|
||
raise SystemExit('\n' + str(e))
|
||
|
||
|
||
def build_docs(args):
|
||
tasks = []
|
||
for lang in args.lang.split(','):
|
||
if lang:
|
||
tasks.append((lang, args,))
|
||
util.run_function_in_parallel(build_for_lang, tasks, threads=False)
|
||
if not args.version_prefix:
|
||
redirects.build_docs_redirects(args)
|
||
|
||
|
||
def build(args):
|
||
if os.path.exists(args.output_dir):
|
||
shutil.rmtree(args.output_dir)
|
||
|
||
if not args.skip_website:
|
||
website.build_website(args)
|
||
|
||
if not args.skip_test_templates:
|
||
test.test_templates(args.website_dir)
|
||
|
||
if not args.skip_docs:
|
||
generate_cmake_flags_files(os.path.join(os.path.dirname(__file__), '..', '..'))
|
||
|
||
build_docs(args)
|
||
from github import build_releases
|
||
build_releases(args, build_docs)
|
||
|
||
if not args.skip_blog:
|
||
blog.build_blog(args)
|
||
|
||
if not args.skip_website:
|
||
website.process_benchmark_results(args)
|
||
website.minify_website(args)
|
||
redirects.build_static_redirects(args)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
|
||
website_dir = os.path.join('..', 'website')
|
||
|
||
arg_parser = argparse.ArgumentParser()
|
||
arg_parser.add_argument('--lang', default='en,es,fr,ru,zh,ja,tr,fa')
|
||
arg_parser.add_argument('--blog-lang', default='en,ru')
|
||
arg_parser.add_argument('--docs-dir', default='.')
|
||
arg_parser.add_argument('--theme-dir', default=website_dir)
|
||
arg_parser.add_argument('--website-dir', default=website_dir)
|
||
arg_parser.add_argument('--blog-dir', default=os.path.join(website_dir, 'blog'))
|
||
arg_parser.add_argument('--output-dir', default='build')
|
||
arg_parser.add_argument('--enable-stable-releases', action='store_true')
|
||
arg_parser.add_argument('--stable-releases-limit', type=int, default='3')
|
||
arg_parser.add_argument('--lts-releases-limit', type=int, default='2')
|
||
arg_parser.add_argument('--nav-limit', type=int, default='0')
|
||
arg_parser.add_argument('--version-prefix', type=str, default='')
|
||
arg_parser.add_argument('--is-stable-release', action='store_true')
|
||
arg_parser.add_argument('--skip-multi-page', action='store_true')
|
||
arg_parser.add_argument('--skip-single-page', action='store_true')
|
||
arg_parser.add_argument('--skip-amp', action='store_true')
|
||
arg_parser.add_argument('--skip-pdf', action='store_true')
|
||
arg_parser.add_argument('--skip-website', action='store_true')
|
||
arg_parser.add_argument('--skip-blog', action='store_true')
|
||
arg_parser.add_argument('--skip-git-log', action='store_true')
|
||
arg_parser.add_argument('--skip-docs', action='store_true')
|
||
arg_parser.add_argument('--skip-test-templates', action='store_true')
|
||
arg_parser.add_argument('--test-only', action='store_true')
|
||
arg_parser.add_argument('--minify', action='store_true')
|
||
arg_parser.add_argument('--htmlproofer', action='store_true')
|
||
arg_parser.add_argument('--no-docs-macros', action='store_true')
|
||
arg_parser.add_argument('--save-raw-single-page', type=str)
|
||
arg_parser.add_argument('--livereload', type=int, default='0')
|
||
arg_parser.add_argument('--verbose', action='store_true')
|
||
|
||
args = arg_parser.parse_args()
|
||
|
||
logging.basicConfig(
|
||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||
stream=sys.stderr
|
||
)
|
||
|
||
logging.getLogger('MARKDOWN').setLevel(logging.INFO)
|
||
|
||
args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), 'docs')
|
||
args.blog_output_dir = os.path.join(os.path.abspath(args.output_dir), 'blog')
|
||
|
||
from github import choose_latest_releases, get_events
|
||
args.stable_releases = choose_latest_releases(args) if args.enable_stable_releases else []
|
||
args.rev = subprocess.check_output('git rev-parse HEAD', shell=True).decode('utf-8').strip()
|
||
args.rev_short = subprocess.check_output('git rev-parse --short HEAD', shell=True).decode('utf-8').strip()
|
||
args.rev_url = f'https://github.com/ClickHouse/ClickHouse/commit/{args.rev}'
|
||
args.events = get_events(args)
|
||
|
||
if args.test_only:
|
||
args.skip_multi_page = True
|
||
args.skip_blog = True
|
||
args.skip_website = True
|
||
args.skip_pdf = True
|
||
args.skip_amp = True
|
||
|
||
if args.skip_git_log or args.skip_amp:
|
||
mdx_clickhouse.PatchedMacrosPlugin.skip_git_log = True
|
||
|
||
from build import build
|
||
build(args)
|
||
|
||
if args.livereload:
|
||
new_args = [arg for arg in sys.argv if not arg.startswith('--livereload')]
|
||
new_args = sys.executable + ' ' + ' '.join(new_args)
|
||
|
||
server = livereload.Server()
|
||
server.watch(args.docs_dir + '**/*', livereload.shell(new_args, cwd='tools', shell=True))
|
||
server.watch(args.website_dir + '**/*', livereload.shell(new_args, cwd='tools', shell=True))
|
||
server.serve(
|
||
root=args.output_dir,
|
||
host='0.0.0.0',
|
||
port=args.livereload
|
||
)
|
||
sys.exit(0)
|