2018-07-09 19:59:07 +00:00
|
|
|
|
#!/usr/bin/env python
|
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import contextlib
|
2018-08-10 14:44:49 +00:00
|
|
|
|
import datetime
|
2018-07-09 19:59:07 +00:00
|
|
|
|
import logging
|
|
|
|
|
import os
|
|
|
|
|
import shutil
|
2018-10-12 15:43:16 +00:00
|
|
|
|
import subprocess
|
2018-07-09 19:59:07 +00:00
|
|
|
|
import sys
|
|
|
|
|
import tempfile
|
2018-10-16 10:47:17 +00:00
|
|
|
|
import time
|
|
|
|
|
|
|
|
|
|
import markdown.extensions
|
|
|
|
|
import markdown.util
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
|
|
|
|
from mkdocs import config
|
|
|
|
|
from mkdocs import exceptions
|
|
|
|
|
from mkdocs.commands import build as mkdocs_build
|
|
|
|
|
|
|
|
|
|
from concatenate import concatenate
|
2018-12-12 17:28:00 +00:00
|
|
|
|
import mdx_clickhouse
|
2018-12-18 11:32:08 +00:00
|
|
|
|
import test
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
|
|
|
|
@contextlib.contextmanager
|
|
|
|
|
def temp_dir():
|
|
|
|
|
path = tempfile.mkdtemp(dir=os.environ.get('TEMP'))
|
|
|
|
|
try:
|
|
|
|
|
yield path
|
|
|
|
|
finally:
|
|
|
|
|
shutil.rmtree(path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@contextlib.contextmanager
|
|
|
|
|
def autoremoved_file(path):
|
|
|
|
|
try:
|
|
|
|
|
with open(path, 'w') as handle:
|
|
|
|
|
yield handle
|
|
|
|
|
finally:
|
|
|
|
|
os.unlink(path)
|
|
|
|
|
|
2018-10-16 10:47:17 +00:00
|
|
|
|
class ClickHouseMarkdown(markdown.extensions.Extension):
|
|
|
|
|
class ClickHousePreprocessor(markdown.util.Processor):
|
|
|
|
|
def run(self, lines):
|
|
|
|
|
for line in lines:
|
|
|
|
|
if '<!--hide-->' not in line:
|
|
|
|
|
yield line
|
|
|
|
|
|
|
|
|
|
def extendMarkdown(self, md):
|
|
|
|
|
md.preprocessors.register(self.ClickHousePreprocessor(), 'clickhouse_preprocessor', 31)
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
2018-10-16 10:47:17 +00:00
|
|
|
|
markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown
|
2018-10-16 06:58:03 +00:00
|
|
|
|
|
2018-07-09 19:59:07 +00:00
|
|
|
|
def build_for_lang(lang, args):
|
|
|
|
|
logging.info('Building %s docs' % lang)
|
2018-12-12 17:28:00 +00:00
|
|
|
|
os.environ['SINGLE_PAGE'] = '0'
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
2018-07-18 09:22:13 +00:00
|
|
|
|
config_path = os.path.join(args.docs_dir, 'toc_%s.yml' % lang)
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
theme_cfg = {
|
2018-08-10 14:44:49 +00:00
|
|
|
|
'name': 'mkdocs',
|
|
|
|
|
'custom_dir': os.path.join(os.path.dirname(__file__), args.theme_dir),
|
2018-07-09 19:59:07 +00:00
|
|
|
|
'language': lang,
|
2018-09-06 10:22:06 +00:00
|
|
|
|
'direction': 'rtl' if lang == 'fa' else 'ltr',
|
2018-07-09 19:59:07 +00:00
|
|
|
|
'feature': {
|
|
|
|
|
'tabs': False
|
|
|
|
|
},
|
|
|
|
|
'palette': {
|
|
|
|
|
'primary': 'white',
|
|
|
|
|
'accent': 'white'
|
|
|
|
|
},
|
|
|
|
|
'font': False,
|
|
|
|
|
'logo': 'images/logo.svg',
|
|
|
|
|
'favicon': 'assets/images/favicon.ico',
|
|
|
|
|
'include_search_page': False,
|
|
|
|
|
'search_index_only': True,
|
|
|
|
|
'static_templates': ['404.html'],
|
|
|
|
|
'extra': {
|
|
|
|
|
'single_page': False,
|
2018-10-16 10:47:17 +00:00
|
|
|
|
'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching
|
2018-07-09 19:59:07 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-08-10 14:44:49 +00:00
|
|
|
|
|
2018-10-08 15:45:18 +00:00
|
|
|
|
site_names = {
|
|
|
|
|
'en': 'ClickHouse Documentation',
|
|
|
|
|
'ru': 'Документация ClickHouse',
|
|
|
|
|
'zh': 'ClickHouse文档',
|
|
|
|
|
'fa': 'مستندات ClickHouse'
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-09 19:59:07 +00:00
|
|
|
|
cfg = config.load_config(
|
|
|
|
|
config_file=config_path,
|
2018-10-08 15:45:18 +00:00
|
|
|
|
site_name=site_names.get(lang, site_names['en']),
|
2018-09-06 18:24:00 +00:00
|
|
|
|
site_url='https://clickhouse.yandex/docs/%s/' % lang,
|
2018-07-09 19:59:07 +00:00
|
|
|
|
docs_dir=os.path.join(args.docs_dir, lang),
|
|
|
|
|
site_dir=os.path.join(args.output_dir, lang),
|
|
|
|
|
strict=True,
|
|
|
|
|
theme=theme_cfg,
|
|
|
|
|
copyright='©2016–2018 Yandex LLC',
|
|
|
|
|
use_directory_urls=True,
|
|
|
|
|
repo_name='yandex/ClickHouse',
|
|
|
|
|
repo_url='https://github.com/yandex/ClickHouse/',
|
|
|
|
|
edit_uri='edit/master/docs/%s' % lang,
|
2018-10-16 10:47:17 +00:00
|
|
|
|
extra_css=['assets/stylesheets/custom.css'],
|
2018-07-20 09:34:42 +00:00
|
|
|
|
markdown_extensions=[
|
2018-10-16 10:47:17 +00:00
|
|
|
|
'clickhouse',
|
2018-07-20 09:34:42 +00:00
|
|
|
|
'admonition',
|
|
|
|
|
'attr_list',
|
2018-09-06 10:22:06 +00:00
|
|
|
|
'codehilite',
|
2018-12-12 17:28:00 +00:00
|
|
|
|
'extra',
|
|
|
|
|
{
|
|
|
|
|
'toc': {
|
|
|
|
|
'permalink': True,
|
|
|
|
|
'slugify': mdx_clickhouse.slugify
|
|
|
|
|
}
|
|
|
|
|
}
|
2018-08-10 14:44:49 +00:00
|
|
|
|
],
|
|
|
|
|
plugins=[{
|
|
|
|
|
'search': {
|
2018-09-06 18:24:00 +00:00
|
|
|
|
'lang': ['en', 'ru'] if lang == 'ru' else ['en']
|
2018-08-10 14:44:49 +00:00
|
|
|
|
}
|
|
|
|
|
}],
|
|
|
|
|
extra={
|
|
|
|
|
'search': {
|
2018-09-06 18:24:00 +00:00
|
|
|
|
'language': 'en,ru' if lang == 'ru' else 'en'
|
2018-08-10 14:44:49 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
2018-07-09 19:59:07 +00:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
mkdocs_build.build(cfg)
|
|
|
|
|
|
|
|
|
|
if not args.skip_single_page:
|
|
|
|
|
build_single_page_version(lang, args, cfg)
|
|
|
|
|
|
|
|
|
|
except exceptions.ConfigurationError as e:
|
|
|
|
|
raise SystemExit('\n' + str(e))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def build_single_page_version(lang, args, cfg):
|
|
|
|
|
logging.info('Building single page version for ' + lang)
|
2018-12-12 17:28:00 +00:00
|
|
|
|
os.environ['SINGLE_PAGE'] = '1'
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
|
|
|
|
with autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md:
|
|
|
|
|
concatenate(lang, args.docs_dir, single_md)
|
|
|
|
|
|
2018-08-10 14:44:49 +00:00
|
|
|
|
with temp_dir() as site_temp:
|
|
|
|
|
with temp_dir() as docs_temp:
|
|
|
|
|
docs_temp_lang = os.path.join(docs_temp, lang)
|
|
|
|
|
shutil.copytree(os.path.join(args.docs_dir, lang), docs_temp_lang)
|
|
|
|
|
for root, _, filenames in os.walk(docs_temp_lang):
|
|
|
|
|
for filename in filenames:
|
|
|
|
|
if filename != 'single.md' and filename.endswith('.md'):
|
|
|
|
|
os.unlink(os.path.join(root, filename))
|
|
|
|
|
|
|
|
|
|
cfg.load_dict({
|
|
|
|
|
'docs_dir': docs_temp_lang,
|
|
|
|
|
'site_dir': site_temp,
|
|
|
|
|
'extra': {
|
2018-10-12 13:22:08 +00:00
|
|
|
|
'single_page': True
|
2018-08-10 14:44:49 +00:00
|
|
|
|
},
|
|
|
|
|
'nav': [
|
|
|
|
|
{cfg.data.get('site_name'): 'single.md'}
|
|
|
|
|
]
|
|
|
|
|
})
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
2018-08-10 14:44:49 +00:00
|
|
|
|
mkdocs_build.build(cfg)
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
2018-08-10 14:44:49 +00:00
|
|
|
|
single_page_output_path = os.path.join(args.docs_dir, args.output_dir, lang, 'single')
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
2018-08-10 14:44:49 +00:00
|
|
|
|
if os.path.exists(single_page_output_path):
|
|
|
|
|
shutil.rmtree(single_page_output_path)
|
|
|
|
|
|
|
|
|
|
shutil.copytree(
|
|
|
|
|
os.path.join(site_temp, 'single'),
|
|
|
|
|
single_page_output_path
|
|
|
|
|
)
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
2018-10-12 15:43:16 +00:00
|
|
|
|
single_page_index_html = os.path.abspath(os.path.join(single_page_output_path, 'index.html'))
|
|
|
|
|
single_page_pdf = single_page_index_html.replace('index.html', 'clickhouse_%s.pdf' % lang)
|
|
|
|
|
create_pdf_command = ['wkhtmltopdf', '--print-media-type', single_page_index_html, single_page_pdf]
|
|
|
|
|
logging.debug(' '.join(create_pdf_command))
|
|
|
|
|
subprocess.check_call(' '.join(create_pdf_command), shell=True)
|
|
|
|
|
|
2018-12-18 11:32:08 +00:00
|
|
|
|
with temp_dir() as test_dir:
|
|
|
|
|
cfg.load_dict({
|
|
|
|
|
'docs_dir': docs_temp_lang,
|
|
|
|
|
'site_dir': test_dir,
|
|
|
|
|
'extra': {
|
|
|
|
|
'single_page': False
|
|
|
|
|
},
|
|
|
|
|
'nav': [
|
|
|
|
|
{cfg.data.get('site_name'): 'single.md'}
|
|
|
|
|
]
|
|
|
|
|
})
|
|
|
|
|
mkdocs_build.build(cfg)
|
|
|
|
|
test.test_single_page(os.path.join(test_dir, 'single', 'index.html'), lang)
|
|
|
|
|
if args.save_raw_single_page:
|
|
|
|
|
shutil.copytree(test_dir, args.save_raw_single_page)
|
|
|
|
|
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
|
|
|
|
def build_redirects(args):
|
|
|
|
|
lang_re_fragment = args.lang.replace(',', '|')
|
|
|
|
|
rewrites = []
|
|
|
|
|
with open(os.path.join(args.docs_dir, 'redirects.txt'), 'r') as f:
|
|
|
|
|
for line in f:
|
|
|
|
|
from_path, to_path = line.split(' ', 1)
|
|
|
|
|
from_path = '^/docs/(' + lang_re_fragment + ')/' + from_path.replace('.md', '/?') + '$'
|
|
|
|
|
to_path = '/docs/$1/' + to_path.replace('.md', '/')
|
|
|
|
|
rewrites.append(' '.join(['rewrite', from_path, to_path, 'permanent;']))
|
|
|
|
|
|
|
|
|
|
with open(os.path.join(args.output_dir, 'redirects.conf'), 'w') as f:
|
|
|
|
|
f.write('\n'.join(rewrites))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def build(args):
|
|
|
|
|
for lang in args.lang.split(','):
|
|
|
|
|
build_for_lang(lang, args)
|
|
|
|
|
|
|
|
|
|
build_redirects(args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
arg_parser = argparse.ArgumentParser()
|
2018-10-08 10:24:34 +00:00
|
|
|
|
arg_parser.add_argument('--lang', default='en,ru,zh,fa')
|
2018-08-10 14:44:49 +00:00
|
|
|
|
arg_parser.add_argument('--docs-dir', default='.')
|
2018-07-09 19:59:07 +00:00
|
|
|
|
arg_parser.add_argument('--theme-dir', default='mkdocs-material-theme')
|
2018-08-10 14:44:49 +00:00
|
|
|
|
arg_parser.add_argument('--output-dir', default='build')
|
2018-07-09 19:59:07 +00:00
|
|
|
|
arg_parser.add_argument('--skip-single-page', action='store_true')
|
2018-12-18 11:32:08 +00:00
|
|
|
|
arg_parser.add_argument('--save-raw-single-page', type=str)
|
2018-07-09 19:59:07 +00:00
|
|
|
|
arg_parser.add_argument('--verbose', action='store_true')
|
|
|
|
|
|
|
|
|
|
args = arg_parser.parse_args()
|
2018-08-10 14:44:49 +00:00
|
|
|
|
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
|
2018-07-09 19:59:07 +00:00
|
|
|
|
|
|
|
|
|
logging.basicConfig(
|
|
|
|
|
level=logging.DEBUG if args.verbose else logging.INFO,
|
|
|
|
|
stream=sys.stderr
|
|
|
|
|
)
|
|
|
|
|
|
2018-08-10 14:44:49 +00:00
|
|
|
|
logging.getLogger('MARKDOWN').setLevel(logging.INFO)
|
|
|
|
|
|
2018-07-09 19:59:07 +00:00
|
|
|
|
from build import build
|
|
|
|
|
build(args)
|