From 87f6889ae66a25900fbb300e8a86cef8a4210b5b Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 8 Apr 2019 19:01:54 +0300 Subject: [PATCH] Support building docs for stable releases in addition to docs from master (#4940) --- docs/tools/build.py | 89 +++++++++++-------- docs/tools/concatenate.py | 71 +++++++-------- docs/tools/github.py | 44 +++++++++ .../assets/stylesheets/custom.css | 32 +++++++ docs/tools/mkdocs-material-theme/base.html | 11 +++ .../mkdocs-material-theme/partials/flags.html | 1 - .../partials/language/en.html | 2 + .../partials/language/fa.html | 2 + .../partials/language/ru.html | 2 + .../partials/language/zh.html | 2 + .../mkdocs-material-theme/partials/nav.html | 15 ++++ docs/tools/release.sh | 2 +- docs/tools/util.py | 22 +++++ website/index.html | 5 -- 14 files changed, 214 insertions(+), 86 deletions(-) create mode 100644 docs/tools/github.py create mode 100644 docs/tools/util.py diff --git a/docs/tools/build.py b/docs/tools/build.py index ff89b437ffc..72a16839bef 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -3,14 +3,12 @@ from __future__ import unicode_literals import argparse -import contextlib import datetime import logging import os import shutil import subprocess import sys -import tempfile import time import markdown.extensions @@ -21,26 +19,12 @@ from mkdocs import exceptions from mkdocs.commands import build as mkdocs_build from concatenate import concatenate + from website import build_website, minify_website import mdx_clickhouse import test +import util -@contextlib.contextmanager -def temp_dir(): - path = tempfile.mkdtemp(dir=os.environ.get('TEMP')) - try: - yield path - finally: - shutil.rmtree(path) - - -@contextlib.contextmanager -def autoremoved_file(path): - try: - with open(path, 'w') as handle: - yield handle - finally: - os.unlink(path) class ClickHouseMarkdown(markdown.extensions.Extension): class ClickHousePreprocessor(markdown.util.Processor): @@ -52,8 +36,10 @@ class ClickHouseMarkdown(markdown.extensions.Extension): def extendMarkdown(self, md): md.preprocessors.register(self.ClickHousePreprocessor(), 'clickhouse_preprocessor', 31) + markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown + def build_for_lang(lang, args): logging.info('Building %s docs' % lang) os.environ['SINGLE_PAGE'] = '0' @@ -80,25 +66,29 @@ def build_for_lang(lang, args): 'search_index_only': True, 'static_templates': ['404.html'], 'extra': { - 'single_page': False, 'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching } } site_names = { - 'en': 'ClickHouse Documentation', - 'ru': 'Документация ClickHouse', - 'zh': 'ClickHouse文档', - 'fa': 'مستندات ClickHouse' + 'en': 'ClickHouse %s Documentation', + 'ru': 'Документация ClickHouse %s', + 'zh': 'ClickHouse文档 %s', + 'fa': 'مستندات %sClickHouse' } + if args.version_prefix: + site_dir = os.path.join(args.docs_output_dir, args.version_prefix, lang) + else: + site_dir = os.path.join(args.docs_output_dir, lang) + cfg = config.load_config( config_file=config_path, - site_name=site_names.get(lang, site_names['en']), + site_name=site_names.get(lang, site_names['en']) % args.version_prefix, site_url='https://clickhouse.yandex/docs/%s/' % lang, docs_dir=os.path.join(args.docs_dir, lang), - site_dir=os.path.join(args.docs_output_dir, lang), - strict=True, + site_dir=site_dir, + strict=not args.version_prefix, theme=theme_cfg, copyright='©2016–2019 Yandex LLC', use_directory_urls=True, @@ -127,7 +117,9 @@ def build_for_lang(lang, args): extra={ 'search': { 'language': 'en,ru' if lang == 'ru' else 'en' - } + }, + 'stable_releases': args.stable_releases, + 'version_prefix': args.version_prefix } ) @@ -144,13 +136,14 @@ def build_single_page_version(lang, args, cfg): logging.info('Building single page version for ' + lang) os.environ['SINGLE_PAGE'] = '1' - with autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md: + with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md: concatenate(lang, args.docs_dir, single_md) - with temp_dir() as site_temp: - with temp_dir() as docs_temp: + with util.temp_dir() as site_temp: + with util.temp_dir() as docs_temp: + docs_src_lang = os.path.join(args.docs_dir, lang) docs_temp_lang = os.path.join(docs_temp, lang) - shutil.copytree(os.path.join(args.docs_dir, lang), docs_temp_lang) + shutil.copytree(docs_src_lang, docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: if filename != 'single.md' and filename.endswith('.md'): @@ -169,7 +162,10 @@ def build_single_page_version(lang, args, cfg): mkdocs_build.build(cfg) - single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single') + if args.version_prefix: + single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, args.version_prefix, lang, 'single') + else: + single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single') if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) @@ -186,7 +182,7 @@ def build_single_page_version(lang, args, cfg): logging.debug(' '.join(create_pdf_command)) subprocess.check_call(' '.join(create_pdf_command), shell=True) - with temp_dir() as test_dir: + with util.temp_dir() as test_dir: cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': test_dir, @@ -198,7 +194,8 @@ def build_single_page_version(lang, args, cfg): ] }) mkdocs_build.build(cfg) - test.test_single_page(os.path.join(test_dir, 'single', 'index.html'), lang) + if not args.version_prefix: # maybe enable in future + test.test_single_page(os.path.join(test_dir, 'single', 'index.html'), lang) if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page) @@ -217,6 +214,11 @@ def build_redirects(args): f.write('\n'.join(rewrites)) +def build_docs(args): + for lang in args.lang.split(','): + build_for_lang(lang, args) + + def build(args): if os.path.exists(args.output_dir): shutil.rmtree(args.output_dir) @@ -224,21 +226,28 @@ def build(args): if not args.skip_website: build_website(args) - for lang in args.lang.split(','): - build_for_lang(lang, args) + build_docs(args) + + from github import build_releases + build_releases(args, build_docs) build_redirects(args) if not args.skip_website: minify_website(args) + if __name__ == '__main__': + os.chdir(os.path.join(os.path.dirname(__file__), '..')) + arg_parser = argparse.ArgumentParser() arg_parser.add_argument('--lang', default='en,ru,zh,fa') arg_parser.add_argument('--docs-dir', default='.') arg_parser.add_argument('--theme-dir', default='mkdocs-material-theme') arg_parser.add_argument('--website-dir', default=os.path.join('..', 'website')) arg_parser.add_argument('--output-dir', default='build') + arg_parser.add_argument('--enable-stable-releases', action='store_true') + arg_parser.add_argument('--version-prefix', type=str, default='') arg_parser.add_argument('--skip-single-page', action='store_true') arg_parser.add_argument('--skip-pdf', action='store_true') arg_parser.add_argument('--skip-website', action='store_true') @@ -246,8 +255,12 @@ if __name__ == '__main__': arg_parser.add_argument('--verbose', action='store_true') args = arg_parser.parse_args() - args.docs_output_dir = os.path.join(args.output_dir, 'docs') - os.chdir(os.path.join(os.path.dirname(__file__), '..')) + args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), 'docs') + + from github import choose_latest_releases + args.stable_releases = choose_latest_releases() if args.enable_stable_releases else [] + + logging.basicConfig( level=logging.DEBUG if args.verbose else logging.INFO, diff --git a/docs/tools/concatenate.py b/docs/tools/concatenate.py index b916579d1cc..bddf63cbc23 100755 --- a/docs/tools/concatenate.py +++ b/docs/tools/concatenate.py @@ -1,17 +1,5 @@ # -*- coding: utf-8 -*- -# - Single-page document. -# - Requirements to the md-souces: -# - Don't use links without anchors. It means, that you can not just link file. You should specify an anchor at the top of the file and then link to this anchor -# - Anchors should be unique through whole document. -# - Implementation: -# - Script gets list of the file from the `pages` section of `mkdocs.yml`. It gets commented files too, and it right. -# - Files are concatenated by order with incrementing level of headers in all files except the first one -# - Script converts links to other files into inside page links. -# - Skipping links started with 'http' -# - Not http-links with anchor are cutted to the anchor sign (#). -# - For not http-links without anchor script logs an error and cuts them from the resulting single-page document. - import logging import re import os @@ -34,35 +22,36 @@ def concatenate(lang, docs_path, single_page_file): ' files will be concatenated into single md-file.') logging.debug('Concatenating: ' + ', '.join(files_to_concatenate)) - first_file = True - for path in files_to_concatenate: - with open(os.path.join(lang_path, path)) as f: - anchors = set() - tmp_path = path.replace('/index.md', '/').replace('.md', '/') - prefixes = ['', '../', '../../', '../../../'] - parts = tmp_path.split('/') - anchors.add(parts[-2] + '/') - anchors.add('/'.join(parts[1:])) - - for part in parts[0:-2] if len(parts) > 2 else parts: - for prefix in prefixes: - anchor = prefix + tmp_path - if anchor: - anchors.add(anchor) - anchors.add('../' + anchor) - anchors.add('../../' + anchor) - tmp_path = tmp_path.replace(part, '..') - - for anchor in anchors: - if re.search(az_re, anchor): - single_page_file.write('\n' % anchor) - - single_page_file.write('\n\n') - - for l in f: - if l.startswith('#'): - l = '#' + l - single_page_file.write(l) + try: + with open(os.path.join(lang_path, path)) as f: + anchors = set() + tmp_path = path.replace('/index.md', '/').replace('.md', '/') + prefixes = ['', '../', '../../', '../../../'] + parts = tmp_path.split('/') + anchors.add(parts[-2] + '/') + anchors.add('/'.join(parts[1:])) + + for part in parts[0:-2] if len(parts) > 2 else parts: + for prefix in prefixes: + anchor = prefix + tmp_path + if anchor: + anchors.add(anchor) + anchors.add('../' + anchor) + anchors.add('../../' + anchor) + tmp_path = tmp_path.replace(part, '..') + + for anchor in anchors: + if re.search(az_re, anchor): + single_page_file.write('\n' % anchor) + + single_page_file.write('\n\n') + + for l in f: + if l.startswith('#'): + l = '#' + l + single_page_file.write(l) + except IOError as e: + logging.warning(str(e)) single_page_file.flush() diff --git a/docs/tools/github.py b/docs/tools/github.py new file mode 100644 index 00000000000..13aa4984fdf --- /dev/null +++ b/docs/tools/github.py @@ -0,0 +1,44 @@ +import collections +import copy +import io +import logging +import os +import tarfile + +import requests + +import util + + +def choose_latest_releases(): + seen = collections.OrderedDict() + candidates = requests.get('https://api.github.com/repos/yandex/ClickHouse/tags?per_page=100').json() + for tag in candidates: + name = tag.get('name', '') + if 'v18' in name or 'stable' not in name: + continue + major_version = '.'.join((name.split('.', 2))[:2]) + if major_version not in seen: + seen[major_version] = (name, tag.get('tarball_url'),) + + return seen.items() + + +def process_release(args, callback, release): + name, (full_name, tarball_url,) = release + logging.info('Building docs for %s', full_name) + buf = io.BytesIO(requests.get(tarball_url).content) + tar = tarfile.open(mode='r:gz', fileobj=buf) + with util.temp_dir() as base_dir: + tar.extractall(base_dir) + args = copy.deepcopy(args) + args.version_prefix = name + args.docs_dir = os.path.join(base_dir, os.listdir(base_dir)[0], 'docs') + callback(args) + + +def build_releases(args, callback): + for release in args.stable_releases: + process_release(args, callback, release) + + diff --git a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css index 97f2d3e01d3..e452e88ad66 100644 --- a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css +++ b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css @@ -166,6 +166,38 @@ h1, h2, h3, .md-logo { text-decoration: none; } +.md-selector { + border: 1px solid #ccc; + padding: 2px; + border-radius: 3px; + overflow: hidden; + background: #fff; +} + +.md-selector select { + padding: 8px 8px; + border: none; + box-shadow: none; + background: transparent; + background-image: none; + -webkit-appearance: none; + display: inline-block; + padding-right: 20px; + position: relative; + z-index: 2; +} + +#releases-selector-after { + padding: 0 4px; + margin-left: -20px; + position: relative; + z-index: 1; +} + +.md-selector select:focus { + outline: none; +} + @media only screen and (min-width: 60em) { #md-sidebar-flags { display: none; diff --git a/docs/tools/mkdocs-material-theme/base.html b/docs/tools/mkdocs-material-theme/base.html index 91a10d8e6c3..8b04617c721 100644 --- a/docs/tools/mkdocs-material-theme/base.html +++ b/docs/tools/mkdocs-material-theme/base.html @@ -256,6 +256,17 @@ feedback_email[i].setAttribute('href', 'mailto:' + feedback_address); feedback_email[i].innerHTML = feedback_address; } + + document.getElementById("releases-selector").addEventListener( + "change", + function() { + var target = window.location.href.split('docs')[0] + 'docs/'; + if (this.value) { + target = target + this.value + '/'; + } + window.location = target + '{{ config.theme.language }}'; + } + ) }); diff --git a/docs/tools/mkdocs-material-theme/partials/flags.html b/docs/tools/mkdocs-material-theme/partials/flags.html index ae0200b602b..26d6cdd8f9f 100644 --- a/docs/tools/mkdocs-material-theme/partials/flags.html +++ b/docs/tools/mkdocs-material-theme/partials/flags.html @@ -6,4 +6,3 @@ {% include "assets/flags/" + alt_lang + ".svg" %} {% endfor %} - diff --git a/docs/tools/mkdocs-material-theme/partials/language/en.html b/docs/tools/mkdocs-material-theme/partials/language/en.html index e84e566446a..bcd2debaac5 100644 --- a/docs/tools/mkdocs-material-theme/partials/language/en.html +++ b/docs/tools/mkdocs-material-theme/partials/language/en.html @@ -7,8 +7,10 @@ "footer.next": "Next", "meta.comments": "Comments", "meta.source": "Source", + "nav.latest": "latest", "nav.multi_page": "Multi page version", "nav.pdf": "PDF version", + "nav.release": "Release", "nav.single_page": "Single page version", "nav.source": "ClickHouse source code", "search.placeholder": "Search", diff --git a/docs/tools/mkdocs-material-theme/partials/language/fa.html b/docs/tools/mkdocs-material-theme/partials/language/fa.html index 71f50043932..b321e1319b8 100644 --- a/docs/tools/mkdocs-material-theme/partials/language/fa.html +++ b/docs/tools/mkdocs-material-theme/partials/language/fa.html @@ -8,8 +8,10 @@ "footer.next": "بعدی", "meta.comments": "نظرات", "meta.source": "منبع", + "nav.latest": "آخرین", "nav.multi_page": "نسخه چند صفحه ای", "nav.pdf": "نسخه PDF", + "nav.release": "رهایی", "nav.single_page": "نسخه تک صفحه", "nav.source": "کد منبع کلیک", "search.language": "", diff --git a/docs/tools/mkdocs-material-theme/partials/language/ru.html b/docs/tools/mkdocs-material-theme/partials/language/ru.html index b8baaea5eff..eb8a31e86a4 100644 --- a/docs/tools/mkdocs-material-theme/partials/language/ru.html +++ b/docs/tools/mkdocs-material-theme/partials/language/ru.html @@ -7,8 +7,10 @@ "footer.next": "Вперед", "meta.comments": "Комментарии", "meta.source": "Исходный код", + "nav.latest": "последний", "nav.multi_page": "Многостраничная версия", "nav.pdf": "PDF версия", + "nav.release": "Релиз", "nav.single_page": "Одностраничная версия", "nav.source": "Исходный код ClickHouse", "search.placeholder": "Поиск", diff --git a/docs/tools/mkdocs-material-theme/partials/language/zh.html b/docs/tools/mkdocs-material-theme/partials/language/zh.html index 89c94ee2a98..36f681c8a0e 100644 --- a/docs/tools/mkdocs-material-theme/partials/language/zh.html +++ b/docs/tools/mkdocs-material-theme/partials/language/zh.html @@ -7,8 +7,10 @@ "footer.next": "前进", "meta.comments": "评论", "meta.source": "来源", + "nav.latest": "最新", "nav.multi_page": "多页版本", "nav.pdf": "PDF版本", + "nav.release": "发布", "nav.single_page": "单页版本", "nav.source": "ClickHouse源代码", "search.placeholder": "搜索", diff --git a/docs/tools/mkdocs-material-theme/partials/nav.html b/docs/tools/mkdocs-material-theme/partials/nav.html index cf58c0c4130..cdc09492e54 100644 --- a/docs/tools/mkdocs-material-theme/partials/nav.html +++ b/docs/tools/mkdocs-material-theme/partials/nav.html @@ -14,6 +14,21 @@ {% endif %}