thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support building docs for stable releases in addition to docs from master (#4940)

Browse Source
This commit is contained in:
Ivan Blinkov 2019-04-08 19:01:54 +03:00 committed by GitHub
parent 80a235fdf9
commit 87f6889ae6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 214 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
docs/tools/build.py
View File

@ -3,14 +3,12 @@
from __future__ import unicode_literals
import argparse
import contextlib
import datetime
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import time
import markdown.extensions
@ -21,26 +19,12 @@ from mkdocs import exceptions
from mkdocs.commands import build as mkdocs_build
from concatenate import concatenate
from website import build_website, minify_website
import mdx_clickhouse
import test
import util
@contextlib.contextmanager
def temp_dir():
path = tempfile.mkdtemp(dir=os.environ.get('TEMP'))
try:
yield path
finally:
shutil.rmtree(path)
@contextlib.contextmanager
def autoremoved_file(path):
try:
with open(path, 'w') as handle:
yield handle
finally:
os.unlink(path)
class ClickHouseMarkdown(markdown.extensions.Extension):
class ClickHousePreprocessor(markdown.util.Processor):
@ -52,8 +36,10 @@ class ClickHouseMarkdown(markdown.extensions.Extension):
def extendMarkdown(self, md):
md.preprocessors.register(self.ClickHousePreprocessor(), 'clickhouse_preprocessor', 31)
markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown
def build_for_lang(lang, args):
logging.info('Building %s docs' % lang)
os.environ['SINGLE_PAGE'] = '0'
@ -80,25 +66,29 @@ def build_for_lang(lang, args):
'search_index_only': True,
'static_templates': ['404.html'],
'extra': {
'single_page': False,
'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching
}
}
site_names = {
'en': 'ClickHouse Documentation',
'ru': 'Документация ClickHouse',
'zh': 'ClickHouse文档',
'fa': 'مستندات ClickHouse'
'en': 'ClickHouse %s Documentation',
'ru': 'Документация ClickHouse %s',
'zh': 'ClickHouse文档 %s',
'fa': 'مستندات %sClickHouse'
}
if args.version_prefix:
site_dir = os.path.join(args.docs_output_dir, args.version_prefix, lang)
else:
site_dir = os.path.join(args.docs_output_dir, lang)
cfg = config.load_config(
config_file=config_path,
site_name=site_names.get(lang, site_names['en']),
site_name=site_names.get(lang, site_names['en']) % args.version_prefix,
site_url='https://clickhouse.yandex/docs/%s/' % lang,
docs_dir=os.path.join(args.docs_dir, lang),
site_dir=os.path.join(args.docs_output_dir, lang),
strict=True,
site_dir=site_dir,
strict=not args.version_prefix,
theme=theme_cfg,
copyright='©2016–2019 Yandex LLC',
use_directory_urls=True,
@ -127,7 +117,9 @@ def build_for_lang(lang, args):
extra={
'search': {
'language': 'en,ru' if lang == 'ru' else 'en'
}
},
'stable_releases': args.stable_releases,
'version_prefix': args.version_prefix
}
)
@ -144,13 +136,14 @@ def build_single_page_version(lang, args, cfg):
logging.info('Building single page version for ' + lang)
os.environ['SINGLE_PAGE'] = '1'
with autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md:
with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md:
concatenate(lang, args.docs_dir, single_md)
with temp_dir() as site_temp:
with temp_dir() as docs_temp:
with util.temp_dir() as site_temp:
with util.temp_dir() as docs_temp:
docs_src_lang = os.path.join(args.docs_dir, lang)
docs_temp_lang = os.path.join(docs_temp, lang)
shutil.copytree(os.path.join(args.docs_dir, lang), docs_temp_lang)
shutil.copytree(docs_src_lang, docs_temp_lang)
for root, _, filenames in os.walk(docs_temp_lang):
for filename in filenames:
if filename != 'single.md' and filename.endswith('.md'):
@ -169,7 +162,10 @@ def build_single_page_version(lang, args, cfg):
mkdocs_build.build(cfg)
single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single')
if args.version_prefix:
single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, args.version_prefix, lang, 'single')
else:
single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single')
if os.path.exists(single_page_output_path):
shutil.rmtree(single_page_output_path)
@ -186,7 +182,7 @@ def build_single_page_version(lang, args, cfg):
logging.debug(' '.join(create_pdf_command))
subprocess.check_call(' '.join(create_pdf_command), shell=True)
with temp_dir() as test_dir:
with util.temp_dir() as test_dir:
cfg.load_dict({
'docs_dir': docs_temp_lang,
'site_dir': test_dir,
@ -198,7 +194,8 @@ def build_single_page_version(lang, args, cfg):
]
})
mkdocs_build.build(cfg)
test.test_single_page(os.path.join(test_dir, 'single', 'index.html'), lang)
if not args.version_prefix: # maybe enable in future
test.test_single_page(os.path.join(test_dir, 'single', 'index.html'), lang)
if args.save_raw_single_page:
shutil.copytree(test_dir, args.save_raw_single_page)
@ -217,6 +214,11 @@ def build_redirects(args):
f.write('\n'.join(rewrites))
def build_docs(args):
for lang in args.lang.split(','):
build_for_lang(lang, args)
def build(args):
if os.path.exists(args.output_dir):
shutil.rmtree(args.output_dir)
@ -224,21 +226,28 @@ def build(args):
if not args.skip_website:
build_website(args)
for lang in args.lang.split(','):
build_for_lang(lang, args)
build_docs(args)
from github import build_releases
build_releases(args, build_docs)
build_redirects(args)
if not args.skip_website:
minify_website(args)
if __name__ == '__main__':
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--lang', default='en,ru,zh,fa')
arg_parser.add_argument('--docs-dir', default='.')
arg_parser.add_argument('--theme-dir', default='mkdocs-material-theme')
arg_parser.add_argument('--website-dir', default=os.path.join('..', 'website'))
arg_parser.add_argument('--output-dir', default='build')
arg_parser.add_argument('--enable-stable-releases', action='store_true')
arg_parser.add_argument('--version-prefix', type=str, default='')
arg_parser.add_argument('--skip-single-page', action='store_true')
arg_parser.add_argument('--skip-pdf', action='store_true')
arg_parser.add_argument('--skip-website', action='store_true')
@ -246,8 +255,12 @@ if __name__ == '__main__':
arg_parser.add_argument('--verbose', action='store_true')
args = arg_parser.parse_args()
args.docs_output_dir = os.path.join(args.output_dir, 'docs')
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), 'docs')
from github import choose_latest_releases
args.stable_releases = choose_latest_releases() if args.enable_stable_releases else []
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,

71
docs/tools/concatenate.py
View File

@ -1,17 +1,5 @@
# -*- coding: utf-8 -*-
# - Single-page document.
# - Requirements to the md-souces:
# - Don't use links without anchors. It means, that you can not just link file. You should specify an anchor at the top of the file and then link to this anchor
# - Anchors should be unique through whole document.
# - Implementation:
# - Script gets list of the file from the `pages` section of `mkdocs.yml`. It gets commented files too, and it right.
# - Files are concatenated by order with incrementing level of headers in all files except the first one
# - Script converts links to other files into inside page links.
# - Skipping links started with 'http'
# - Not http-links with anchor are cutted to the anchor sign (#).
# - For not http-links without anchor script logs an error and cuts them from the resulting single-page document.
import logging
import re
import os
@ -34,35 +22,36 @@ def concatenate(lang, docs_path, single_page_file):
' files will be concatenated into single md-file.')
logging.debug('Concatenating: ' + ', '.join(files_to_concatenate))
first_file = True
for path in files_to_concatenate:
with open(os.path.join(lang_path, path)) as f:
anchors = set()
tmp_path = path.replace('/index.md', '/').replace('.md', '/')
prefixes = ['', '../', '../../', '../../../']
parts = tmp_path.split('/')
anchors.add(parts[-2] + '/')
anchors.add('/'.join(parts[1:]))
for part in parts[0:-2] if len(parts) > 2 else parts:
for prefix in prefixes:
anchor = prefix + tmp_path
if anchor:
anchors.add(anchor)
anchors.add('../' + anchor)
anchors.add('../../' + anchor)
tmp_path = tmp_path.replace(part, '..')
for anchor in anchors:
if re.search(az_re, anchor):
single_page_file.write('<a name="%s"></a>\n' % anchor)
single_page_file.write('\n\n')
for l in f:
if l.startswith('#'):
l = '#' + l
single_page_file.write(l)
try:
with open(os.path.join(lang_path, path)) as f:
anchors = set()
tmp_path = path.replace('/index.md', '/').replace('.md', '/')
prefixes = ['', '../', '../../', '../../../']
parts = tmp_path.split('/')
anchors.add(parts[-2] + '/')
anchors.add('/'.join(parts[1:]))
for part in parts[0:-2] if len(parts) > 2 else parts:
for prefix in prefixes:
anchor = prefix + tmp_path
if anchor:
anchors.add(anchor)
anchors.add('../' + anchor)
anchors.add('../../' + anchor)
tmp_path = tmp_path.replace(part, '..')
for anchor in anchors:
if re.search(az_re, anchor):
single_page_file.write('<a name="%s"></a>\n' % anchor)
single_page_file.write('\n\n')
for l in f:
if l.startswith('#'):
l = '#' + l
single_page_file.write(l)
except IOError as e:
logging.warning(str(e))
single_page_file.flush()

44
docs/tools/github.py Normal file
View File

@ -0,0 +1,44 @@
import collections
import copy
import io
import logging
import os
import tarfile
import requests
import util
def choose_latest_releases():
seen = collections.OrderedDict()
candidates = requests.get('https://api.github.com/repos/yandex/ClickHouse/tags?per_page=100').json()
for tag in candidates:
name = tag.get('name', '')
if 'v18' in name or 'stable' not in name:
continue
major_version = '.'.join((name.split('.', 2))[:2])
if major_version not in seen:
seen[major_version] = (name, tag.get('tarball_url'),)
return seen.items()
def process_release(args, callback, release):
name, (full_name, tarball_url,) = release
logging.info('Building docs for %s', full_name)
buf = io.BytesIO(requests.get(tarball_url).content)
tar = tarfile.open(mode='r:gz', fileobj=buf)
with util.temp_dir() as base_dir:
tar.extractall(base_dir)
args = copy.deepcopy(args)
args.version_prefix = name
args.docs_dir = os.path.join(base_dir, os.listdir(base_dir)[0], 'docs')
callback(args)
def build_releases(args, callback):
for release in args.stable_releases:
process_release(args, callback, release)

32
docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css
View File

@ -166,6 +166,38 @@ h1, h2, h3, .md-logo {
text-decoration: none;
}
.md-selector {
border: 1px solid #ccc;
padding: 2px;
border-radius: 3px;
overflow: hidden;
background: #fff;
}
.md-selector select {
padding: 8px 8px;
border: none;
box-shadow: none;
background: transparent;
background-image: none;
-webkit-appearance: none;
display: inline-block;
padding-right: 20px;
position: relative;
z-index: 2;
}
#releases-selector-after {
padding: 0 4px;
margin-left: -20px;
position: relative;
z-index: 1;
}
.md-selector select:focus {
outline: none;
}
@media only screen and (min-width: 60em) {
#md-sidebar-flags {
display: none;

11
docs/tools/mkdocs-material-theme/base.html
View File

@ -256,6 +256,17 @@
feedback_email[i].setAttribute('href', 'mailto:' + feedback_address);
feedback_email[i].innerHTML = feedback_address;
}
document.getElementById("releases-selector").addEventListener(
"change",
function() {
var target = window.location.href.split('docs')[0] + 'docs/';
if (this.value) {
target = target + this.value + '/';
}
window.location = target + '{{ config.theme.language }}';
}
)
});
</script>
<!-- Yandex.Metrika counter -->

1
docs/tools/mkdocs-material-theme/partials/flags.html
View File

@ -6,4 +6,3 @@
{% include "assets/flags/" + alt_lang + ".svg" %}
</a>
{% endfor %}

2
docs/tools/mkdocs-material-theme/partials/language/en.html
View File

@ -7,8 +7,10 @@
"footer.next": "Next",
"meta.comments": "Comments",
"meta.source": "Source",
"nav.latest": "latest",
"nav.multi_page": "Multi page version",
"nav.pdf": "PDF version",
"nav.release": "Release",
"nav.single_page": "Single page version",
"nav.source": "ClickHouse source code",
"search.placeholder": "Search",

2
docs/tools/mkdocs-material-theme/partials/language/fa.html
View File

@ -8,8 +8,10 @@
"footer.next": "بعدی",
"meta.comments": "نظرات",
"meta.source": "منبع",
"nav.latest": "آخرین",
"nav.multi_page": "نسخه چند صفحه ای",
"nav.pdf": "نسخه PDF",
"nav.release": "رهایی",
"nav.single_page": "نسخه تک صفحه",
"nav.source": "کد منبع کلیک",
"search.language": "",

2
docs/tools/mkdocs-material-theme/partials/language/ru.html
View File

@ -7,8 +7,10 @@
"footer.next": "Вперед",
"meta.comments": "Комментарии",
"meta.source": "Исходный код",
"nav.latest": "последний",
"nav.multi_page": "Многостраничная версия",
"nav.pdf": "PDF версия",
"nav.release": "Релиз",
"nav.single_page": "Одностраничная версия",
"nav.source": "Исходный код ClickHouse",
"search.placeholder": "Поиск",

2
docs/tools/mkdocs-material-theme/partials/language/zh.html
View File

@ -7,8 +7,10 @@
"footer.next": "前进",
"meta.comments": "评论",
"meta.source": "来源",
"nav.latest": "最新",
"nav.multi_page": "多页版本",
"nav.pdf": "PDF版本",
"nav.release": "发布",
"nav.single_page": "单页版本",
"nav.source": "ClickHouse源代码",
"search.placeholder": "搜索",

15
docs/tools/mkdocs-material-theme/partials/nav.html
View File

@ -14,6 +14,21 @@
{% endif %}
<ul id="md-extra-nav" class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--active">
{{ lang.t("nav.release") }}:
<span class="md-selector">
<select id="releases-selector">
<option value="">{{ lang.t("nav.latest") }}</option>
{% for release in config.extra.stable_releases %}
<option value="{{ release.0 }}"
{% if release.0 == config.extra.version_prefix %}
selected="selected"
{% endif %}
>{{ release.0 }}</option>
{% endfor %}
</select><span id="releases-selector-after">&nbsp;▾</span>
</span>
</li>
<li class="md-nav__item md-nav__item--active">
{% if config.extra.single_page %}
<a href="{{ base_url }}" class="md-nav__link md-nav__link--active">{{ lang.t("nav.multi_page") }}</a>

2
docs/tools/release.sh
View File

@ -15,7 +15,7 @@ DOCKER_HASH="$2"
if [[ -z "$1" ]]
then
source "${BASE_DIR}/venv/bin/activate"
python "${BASE_DIR}/build.py"
python "${BASE_DIR}/build.py" "--enable-stable-releases"
cd "${BUILD_DIR}"
docker build -t "${FULL_NAME}" "${BUILD_DIR}"
docker tag "${FULL_NAME}" "${REMOTE_NAME}"

22
docs/tools/util.py Normal file
View File

@ -0,0 +1,22 @@
import contextlib
import os
import shutil
import tempfile
@contextlib.contextmanager
def temp_dir():
path = tempfile.mkdtemp(dir=os.environ.get('TEMP'))
try:
yield path
finally:
shutil.rmtree(path)
@contextlib.contextmanager
def autoremoved_file(path):
try:
with open(path, 'w') as handle:
yield handle
finally:
os.unlink(path)

5
website/index.html
View File

@ -92,11 +92,6 @@
<div class="clear"></div>
</div>
</div>
<div id="announcement" class="colored-block">
<div class="page">
Upcoming ClickHouse Community Meetups: <a class="announcement-link" href="https://www.eventbrite.com/e/clickhouse-meetup-in-madrid-registration-55376746339" rel="external nofollow" target="_blank">Madrid</a> on April 2
</div>
</div>
<div class="page">
<h2 id="slogan">ClickHouse. Just makes you think faster.</h2>