mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Support building docs for stable releases in addition to docs from master (#4940)
This commit is contained in:
Ivan Blinkov
GitHub
parent
80a235fdf9
commit
87f6889ae6
@ -3,14 +3,12 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import markdown.extensions
|
||||
@ -21,26 +19,12 @@ from mkdocs import exceptions
|
||||
from mkdocs.commands import build as mkdocs_build
|
||||
|
||||
from concatenate import concatenate
|
||||
|
||||
from website import build_website, minify_website
|
||||
import mdx_clickhouse
|
||||
import test
|
||||
import util
|
||||
|
||||
@contextlib.contextmanager
|
||||
def temp_dir():
|
||||
path = tempfile.mkdtemp(dir=os.environ.get('TEMP'))
|
||||
try:
|
||||
yield path
|
||||
finally:
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def autoremoved_file(path):
|
||||
try:
|
||||
with open(path, 'w') as handle:
|
||||
yield handle
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
class ClickHouseMarkdown(markdown.extensions.Extension):
|
||||
class ClickHousePreprocessor(markdown.util.Processor):
|
||||
@ -52,8 +36,10 @@ class ClickHouseMarkdown(markdown.extensions.Extension):
|
||||
def extendMarkdown(self, md):
|
||||
md.preprocessors.register(self.ClickHousePreprocessor(), 'clickhouse_preprocessor', 31)
|
||||
|
||||
|
||||
markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown
|
||||
|
||||
|
||||
def build_for_lang(lang, args):
|
||||
logging.info('Building %s docs' % lang)
|
||||
os.environ['SINGLE_PAGE'] = '0'
|
||||
@ -80,25 +66,29 @@ def build_for_lang(lang, args):
|
||||
'search_index_only': True,
|
||||
'static_templates': ['404.html'],
|
||||
'extra': {
|
||||
'single_page': False,
|
||||
'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching
|
||||
}
|
||||
}
|
||||
|
||||
site_names = {
|
||||
'en': 'ClickHouse Documentation',
|
||||
'ru': 'Документация ClickHouse',
|
||||
'zh': 'ClickHouse文档',
|
||||
'fa': 'مستندات ClickHouse'
|
||||
'en': 'ClickHouse %s Documentation',
|
||||
'ru': 'Документация ClickHouse %s',
|
||||
'zh': 'ClickHouse文档 %s',
|
||||
'fa': 'مستندات %sClickHouse'
|
||||
}
|
||||
|
||||
if args.version_prefix:
|
||||
site_dir = os.path.join(args.docs_output_dir, args.version_prefix, lang)
|
||||
else:
|
||||
site_dir = os.path.join(args.docs_output_dir, lang)
|
||||
|
||||
cfg = config.load_config(
|
||||
config_file=config_path,
|
||||
site_name=site_names.get(lang, site_names['en']),
|
||||
site_name=site_names.get(lang, site_names['en']) % args.version_prefix,
|
||||
site_url='https://clickhouse.yandex/docs/%s/' % lang,
|
||||
docs_dir=os.path.join(args.docs_dir, lang),
|
||||
site_dir=os.path.join(args.docs_output_dir, lang),
|
||||
strict=True,
|
||||
site_dir=site_dir,
|
||||
strict=not args.version_prefix,
|
||||
theme=theme_cfg,
|
||||
copyright='©2016–2019 Yandex LLC',
|
||||
use_directory_urls=True,
|
||||
@ -127,7 +117,9 @@ def build_for_lang(lang, args):
|
||||
extra={
|
||||
'search': {
|
||||
'language': 'en,ru' if lang == 'ru' else 'en'
|
||||
}
|
||||
},
|
||||
'stable_releases': args.stable_releases,
|
||||
'version_prefix': args.version_prefix
|
||||
}
|
||||
)
|
||||
|
||||
@ -144,13 +136,14 @@ def build_single_page_version(lang, args, cfg):
|
||||
logging.info('Building single page version for ' + lang)
|
||||
os.environ['SINGLE_PAGE'] = '1'
|
||||
|
||||
with autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md:
|
||||
with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md:
|
||||
concatenate(lang, args.docs_dir, single_md)
|
||||
|
||||
with temp_dir() as site_temp:
|
||||
with temp_dir() as docs_temp:
|
||||
with util.temp_dir() as site_temp:
|
||||
with util.temp_dir() as docs_temp:
|
||||
docs_src_lang = os.path.join(args.docs_dir, lang)
|
||||
docs_temp_lang = os.path.join(docs_temp, lang)
|
||||
shutil.copytree(os.path.join(args.docs_dir, lang), docs_temp_lang)
|
||||
shutil.copytree(docs_src_lang, docs_temp_lang)
|
||||
for root, _, filenames in os.walk(docs_temp_lang):
|
||||
for filename in filenames:
|
||||
if filename != 'single.md' and filename.endswith('.md'):
|
||||
@ -169,6 +162,9 @@ def build_single_page_version(lang, args, cfg):
|
||||
|
||||
mkdocs_build.build(cfg)
|
||||
|
||||
if args.version_prefix:
|
||||
single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, args.version_prefix, lang, 'single')
|
||||
else:
|
||||
single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single')
|
||||
|
||||
if os.path.exists(single_page_output_path):
|
||||
@ -186,7 +182,7 @@ def build_single_page_version(lang, args, cfg):
|
||||
logging.debug(' '.join(create_pdf_command))
|
||||
subprocess.check_call(' '.join(create_pdf_command), shell=True)
|
||||
|
||||
with temp_dir() as test_dir:
|
||||
with util.temp_dir() as test_dir:
|
||||
cfg.load_dict({
|
||||
'docs_dir': docs_temp_lang,
|
||||
'site_dir': test_dir,
|
||||
@ -198,6 +194,7 @@ def build_single_page_version(lang, args, cfg):
|
||||
]
|
||||
})
|
||||
mkdocs_build.build(cfg)
|
||||
if not args.version_prefix: # maybe enable in future
|
||||
test.test_single_page(os.path.join(test_dir, 'single', 'index.html'), lang)
|
||||
if args.save_raw_single_page:
|
||||
shutil.copytree(test_dir, args.save_raw_single_page)
|
||||
@ -217,6 +214,11 @@ def build_redirects(args):
|
||||
f.write('\n'.join(rewrites))
|
||||
|
||||
|
||||
def build_docs(args):
|
||||
for lang in args.lang.split(','):
|
||||
build_for_lang(lang, args)
|
||||
|
||||
|
||||
def build(args):
|
||||
if os.path.exists(args.output_dir):
|
||||
shutil.rmtree(args.output_dir)
|
||||
@ -224,21 +226,28 @@ def build(args):
|
||||
if not args.skip_website:
|
||||
build_website(args)
|
||||
|
||||
for lang in args.lang.split(','):
|
||||
build_for_lang(lang, args)
|
||||
build_docs(args)
|
||||
|
||||
from github import build_releases
|
||||
build_releases(args, build_docs)
|
||||
|
||||
build_redirects(args)
|
||||
|
||||
if not args.skip_website:
|
||||
minify_website(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
|
||||
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
arg_parser.add_argument('--lang', default='en,ru,zh,fa')
|
||||
arg_parser.add_argument('--docs-dir', default='.')
|
||||
arg_parser.add_argument('--theme-dir', default='mkdocs-material-theme')
|
||||
arg_parser.add_argument('--website-dir', default=os.path.join('..', 'website'))
|
||||
arg_parser.add_argument('--output-dir', default='build')
|
||||
arg_parser.add_argument('--enable-stable-releases', action='store_true')
|
||||
arg_parser.add_argument('--version-prefix', type=str, default='')
|
||||
arg_parser.add_argument('--skip-single-page', action='store_true')
|
||||
arg_parser.add_argument('--skip-pdf', action='store_true')
|
||||
arg_parser.add_argument('--skip-website', action='store_true')
|
||||
@ -246,8 +255,12 @@ if __name__ == '__main__':
|
||||
arg_parser.add_argument('--verbose', action='store_true')
|
||||
|
||||
args = arg_parser.parse_args()
|
||||
args.docs_output_dir = os.path.join(args.output_dir, 'docs')
|
||||
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
|
||||
args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), 'docs')
|
||||
|
||||
from github import choose_latest_releases
|
||||
args.stable_releases = choose_latest_releases() if args.enable_stable_releases else []
|
||||
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
|
||||
@ -1,17 +1,5 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# - Single-page document.
|
||||
# - Requirements to the md-souces:
|
||||
# - Don't use links without anchors. It means, that you can not just link file. You should specify an anchor at the top of the file and then link to this anchor
|
||||
# - Anchors should be unique through whole document.
|
||||
# - Implementation:
|
||||
# - Script gets list of the file from the `pages` section of `mkdocs.yml`. It gets commented files too, and it right.
|
||||
# - Files are concatenated by order with incrementing level of headers in all files except the first one
|
||||
# - Script converts links to other files into inside page links.
|
||||
# - Skipping links started with 'http'
|
||||
# - Not http-links with anchor are cutted to the anchor sign (#).
|
||||
# - For not http-links without anchor script logs an error and cuts them from the resulting single-page document.
|
||||
|
||||
import logging
|
||||
import re
|
||||
import os
|
||||
@ -34,9 +22,8 @@ def concatenate(lang, docs_path, single_page_file):
|
||||
' files will be concatenated into single md-file.')
|
||||
logging.debug('Concatenating: ' + ', '.join(files_to_concatenate))
|
||||
|
||||
first_file = True
|
||||
|
||||
for path in files_to_concatenate:
|
||||
try:
|
||||
with open(os.path.join(lang_path, path)) as f:
|
||||
anchors = set()
|
||||
tmp_path = path.replace('/index.md', '/').replace('.md', '/')
|
||||
@ -64,5 +51,7 @@ def concatenate(lang, docs_path, single_page_file):
|
||||
if l.startswith('#'):
|
||||
l = '#' + l
|
||||
single_page_file.write(l)
|
||||
except IOError as e:
|
||||
logging.warning(str(e))
|
||||
|
||||
single_page_file.flush()
|
||||
|
||||
44
docs/tools/github.py
Normal file
44
docs/tools/github.py
Normal file
@ -0,0 +1,44 @@
|
||||
import collections
|
||||
import copy
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tarfile
|
||||
|
||||
import requests
|
||||
|
||||
import util
|
||||
|
||||
|
||||
def choose_latest_releases():
|
||||
seen = collections.OrderedDict()
|
||||
candidates = requests.get('https://api.github.com/repos/yandex/ClickHouse/tags?per_page=100').json()
|
||||
for tag in candidates:
|
||||
name = tag.get('name', '')
|
||||
if 'v18' in name or 'stable' not in name:
|
||||
continue
|
||||
major_version = '.'.join((name.split('.', 2))[:2])
|
||||
if major_version not in seen:
|
||||
seen[major_version] = (name, tag.get('tarball_url'),)
|
||||
|
||||
return seen.items()
|
||||
|
||||
|
||||
def process_release(args, callback, release):
|
||||
name, (full_name, tarball_url,) = release
|
||||
logging.info('Building docs for %s', full_name)
|
||||
buf = io.BytesIO(requests.get(tarball_url).content)
|
||||
tar = tarfile.open(mode='r:gz', fileobj=buf)
|
||||
with util.temp_dir() as base_dir:
|
||||
tar.extractall(base_dir)
|
||||
args = copy.deepcopy(args)
|
||||
args.version_prefix = name
|
||||
args.docs_dir = os.path.join(base_dir, os.listdir(base_dir)[0], 'docs')
|
||||
callback(args)
|
||||
|
||||
|
||||
def build_releases(args, callback):
|
||||
for release in args.stable_releases:
|
||||
process_release(args, callback, release)
|
||||
|
||||
|
||||
@ -166,6 +166,38 @@ h1, h2, h3, .md-logo {
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.md-selector {
|
||||
border: 1px solid #ccc;
|
||||
padding: 2px;
|
||||
border-radius: 3px;
|
||||
overflow: hidden;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.md-selector select {
|
||||
padding: 8px 8px;
|
||||
border: none;
|
||||
box-shadow: none;
|
||||
background: transparent;
|
||||
background-image: none;
|
||||
-webkit-appearance: none;
|
||||
display: inline-block;
|
||||
padding-right: 20px;
|
||||
position: relative;
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
#releases-selector-after {
|
||||
padding: 0 4px;
|
||||
margin-left: -20px;
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
.md-selector select:focus {
|
||||
outline: none;
|
||||
}
|
||||
|
||||
@media only screen and (min-width: 60em) {
|
||||
#md-sidebar-flags {
|
||||
display: none;
|
||||
|
||||
@ -256,6 +256,17 @@
|
||||
feedback_email[i].setAttribute('href', 'mailto:' + feedback_address);
|
||||
feedback_email[i].innerHTML = feedback_address;
|
||||
}
|
||||
|
||||
document.getElementById("releases-selector").addEventListener(
|
||||
"change",
|
||||
function() {
|
||||
var target = window.location.href.split('docs')[0] + 'docs/';
|
||||
if (this.value) {
|
||||
target = target + this.value + '/';
|
||||
}
|
||||
window.location = target + '{{ config.theme.language }}';
|
||||
}
|
||||
)
|
||||
});
|
||||
</script>
|
||||
<!-- Yandex.Metrika counter -->
|
||||
|
||||
@ -6,4 +6,3 @@
|
||||
{% include "assets/flags/" + alt_lang + ".svg" %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
|
||||
|
||||
@ -7,8 +7,10 @@
|
||||
"footer.next": "Next",
|
||||
"meta.comments": "Comments",
|
||||
"meta.source": "Source",
|
||||
"nav.latest": "latest",
|
||||
"nav.multi_page": "Multi page version",
|
||||
"nav.pdf": "PDF version",
|
||||
"nav.release": "Release",
|
||||
"nav.single_page": "Single page version",
|
||||
"nav.source": "ClickHouse source code",
|
||||
"search.placeholder": "Search",
|
||||
|
||||
@ -8,8 +8,10 @@
|
||||
"footer.next": "بعدی",
|
||||
"meta.comments": "نظرات",
|
||||
"meta.source": "منبع",
|
||||
"nav.latest": "آخرین",
|
||||
"nav.multi_page": "نسخه چند صفحه ای",
|
||||
"nav.pdf": "نسخه PDF",
|
||||
"nav.release": "رهایی",
|
||||
"nav.single_page": "نسخه تک صفحه",
|
||||
"nav.source": "کد منبع کلیک",
|
||||
"search.language": "",
|
||||
|
||||
@ -7,8 +7,10 @@
|
||||
"footer.next": "Вперед",
|
||||
"meta.comments": "Комментарии",
|
||||
"meta.source": "Исходный код",
|
||||
"nav.latest": "последний",
|
||||
"nav.multi_page": "Многостраничная версия",
|
||||
"nav.pdf": "PDF версия",
|
||||
"nav.release": "Релиз",
|
||||
"nav.single_page": "Одностраничная версия",
|
||||
"nav.source": "Исходный код ClickHouse",
|
||||
"search.placeholder": "Поиск",
|
||||
|
||||
@ -7,8 +7,10 @@
|
||||
"footer.next": "前进",
|
||||
"meta.comments": "评论",
|
||||
"meta.source": "来源",
|
||||
"nav.latest": "最新",
|
||||
"nav.multi_page": "多页版本",
|
||||
"nav.pdf": "PDF版本",
|
||||
"nav.release": "发布",
|
||||
"nav.single_page": "单页版本",
|
||||
"nav.source": "ClickHouse源代码",
|
||||
"search.placeholder": "搜索",
|
||||
|
||||
@ -14,6 +14,21 @@
|
||||
{% endif %}
|
||||
|
||||
<ul id="md-extra-nav" class="md-nav__list" data-md-scrollfix>
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
{{ lang.t("nav.release") }}:
|
||||
<span class="md-selector">
|
||||
<select id="releases-selector">
|
||||
<option value="">{{ lang.t("nav.latest") }}</option>
|
||||
{% for release in config.extra.stable_releases %}
|
||||
<option value="{{ release.0 }}"
|
||||
{% if release.0 == config.extra.version_prefix %}
|
||||
selected="selected"
|
||||
{% endif %}
|
||||
>{{ release.0 }}</option>
|
||||
{% endfor %}
|
||||
</select><span id="releases-selector-after"> ▾</span>
|
||||
</span>
|
||||
</li>
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
{% if config.extra.single_page %}
|
||||
<a href="{{ base_url }}" class="md-nav__link md-nav__link--active">{{ lang.t("nav.multi_page") }}</a>
|
||||
|
||||
@ -15,7 +15,7 @@ DOCKER_HASH="$2"
|
||||
if [[ -z "$1" ]]
|
||||
then
|
||||
source "${BASE_DIR}/venv/bin/activate"
|
||||
python "${BASE_DIR}/build.py"
|
||||
python "${BASE_DIR}/build.py" "--enable-stable-releases"
|
||||
cd "${BUILD_DIR}"
|
||||
docker build -t "${FULL_NAME}" "${BUILD_DIR}"
|
||||
docker tag "${FULL_NAME}" "${REMOTE_NAME}"
|
||||
|
||||
22
docs/tools/util.py
Normal file
22
docs/tools/util.py
Normal file
@ -0,0 +1,22 @@
|
||||
import contextlib
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def temp_dir():
|
||||
path = tempfile.mkdtemp(dir=os.environ.get('TEMP'))
|
||||
try:
|
||||
yield path
|
||||
finally:
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def autoremoved_file(path):
|
||||
try:
|
||||
with open(path, 'w') as handle:
|
||||
yield handle
|
||||
finally:
|
||||
os.unlink(path)
|
||||
@ -92,11 +92,6 @@
|
||||
<div class="clear"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="announcement" class="colored-block">
|
||||
<div class="page">
|
||||
Upcoming ClickHouse Community Meetups: <a class="announcement-link" href="https://www.eventbrite.com/e/clickhouse-meetup-in-madrid-registration-55376746339" rel="external nofollow" target="_blank">Madrid</a> on April 2
|
||||
</div>
|
||||
</div>
|
||||
<div class="page">
|
||||
<h2 id="slogan">ClickHouse. Just makes you think faster.</h2>
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user