Fix issues with docs

docs/tools/README.md

@@ -28,8 +28,8 @@ Follow the instructions on it's official website: <https://wkhtmltopdf.org/downl
 
 #### 2. Install CLI tools from npm
 
-1. `apt-get install npm` for Debian/Ubuntu or `brew install npm` on Mac OS X.
+1. `sudo apt-get install npm` for Debian/Ubuntu or `brew install npm` on Mac OS X.
-2. `npm install -g purifycss amphtml-validator`.
+2. `sudo npm install -g purify-css amphtml-validator`.
 
 #### 3. Set up virtualenv
 

docs/tools/build.py

@@ -48,11 +48,6 @@ def build_for_lang(lang, args):
     logging.info(f'Building {lang} docs')
     os.environ['SINGLE_PAGE'] = '0'
 
-    config_path = os.path.join(args.docs_dir, f'toc_{lang}.yml')
-    if args.is_stable_release and not os.path.exists(config_path):
-        logging.warning(f'Skipping {lang} docs, because {config} does not exist')
-        return
-
     try:
         theme_cfg = {
             'name': None,
@@ -73,9 +68,7 @@ def build_for_lang(lang, args):
             'es': 'Español',
             'fr': 'Français',
             'ru': 'Русский',
-            'ja': '日本語',
+            'ja': '日本語'
-            'tr': 'Türkçe',
-            'fa': 'فارسی'
         }
 
         site_names = {
@@ -84,16 +77,11 @@ def build_for_lang(lang, args):
             'es': 'Documentación de ClickHouse %s',
             'fr': 'Documentation ClickHouse %s',
             'ru': 'Документация ClickHouse %s',
-            'ja': 'ClickHouseドキュメント %s',
+            'ja': 'ClickHouseドキュメント %s'
-            'tr': 'ClickHouse Belgeleri %s',
-            'fa': 'مستندات %sClickHouse'
         }
 
         assert len(site_names) == len(languages)
 
-        if args.version_prefix:
-            site_dir = os.path.join(args.docs_output_dir, args.version_prefix, lang)
-        else:
         site_dir = os.path.join(args.docs_output_dir, lang)
 
         plugins = ['macros']
@@ -101,14 +89,14 @@ def build_for_lang(lang, args):
             plugins.append('htmlproofer')
 
         website_url = 'https://clickhouse.tech'
-        site_name = site_names.get(lang, site_names['en']) % args.version_prefix
+        site_name = site_names.get(lang, site_names['en']) % ''
         site_name = site_name.replace('  ', ' ')
         raw_config = dict(
             site_name=site_name,
             site_url=f'{website_url}/docs/{lang}/',
             docs_dir=os.path.join(args.docs_dir, lang),
             site_dir=site_dir,
-            strict=not args.version_prefix,
+            strict=True,
             theme=theme_cfg,
             copyright='©2016–2020 Yandex LLC',
             use_directory_urls=True,
@@ -119,8 +107,6 @@ def build_for_lang(lang, args):
             plugins=plugins,
             extra=dict(
                 now=datetime.datetime.now().isoformat(),
-                stable_releases=args.stable_releases,
-                version_prefix=args.version_prefix,
                 single_page=False,
                 rev=args.rev,
                 rev_short=args.rev_short,
@@ -134,23 +120,14 @@ def build_for_lang(lang, args):
             )
         )
 
-        if os.path.exists(config_path):
-            raw_config['config_file'] = config_path
-        else:
         raw_config['nav'] = nav.build_docs_nav(lang, args)
 
         cfg = config.load_config(**raw_config)
 
         if not args.skip_multi_page:
-            try:
-                mkdocs.commands.build.build(cfg)
-            except jinja2.exceptions.TemplateError:
-                if not args.version_prefix:
-                    raise
-                mdx_clickhouse.PatchedMacrosPlugin.disabled = True
             mkdocs.commands.build.build(cfg)
 
-        if not (args.skip_amp or args.version_prefix):
+        if not args.skip_amp:
             amp.build_amp(lang, args, cfg)
 
         if not args.skip_single_page:
@@ -170,7 +147,6 @@ def build_docs(args):
         if lang:
             tasks.append((lang, args,))
     util.run_function_in_parallel(build_for_lang, tasks, threads=False)
-    if not args.version_prefix:
     redirects.build_docs_redirects(args)
 
 
@@ -188,8 +164,6 @@ def build(args):
         generate_cmake_flags_files()
 
         build_docs(args)
-        from github import build_releases
-        build_releases(args, build_docs)
 
     if not args.skip_blog:
         blog.build_blog(args)
@@ -209,7 +183,7 @@ if __name__ == '__main__':
     website_dir = os.path.join(src_dir, 'website')
 
     arg_parser = argparse.ArgumentParser()
-    arg_parser.add_argument('--lang', default='en,es,fr,ru,zh,ja,tr,fa')
+    arg_parser.add_argument('--lang', default='en,es,fr,ru,zh,ja')
     arg_parser.add_argument('--blog-lang', default='en,ru')
     arg_parser.add_argument('--docs-dir', default='.')
     arg_parser.add_argument('--theme-dir', default=website_dir)
@@ -217,12 +191,7 @@ if __name__ == '__main__':
     arg_parser.add_argument('--src-dir', default=src_dir)
     arg_parser.add_argument('--blog-dir', default=os.path.join(website_dir, 'blog'))
     arg_parser.add_argument('--output-dir', default='build')
-    arg_parser.add_argument('--enable-stable-releases', action='store_true')
-    arg_parser.add_argument('--stable-releases-limit', type=int, default='3')
-    arg_parser.add_argument('--lts-releases-limit', type=int, default='2')
     arg_parser.add_argument('--nav-limit', type=int, default='0')
-    arg_parser.add_argument('--version-prefix', type=str, default='')
-    arg_parser.add_argument('--is-stable-release', action='store_true')
     arg_parser.add_argument('--skip-multi-page', action='store_true')
     arg_parser.add_argument('--skip-single-page', action='store_true')
     arg_parser.add_argument('--skip-amp', action='store_true')
@@ -252,8 +221,7 @@ if __name__ == '__main__':
     args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), 'docs')
     args.blog_output_dir = os.path.join(os.path.abspath(args.output_dir), 'blog')
 
-    from github import choose_latest_releases, get_events
+    from github import get_events
-    args.stable_releases = choose_latest_releases(args) if args.enable_stable_releases else []
     args.rev = subprocess.check_output('git rev-parse HEAD', shell=True).decode('utf-8').strip()
     args.rev_short = subprocess.check_output('git rev-parse --short HEAD', shell=True).decode('utf-8').strip()
     args.rev_url = f'https://github.com/ClickHouse/ClickHouse/commit/{args.rev}'

docs/tools/github.py

@@ -13,88 +13,6 @@ import requests
 import util
 
 
-def yield_candidates():
-    for page in range(1, 100):
-        url = f'https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100&page={page}'
-        github_token = os.getenv('GITHUB_TOKEN')
-        if github_token:
-            headers = {'authorization': f'OAuth {github_token}'}
-        else:
-            headers = {}
-        for candidate in requests.get(url, headers=headers).json():
-            yield candidate
-    time.sleep(random.random() * 3)
-
-
-def choose_latest_releases(args):
-    logging.info('Collecting release candidates')
-    seen_stable = collections.OrderedDict()
-    seen_lts = collections.OrderedDict()
-    candidates = []
-    stable_count = 0
-    lts_count = 0
-
-    for tag in yield_candidates():
-        if isinstance(tag, dict):
-            name = tag.get('name', '')
-            is_stable = 'stable' in name
-            is_lts = 'lts' in name
-            is_unstable = not (is_stable or is_lts)
-            is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name)
-            if is_unstable or is_in_blacklist:
-                continue
-            major_version = '.'.join((name.split('.', 2))[:2])
-            if major_version not in seen_lts:
-                if (stable_count >= args.stable_releases_limit) and (lts_count >= args.lts_releases_limit):
-                    break
-
-                payload = (name, tag.get('tarball_url'), is_lts,)
-                logging.debug(payload)
-                if is_lts:
-                    if lts_count < args.lts_releases_limit:
-                        seen_lts[major_version] = payload
-                        try:
-                            del seen_stable[major_version]
-                        except KeyError:
-                            pass
-                    lts_count += 1
-                else:
-                    if stable_count < args.stable_releases_limit:
-                        if major_version not in seen_stable:
-                            seen_stable[major_version] = payload
-                            stable_count += 1
-
-            logging.debug(
-                f'Stables: {stable_count}/{args.stable_releases_limit} LTS: {lts_count}/{args.lts_releases_limit}'
-            )
-        else:
-            logging.fatal('Unexpected GitHub response: %s', str(candidates))
-            sys.exit(1)
-
-    logging.info('Found LTS releases: %s', ', '.join(list(seen_lts.keys())))
-    logging.info('Found stable releases: %s', ', '.join(list(seen_stable.keys())))
-    return sorted(list(seen_lts.items()) + list(seen_stable.items()))
-
-
-def process_release(args, callback, release):
-    name, (full_name, tarball_url, is_lts,) = release
-    logging.info(f'Building docs for {full_name}')
-    buf = io.BytesIO(requests.get(tarball_url).content)
-    tar = tarfile.open(mode='r:gz', fileobj=buf)
-    with util.temp_dir() as base_dir:
-        tar.extractall(base_dir)
-        args = copy.copy(args)
-        args.version_prefix = name
-        args.is_stable_release = True
-        args.docs_dir = os.path.join(base_dir, os.listdir(base_dir)[0], 'docs')
-        callback(args)
-
-
-def build_releases(args, callback):
-    for release in args.stable_releases:
-        process_release(args, callback, release)
-
-
 def get_events(args):
     events = []
     skip = True
@@ -118,12 +36,7 @@ def get_events(args):
 
 
 if __name__ == '__main__':
-    class DummyArgs(object):
-        lts_releases_limit = 1
-        stable_releases_limit = 3
     logging.basicConfig(
         level=logging.DEBUG,
         stream=sys.stderr
     )
-    for item in choose_latest_releases(DummyArgs()):
-        print(item)

docs/tools/mdx_clickhouse.py

@@ -145,24 +145,9 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin):
         if self.skip_git_log:
             return markdown
         src_path = page.file.abs_src_path
-        try:
+
-            git_log = subprocess.check_output(f'git log --follow --date=iso8601 "{src_path}"', shell=True)
+        # There was a code that determined the minimum and maximum modification dates for a page.
-        except subprocess.CalledProcessError:
+        # It was removed due to being obnoxiously slow.
-            return markdown
-        max_date = None
-        min_date = None
-        for line in git_log.decode('utf-8').split('\n'):
-            if line.startswith('Date:'):
-                line = line.replace('Date:', '').strip().replace(' ', 'T', 1).replace(' ', '')
-                current_date = datetime.datetime.fromisoformat(line[:-2] + ':' + line[-2:])
-                if (not max_date) or current_date > max_date:
-                    max_date = current_date
-                if (not min_date) or current_date < min_date:
-                    min_date = current_date
-        if min_date:
-            page.meta['published_date'] = min_date
-        if max_date:
-            page.meta['modified_date'] = max_date
         return markdown
 
     def render_impl(self, markdown):

docs/tools/redirects.py

@@ -30,9 +30,8 @@ def build_redirect_html(args, base_prefix, lang, output_dir, from_path, to_path)
         output_dir, lang,
         from_path.replace('/index.md', '/index.html').replace('.md', '/index.html')
     )
-    version_prefix = f'/{args.version_prefix}/' if args.version_prefix else '/'
     target_path = to_path.replace('/index.md', '/').replace('.md', '/')
-    to_url = f'/{base_prefix}{version_prefix}{lang}/{target_path}'
+    to_url = f'/{base_prefix}/{lang}/{target_path}'
     to_url = to_url.strip()
     write_redirect_html(out_path, to_url)
 

docs/tools/release.sh

@@ -7,8 +7,7 @@ PUBLISH_DIR="${BASE_DIR}/../publish"
 BASE_DOMAIN="${BASE_DOMAIN:-content.clickhouse.tech}"
 GIT_TEST_URI="${GIT_TEST_URI:-git@github.com:ClickHouse/clickhouse-website-content.git}"
 GIT_PROD_URI="git@github.com:ClickHouse/clickhouse-website-content.git"
-EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:---enable-stable-releases --minify --verbose}"
+EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:---minify --verbose}"
-HISTORY_SIZE="${HISTORY_SIZE:-5}"
 
 if [[ -z "$1" ]]
 then

docs/tools/single_page.py

@@ -111,9 +111,6 @@ def build_single_page_version(lang, args, nav, cfg):
                 if not args.test_only:
                     mkdocs.commands.build.build(cfg)
 
-                    if args.version_prefix:
-                        single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, args.version_prefix, lang, 'single')
-                    else:
                     single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single')
 
                     if os.path.exists(single_page_output_path):
@@ -157,7 +154,6 @@ def build_single_page_version(lang, args, nav, cfg):
                     if args.save_raw_single_page:
                         shutil.copytree(test_dir, args.save_raw_single_page)
 
-                    if not args.version_prefix:  # maybe enable in future
                     logging.info(f'Running tests for {lang}')
                     test.test_single_page(
                         os.path.join(test_dir, 'single', 'index.html'), lang)

docs/tools/translate/translate.py

@@ -11,8 +11,6 @@ import googletrans
 import requests
 import yaml
 
-import typograph_ru
-
 
 translator = googletrans.Translator()
 default_target_language = os.environ.get('TARGET_LANGUAGE', 'ru')
@@ -25,8 +23,6 @@ def translate_impl(text, target_language=None):
     target_language = target_language or default_target_language
     if target_language == 'en':
         return text
-    elif target_language == 'typograph_ru':
-        return typograph_ru.typograph(text)
     elif is_yandex:
         text = text.replace('‘', '\'')
         text = text.replace('’', '\'')
@@ -59,25 +55,10 @@ def translate(text, target_language=None):
     )
 
 
-def translate_toc(root, lang):
-    global is_yandex
-    is_yandex = True
-    if isinstance(root, dict):
-        result = []
-        for key, value in root.items():
-            key = translate(key, lang) if key != 'hidden' and not key.isupper() else key
-            result.append((key, translate_toc(value, lang),))
-        return dict(result)
-    elif isinstance(root, list):
-        return [translate_toc(item, lang) for item in root]
-    elif isinstance(root, str):
-        return root
-
-
 def translate_po():
     import babel.messages.pofile
     base_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'website', 'locale')
-    for lang in ['en', 'zh', 'es', 'fr', 'ru', 'ja', 'tr', 'fa']:
+    for lang in ['en', 'zh', 'es', 'fr', 'ru', 'ja']:
         po_path = os.path.join(base_dir, lang, 'LC_MESSAGES', 'messages.po')
         with open(po_path, 'r') as f:
             po_file = babel.messages.pofile.read_po(f, locale=lang, domain='messages')

docs/tools/website.py

@@ -232,6 +232,7 @@ def minify_website(args):
             f"'{args.output_dir}/docs/en/**/*.html' '{args.website_dir}/js/**/*.js' > {css_out}"
     else:
         command = f'cat {css_in} > {css_out}'
+
     logging.info(command)
     output = subprocess.check_output(command, shell=True)
     logging.debug(output)

website/README.md

@@ -6,6 +6,12 @@ ClickHouse website is built alongside it's documentation via [docs/tools](https:
 cd ../docs/tools
 sudo apt install python-3 pip
 pip3 install -r requirements.txt
+
+# This is needed only when documentation is included
+sudo npm install -g purify-css amphtml-validator
+sudo apt install wkhtmltopdf
+virtualenv build
+
 ./build.py --skip-multi-page --skip-single-page --skip-amp --skip-pdf --skip-blog --skip-git-log --skip-docs --skip-test-templates --livereload 8080
 
 # Open the web browser and go to http://localhost:8080/

website/main.html

@@ -31,7 +31,7 @@
     {% set description = description.replace('¶','')[0:120] %}
 {% endif %}
 
-{% set data_version = config.extra.version_prefix or 'master' %}
+{% set data_version = 'master' %}
 {% set data_single_page = 'true' if config.extra.single_page else 'false' %}
 
 {% if is_amp %}

website/templates/common_meta.html

@@ -32,15 +32,15 @@
 <meta name="keywords"
       content="ClickHouse, DBMS, OLAP, SQL, {{ _('open-source') }}, {{ _('relational') }}, {{ _('analytics') }}, {{ _('analytical') }}, {{ _('Big Data') }}, {{ _('web-analytics') }}" />
 {% endif %}
-{% if config and (config.extra.single_page or config.extra.version_prefix) %}
+{% if config and config.extra.single_page %}
 <meta name="robots" content="noindex,follow" />
 {% endif %}
 
 {% if config and page and not is_blog %}
     {% for code, name in config.extra.languages.items() %}
-    <link rel="alternate" hreflang="{{ code }}" href="{{ config.extra.website_url }}/docs/{% if config.extra.version_prefix %}{{ config.extra.version_prefix }}/{% endif %}{{ code }}/{{ page.url }}" />
+    <link rel="alternate" hreflang="{{ code }}" href="{{ config.extra.website_url }}/docs/{{ code }}/{{ page.url }}" />
     {% endfor %}
-    <link rel="alternate" hreflang="x-default" href="{{ config.extra.website_url }}/docs/{% if config.extra.version_prefix %}{{ config.extra.version_prefix }}/{% endif %}en/{{ page.url }}" />
+    <link rel="alternate" hreflang="x-default" href="{{ config.extra.website_url }}/docs/en/{{ page.url }}" />
 {% endif %}
 
 {% for prefetch_item in prefetch_items %}

website/templates/docs/amp.html

@@ -7,7 +7,7 @@
     <link rel="canonical" href="{{ canonical_url or 'https://clickhouse.tech/' }}" />
     <meta name="viewport" content="width=device-width,minimum-scale=1,initial-scale=1">
     {% include "templates/docs/ld_json.html" %}
-    {% if config.extra.single_page or config.extra.version_prefix %}
+    {% if config.extra.single_page %}
       <meta name="robots" content="noindex,follow" />
     {% endif %}
     <style amp-boilerplate>body{-webkit-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-moz-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-ms-animation:-amp-start 8s steps(1,end) 0s 1 normal both;animation:-amp-start 8s steps(1,end) 0s 1 normal both}@-webkit-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-moz-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-ms-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-o-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}</style><noscript><style amp-boilerplate>body{-webkit-animation:none;-moz-animation:none;-ms-animation:none;animation:none}</style></noscript>

website/templates/docs/nav.html

@@ -40,7 +40,7 @@
                         <div class="dropdown-menu bg-dark" aria-labelledby="lang-dropdown">
                             {% for code, name in config.extra.languages.items() %}
                             <a class="dropdown-item{% if language == code %} disabled{% endif %}"
-                               href="/docs/{% if config.extra.version_prefix %}{{ config.extra.version_prefix }}/{% endif %}{{ code }}/{{ page.url }}">
+                               href="/docs/{{ code }}/{{ page.url }}">
                                 <img src="/images/flags/{{ code }}.svg" alt="" title="" width="32" class="d-inline-block mr-2" />{{ name }}
                             </a>
                             {% endfor %}

website/templates/docs/sidebar.html

@@ -8,23 +8,6 @@
                 <img src="/images/mkdocs/single.svg" alt="{{ _('Single-page version') }}" title="{{ _('Single-page version') }}" />
             </a>
         </div>
-        <div class="dropdown mb-3">
-            <a class="btn btn-dark dropdown-toggle text-muted" href="#" role="button" id="select-version" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-                {{ _('Version') }}: <span class="text-light">{{ config.extra.version_prefix or _('latest') }}</span>
-            </a>
-
-            <div class="dropdown-menu bg-dark text-muted p-1" aria-labelledby="select-version">
-                <a class="dropdown-item{% if not config.extra.version_prefix %} disable{% endif %}"
-                   href="/docs/{{ language }}/">
-                    <span class="text-light">{{ _('latest') }}</span>
-                </a>
-
-                {% for release in config.extra.stable_releases %}
-                    <a class="dropdown-item{% if release.0 == config.extra.version_prefix %} disable{% endif %}"
-                       href="/docs/{{ release.0 }}/{{ language }}/">{{ release.0 }}{% if release.1.2 %} LTS{% endif %}</a>
-                {% endfor %}
-            </div>
-        </div>
         {% if not single_page %}
         <nav id="sidebar-nav" class="nav flex-column mb-5">
             {% for nav_item in nav %}