Merge pull request #8895 from blinkov/master

Documentation infrastructure improvements

docs/tools/build.py

@@ -11,7 +11,6 @@ import subprocess
 import sys
 import time
 
-import markdown.extensions
 import markdown.util
 
 from mkdocs import config
@@ -212,8 +211,10 @@ def build_redirects(args):
 
 
 def build_docs(args):
+    tasks = []
     for lang in args.lang.split(','):
-        build_for_lang(lang, args)
+        tasks.append((lang, args,))
+    util.run_function_in_parallel(build_for_lang, tasks, threads=True)
 
 
 def build(args):

docs/tools/github.py

@@ -3,6 +3,7 @@ import copy
 import io
 import logging
 import os
+import sys
 import tarfile
 
 import requests
@@ -18,17 +19,22 @@ def choose_latest_releases():
         candidates += requests.get(url).json()
 
     for tag in candidates:
-        name = tag.get('name', '')
-        is_unstable = ('stable' not in name) and ('lts' not in name)
-        is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name)
-        if is_unstable or is_in_blacklist:
-            continue
-        major_version = '.'.join((name.split('.', 2))[:2])
-        if major_version not in seen:
-            seen[major_version] = (name, tag.get('tarball_url'),)
-            if len(seen) > 10:
-                break
-            
+        if isinstance(tag, dict):
+            name = tag.get('name', '')
+            is_unstable = ('stable' not in name) and ('lts' not in name)
+            is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name)
+            if is_unstable or is_in_blacklist:
+                continue
+            major_version = '.'.join((name.split('.', 2))[:2])
+            if major_version not in seen:
+                seen[major_version] = (name, tag.get('tarball_url'),)
+                if len(seen) > 10:
+                    break
+        else:
+            logging.fatal('Unexpected GitHub response: %s', str(candidates))
+            sys.exit(1)
+
+    logging.info('Found stable releases: %s', str(seen.keys()))
     return seen.items()
     
 
@@ -47,7 +53,9 @@ def process_release(args, callback, release):
 
 
 def build_releases(args, callback):
+    tasks = []
     for release in args.stable_releases:
-        process_release(args, callback, release)
+        tasks.append((args, callback, release,))
+    util.run_function_in_parallel(process_release, tasks)
 
 

docs/tools/release.sh

@@ -22,6 +22,8 @@ then
     rm -rf "${PUBLISH_DIR}" || true
     git clone git@github.com:ClickHouse/clickhouse.github.io.git "${PUBLISH_DIR}"
     cd "${PUBLISH_DIR}"
+    git config user.email "robot-clickhouse@yandex-team.ru"
+    git config user.name "robot-clickhouse"
     git rm -rf *
     git commit -a -m "wipe old release"
     cp -R "${BUILD_DIR}"/* .

docs/tools/util.py

@@ -1,7 +1,9 @@
 import contextlib
+import multiprocessing
 import os
 import shutil
 import tempfile
+import threading
 
 
 @contextlib.contextmanager
@@ -20,3 +22,13 @@ def autoremoved_file(path):
             yield handle
     finally:
         os.unlink(path)
+
+
+def run_function_in_parallel(func, args_list, threads=False):
+    processes = []
+    for task in args_list:
+        cls = threading.Thread if threads else multiprocessing.Process
+        processes.append(cls(target=func, args=task))
+        processes[-1].start()
+    for process in processes:
+        process.join()