import collections import contextlib import datetime import multiprocessing import os import shutil import sys import socket import tempfile import threading import jinja2 import yaml @contextlib.contextmanager def temp_dir(): path = tempfile.mkdtemp(dir=os.environ.get("TEMP")) try: yield path finally: shutil.rmtree(path) @contextlib.contextmanager def cd(new_cwd): old_cwd = os.getcwd() os.chdir(new_cwd) try: yield finally: os.chdir(old_cwd) def get_free_port(): with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(("", 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) return s.getsockname()[1] def run_function_in_parallel(func, args_list, threads=False): processes = [] exit_code = 0 for task in args_list: cls = threading.Thread if threads else multiprocessing.Process processes.append(cls(target=func, args=task)) processes[-1].start() for process in processes: process.join() if not threads: if process.exitcode and not exit_code: exit_code = process.exitcode if exit_code: sys.exit(exit_code) def read_md_file(path): in_meta = False meta = {} meta_text = [] content = [] if os.path.exists(path): with open(path, "r") as f: for line in f: if line.startswith("---"): if in_meta: in_meta = False meta = yaml.full_load("".join(meta_text)) else: in_meta = True else: if in_meta: meta_text.append(line) else: content.append(line) return meta, "".join(content) def write_md_file(path, meta, content): dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname) with open(path, "w") as f: if meta: print("---", file=f) yaml.dump(meta, f) print("---", file=f) if not content.startswith("\n"): print("", file=f) f.write(content) def represent_ordereddict(dumper, data): value = [] for item_key, item_value in data.items(): node_key = dumper.represent_data(item_key) node_value = dumper.represent_data(item_value) value.append((node_key, node_value)) return yaml.nodes.MappingNode("tag:yaml.org,2002:map", value) yaml.add_representer(collections.OrderedDict, represent_ordereddict) def init_jinja2_filters(env): import amp import website chunk_size = 10240 env.filters["chunks"] = lambda line: [ line[i : i + chunk_size] for i in range(0, len(line), chunk_size) ] env.filters["html_to_amp"] = amp.html_to_amp env.filters["adjust_markdown_html"] = website.adjust_markdown_html env.filters["to_rfc882"] = lambda d: datetime.datetime.strptime( d, "%Y-%m-%d" ).strftime("%a, %d %b %Y %H:%M:%S GMT") def init_jinja2_env(args): import mdx_clickhouse env = jinja2.Environment( loader=jinja2.FileSystemLoader( [args.website_dir, os.path.join(args.docs_dir, "_includes")] ), extensions=["jinja2.ext.i18n", "jinja2_highlight.HighlightExtension"], ) env.extend(jinja2_highlight_cssclass="syntax p-3 my-3") translations_dir = os.path.join(args.website_dir, "locale") env.install_gettext_translations( mdx_clickhouse.get_translations(translations_dir, "en"), newstyle=True ) init_jinja2_filters(env) return env