ClickHouse/docs/tools/concatenate.py

73 lines
2.7 KiB
Python
Raw Normal View History

2017-12-29 12:43:05 +00:00
# -*- coding: utf-8 -*-
# - Single-page document.
# - Requirements to the md-souces:
# - Don't use links without anchors. It means, that you can not just link file. You should specify an anchor at the top of the file and then link to this anchor
# - Anchors should be unique through whole document.
# - Implementation:
# - Script gets list of the file from the `pages` section of `mkdocs.yml`. It gets commented files too, and it right.
# - Files are concatenated by order with incrementing level of headers in all files except the first one
# - Script converts links to other files into inside page links.
# - Skipping links started with 'http'
# - Not http-links with anchor are cutted to the anchor sign (#).
# - For not http-links without anchor script logs an error and cuts them from the resulting single-page document.
2017-12-29 12:43:05 +00:00
import logging
2017-12-29 12:43:05 +00:00
import re
import os
def concatenate(lang, docs_path, single_page_file):
2017-12-29 12:43:05 +00:00
proj_config = os.path.join(docs_path, 'mkdocs_%s.yml' % lang)
lang_path = os.path.join(docs_path, lang)
2017-12-29 12:43:05 +00:00
with open(proj_config) as cfg_file:
2018-07-09 12:35:19 +00:00
files_to_concatenate = []
for l in cfg_file:
if '.md' in l and 'single_page' not in l:
path = (l[l.index(':') + 1:]).strip(" '\n")
files_to_concatenate.append(path)
2017-12-29 12:43:05 +00:00
2018-07-11 08:14:23 +00:00
logging.info(
str(len(files_to_concatenate)) +
2018-07-11 08:17:36 +00:00
' files will be concatenated into single md-file.')
logging.debug('Concatenating: ' + ', '.join(files_to_concatenate))
2017-12-29 12:43:05 +00:00
first_file = True
2017-12-29 12:43:05 +00:00
for path in files_to_concatenate:
2017-12-29 12:43:05 +00:00
single_page_file.write('\n\n')
2017-12-29 12:43:05 +00:00
with open(os.path.join(lang_path, path)) as f:
2017-12-29 12:43:05 +00:00
# function is passed into re.sub() to process links
def link_proc(matchObj):
text, link = matchObj.group().strip('[)').split('](')
if link.startswith('http'):
return '[' + text + '](' + link + ')'
else:
sharp_pos = link.find('#')
if sharp_pos > -1:
return '[' + text + '](' + link[sharp_pos:] + ')'
2018-07-09 12:35:19 +00:00
else:
raise RuntimeError(
2018-07-11 08:14:23 +00:00
'ERROR: Link [' + text + '](' + link + ') in file ' +
path + ' has no anchor. Please provide it.')
2017-12-29 12:43:05 +00:00
for l in f:
# Processing links in a string
l = re.sub(r'\[.+?\]\(.+?\)', link_proc, l)
2017-12-29 12:43:05 +00:00
# Correcting headers levels
if not first_file:
if l.startswith('#'):
l = '#' + l
else:
first_file = False
2017-12-29 12:43:05 +00:00
single_page_file.write(l)
2017-12-29 12:43:05 +00:00
single_page_file.flush()