ClickHouse/docs/concatenate.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# - Single-page document.
#   - Requirements to the md-souces:
#       - Don't use links without anchors. It means, that you can not just link file. You should specify an anchor at the top of the file and then link to this anchor
#       - Anchors should be unique through whole document.
#   - Implementation:
#       - Script gets list of the file from the `pages` section of `mkdocs.yml`. It gets commented files too, and it right.
#       - Files are concatenated by order with incrementing level of headers in all files except the first one
#       - Script converts links to other files into inside page links.
#         - Skipping links started with 'http'
#         - Not http-links with anchor are cutted to the anchor sign (#).
#         - For not http-links without anchor script logs an error and cuts them from the resulting single-page document.


import codecs
import sys
import re
import os

if len(sys.argv) < 2:
    print "Usage: concatenate.py language_dir"
    print "Example: concatenate.py ru"
    sys.exit(1)

if not os.path.exists(sys.argv[1]):
    print "Pass language_dir correctly. For example, 'ru'."
    sys.exit(2)

# Configuration
PROJ_CONFIG = 'mkdocs_' + sys.argv[1] + '.yml'
SINGLE_PAGE = sys.argv[1] + '_single_page/index.md'
DOCS_DIR = sys.argv[1] + '/'

# 1. Open mkdocs.yml file and read `pages` configuration to get an ordered list of files
cfg_file = open(PROJ_CONFIG)

files_to_concatenate = []

for l in cfg_file:
    if('.md' in l) and ('single_page' not in l):
        path = (l[l.index(':') + 1:]).strip(" '\n")
        files_to_concatenate.append(path)

print str(len(files_to_concatenate)) + " files will be concatenated into single md-file.\nFiles:"
print files_to_concatenate

# 2. Concatenate all of the files in the list

single_page_file = open(SINGLE_PAGE, 'w')

first_file = True

for path in files_to_concatenate:

    single_page_file.write('\n\n')

    file = open(DOCS_DIR + path)

    # function is passed into re.sub() to process links
    def link_proc(matchObj):
        text, link = matchObj.group().strip('[)').split('](')
        if link.startswith('http'):
            return '[' + text + '](' + link + ')'
        else:
            sharp_pos = link.find('#')
            if sharp_pos > -1:
                return '[' + text + '](' + link[sharp_pos:] + ')'
            else:
                print 'ERROR: Link [' + text + '](' + link + ') in file ' + path + ' has no anchor. Please provide it.'
                # return '['+text+'](#'+link.replace('/','-')+')'

    for l in file:
        # Processing links in a string
        l = re.sub(r'\[.+?\]\(.+?\)', link_proc, l)

        # Correcting headers levels
        if not first_file:
            if(l.startswith('#')):
                l = '#' + l
        else:
            first_file = False

        single_page_file.write(l)

single_page_file.close()