ClickHouse/docs/tools/easy_diff.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys
import argparse
import subprocess
import contextlib
from git import cmd
from tempfile import NamedTemporaryFile

SCRIPT_DESCRIPTION = '''
    usage: ./easy_diff.py language/document path

    Show the difference between a language document and an English document.

    This script is based on the assumption that documents in other languages are fully synchronized with the en document at a commit.

    For example:
        Execute:
            ./easy_diff.py --no-pager zh/data_types
        Output:
            Need translate document:~/ClickHouse/docs/en/data_types/uuid.md
            Need link document:~/ClickHouse/docs/en/data_types/decimal.md to ~/ClickHouse/docs/zh/data_types/decimal.md
            diff --git a/docs/en/data_types/domains/ipv6.md b/docs/en/data_types/domains/ipv6.md
            index 1bfbe3400b..e2abaff017 100644
            --- a/docs/en/data_types/domains/ipv6.md
            +++ b/docs/en/data_types/domains/ipv6.md
            @@ -4,13 +4,13 @@

             ### Basic Usage

            -``` sql
            +```sql
             CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY url;

             DESCRIBE TABLE hits;
             ```

            -```
            +```text
             ┌─name─┬─type───┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┐
             │ url  │ String │              │                    │         │                  │
             │ from │ IPv6   │              │                    │         │                  │
            @@ -19,19 +19,19 @@ DESCRIBE TABLE hits;

             OR you can use `IPv6` domain as a key:

            -``` sql
            +```sql
             CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from;
            ... MORE

    OPTIONS:
        -h, --help  show this help message and exit
        --no-pager  use stdout as difference result output
'''

SCRIPT_PATH = os.path.abspath(__file__)
CLICKHOUSE_REPO_HOME = os.path.join(os.path.dirname(SCRIPT_PATH), '..', '..')
SCRIPT_COMMAND_EXECUTOR = cmd.Git(CLICKHOUSE_REPO_HOME)

SCRIPT_COMMAND_PARSER = argparse.ArgumentParser(add_help=False)
SCRIPT_COMMAND_PARSER.add_argument('path', type=bytes, nargs='?', default=None)
SCRIPT_COMMAND_PARSER.add_argument('--no-pager', action='store_true', default=False)
SCRIPT_COMMAND_PARSER.add_argument('-h', '--help', action='store_true', default=False)


def execute(commands):
    return SCRIPT_COMMAND_EXECUTOR.execute(commands)


def get_hash(file_name):
    return execute(['git', 'log', '-n', '1', '--pretty=format:"%H"', file_name])


def diff_file(reference_file, working_file, out):
    if not os.path.exists(reference_file):
        raise RuntimeError('reference file [' + os.path.abspath(reference_file) + '] is not exists.')

    if os.path.islink(working_file):
        out.writelines(["Need translate document:" + os.path.abspath(reference_file)])
    elif not os.path.exists(working_file):
        out.writelines(['Need link document ' + os.path.abspath(reference_file) + ' to ' + os.path.abspath(working_file)])
    elif get_hash(working_file) != get_hash(reference_file):
        out.writelines([(execute(['git', 'diff', get_hash(working_file).strip('"'), reference_file]).encode('utf-8'))])

    return 0


def diff_directory(reference_directory, working_directory, out):
    if not os.path.isdir(reference_directory):
        return diff_file(reference_directory, working_directory, out)

    for list_item in os.listdir(reference_directory):
        working_item = os.path.join(working_directory, list_item)
        reference_item = os.path.join(reference_directory, list_item)
        if diff_file(reference_item, working_item, out) if os.path.isfile(reference_item) else diff_directory(reference_item, working_item, out) != 0:
            return 1

    return 0


def find_language_doc(custom_document, other_language='en', children=[]):
    if len(custom_document) == 0:
        raise RuntimeError('The ' + os.path.join(custom_document, *children) + " is not in docs directory.")

    if os.path.samefile(os.path.join(CLICKHOUSE_REPO_HOME, 'docs'), custom_document):
        return os.path.join(CLICKHOUSE_REPO_HOME, 'docs', other_language, *children[1:])
    children.insert(0, os.path.split(custom_document)[1])
    return find_language_doc(os.path.split(custom_document)[0], other_language, children)


class ToPager:
    def __init__(self, temp_named_file):
        self.temp_named_file = temp_named_file

    def writelines(self, lines):
        self.temp_named_file.writelines(lines)

    def close(self):
        self.temp_named_file.flush()
        git_pager = execute(['git', 'var', 'GIT_PAGER'])
        subprocess.check_call([git_pager, self.temp_named_file.name])
        self.temp_named_file.close()


class ToStdOut:
    def writelines(self, lines):
        self.system_stdout_stream.writelines(lines)

    def close(self):
        self.system_stdout_stream.flush()

    def __init__(self, system_stdout_stream):
        self.system_stdout_stream = system_stdout_stream


if __name__ == '__main__':
    arguments = SCRIPT_COMMAND_PARSER.parse_args()
    if arguments.help or not arguments.path:
        sys.stdout.write(SCRIPT_DESCRIPTION)
        sys.exit(0)

    working_language = os.path.join(CLICKHOUSE_REPO_HOME, 'docs', arguments.path)
    with contextlib.closing(ToStdOut(sys.stdout) if arguments.no_pager else ToPager(NamedTemporaryFile('r+'))) as writer:
        exit(diff_directory(find_language_doc(working_language), working_language, writer))