ClickHouse/docs/tools/cmake_in_clickhouse_generator.py

170 lines
6.1 KiB
Python
Raw Normal View History

2020-09-15 19:32:42 +00:00
import re
2020-09-19 17:14:37 +00:00
import os
2020-09-15 21:35:06 +00:00
from typing import TextIO, List, Tuple, Optional, Dict
2020-09-15 19:32:42 +00:00
2020-09-19 16:42:36 +00:00
# name, default value, description
2020-09-15 19:32:42 +00:00
Entity = Tuple[str, str, str]
2020-09-19 16:42:36 +00:00
# https://regex101.com/r/R6iogw/12
2020-09-15 19:32:42 +00:00
cmake_option_regex: str = r"^\s*option\s*\(([A-Z_0-9${}]+)\s*(?:\"((?:.|\n)*?)\")?\s*(.*)?\).*$"
ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/"
name_str: str = "<a name=\"{anchor}\"></a>[`{name}`](" + ch_master_url + "{path}#L{line})"
default_anchor_str: str = "[`{name}`](#{anchor})"
2020-09-15 19:32:42 +00:00
2020-09-19 16:42:36 +00:00
comment_var_regex: str = r"\${(.+)}"
comment_var_replace: str = "`\\1`"
table_header: str = """
| Name | Default value | Description | Comment |
|------|---------------|-------------|---------|
"""
2020-09-15 21:35:06 +00:00
# Needed to detect conditional variables (those which are defined twice)
# name -> (path, values)
entities: Dict[str, Tuple[str, str]] = {}
2020-09-15 21:35:06 +00:00
def make_anchor(t: str) -> str:
return "".join(["-" if i == "_" else i.lower() for i in t if i.isalpha() or i == "_"])
2020-09-19 16:42:36 +00:00
def process_comment(comment: str) -> str:
return re.sub(comment_var_regex, comment_var_replace, comment, flags=re.MULTILINE)
def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> None:
2020-09-15 19:32:42 +00:00
(line, comment) = line_comment
2020-09-19 16:42:36 +00:00
(name, description, default) = entity
2020-09-15 19:32:42 +00:00
2020-09-19 16:42:36 +00:00
if name in entities:
2020-09-15 21:35:06 +00:00
return
2020-09-15 19:32:42 +00:00
2020-09-23 13:53:54 +00:00
# cannot escape the { in macro option description -> invalid AMP html
2020-09-23 13:58:16 +00:00
# Skipping "USE_INTERNAL_${LIB_NAME_UC}_LIBRARY"
if "LIB_NAME_UC" in name:
2020-09-23 13:53:54 +00:00
return
2020-09-15 21:12:37 +00:00
if len(default) == 0:
2020-09-19 16:42:36 +00:00
formatted_default: str = "`OFF`"
2020-09-15 19:32:42 +00:00
elif default[0] == "$":
2020-09-23 11:47:55 +00:00
formatted_default: str = "`{}`".format(default[2:-1])
2020-09-15 19:32:42 +00:00
else:
2020-09-19 16:42:36 +00:00
formatted_default: str = "`" + default + "`"
2020-09-15 19:32:42 +00:00
2020-09-19 16:42:36 +00:00
formatted_name: str = name_str.format(
anchor=make_anchor(name),
name=name,
2020-09-15 19:32:42 +00:00
path=path,
2020-09-29 08:59:40 +00:00
line=line)
2020-09-15 19:32:42 +00:00
2020-09-19 16:42:36 +00:00
formatted_description: str = "".join(description.split("\n"))
formatted_comment: str = process_comment(comment)
2020-09-19 16:42:36 +00:00
formatted_entity: str = "| {} | {} | {} | {} |".format(
formatted_name, formatted_default, formatted_description, formatted_comment)
2020-09-15 19:32:42 +00:00
2020-09-19 16:42:36 +00:00
entities[name] = path, formatted_entity
2020-09-28 13:28:36 +00:00
def process_file(root_path: str, file_path: str, file_name: str) -> None:
with open(os.path.join(file_path, file_name), 'r') as cmake_file:
2020-09-15 19:32:42 +00:00
contents: str = cmake_file.read()
def get_line_and_comment(target: str) -> Tuple[int, str]:
contents_list: List[str] = contents.split("\n")
comment: str = ""
for n, line in enumerate(contents_list):
2020-09-29 08:59:40 +00:00
if 'option' not in line.lower() or target not in line:
2020-09-15 19:32:42 +00:00
continue
for maybe_comment_line in contents_list[n - 1::-1]:
if not re.match("\s*#\s*", maybe_comment_line):
2020-09-15 19:32:42 +00:00
break
2020-09-17 15:46:55 +00:00
comment = re.sub("\s*#\s*", "", maybe_comment_line) + " " + comment
2020-09-29 08:59:40 +00:00
# line numbering starts with 1
return n + 1, comment
2020-09-15 19:32:42 +00:00
matches: Optional[List[Entity]] = re.findall(cmake_option_regex, contents, re.MULTILINE)
2020-09-29 08:59:40 +00:00
file_rel_path_with_name: str = os.path.join(file_path[len(root_path):], file_name)
if file_rel_path_with_name.startswith('/'):
file_rel_path_with_name = file_rel_path_with_name[1:]
2020-09-28 13:28:36 +00:00
2020-09-15 19:32:42 +00:00
if matches:
for entity in matches:
2020-09-28 13:28:36 +00:00
build_entity(file_rel_path_with_name, entity, get_line_and_comment(entity[0]))
2020-09-15 19:32:42 +00:00
2020-09-28 13:28:36 +00:00
def process_folder(root_path: str, name: str) -> None:
2020-09-23 11:47:55 +00:00
for root, _, files in os.walk(os.path.join(root_path, name)):
2020-09-15 19:32:42 +00:00
for f in files:
if f == "CMakeLists.txt" or ".cmake" in f:
2020-09-28 13:28:36 +00:00
process_file(root_path, root, f)
2020-09-15 19:32:42 +00:00
2020-09-28 12:47:10 +00:00
def generate_cmake_flags_files() -> None:
root_path: str = os.path.join(os.path.dirname(__file__), '..', '..')
2020-09-22 15:08:48 +00:00
output_file_name: str = os.path.join(root_path, "docs/en/development/cmake-in-clickhouse.md")
header_file_name: str = os.path.join(root_path, "docs/_includes/cmake_in_clickhouse_header.md")
footer_file_name: str = os.path.join(root_path, "docs/_includes/cmake_in_clickhouse_footer.md")
2020-09-15 21:35:06 +00:00
2020-09-28 13:28:36 +00:00
process_file(root_path, root_path, "CMakeLists.txt")
process_file(root_path, os.path.join(root_path, "programs"), "CMakeLists.txt")
2020-09-19 17:14:37 +00:00
2020-09-19 18:41:38 +00:00
process_folder(root_path, "base")
process_folder(root_path, "cmake")
process_folder(root_path, "src")
2020-09-15 21:35:06 +00:00
2020-09-15 19:32:42 +00:00
with open(output_file_name, "w") as f:
with open(header_file_name, "r") as header:
f.write(header.read())
sorted_keys: List[str] = sorted(entities.keys())
ignored_keys: List[str] = []
f.write("### ClickHouse modes\n" + table_header)
for k in sorted_keys:
if k.startswith("ENABLE_CLICKHOUSE_"):
f.write(entities[k][1] + "\n")
ignored_keys.append(k)
2020-09-23 11:47:55 +00:00
f.write("\n### External libraries\nNote that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.\n" +
2020-09-19 16:51:21 +00:00
table_header)
2020-09-19 16:42:36 +00:00
for k in sorted_keys:
2020-09-23 11:47:55 +00:00
if k.startswith("ENABLE_") and ".cmake" in entities[k][0]:
f.write(entities[k][1] + "\n")
ignored_keys.append(k)
2020-09-28 12:47:10 +00:00
f.write("\n\n### External libraries system/bundled mode\n" + table_header)
for k in sorted_keys:
if k.startswith("USE_INTERNAL_"):
f.write(entities[k][1] + "\n")
ignored_keys.append(k)
2020-09-28 12:47:10 +00:00
f.write("\n\n### Other flags\n" + table_header)
for k in sorted(set(sorted_keys).difference(set(ignored_keys))):
f.write(entities[k][1] + "\n")
2020-09-15 19:32:42 +00:00
with open(footer_file_name, "r") as footer:
f.write(footer.read())
2020-09-19 18:41:38 +00:00
2021-09-21 07:04:21 +00:00
other_languages = ["docs/ja/development/cmake-in-clickhouse.md",
"docs/zh/development/cmake-in-clickhouse.md",
"docs/ru/development/cmake-in-clickhouse.md"]
for lang in other_languages:
2021-10-12 20:15:49 +00:00
other_file_name = os.path.join(root_path, lang)
if os.path.exists(other_file_name):
os.unlink(other_file_name)
os.symlink(output_file_name, other_file_name)
2020-09-19 18:41:38 +00:00
2020-09-19 18:45:10 +00:00
if __name__ == '__main__':
2020-09-28 12:47:10 +00:00
generate_cmake_flags_files()