Merge pull request #33175 from Alex-Burmak/clickhouse-diagnostics

ClickHouse diagnostics tool

utils/clickhouse-diagnostics/clickhouse-diagnostics

@@ -0,0 +1,960 @@
+#!/usr/bin/env python3
+
+import argparse
+import gzip
+import io
+import json
+import socket
+import subprocess
+import sys
+from copy import deepcopy
+from datetime import datetime
+from typing import MutableMapping
+
+import jinja2
+import requests
+import sqlparse
+import tenacity
+import xmltodict
+import yaml
+
+SELECT_VERSION = r'SELECT version()'
+
+SELECT_UPTIME = r'''
+{% if version_ge('21.3') -%}
+SELECT formatReadableTimeDelta(uptime())
+{% else -%}
+SELECT
+    toString(floor(uptime() / 3600 / 24)) || ' days ' ||
+    toString(floor(uptime() % (24 * 3600) / 3600, 1)) || ' hours'
+{% endif -%}
+'''
+
+SELECT_SYSTEM_TABLES = "SELECT name FROM system.tables WHERE database = 'system'"
+
+SELECT_DATABASE_ENGINES = r'''SELECT
+    engine,
+    count() "count"
+FROM system.databases
+GROUP BY engine
+'''
+
+SELECT_DATABASES = r'''SELECT
+    name,
+    engine,
+    tables,
+    partitions,
+    parts,
+    formatReadableSize(bytes_on_disk) "disk_size"
+FROM system.databases db
+LEFT JOIN
+(
+    SELECT
+        database,
+        uniq(table) "tables",
+        uniq(table, partition) "partitions",
+        count() AS parts,
+        sum(bytes_on_disk) "bytes_on_disk"
+    FROM system.parts
+    WHERE active
+    GROUP BY database
+) AS db_stats ON db.name = db_stats.database
+ORDER BY bytes_on_disk DESC
+LIMIT 10
+'''
+
+SELECT_TABLE_ENGINES = r'''SELECT
+    engine,
+    count() "count"
+FROM system.tables
+WHERE database != 'system'
+GROUP BY engine
+'''
+
+SELECT_DICTIONARIES = r'''SELECT
+    source,
+    type,
+    status,
+    count() "count"
+FROM system.dictionaries
+GROUP BY source, type, status
+ORDER BY status DESC, source
+'''
+
+SELECT_ACCESS = "SHOW ACCESS"
+
+SELECT_QUOTA_USAGE = "SHOW QUOTA"
+
+SELECT_REPLICAS = r'''SELECT
+    database,
+    table,
+    is_leader,
+    is_readonly,
+    absolute_delay,
+    queue_size,
+    inserts_in_queue,
+    merges_in_queue
+FROM system.replicas
+ORDER BY absolute_delay DESC
+LIMIT 10
+'''
+
+SELECT_REPLICATION_QUEUE = r'''SELECT
+    database,
+    table,
+    replica_name,
+    position,
+    node_name,
+    type,
+    source_replica,
+    parts_to_merge,
+    new_part_name,
+    create_time,
+    required_quorum,
+    is_detach,
+    is_currently_executing,
+    num_tries,
+    last_attempt_time,
+    last_exception,
+    concat('time: ', toString(last_postpone_time), ', number: ', toString(num_postponed), ', reason: ', postpone_reason) postpone
+FROM system.replication_queue
+ORDER BY create_time ASC
+LIMIT 20
+'''
+
+SELECT_REPLICATED_FETCHES = r'''SELECT
+    database,
+    table,
+    round(elapsed, 1) "elapsed",
+    round(100 * progress, 1) "progress",
+    partition_id,
+    result_part_name,
+    result_part_path,
+    total_size_bytes_compressed,
+    bytes_read_compressed,
+    source_replica_path,
+    source_replica_hostname,
+    source_replica_port,
+    interserver_scheme,
+    to_detached,
+    thread_id
+FROM system.replicated_fetches
+'''
+
+SELECT_PARTS_PER_TABLE = r'''SELECT
+        database,
+    table,
+    count() "partitions",
+    sum(part_count) "parts",
+    max(part_count) "max_parts_per_partition"
+FROM
+(
+    SELECT
+        database,
+        table,
+        partition,
+        count() "part_count"
+    FROM system.parts
+    WHERE active
+    GROUP BY database, table, partition
+) partitions
+GROUP BY database, table
+ORDER BY max_parts_per_partition DESC
+LIMIT 10
+'''
+
+SELECT_MERGES = r'''SELECT
+    database,
+    table,
+    round(elapsed, 1) "elapsed",
+    round(100 * progress, 1) "progress",
+    is_mutation,
+    partition_id,
+{% if version_ge('20.3') -%}
+    result_part_path,
+    source_part_paths,
+{% endif -%}
+    num_parts,
+    formatReadableSize(total_size_bytes_compressed) "total_size_compressed",
+    formatReadableSize(bytes_read_uncompressed) "read_uncompressed",
+    formatReadableSize(bytes_written_uncompressed) "written_uncompressed",
+    columns_written,
+{% if version_ge('20.3') -%}
+    formatReadableSize(memory_usage) "memory_usage",
+    thread_id
+{% else -%}
+    formatReadableSize(memory_usage) "memory_usage"
+{% endif -%}
+FROM system.merges
+'''
+
+SELECT_MUTATIONS = r'''SELECT
+    database,
+    table,
+    mutation_id,
+    command,
+    create_time,
+{% if version_ge('20.3') -%}
+    parts_to_do_names,
+{% endif -%}
+    parts_to_do,
+    is_done,
+    latest_failed_part,
+    latest_fail_time,
+    latest_fail_reason
+FROM system.mutations
+WHERE NOT is_done
+ORDER BY create_time DESC
+'''
+
+SELECT_RECENT_DATA_PARTS = r'''SELECT
+    database,
+    table,
+    engine,
+    partition_id,
+    name,
+{% if version_ge('20.3') -%}
+    part_type,
+{% endif -%}
+    active,
+    level,
+{% if version_ge('20.3') -%}
+    disk_name,
+{% endif -%}
+    path,
+    marks,
+    rows,
+    bytes_on_disk,
+    data_compressed_bytes,
+    data_uncompressed_bytes,
+    marks_bytes,
+    modification_time,
+    remove_time,
+    refcount,
+    is_frozen,
+    min_date,
+    max_date,
+    min_time,
+    max_time,
+    min_block_number,
+    max_block_number
+FROM system.parts
+WHERE modification_time > now() - INTERVAL 3 MINUTE
+ORDER BY modification_time DESC
+'''
+
+SELECT_DETACHED_DATA_PARTS = r'''SELECT
+    database,
+    table,
+    partition_id,
+    name,
+    disk,
+    reason,
+    min_block_number,
+    max_block_number,
+    level
+FROM system.detached_parts
+'''
+
+SELECT_PROCESSES = r'''SELECT
+    elapsed,
+    query_id,
+    {% if normalize_queries -%}
+    normalizeQuery(query) AS normalized_query,
+    {% else -%}
+    query,
+    {% endif -%}
+    is_cancelled,
+    concat(toString(read_rows), ' rows / ', formatReadableSize(read_bytes)) AS read,
+    concat(toString(written_rows), ' rows / ', formatReadableSize(written_bytes)) AS written,
+    formatReadableSize(memory_usage) AS "memory usage",
+    user,
+    multiIf(empty(client_name), http_user_agent, concat(client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch))) AS client,
+    {% if version_ge('21.3') -%}
+    thread_ids,
+    {% endif -%}
+    {% if version_ge('21.8') -%}
+    ProfileEvents,
+    Settings
+    {% else -%}
+    ProfileEvents.Names,
+    ProfileEvents.Values,
+    Settings.Names,
+    Settings.Values
+    {% endif -%}
+FROM system.processes
+ORDER BY elapsed DESC
+'''
+
+SELECT_TOP_QUERIES_BY_DURATION = r'''SELECT
+    type,
+    query_start_time,
+    query_duration_ms,
+    query_id,
+    query_kind,
+    is_initial_query,
+    {% if normalize_queries -%}
+    normalizeQuery(query) AS normalized_query,
+    {% else -%}
+    query,
+    {% endif -%}
+    concat(toString(read_rows), ' rows / ', formatReadableSize(read_bytes)) AS read,
+    concat(toString(written_rows), ' rows / ', formatReadableSize(written_bytes)) AS written,
+    concat(toString(result_rows), ' rows / ', formatReadableSize(result_bytes)) AS result,
+    formatReadableSize(memory_usage) AS "memory usage",
+    exception,
+    '\n' || stack_trace AS stack_trace,
+    user,
+    initial_user,
+    multiIf(empty(client_name), http_user_agent, concat(client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch))) AS client,
+    client_hostname,
+    {% if version_ge('21.3') -%}
+    databases,
+    tables,
+    columns,
+    used_aggregate_functions,
+    used_aggregate_function_combinators,
+    used_database_engines,
+    used_data_type_families,
+    used_dictionaries,
+    used_formats,
+    used_functions,
+    used_storages,
+    used_table_functions,
+    thread_ids,
+    {% endif -%}
+    {% if version_ge('21.8') -%}
+    ProfileEvents,
+    Settings
+    {% else -%}
+    ProfileEvents.Names,
+    ProfileEvents.Values,
+    Settings.Names,
+    Settings.Values
+    {% endif -%}
+FROM system.query_log
+WHERE type != 'QueryStart'
+  AND event_date >= today() - 1
+  AND event_time >= now() - INTERVAL 1 DAY
+ORDER BY query_duration_ms DESC
+LIMIT 10
+'''
+
+SELECT_TOP_QUERIES_BY_MEMORY_USAGE = r'''SELECT
+    type,
+    query_start_time,
+    query_duration_ms,
+    query_id,
+    query_kind,
+    is_initial_query,
+    {% if normalize_queries -%}
+    normalizeQuery(query) AS normalized_query,
+    {% else -%}
+    query,
+    {% endif -%}
+    concat(toString(read_rows), ' rows / ', formatReadableSize(read_bytes)) AS read,
+    concat(toString(written_rows), ' rows / ', formatReadableSize(written_bytes)) AS written,
+    concat(toString(result_rows), ' rows / ', formatReadableSize(result_bytes)) AS result,
+    formatReadableSize(memory_usage) AS "memory usage",
+    exception,
+    '\n' || stack_trace AS stack_trace,
+    user,
+    initial_user,
+    multiIf(empty(client_name), http_user_agent, concat(client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch))) AS client,
+    client_hostname,
+    {% if version_ge('21.3') -%}
+    databases,
+    tables,
+    columns,
+    used_aggregate_functions,
+    used_aggregate_function_combinators,
+    used_database_engines,
+    used_data_type_families,
+    used_dictionaries,
+    used_formats,
+    used_functions,
+    used_storages,
+    used_table_functions,
+    thread_ids,
+    {% endif -%}
+    {% if version_ge('21.8') -%}
+    ProfileEvents,
+    Settings
+    {% else -%}
+    ProfileEvents.Names,
+    ProfileEvents.Values,
+    Settings.Names,
+    Settings.Values
+    {% endif -%}
+FROM system.query_log
+WHERE type != 'QueryStart'
+  AND event_date >= today() - 1
+  AND event_time >= now() - INTERVAL 1 DAY
+ORDER BY memory_usage DESC
+LIMIT 10
+'''
+
+SELECT_FAILED_QUERIES = r'''SELECT
+    type,
+    query_start_time,
+    query_duration_ms,
+    query_id,
+    query_kind,
+    is_initial_query,
+    {% if normalize_queries -%}
+    normalizeQuery(query) AS normalized_query,
+    {% else -%}
+    query,
+    {% endif -%}
+    concat(toString(read_rows), ' rows / ', formatReadableSize(read_bytes)) AS read,
+    concat(toString(written_rows), ' rows / ', formatReadableSize(written_bytes)) AS written,
+    concat(toString(result_rows), ' rows / ', formatReadableSize(result_bytes)) AS result,
+    formatReadableSize(memory_usage) AS "memory usage",
+    exception,
+    '\n' || stack_trace AS stack_trace,
+    user,
+    initial_user,
+    multiIf(empty(client_name), http_user_agent, concat(client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch))) AS client,
+    client_hostname,
+    {% if version_ge('21.3') -%}
+    databases,
+    tables,
+    columns,
+    used_aggregate_functions,
+    used_aggregate_function_combinators,
+    used_database_engines,
+    used_data_type_families,
+    used_dictionaries,
+    used_formats,
+    used_functions,
+    used_storages,
+    used_table_functions,
+    thread_ids,
+    {% endif -%}
+    {% if version_ge('21.8') -%}
+    ProfileEvents,
+    Settings
+    {% else -%}
+    ProfileEvents.Names,
+    ProfileEvents.Values,
+    Settings.Names,
+    Settings.Values
+    {% endif -%}
+FROM system.query_log
+WHERE type != 'QueryStart'
+  AND event_date >= today() - 1
+  AND event_time >= now() - INTERVAL 1 DAY
+  AND exception != ''
+ORDER BY query_start_time DESC
+LIMIT 10
+'''
+
+SELECT_STACK_TRACES = r'''SELECT
+    '\n' || arrayStringConcat(
+       arrayMap(
+           x,
+           y -> concat(x, ': ', y),
+           arrayMap(x -> addressToLine(x), trace),
+           arrayMap(x -> demangle(addressToSymbol(x)), trace)),
+       '\n') AS trace
+FROM system.stack_trace
+'''
+
+SELECT_CRASH_LOG = r'''SELECT
+    event_time,
+    signal,
+    thread_id,
+    query_id,
+    '\n' || arrayStringConcat(trace_full, '\n') AS trace,
+    version
+FROM system.crash_log
+ORDER BY event_time DESC
+'''
+
+
+def retry(exception_types, max_attempts=5, max_interval=5):
+    """
+    Function decorator that retries wrapped function on failures.
+    """
+    return tenacity.retry(
+        retry=tenacity.retry_if_exception_type(exception_types),
+        wait=tenacity.wait_random_exponential(multiplier=0.5, max=max_interval),
+        stop=tenacity.stop_after_attempt(max_attempts),
+        reraise=True)
+
+
+class ClickhouseError(Exception):
+    """
+    ClickHouse interaction error.
+    """
+
+    def __init__(self, response):
+        self.response = response
+        super().__init__(self.response.text.strip())
+
+
+class ClickhouseClient:
+    """
+    ClickHouse client.
+    """
+
+    def __init__(self, *, host, port=8123, user=None):
+        self._session = requests.Session()
+        if user:
+            self._session.headers['X-ClickHouse-User'] = user
+        self._url = f'http://{host}:{port}'
+        self._timeout = 60
+        self._ch_version = None
+
+    @property
+    def clickhouse_version(self):
+        if self._ch_version is None:
+            self._ch_version = self.query(SELECT_VERSION)
+
+        return self._ch_version
+
+    @retry(requests.exceptions.ConnectionError)
+    def query(self, query, query_args=None, format=None, post_data=None, timeout=None, echo=False, dry_run=False):
+        """
+        Execute query.
+        """
+        if query_args:
+            query = self.render_query(query, **query_args)
+
+        if format:
+            query += f' FORMAT {format}'
+
+        if timeout is None:
+            timeout = self._timeout
+
+        if echo:
+            print(sqlparse.format(query, reindent=True), '\n')
+
+        if dry_run:
+            return None
+
+        try:
+            response = self._session.post(self._url,
+                                          params={
+                                              'query': query,
+                                          },
+                                          json=post_data,
+                                          timeout=timeout)
+
+            response.raise_for_status()
+
+            if format in ('JSON', 'JSONCompact'):
+                return response.json()
+
+            return response.text.strip()
+        except requests.exceptions.HTTPError as e:
+            raise ClickhouseError(e.response) from None
+
+    def render_query(self, query, **kwargs):
+        env = jinja2.Environment()
+
+        env.globals['version_ge'] = lambda version: version_ge(self.clickhouse_version, version)
+
+        template = env.from_string(query)
+        return template.render(kwargs)
+
+
+class ClickhouseConfig:
+    """
+    ClickHouse server configuration.
+    """
+
+    def __init__(self, config):
+        self._config = config
+
+    def dump(self, mask_secrets=True):
+        config = deepcopy(self._config)
+        if mask_secrets:
+            self._mask_secrets(config)
+
+        return xmltodict.unparse(config, pretty=True)
+
+    @classmethod
+    def load(cls):
+        return ClickhouseConfig(cls._load_config('/var/lib/clickhouse/preprocessed_configs/config.xml'))
+
+    @staticmethod
+    def _load_config(config_path):
+        with open(config_path, 'r') as file:
+            return xmltodict.parse(file.read())
+
+    @classmethod
+    def _mask_secrets(cls, config):
+        if isinstance(config, MutableMapping):
+            for key, value in list(config.items()):
+                if isinstance(value, MutableMapping):
+                    cls._mask_secrets(config[key])
+                elif key in ('password', 'secret_access_key', 'header', 'identity'):
+                    config[key] = '*****'
+
+
+class DiagnosticsData:
+    """
+    Diagnostics data.
+    """
+
+    def __init__(self, args, host):
+        self.args = args
+        self.host = host
+        self._sections = [{'section': None, 'data': {}}]
+
+    def add_string(self, name, value, section=None):
+        self._section(section)[name] = {
+            'type': 'string',
+            'value': value,
+        }
+
+    def add_xml_document(self, name, document, section=None):
+        self._section(section)[name] = {
+            'type': 'xml',
+            'value': document,
+        }
+
+    def add_query(self, name, query, result, section=None):
+        self._section(section)[name] = {
+            'type': 'query',
+            'query': query,
+            'result': result,
+        }
+
+    def add_command(self, name, command, result, section=None):
+        self._section(section)[name] = {
+            'type': 'command',
+            'command': command,
+            'result': result,
+        }
+
+    def dump(self, format):
+        if format.startswith('json'):
+            result = self._dump_json()
+        elif format.startswith('yaml'):
+            result = self._dump_yaml()
+        else:
+            result = self._dump_wiki()
+
+        if format.endswith('.gz'):
+            compressor = gzip.GzipFile(mode='wb', fileobj=sys.stdout.buffer)
+            compressor.write(result.encode())
+        else:
+            print(result)
+
+    def _section(self, name=None):
+        if self._sections[-1]['section'] != name:
+            self._sections.append({'section': name, 'data': {}})
+
+        return self._sections[-1]['data']
+
+    def _dump_json(self):
+        """
+        Dump diagnostic data in JSON format.
+        """
+        return json.dumps(self._sections, indent=2, ensure_ascii=False)
+
+    def _dump_yaml(self):
+        """
+        Dump diagnostic data in YAML format.
+        """
+        return yaml.dump(self._sections, default_flow_style=False, allow_unicode=True)
+
+    def _dump_wiki(self):
+        """
+        Dump diagnostic data in Yandex wiki format.
+        """
+
+        def _write_title(buffer, value):
+            buffer.write(f'### {value}\n')
+
+        def _write_subtitle(buffer, value):
+            buffer.write(f'#### {value}\n')
+
+        def _write_string_item(buffer, name, item):
+            value = item['value']
+            if value != '':
+                value = f'**{value}**'
+            buffer.write(f'{name}: {value}\n')
+
+        def _write_xml_item(buffer, section_name, name, item):
+            if section_name:
+                buffer.write(f'##### {name}\n')
+            else:
+                _write_subtitle(buffer, name)
+
+            _write_result(buffer, item['value'], format='XML')
+
+        def _write_query_item(buffer, section_name, name, item):
+            if section_name:
+                buffer.write(f'##### {name}\n')
+            else:
+                _write_subtitle(buffer, name)
+
+            _write_query(buffer, item['query'])
+            _write_result(buffer, item['result'])
+
+        def _write_command_item(buffer, section_name, name, item):
+            if section_name:
+                buffer.write(f'##### {name}\n')
+            else:
+                _write_subtitle(buffer, name)
+
+            _write_command(buffer, item['command'])
+            _write_result(buffer, item['result'])
+
+        def _write_unknown_item(buffer, section_name, name, item):
+            if section_name:
+                buffer.write(f'**{name}**\n')
+            else:
+                _write_subtitle(buffer, name)
+
+            json.dump(item, buffer, indent=2)
+
+        def _write_query(buffer, query):
+            buffer.write('**query**\n')
+            buffer.write('```sql\n')
+            buffer.write(query)
+            buffer.write('\n```\n')
+
+        def _write_command(buffer, command):
+            buffer.write('**command**\n')
+            buffer.write('```\n')
+            buffer.write(command)
+            buffer.write('\n```\n')
+
+        def _write_result(buffer, result, format=None):
+            buffer.write('**result**\n')
+            buffer.write(f'```{format}\n' if format else '```\n')
+            buffer.write(result)
+            buffer.write('\n```\n')
+
+        buffer = io.StringIO()
+
+        _write_title(buffer, f'Diagnostics data for host {self.host}')
+        for section in self._sections:
+            section_name = section['section']
+            if section_name:
+                _write_subtitle(buffer, section_name)
+
+            for name, item in section['data'].items():
+                if item['type'] == 'string':
+                    _write_string_item(buffer, name, item)
+                elif item['type'] == 'query':
+                    _write_query_item(buffer, section_name, name, item)
+                elif item['type'] == 'command':
+                    _write_command_item(buffer, section_name, name, item)
+                elif item['type'] == 'xml':
+                    _write_xml_item(buffer, section_name, name, item)
+                else:
+                    _write_unknown_item(buffer, section_name, name, item)
+
+        return buffer.getvalue()
+
+
+def main():
+    """
+    Program entry point.
+    """
+    args = parse_args()
+
+    host = socket.getfqdn()
+    timestamp = datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')
+    client = ClickhouseClient(host=host)
+    ch_config = ClickhouseConfig.load()
+    version = client.clickhouse_version
+    system_tables = [row[0] for row in execute_query(client, SELECT_SYSTEM_TABLES, format='JSONCompact')['data']]
+
+    diagnostics = DiagnosticsData(args, host)
+    diagnostics.add_string('Version', version)
+    diagnostics.add_string('Timestamp', timestamp)
+    diagnostics.add_string('Uptime', execute_query(client, SELECT_UPTIME))
+
+    diagnostics.add_xml_document('ClickHouse configuration', ch_config.dump())
+
+    if version_ge(version, '20.8'):
+        add_query(diagnostics, 'Access configuration',
+                  client=client,
+                  query=SELECT_ACCESS,
+                  format='TSVRaw')
+        add_query(diagnostics, 'Quotas',
+                  client=client,
+                  query=SELECT_QUOTA_USAGE,
+                  format='Vertical')
+
+    add_query(diagnostics, 'Database engines',
+              client=client,
+              query=SELECT_DATABASE_ENGINES,
+              format='PrettyCompactNoEscapes',
+              section='Schema')
+    add_query(diagnostics, 'Databases (top 10 by size)',
+              client=client,
+              query=SELECT_DATABASES,
+              format='PrettyCompactNoEscapes',
+              section='Schema')
+    add_query(diagnostics, 'Table engines',
+              client=client,
+              query=SELECT_TABLE_ENGINES,
+              format='PrettyCompactNoEscapes',
+              section='Schema')
+    add_query(diagnostics, 'Dictionaries',
+              client=client,
+              query=SELECT_DICTIONARIES,
+              format='PrettyCompactNoEscapes',
+              section='Schema')
+
+    add_query(diagnostics, 'Replicated tables (top 10 by absolute delay)',
+              client=client,
+              query=SELECT_REPLICAS,
+              format='PrettyCompactNoEscapes',
+              section='Replication')
+    add_query(diagnostics, 'Replication queue (top 20 oldest tasks)',
+              client=client,
+              query=SELECT_REPLICATION_QUEUE,
+              format='Vertical',
+              section='Replication')
+    if version_ge(version, '21.3'):
+        add_query(diagnostics, 'Replicated fetches',
+                  client=client,
+                  query=SELECT_REPLICATED_FETCHES,
+                  format='Vertical',
+                  section='Replication')
+
+    add_query(diagnostics, 'Top 10 tables by max parts per partition',
+              client=client,
+              query=SELECT_PARTS_PER_TABLE,
+              format='PrettyCompactNoEscapes')
+    add_query(diagnostics, 'Merges in progress',
+              client=client,
+              query=SELECT_MERGES,
+              format='Vertical')
+    add_query(diagnostics, 'Mutations in progress',
+              client=client,
+              query=SELECT_MUTATIONS,
+              format='Vertical')
+    add_query(diagnostics, 'Recent data parts (modification time within last 3 minutes)',
+              client=client,
+              query=SELECT_RECENT_DATA_PARTS,
+              format='Vertical')
+
+    add_query(diagnostics, 'system.detached_parts',
+              client=client,
+              query=SELECT_DETACHED_DATA_PARTS,
+              format='PrettyCompactNoEscapes',
+              section='Detached data')
+    add_command(diagnostics, 'Disk space usage',
+                command='du -sh -L -c /var/lib/clickhouse/data/*/*/detached/* | sort -rsh',
+                section='Detached data')
+
+    add_query(diagnostics, 'Queries in progress (process list)',
+              client=client,
+              query=SELECT_PROCESSES,
+              format='Vertical',
+              section='Queries')
+    add_query(diagnostics, 'Top 10 queries by duration',
+              client=client,
+              query=SELECT_TOP_QUERIES_BY_DURATION,
+              format='Vertical',
+              section='Queries')
+    add_query(diagnostics, 'Top 10 queries by memory usage',
+              client=client,
+              query=SELECT_TOP_QUERIES_BY_MEMORY_USAGE,
+              format='Vertical',
+              section='Queries')
+    add_query(diagnostics, 'Last 10 failed queries',
+              client=client,
+              query=SELECT_FAILED_QUERIES,
+              format='Vertical',
+              section='Queries')
+
+    add_query(diagnostics, 'Stack traces',
+              client=client,
+              query=SELECT_STACK_TRACES,
+              format='Vertical')
+
+    if 'crash_log' in system_tables:
+        add_query(diagnostics, 'Crash log',
+                  client=client,
+                  query=SELECT_CRASH_LOG,
+                  format='Vertical')
+
+    add_command(diagnostics, 'uname', 'uname -a')
+
+    diagnostics.dump(args.format)
+
+
+def parse_args():
+    """
+    Parse command-line arguments.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--format',
+                        choices=['json', 'yaml', 'json.gz', 'yaml.gz', 'wiki', 'wiki.gz'],
+                        default='wiki')
+    parser.add_argument('--normalize-queries',
+                        action='store_true',
+                        default=False)
+    return parser.parse_args()
+
+
+def add_query(diagnostics, name, client, query, format, section=None):
+    query_args = {
+        'normalize_queries': diagnostics.args.normalize_queries,
+    }
+    query = client.render_query(query, **query_args)
+    diagnostics.add_query(
+        name=name,
+        query=query,
+        result=execute_query(client, query, render_query=False, format=format),
+        section=section)
+
+
+def execute_query(client, query, render_query=True, format=None):
+    if render_query:
+        query = client.render_query(query)
+
+    try:
+        return client.query(query, format=format)
+    except Exception as e:
+        return repr(e)
+
+
+def add_command(diagnostics, name, command, section=None):
+    diagnostics.add_command(
+        name=name,
+        command=command,
+        result=execute_command(command),
+        section=section)
+
+
+def execute_command(command, input=None):
+    proc = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    if isinstance(input, str):
+        input = input.encode()
+
+    stdout, stderr = proc.communicate(input=input)
+
+    if proc.returncode:
+        return f'failed with exit code {proc.returncode}\n{stderr.decode()}'
+
+    return stdout.decode()
+
+
+def version_ge(version1, version2):
+    """
+    Return True if version1 is greater or equal than version2.
+    """
+    return parse_version(version1) >= parse_version(version2)
+
+
+def parse_version(version):
+    """
+    Parse version string.
+    """
+    return [int(x) for x in version.strip().split('.')]
+
+
+if __name__ == '__main__':
+    main()

utils/clickhouse-diagnostics/requirements.txt

@@ -0,0 +1,6 @@
+Jinja2
+PyYAML
+requests
+sqlparse
+tenacity
+xmltodict