ClickHouse/docs/tools/github.py

import collections
import copy
import io
import logging
import os
import random
import sys
import tarfile
import time

import requests

import util


def yield_candidates():
    for page in range(1, 100):
        url = f'https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100&page={page}'
        github_token = os.getenv('GITHUB_TOKEN')
        if github_token:
            headers = {'authorization': f'OAuth {github_token}'}
        else:
            headers = {}
        for candidate in requests.get(url, headers=headers).json():
            yield candidate
    time.sleep(random.random() * 3)


def choose_latest_releases(args):
    logging.info('Collecting release candidates')
    seen_stable = collections.OrderedDict()
    seen_lts = collections.OrderedDict()
    candidates = []
    stable_count = 0
    lts_count = 0

    for tag in yield_candidates():
        if isinstance(tag, dict):
            name = tag.get('name', '')
            is_stable = 'stable' in name
            is_lts = 'lts' in name
            is_unstable = not (is_stable or is_lts)
            is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name)
            if is_unstable or is_in_blacklist:
                continue
            major_version = '.'.join((name.split('.', 2))[:2])
            if major_version not in seen_lts:
                if (stable_count >= args.stable_releases_limit) and (lts_count >= args.lts_releases_limit):
                    break

                payload = (name, tag.get('tarball_url'), is_lts,)
                logging.debug(payload)
                if is_lts:
                    if lts_count < args.lts_releases_limit:
                        seen_lts[major_version] = payload
                        try:
                            del seen_stable[major_version]
                        except KeyError:
                            pass
                    lts_count += 1
                else:
                    if stable_count < args.stable_releases_limit:
                        if major_version not in seen_stable:
                            seen_stable[major_version] = payload
                            stable_count += 1

            logging.debug(
                f'Stables: {stable_count}/{args.stable_releases_limit} LTS: {lts_count}/{args.lts_releases_limit}'
            )
        else:
            logging.fatal('Unexpected GitHub response: %s', str(candidates))
            sys.exit(1)

    logging.info('Found LTS releases: %s', ', '.join(list(seen_lts.keys())))
    logging.info('Found stable releases: %s', ', '.join(list(seen_stable.keys())))
    return sorted(list(seen_lts.items()) + list(seen_stable.items()))


def process_release(args, callback, release):
    name, (full_name, tarball_url, is_lts,) = release
    logging.info(f'Building docs for {full_name}')
    buf = io.BytesIO(requests.get(tarball_url).content)
    tar = tarfile.open(mode='r:gz', fileobj=buf)
    with util.temp_dir() as base_dir:
        tar.extractall(base_dir)
        args = copy.copy(args)
        args.version_prefix = name
        args.is_stable_release = True
        args.docs_dir = os.path.join(base_dir, os.listdir(base_dir)[0], 'docs')
        callback(args)


def build_releases(args, callback):
    for release in args.stable_releases:
        process_release(args, callback, release)


def get_events(args):
    events = []
    skip = True
    with open(os.path.join(args.docs_dir, '..', 'README.md')) as f:
        for line in f:
            if skip:
                if 'Upcoming Events' in line:
                    skip = False
            else:
                if not line:
                    continue
                line = line.strip().split('](')
                if len(line) == 2:
                    tail = line[1].split(') ')
                    events.append({
                        'signup_link': tail[0],
                        'event_name':  line[0].replace('* [', ''),
                        'event_date':  tail[1].replace('on ', '').replace('.', '')
                    })
    return events


if __name__ == '__main__':
    class DummyArgs(object):
        lts_releases_limit = 1
        stable_releases_limit = 3
    logging.basicConfig(
        level=logging.DEBUG,
        stream=sys.stderr
    )
    for item in choose_latest_releases(DummyArgs()):
        print(item)