ClickHouse/docs/tools/github.py

114 lines
3.6 KiB
Python
Raw Normal View History

import collections
import copy
import io
import logging
import os
import sys
import tarfile
import requests
import util
def yield_candidates():
for page in range(1, 100):
url = 'https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100&page=%d' % page
for candidate in requests.get(url).json():
yield candidate
2020-03-30 08:25:29 +00:00
def choose_latest_releases(args):
2020-03-19 06:28:58 +00:00
logging.info('Collecting release candidates')
seen = collections.OrderedDict()
candidates = []
stable_count = 0
lts_count = 0
for tag in yield_candidates():
if isinstance(tag, dict):
name = tag.get('name', '')
is_stable = 'stable' in name
is_lts = 'lts' in name
is_unstable = not (is_stable or is_lts)
is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name)
if is_unstable or is_in_blacklist:
continue
major_version = '.'.join((name.split('.', 2))[:2])
if major_version not in seen:
if (stable_count >= args.stable_releases_limit) and (lts_count >= args.lts_releases_limit):
break
payload = (name, tag.get('tarball_url'), is_lts,)
if is_lts:
if lts_count < args.lts_releases_limit:
seen[major_version] = payload
lts_count += 1
else:
if stable_count < args.stable_releases_limit:
seen[major_version] = payload
stable_count += 1
logging.debug(
f'Stables: {stable_count}/{args.stable_releases_limit} LTS: {lts_count}/{args.lts_releases_limit}'
)
else:
logging.fatal('Unexpected GitHub response: %s', str(candidates))
sys.exit(1)
logging.info('Found stable releases: %s', ', '.join(seen.keys()))
return seen.items()
def process_release(args, callback, release):
name, (full_name, tarball_url, is_lts,) = release
logging.info(f'Building docs for {full_name}')
buf = io.BytesIO(requests.get(tarball_url).content)
tar = tarfile.open(mode='r:gz', fileobj=buf)
with util.temp_dir() as base_dir:
tar.extractall(base_dir)
2020-03-19 06:28:58 +00:00
args = copy.copy(args)
args.version_prefix = name
2019-12-10 08:15:38 +00:00
args.is_stable_release = True
args.docs_dir = os.path.join(base_dir, os.listdir(base_dir)[0], 'docs')
2020-03-30 08:25:29 +00:00
callback(args)
def build_releases(args, callback):
for release in args.stable_releases:
2020-01-29 16:03:20 +00:00
process_release(args, callback, release)
def get_events(args):
events = []
skip = True
with open(os.path.join(args.docs_dir, '..', 'README.md')) as f:
for line in f:
if skip:
if 'Upcoming Events' in line:
skip = False
else:
if not line:
continue
line = line.strip().split('](')
if len(line) == 2:
tail = line[1].split(') ')
events.append({
'signup_link': tail[0],
'event_name': line[0].replace('* [', ''),
'event_date': tail[1].replace('on ', '').replace('.', '')
})
return events
if __name__ == '__main__':
class DummyArgs(object):
lts_releases_limit = 1
stable_releases_limit = 3
logging.basicConfig(
level=logging.DEBUG,
stream=sys.stderr
)
for item in choose_latest_releases(DummyArgs()):
print(item)