Update changelog script.

* support python3
* synchronize with Description Check

utils/make_changelog.py

@@ -217,49 +217,61 @@ def process_unknown_commits(commits, commits_info, users):
     text = 'Commits which are not from any pull request:\n\n'
     return text + '\n\n'.join(texts)
 
+# This function mirrors the PR description checks in ClickhousePullRequestTrigger.
+# Returns False if the PR should not be mentioned changelog.
+def parse_one_pull_request(item):
+    description = item['description']
+    lines = [line for line in map(lambda x: x.strip(), description.split('\n')) if line]
+    lines = [re.sub(r'\s+', ' ', l) for l in lines]
+
+    cat_pos = None
+    short_descr_pos = None
+    long_descr_pos = None
+
+    if lines:
+        for i in range(len(lines) - 1):
+            if re.match(r'(?i)^\**Category', lines[i]):
+                cat_pos = i
+            if re.match(r'(?i)^\**\s*(Short description|Change\s*log entry)', lines[i]):
+                short_descr_pos = i
+            if re.match(r'(?i)^\**\s*Detailed description', lines[i]):
+                long_descr_pos = i
+
+    if cat_pos is None:
+        return False
+    cat = lines[cat_pos + 1]
+    cat = re.sub(r'^[-*\s]*', '', cat)
+
+    # Filter out the PR categories that are not for changelog.
+    if re.match(r'(?i)doc|((non|in|not|un)[-\s]*significant)', cat):
+        return False
+
+    short_descr = ''
+    if short_descr_pos:
+        short_descr_end = long_descr_pos or len(lines)
+        short_descr = lines[short_descr_pos + 1]
+        if short_descr_pos + 2 != short_descr_end:
+            short_descr += ' ...'
+
+    # If we have nothing meaningful
+    if not re.match('\w', short_descr):
+        short_descr = item['title']
+
+    # TODO: Add detailed description somewhere
+
+    item['entry'] = short_descr
+    item['category'] = cat
+
+    return True
+
 
 # List of pull requests -> text description.
 def process_pull_requests(pull_requests, users, repo):
     groups = {}
 
     for id, item in pull_requests.items():
-        lines = list(filter(len, map(lambda x: x.strip(), item['description'].split('\n'))))
-
-        cat_pos = None
-        short_descr_pos = None
-        long_descr_pos = None
-
-        if lines:
-            for i in range(len(lines) - 1):
-                if re.match('^\**Category', lines[i]):
-                    cat_pos = i
-                if re.match('^\**\s*Short description', lines[i]):
-                    short_descr_pos = i
-                if re.match('^\**\s*Detailed description', lines[i]):
-                    long_descr_pos = i
-
-        cat = ''
-        if cat_pos is not None:
-            # TODO: Sometimes have more than one
-            cat = lines[cat_pos + 1]
-        cat = cat.strip().lstrip('-').strip()
-
-        # We are not interested in documentation PRs in changelog.
-        if re.match('^\**\s*(?:Documentation|Doc\s)', cat):
-            continue;
-
-        short_descr = ''
-        if short_descr_pos:
-            short_descr_end = long_descr_pos or len(lines)
-            short_descr = lines[short_descr_pos + 1]
-            if short_descr_pos + 2 != short_descr_end:
-                short_descr += ' ...'
-
-        # If we have nothing meaningful
-        if not re.match('\w', short_descr):
-            short_descr = item['title']
-
-        # TODO: Add detailed description somewhere
+        if not parse_one_pull_request(item):
+            continue
 
         pattern = u"{} [#{}]({}) ({})"
         link = 'https://github.com/{}/pull/{}'.format(repo, id)
@@ -269,20 +281,21 @@ def process_pull_requests(pull_requests, users, repo):
             user = users[item['user']]
             author = u'[{}]({})'.format(user['name'] or user['login'], user['html_url'])
 
+        cat = item['category']
         if cat not in groups:
             groups[cat] = []
-        groups[cat].append(pattern.format(short_descr, id, link, author))
+        groups[cat].append(pattern.format(item['entry'], id, link, author))
 
-    categories_preferred_order = ['New Feature', 'Bug Fix', 'Improvement', 'Performance Improvement', 'Build/Testing/Packaging Improvement', 'Backward Incompatible Change', 'Other']
+    categories_preferred_order = ['Backward Incompatible Change', 'New Feature', 'Bug Fix', 'Improvement', 'Performance Improvement', 'Build/Testing/Packaging Improvement', 'Other']
 
     def categories_sort_key(name):
         if name in categories_preferred_order:
-            return categories_preferred_order.index(name)
+            return str(categories_preferred_order.index(name)).zfill(3)
         else:
             return name.lower()
 
     texts = []
-    for group, text in sorted(groups.items(), key = lambda (k, v):  categories_sort_key(k)):
+    for group, text in sorted(groups.items(), key = lambda kv: categories_sort_key(kv[0])):
         items = [u'* {}'.format(pr) for pr in text]
         texts.append(u'### {}\n{}'.format(group if group else u'[No category]', '\n'.join(items)))