Implement auto-labelling for the backporting script (#7001)

utils/github/__main__.py

@@ -20,13 +20,18 @@
 '''
 
 from . import local, query
-
-from termcolor import colored  # `pip install termcolor`
+from . import parser as parse_description
 
 import argparse
 import re
 import sys
 
+try:
+    from termcolor import colored  # `pip install termcolor`
+except ImportError:
+    sys.exit("Package 'termcolor' not found. Try run: `pip3 install termcolor`")
+
+
 CHECK_MARK = colored('🗸', 'green')
 CROSS_MARK = colored('🗙', 'red')
 LABEL_MARK = colored('🏷', 'yellow')
@@ -44,6 +49,8 @@ parser.add_argument('--token', type=str, required=True,
     help='token for Github access')
 parser.add_argument('--login', type=str,
     help='filter authorship by login')
+parser.add_argument('--auto-label', action='store_true', dest='autolabel',
+    help='try to automatically parse PR description and put labels')
 
 args = parser.parse_args()
 
@@ -83,14 +90,23 @@ def print_responsible(pull_request):
 bad_pull_requests = []  # collect and print if not empty
 need_backporting = []
 for pull_request in pull_requests:
-    label_found = False
 
-    for label in github.get_labels(pull_request):
-        if label['name'].startswith('pr-'):
-            label_found = True
-            if label['color'] == 'ff0000':
-                need_backporting.append(pull_request)
-            break
+    def find_label():
+        for label in github.get_labels(pull_request):
+            if label['name'].startswith('pr-'):
+                if label['color'] == 'ff0000':
+                    need_backporting.append(pull_request)
+                return True
+        return False
+
+    label_found = find_label()
+
+    if not label_found and args.autolabel:
+        print(f"Trying to auto-label pull-request: {pull_request['number']}")
+        description = parse_description.Description(pull_request)
+        if description.label_name:
+            github.set_label(pull_request, description.label_name)
+            label_found = find_label()
 
     if not label_found:
         bad_pull_requests.append(pull_request)

utils/github/parser.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+class Description:
+    '''Parsed description representation
+    '''
+    MAP_CATEGORY_TO_LABEL = {
+        'New Feature': 'pr-feature',
+        'Bug Fix': 'pr-bugfix',
+        'Improvement': 'pr-improvement',
+        'Performance Improvement': 'pr-performance',
+        # 'Backward Incompatible Change': doesn't match anything
+        'Build/Testing/Packaging Improvement': 'pr-build',
+        # 'Other': doesn't match anything
+    }
+
+    def __init__(self, pull_request):
+        self.label_name = str()
+        self.legal = False
+
+        self._parse(pull_request['bodyText'])
+
+    def _parse(self, text):
+        lines = text.splitlines()
+        next_category = False
+        category = str()
+
+        for line in lines:
+            stripped = line.strip()
+
+            if not stripped:
+                continue
+
+            if next_category:
+                category = stripped
+                next_category = False
+
+            if stripped == 'I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en':
+                self.legal = True
+
+            if stripped == 'Category (leave one):':
+                next_category = True
+
+        if category in Description.MAP_CATEGORY_TO_LABEL:
+            self.label_name = Description.MAP_CATEGORY_TO_LABEL[category]

utils/github/query.py

@@ -180,10 +180,12 @@ class Query:
                                     totalCount
                                     nodes {{
                                         ... on PullRequest {{
+                                            id
                                             number
                                             author {{
                                                 login
                                             }}
+                                            bodyText
                                             mergedBy {{
                                                 login
                                             }}
@@ -301,7 +303,58 @@ class Query:
         '''
         return self._run(Query._DEFAULT)['repository']['defaultBranchRef']['name']
 
-    def _run(self, query):
+    _GET_LABEL = '''
+        repository(owner: "yandex" name: "ClickHouse") {{
+            labels(first: {max_page_size} {next} query: "{name}") {{
+                pageInfo {{
+                    hasNextPage
+                    endCursor
+                }}
+                nodes {{
+                    id
+                    name
+                    color
+                }}
+            }}
+        }}
+    '''
+    _SET_LABEL = '''
+        addLabelsToLabelable(input: {{ labelableId: "{pr_id}", labelIds: "{label_id}" }}) {{
+            clientMutationId
+        }}
+    '''
+    def set_label(self, pull_request, label_name):
+        '''Set label by name to the pull request
+
+        Args:
+            pull_request: JSON object returned by `get_pull_requests()`
+            label_name (string): label name
+        '''
+        labels = []
+        not_end = True
+        query = Query._GET_LABEL.format(name=label_name,
+                                        max_page_size=self._max_page_size,
+                                        next='')
+
+        while not_end:
+            result = self._run(query)['repository']['labels']
+            not_end = result['pageInfo']['hasNextPage']
+            query = Query._GET_LABEL.format(name=label_name,
+                                            max_page_size=self._max_page_size,
+                                            next=f'after: "{result["pageInfo"]["endCursor"]}"')
+
+            labels += [label for label in result['nodes']]
+
+        if not labels:
+            return
+
+        query = Query._SET_LABEL.format(pr_id = pull_request['id'], label_id = labels[0]['id'])
+        self._run(query, is_mutation=True)
+
+        pull_request['labels']['nodes'].append(labels[0])
+
+
+    def _run(self, query, is_mutation=False):
         from requests.adapters import HTTPAdapter
         from urllib3.util.retry import Retry
 
@@ -325,26 +378,34 @@ class Query:
             return session
 
         headers = {'Authorization': f'bearer {self._token}'}
-        query = f'''
-        {{
-            {query}
-            rateLimit {{
-                cost
-                remaining
+        if is_mutation:
+            query = f'''
+            mutation {{
+                {query}
             }}
-        }}
-        '''
+            '''
+        else:
+            query = f'''
+            query {{
+                {query}
+                rateLimit {{
+                    cost
+                    remaining
+                }}
+            }}
+            '''
         request = requests_retry_session().post('https://api.github.com/graphql', json={'query': query}, headers=headers)
         if request.status_code == 200:
             result = request.json()
             if 'errors' in result:
                 raise Exception(f'Errors occured: {result["errors"]}')
 
-            import inspect
-            caller = inspect.getouterframes(inspect.currentframe(), 2)[1][3]
-            if caller not in self.api_costs.keys():
-                self.api_costs[caller] = 0
-            self.api_costs[caller] += result['data']['rateLimit']['cost']
+            if not is_mutation:
+                import inspect
+                caller = inspect.getouterframes(inspect.currentframe(), 2)[1][3]
+                if caller not in self.api_costs.keys():
+                    self.api_costs[caller] = 0
+                self.api_costs[caller] += result['data']['rateLimit']['cost']
 
             return result['data']
         else: