From 70ebffca56ace571982c23f4e9a4776575d5268c Mon Sep 17 00:00:00 2001 From: Nathan Cutler Date: Wed, 21 Aug 2019 16:49:26 +0200 Subject: [PATCH] script: add backport-resolve-issue This script processes GitHub backport PRs, checking for proper cross-linking with a Redmine Backport tracker issue and, if a PR is merged and properly cross-linked, it can optionally resolve the tracker issue and correctly populate the "Target version" field. The script takes a single positional argument, which is optional. If the argument is an integer, it is assumed to be a GitHub backport PR ID (e.g. "28549"). In this mode ("single PR mode") the script processes a single GitHub backport PR and terminates. If the argument is not an integer, or is missing, it is assumed to be a commit (SHA1 or tag) to start from. If no positional argument is given, it defaults to the tag "BRI", which might have been added by the last run of the script. This mode is called "scan merge commits mode". In both modes, the script scans a local git repo, which is assumed to be in the current working directory. In single PR mode, the script will work only if the PR's merge commit is present in the current branch of the local git repo. In scan merge commits mode, the script starts from the given SHA1 or tag, taking each merge commit in turn and attempting to obtain the GitHub PR number for each. For each GitHub PR, the script interactively displays all relevant information (NOTE: this includes displaying the GitHub PR and Redmine backport issue in web browser tabs!) and prompts the user for her preferred disposition. Signed-off-by: Nathan Cutler --- src/script/backport-resolve-issue | 583 ++++++++++++++++++++++++++++++ 1 file changed, 583 insertions(+) create mode 100755 src/script/backport-resolve-issue diff --git a/src/script/backport-resolve-issue b/src/script/backport-resolve-issue new file mode 100755 index 00000000000..15d7b1184f3 --- /dev/null +++ b/src/script/backport-resolve-issue @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 +# +# backport-resolve-issue +# +# Based on "backport-create-issue", which was itself based on work by +# by Loic Dachary. +# +# +# Introduction +# ============ +# +# This script processes GitHub backport PRs, checking for proper cross-linking +# with a Redmine Backport tracker issue and, if a PR is merged and properly +# cross-linked, it can optionally resolve the tracker issue and correctly +# populate the "Target version" field. +# +# The script takes a single positional argument, which is optional. If the +# argument is an integer, it is assumed to be a GitHub backport PR ID (e.g. "28549"). +# In this mode ("single PR mode") the script processes a single GitHub backport +# PR and terminates. +# +# If the argument is not an integer, or is missing, it is assumed to be a +# commit (SHA1 or tag) to start from. If no positional argument is given, it +# defaults to the tag "BRI-{release}", which might have been added by the last run of the +# script. This mode is called "scan merge commits mode". +# +# In both modes, the script scans a local git repo, which is assumed to be +# in the current working directory. In single PR mode, the script will work +# only if the PR's merge commit is present in the current branch of the local +# git repo. In scan merge commits mode, the script starts from the given SHA1 +# or tag, taking each merge commit in turn and attempting to obtain the GitHub +# PR number for each. +# +# For each GitHub PR, the script interactively displays all relevant information +# (NOTE: this includes displaying the GitHub PR and Redmine backport issue in +# web browser tabs!) and prompts the user for her preferred disposition. +# +# +# Assumptions +# =========== +# +# Among other things, the script assumes: +# +# 1. it is being run in the top-level directory of a Ceph git repo +# 2. the preferred web browser is Firefox and the command to open a browser +# tab is "firefox" +# 3. if Firefox is running and '--no-browser' was not given, the Firefox window +# is visible to the user and the user desires to view GitHub PRs and Tracker +# Issues in the browser +# 4. if Firefox is not running, the user does not want to view PRs and issues +# in a web browser +# +# +# Dependencies +# ============ +# +# To run this script, first install the dependencies +# +# virtualenv v +# source v/bin/activate +# pip install gitpython python-redmine +# +# Then, copy the script from src/script/backport-resolve-issue (in the branch +# "master" - the script is not maintained anywhere else) to somewhere in your +# PATH. +# +# Finally, run the script with appropriate parameters. For example: +# +# backport-resolve-issue --key $MY_REDMINE_KEY +# backport-resolve-issue --user $MY_REDMINE_USER --password $MY_REDMINE_PASSWORD +# +# +# Copyright Notice +# ================ +# +# Copyright (C) 2019, SUSE LLC +# +# Author: Nathan Cutler +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/> +# +import argparse +import logging +import json +import os +import re +import time +from redminelib import Redmine # https://pypi.org/project/python-redmine/ +from git import Repo +from git.exc import GitCommandError + +github_endpoint = "https://github.com/ceph/ceph" +redmine_endpoint = "https://tracker.ceph.com" +project_name = "Ceph" +status2status_id = {} +project_id2project = {} +tracker2tracker_id = {} +version2version_id = {} +delay_seconds = 5 +browser_cmd = "firefox" +no_browser = False +ceph_release = None +dry_run = False +redmine = None +bri_tag = None +github_token = None + +def usage(): + logging.error("Command-line arguments must include either a Redmine key (--key) " + "or a Redmine username and password (via --user and --password). " + "Optionally, one or more issue numbers can be given via positional " + "argument(s). In the absence of positional arguments, the script " + "will loop through all merge commits after the tag \"BRI-{release}\". If there " + "is no such tag in the local branch, one will be created for you.") + exit(-1) + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--key", help="Redmine user key") + parser.add_argument("--user", help="Redmine user") + parser.add_argument("--password", help="Redmine password") + parser.add_argument("--token", help="GitHub token") + parser.add_argument("--debug", help="Show debug-level messages", + action="store_true") + parser.add_argument("--dry-run", help="Do not write anything to Redmine", + action="store_true") + parser.add_argument("--no-browser", help="Do not use web browser even if it is running", + action="store_true") + parser.add_argument("pr_or_commit", nargs='*', help="GitHub PR ID, or last merge commit successfully processed") + return parser.parse_args() + +def set_logging_level(a): + if a.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + return None + +def report_params(a): + global dry_run + global no_browser + global github_token + if a.dry_run: + dry_run = True + logging.warning("Dry run: nothing will be written to Redmine") + if a.no_browser: + no_browser = True + logging.warning("Web browser will not be used even if it is running") + if a.token: + github_token = a.token + logging.info("GitHub token provided on command line; using it") + +def connect_to_redmine(a): + if a.key: + logging.info("Redmine key was provided; using it") + return Redmine(redmine_endpoint, key=a.key) + elif a.user and a.password: + logging.info("Redmine username and password were provided; using them") + return Redmine(redmine_endpoint, username=a.user, password=a.password) + else: + usage() + +def releases(): + return ('argonaut', 'bobtail', 'cuttlefish', 'dumpling', 'emperor', + 'firefly', 'giant', 'hammer', 'infernalis', 'jewel', 'kraken', + 'luminous', 'mimic', 'nautilus') + +def ver_to_release(): + return {'v9.2': 'infernalis', 'v10.2': 'jewel', 'v11.2': 'kraken', + 'v12.2': 'luminous', 'v13.2': 'mimic', 'v14.2': 'nautilus'} + +def populate_status_dict(r): + for status in r.issue_status.all(): + status2status_id[status.name] = status.id + logging.debug("Statuses {}".format(status2status_id)) + return None + +# not used currently, but might be useful +def populate_version_dict(r, p_id): + versions = r.version.filter(project_id=p_id) + for version in versions: + version2version_id[version.name] = version.id + return None + +def populate_tracker_dict(r): + for tracker in r.tracker.all(): + tracker2tracker_id[tracker.name] = tracker.id + logging.debug("Trackers {}".format(tracker2tracker_id)) + return None + +def has_tracker(r, p_id, tracker_name): + for tracker in get_project(r, p_id).trackers: + if tracker['name'] == tracker_name: + return True + return False + +def get_project(r, p_id): + if p_id not in project_id2project: + p_obj = r.project.get(p_id, include='trackers') + project_id2project[p_id] = p_obj + return project_id2project[p_id] + +def ceph_version(repo, sha1=None): + if sha1: + return repo.git.describe('--match', 'v*', sha1).split('-')[0] + return repo.git.describe('--match', 'v*').split('-')[0] + +def populate_ceph_release(repo): + global ceph_release + current_branch = repo.git.rev_parse('--abbrev-ref', 'HEAD') + release_ver_full = ceph_version(repo) + logging.info("Current git branch is {}, {}".format(current_branch, release_ver_full)) + release_ver = release_ver_full.split('.')[0] + '.' + release_ver_full.split('.')[1] + try: + ceph_release = ver_to_release()[release_ver] + except KeyError: + assert False, \ + "Release version {} does not correspond to any known stable release".format(release_ver) + logging.info("Ceph release is {}".format(ceph_release)) + +def ensure_bri_tag_exists(repo, release): + global bri_tag + bri_tag = "BRI-{}".format(release) + bri_tag_exists = '' + try: + bri_tag_exists = repo.git.show_ref(bri_tag) + except GitCommandError as err: + logging.error(err) + logging.debug("git show-ref {} returned ->{}<-".format(bri_tag, bri_tag_exists)) + if not bri_tag_exists: + c_v = ceph_version(repo) + logging.info("No {} tag found: setting it to {}".format(bri_tag, c_v)) + repo.git.tag(bri_tag, c_v) + +def commit_range(args): + global bri_tag + if len(args.pr_or_commit) == 0: + return '{}..HEAD'.format(bri_tag) + elif len(args.pr_or_commit) == 1: + pass + else: + logging.warn("Ignoring positional parameters {}".format(args.pr_or_commit[1:])) + commit = args.pr_or_commit[0] + return '{}..HEAD'.format(commit) + +def tag_sha1(repo, sha1): + global bri_tag + repo.git.tag('--delete', bri_tag) + repo.git.tag(bri_tag, sha1) + +def browser_running(): + global browser_cmd + retval = os.system("pgrep {} >/dev/null".format(browser_cmd)) + if retval == 0: + return True + return False + + +class Backport: + + def __init__(self, repo, merge_commit_string): + ''' + The merge commit string should look something like this: + 27ff851953 Merge pull request #29678 from pdvian/wip-40948-nautilus + ''' + global browser_cmd + global ceph_release + global github_token + self.repo = repo + # + # split merge commit string on first space character + merge_commit_sha1_short, merge_commit_description = merge_commit_string.split(' ', 1) + # + # merge commit SHA1 from merge commit string + p = re.compile('\\S+') + self.merge_commit_sha1_short = p.match(merge_commit_sha1_short).group() + assert self.merge_commit_sha1_short == merge_commit_sha1_short, \ + "Failed to extract merge commit short SHA1 from merge commit string ->{}<-".format(merge_commit_string) + logging.debug("Short merge commit SHA1 is {}".format(self.merge_commit_sha1_short)) + self.merge_commit_sha1 = self.repo.git.rev_list('--max-count=1', self.merge_commit_sha1_short) + logging.debug("Full merge commit SHA1 is {}".format(self.merge_commit_sha1)) + self.merge_commit_gd = repo.git.describe('--match', 'v*', self.merge_commit_sha1) + self.populate_base_version() + self.populate_target_version() + # + # GitHub PR ID from merge commit string + p = re.compile('\\d+') + self.github_pr_id = p.search(merge_commit_description).group() + logging.debug("GitHub PR ID from merge commit string: {}".format(self.github_pr_id)) + self.populate_github_url() + # + # GitHub PR description and merged status from GitHub + cmd = "curl --silent https://api.github.com/repos/ceph/ceph/pulls/{}".format(self.github_pr_id) + # if GitHub token was provided, use it to avoid throttling - + if github_token: + cmd = "{}?access_token={}".format(cmd, github_token) + json_str = os.popen(cmd).read() + github_api_result = json.loads(json_str) + if "title" not in github_api_result: + # rate limiting + assert False, \ + "GitHub API unexpectedly returned ->{}<- (rate limiting? maybe use --token?)".format(github_api_result) + self.github_pr_title = github_api_result["title"] + self.github_pr_desc = github_api_result["body"] + self.mogrify_github_pr_desc() + self.github_pr_merged = github_api_result["merged"] + if not no_browser: + if browser_running(): + os.system("{} {}".format(browser_cmd, self.github_url)) + print('''================================================================ +{} +merge commit {} ({}) +{} +``` +{} +``` +Merged: {} +Ceph version: base {}, target {} +================================================================'''.format(self.github_url, self.merge_commit_sha1, self.merge_commit_gd, self.github_pr_title, self.github_pr_desc, self.github_pr_merged, self.base_version, self.target_version)) + assert self.github_pr_merged, "GitHub PR {} has not been merged!".format(self.github_pr_id) + # + # extract tracker URL from description + p = re.compile('http.*://tracker.ceph.com/issues/\\d+') + try: + tracker_url = p.search(self.github_pr_desc).group() + except AttributeError: + assert False, \ + "GitHub PR description does not contain a Tracker URL!" + p = re.compile('\\d+') + self.tracker_id = p.search(tracker_url).group() + assert self.tracker_id, \ + "Failed to extract tracker ID from tracker URL {}".format(tracker_url) + self.populate_tracker_url() + logging.info("PR {} links to tracker {}" + .format(self.github_url, self.tracker_url)) + # + # we have a Tracker URL, but is it really a backport tracker? + backport_tracker_id = tracker2tracker_id['Backport'] + self.redmine_issue = redmine.issue.get(self.tracker_id) + assert self.redmine_issue.tracker.id == backport_tracker_id, \ + "Tracker {} is a {} (expected ->Backport<-)".format(self.tracker_url, self.redmine_issue.tracker) + # + # does the Backport Tracker description link back to the GitHub PR? + p = re.compile('http.*://github.com/ceph/ceph/pull/\\d+') + self.tracker_description = self.redmine_issue.description + self.github_url_from_tracker = None + try: + self.github_url_from_tracker = p.search(self.tracker_description).group() + except AttributeError: + logging.info("Description of backport tracker {} does not cite GitHub PR URL {}" + .format(self.tracker_url, self.github_url)) + if self.github_url_from_tracker: + p = re.compile('\\d+') + github_id_from_tracker = p.search(self.github_url_from_tracker).group() + logging.debug("GitHub PR from Tracker: URL is ->{}<- and ID is {}" + .format(self.github_url_from_tracker, github_id_from_tracker)) + assert github_id_from_tracker == self.github_pr_id, \ + "GitHub PR ID {} does not match GitHub ID from tracker {}".format(self.github_pr_id, github_id_from_tracker) + logging.info("Tracker {} links to PR {}".format(self.tracker_url, self.github_url)) + else: + logging.info("Tracker {} does not link to PR {} - will update". + format(self.tracker_url, self.github_url)) + # + # does the Backport Tracker's release field match the Ceph release? + tracker_release = self.get_issue_release() + assert ceph_release == tracker_release, \ + "Backport Tracker {} is a {} backport - expected {}".format(self.tracker_id, tracker_release, ceph_release) + # + # is the Backport Tracker's "Target version" custom field populated? + try: + ttv = self.get_tracker_target_version() + except: + logging.info("Backport Tracker {} target version not populated yet!" + .format(self.tracker_id)) + self.set_target_version = True + else: + self.tracker_target_version = ttv + logging.info("Backport Tracker {} target version already populated with correct value {}" + .format(self.tracker_id, self.tracker_target_version)) + self.set_target_version = False + assert self.tracker_target_version == self.target_version, \ + "Tracker target version {} is wrong; should be {}".format(self.tracker_target_version, self.target_version) + # + # is the Backport Tracker's status already set to Resolved? + resolved_id = status2status_id['Resolved'] + if self.redmine_issue.status.id == resolved_id: + logging.info("Backport Tracker {} status is already set to Resolved" + .format(self.tracker_id)) + self.set_tracker_status = False + else: + logging.info("Backport Tracker {} status is currently set to {}" + .format(self.tracker_id, self.redmine_issue.status)) + self.set_tracker_status = True + + def populate_base_version(self): + self.base_version = ceph_version(self.repo, self.merge_commit_sha1) + + def populate_target_version(self): + x, y, z = self.base_version.split('v')[1].split('.') + maybe_stable = "v{}.{}".format(x, y) + assert ver_to_release()[maybe_stable], \ + "SHA1 {} is not based on any known stable release ({})".format(sha1, maybe_stable) + tv = "v{}.{}.{}".format(x, y, int(z) + 1) + assert version2version_id[tv], \ + "Target version {} is not in Redmine".format(tv) + self.target_version = tv + + def mogrify_github_pr_desc(self): + if not self.github_pr_desc: + self.github_pr_desc = '' + p = re.compile('', re.DOTALL) + new_str = p.sub('', self.github_pr_desc) + if new_str == self.github_pr_desc: + logging.debug("GitHub PR description not mogrified") + else: + self.github_pr_desc = new_str + + def populate_github_url(self): + global github_endpoint + self.github_url = "{}/pull/{}".format(github_endpoint, self.github_pr_id) + + def populate_tracker_url(self): + global redmine_endpoint + self.tracker_url = "{}/issues/{}".format(redmine_endpoint, self.tracker_id) + + def get_issue_release(self): + for field in self.redmine_issue.custom_fields: + if field['name'] == 'Release': + return field['value'] + return None + + def get_tracker_target_version(self): + if self.redmine_issue.fixed_version: + logging.debug("Target version: ID {}, name {}" + .format(self.redmine_issue.fixed_version.id, self.redmine_issue.fixed_version.name)) + return self.redmine_issue.fixed_version.name + return None + + def resolve(self): + global delay_seconds + global dry_run + global redmine + kwargs = {} + if self.set_tracker_status: + kwargs['status_id'] = status2status_id['Resolved'] + if self.set_target_version: + kwargs['fixed_version_id'] = version2version_id[self.target_version] + if not self.github_url_from_tracker: + if self.tracker_description: + kwargs['description'] = "{}\n\n---\n\n{}".format(self.github_url, self.tracker_description) + else: + kwargs['description'] = self.github_url + kwargs['notes'] = """This update was made using the script "backport-resolve-issue". +backport PR {} +merge commit {} ({})""".format(self.github_url, self.merge_commit_sha1, self.merge_commit_gd) + my_delay_seconds = delay_seconds + if dry_run: + logging.info("--dry-run was given: NOT updating Redmine") + my_delay_seconds = 0 + else: + redmine.issue.update(self.tracker_id, **kwargs) + if not no_browser: + if browser_running(): + os.system("{} {}".format(browser_cmd, self.tracker_url)) + my_delay_seconds = 3 + logging.debug("Delaying {} seconds to avoid seeming like a spammer".format(my_delay_seconds)) + time.sleep(my_delay_seconds) + + +if __name__ == '__main__': + args = parse_arguments() + set_logging_level(args) + logging.debug(args) + report_params(args) + # + # set up Redmine variables + redmine = connect_to_redmine(args) + project = redmine.project.get(project_name) + ceph_project_id = project.id + logging.debug("Project {} has ID {}".format(project_name, ceph_project_id)) + populate_status_dict(redmine) + pending_backport_status_id = status2status_id["Pending Backport"] + logging.debug("Pending Backport status has ID {}" + .format(pending_backport_status_id)) + populate_tracker_dict(redmine) + populate_version_dict(redmine, ceph_project_id) + # + # construct github Repo object for the current directory + repo = Repo('.') + assert not repo.bare + populate_ceph_release(repo) + # + # if positional argument is an integer, assume it is a GitHub PR + if args.pr_or_commit: + pr_id = args.pr_or_commit[0] + try: + pr_id = int(pr_id) + tag_merge_commits = False + except ValueError: + tag_merge_commits = True + else: + tag_merge_commits = True + # + # get list of merges + if tag_merge_commits: + ensure_bri_tag_exists(repo, ceph_release) + c_r = commit_range(args) + logging.info("Commit range is {}".format(c_r)) + # + # get the list of merge commits, i.e. strings that looks like: + # "27ff851953 Merge pull request #29678 from pdvian/wip-40948-nautilus" + merges_raw_str = repo.git.log(c_r, '--merges', '--oneline', '--no-decorate', '--reverse') + else: + pr_id = args.pr_or_commit[0] + merges_raw_str = repo.git.log('--merges', '--grep=#{}'.format(pr_id), '--oneline', '--no-decorate', '--reverse') + if merges_raw_str: + merges_raw_list = merges_raw_str.split('\n') + else: + merges_raw_list = [] # prevent [''] + merges_remaining = len(merges_raw_list) + logging.info("I see {} merge(s) to process".format(merges_remaining)) + if not merges_remaining: + logging.info("Did you do \"git pull\" before running the script?") + if not tag_merge_commits: + logging.info("Or maybe GitHub PR {} has not been merged yet?".format(pr_id)) + # + # loop over the merge commits + for merge in merges_raw_list: + print("\nMerges remaining to process: {}".format(merges_remaining)) + backport = None + sha1 = merge.split(' ')[0] + print("Merge {}".format(merge)) + possible_to_resolve = True + try: + backport = Backport(repo, merge_commit_string=merge) + except AssertionError as err: + logging.error("Malformed backport due to ->{}<-".format(err)) + possible_to_resolve = False + if tag_merge_commits: + if possible_to_resolve: + prompt = "[a] Abort, [i] Ignore and advance {bri} tag, [u] Update tracker and advance {bri} tag (default 'u') --> ".format(bri=bri_tag) + default_input_val = "u" + else: + prompt = "[a] Abort, [i] Ignore and advance {bri} tag (default 'i') --> ".format(bri=bri_tag) + default_input_val = "i" + else: + if possible_to_resolve: + prompt = "[a] Abort, [i] Ignore, [u] Update tracker (default 'u') --> " + default_input_val = "u" + else: + prompt = "[a] Abort, [i] Ignore --> " + default_input_val = "i" + input_val = input(prompt) + if input_val == '': + input_val = default_input_val + if input_val.lower() == "a": + exit(-1) + elif input_val.lower() == "i": + pass + else: + input_val = "u" + if input_val.lower() == "u": + if backport: + backport.resolve() + else: + logging.warn("Cannot determine which issue to resolve. Ignoring.") + if tag_merge_commits: + if backport: + tag_sha1(repo, backport.merge_commit_sha1) + else: + tag_sha1(repo, sha1) + merges_remaining -= 1 -- 2.39.5