From: Nathan Cutler <ncutler@suse.com>
Date: Wed, 28 Aug 2019 11:57:43 +0000 (+0200)
Subject: script/ceph-backport.sh: wholesale refactor
X-Git-Tag: v15.1.0~1704^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=deca13e83a9974eb8ff572efcba8de5713eda860;p=ceph-ci.git

script/ceph-backport.sh: wholesale refactor

This commit refactors the script to make it more user-friendly and
maintainable. Added features:

* script now determine release/milestone from backport issue instead of
  requiring user to provide it
* script now generates a more verbose PR description
* backport PR title is based on original PR instead of the tracker issue
* improved error handling
* new --debug option that triggers "set -x"
* new --verbose option
* errors and diagnostic messages are now printed to stderr

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---

diff --git a/src/script/ceph-backport.sh b/src/script/ceph-backport.sh
index dbebfc04c61..3be2045eebb 100755
--- a/src/script/ceph-backport.sh
+++ b/src/script/ceph-backport.sh
@@ -65,21 +65,22 @@
 #
 # For simple backports you can just run:
 #
-# ceph-backport.sh 19206 jewel --prepare
-# ceph-backport.sh 19206 jewel
+# ceph-backport.sh 19206 --prepare
+# ceph-backport.sh 19206
 #
-# alternatively, you can prepare the backport manually:
+# Alternatively, instead of running the script with --prepare you can prepare
+# the backport manually:
 #
 # git remote add ceph http://github.com/ceph/ceph.git
 # git fetch ceph
 # git checkout -b wip-19206-jewel ceph/jewel
 # git cherry-pick -x ...
-# ceph-backport.sh 19206 jewel
+# ceph-backport.sh 19206
 #
-# optionally, you can set the component label that will be added to the PR with
+# Optionally, you can set the component label that will be added to the PR with
 # an environment variable:
 #
-# COMPONENT=dashboard ceph-backport.sh 40056 nautilus
+# COMPONENT=dashboard ceph-backport.sh 40056
 #
 #
 # Troubleshooting notes
@@ -101,6 +102,16 @@
 #     $ git cherry-pick --quit
 #
 #
+# Reporting bugs
+# --------------
+#
+# Please report any bugs in this script to https://tracker.ceph.com/projects/ceph/issues/new
+#
+# (Ideally, the bug report would include a typescript obtained while
+# reproducing the bug with the --debug option. To understand what is meant by
+# "typescript", see "man script".)
+#
+#
 # Other backporting resources
 # ---------------------------
 #
@@ -111,14 +122,74 @@
 # Nathan
 #
 source $HOME/bin/backport_common.sh
+this_script=$(basename "$0")
+verbose=
+
+if [[ $* == *--debug* ]]; then
+    set -x
+    verbose="1"
+fi
+
+if [[ $* == *--verbose* ]]; then
+    verbose="1"
+fi
+
+function log {
+    local level="$1"
+    shift
+    local msg="$@"
+    prefix="${this_script}: "
+    verbose_only=
+    case $level in
+        err*)
+            prefix="${prefix}ERROR: "
+            ;;
+        info)
+            :
+            ;;
+        bare)
+            prefix=
+            ;;
+        warn|warning)
+            prefix="${prefix}WARNING: "
+            ;;
+        debug|verbose)
+            prefix="${prefix}DEBUG: "
+            verbose_only="1"
+            ;;
+    esac
+    if [ "$verbose_only" -a ! "$verbose" ] ; then
+        :
+    else
+        msg="${prefix}${msg}"
+        echo "$msg" >&2
+    fi
+}
+
+function error {
+    log error $@
+}
+
+function warning {
+    log warning $@
+}
+
+function info {
+    log info $@
+}
 
-function failed_required_variable_check () {
-    local varname=$1
-    echo "$0: $varname not defined. Did you create $HOME/bin/backport_common.sh?"
-    echo "(For instructions, see comment block at beginning of script)"
+function debug {
+    log debug $@
+}
+
+function failed_required_variable_check {
+    local varname="$1"
+    error "$varname not defined. Did you create $HOME/bin/backport_common.sh?"
+    info "(For detailed instructions, see comment block at the beginning of the script)"
     exit 1
 }
 
+debug Checking mandatory variables
 test "$redmine_key"     || failed_required_variable_check redmine_key
 test "$redmine_user_id" || failed_required_variable_check redmine_user_id
 test "$github_token"    || failed_required_variable_check github_token
@@ -126,100 +197,183 @@ test "$github_user"     || failed_required_variable_check github_user
 : "${github_repo:=origin}"
 : "${ceph_repo:=upstream}"
 redmine_endpoint="https://tracker.ceph.com"
+github_endpoint="https://github.com/ceph/ceph"
+original_issue=
+original_pr=
 
-function usage () {
-    echo "Usage:"
-    echo "   $0 [BACKPORT_TRACKER_ISSUE_NUMBER] [MILESTONE] [--prepare]"
-    echo
-    echo "Example:"
-    echo "   $0 19206 jewel"
-    echo
-    echo "If MILESTONE is not given on the command line, the script will"
-    echo "try to use the value of the MILESTONE environment variable, if set."
-    echo
-    echo "The script must be run from inside the local git clone"
+function usage {
+    log bare
+    log bare "Usage:"
+    log bare "   ${this_script} BACKPORT_TRACKER_ISSUE_NUMBER [--debug] [--prepare] [--verbose]"
+    log bare
+    log bare "Example:"
+    log bare "   ${this_script} 19206 --prepare"
+    log bare
+    log bare "The script must be run from inside a local git clone."
+    log bare
+    log bare "Documentation can be found in the comment block at the top of the script itself."
     exit 1
 }
 
-[[ $1 =~ ^[0-9]+$ ]] || usage
-issue=$1
-echo "Backport issue: $issue"
+function populate_original_issue {
+    if [ -z "$original_issue" ] ; then
+        original_issue=$(curl --silent ${redmine_url}.json?include=relations |
+            jq '.issue.relations[] | select(.relation_type | contains("copied_to")) | .issue_id')
+    fi
+}
 
-milestone=
-test "$2" && milestone="$2"
-if [ -z "$milestone" ] ; then
-    test "$MILESTONE" && milestone="$MILESTONE"
-fi
-test "$milestone" || usage
-echo "Milestone: $milestone"
+function populate_original_pr {
+    if [ "$original_issue" ] ; then
+        if [ -z "$original_pr" ] ; then
+            original_pr=$(curl --silent ${redmine_endpoint}/issues/${original_issue}.json |
+                          jq -r '.issue.custom_fields[] | select(.id | contains(21)) | .value')
+        fi
+    fi
+}
 
-# milestone numbers can be obtained manually with:
-#   curl --verbose -X GET https://api.github.com/repos/ceph/ceph/milestones
+function prepare {
+    populate_original_issue
+    if [ -z "$original_issue" ] ; then
+        error "Could not find original issue"
+        info "Does ${redmine_url} have a \"Copied from\" relation?"
+        exit 1
+    fi
+    info "Parent issue: ${redmine_endpoint}/issues/${original_issue}"
 
-milestone_number=$(curl -s -X GET 'https://api.github.com/repos/ceph/ceph/milestones?access_token='$github_token | jq --arg milestone $milestone '.[] | select(.title==$milestone) | .number')
+    populate_original_pr
+    if [ -z "$original_pr" ]; then
+        error "Could not find original PR"
+        info "Is the \"Pull request ID\" field of ${redmine_endpoint}/issues/${original_issue} populated?"
+        exit 1
+    fi
+    info "Parent issue ostensibly fixed by: ${github_endpoint}/pull/${original_pr}"
 
-if test -n "$milestone_number" ; then
-    target_branch="$milestone"
+    debug "Counting commits in ${github_endpoint}/pull/${original_pr}"
+    number=$(curl --silent https://api.github.com/repos/ceph/ceph/pulls/${original_pr}?access_token=${github_token} | jq .commits)
+    if [ -z "$number" ] ; then
+        error "Could not determine the number of commits in ${github_endpoint}/pull/${original_pr}"
+        return 1
+    fi
+    info "Found $number commits in ${github_endpoint}/pull/${original_pr}"
+
+    git fetch $ceph_repo
+    debug "Fetched latest commits from upstream"
+
+    git checkout $ceph_repo/$milestone -b $local_branch
+
+    git fetch $ceph_repo pull/$original_pr/head:pr-$original_pr
+
+    debug "Cherry picking $number commits from ${github_endpoint}/pull/${original_pr} into local branch $local_branch"
+    debug "If this operation does not succeed, you will need to resolve the conflicts manually"
+    git cherry-pick -x pr-$original_pr~$number..pr-$original_pr
+    info "Cherry picked $number commits from ${github_endpoint}/pull/${original_pr} into local branch $local_branch"
+
+    exit 0
+}
+
+if git show-ref HEAD >/dev/null 2>&1 ; then
+    debug "In a local git clone. Good."
 else
-    echo -n "Unknown Milestone. Please use one of the following ones: "
-    echo $(curl -s -X GET 'https://api.github.com/repos/ceph/ceph/milestones?access_token='$github_token | jq '.[].title')
+    error "This script must be run from inside a local git clone"
     exit 1
 fi
-echo "Milestone is $milestone and milestone number is $milestone_number"
 
-if [ $(curl --silent ${redmine_endpoint}/issues/${issue}.json | jq -r .issue.tracker.name) != "Backport" ]
-then
-    echo "${redmine_endpoint}/issues/${issue} is not a backport (edit and change tracker?)"
-    exit 1
+if [ $verbose ] ; then
+    debug "Redmine user: ${redmine_user_id}"
+    debug "GitHub user:  ${github_user}"
+    debug "Fork remote:  ${github_repo}"
+    git remote -v | egrep ^${github_repo}\\s+
+    debug "Ceph remote:  ${ceph_repo}"
+    git remote -v | egrep ^${ceph_repo}\\s+
 fi
 
-title=$(curl --silent ${redmine_endpoint}/issues/${issue}.json?key=$redmine_key | jq .issue.subject | tr -d '\\"')
-echo "Issue title: $title"
+if [[ $1 =~ ^[0-9]+$ ]] ; then
+    issue=$1
+else
+    error "Invalid or missing argument"
+    usage  # does not return
+fi
 
-function prepare () {
-    related_issue=$(curl --silent ${redmine_endpoint}/issues/${issue}.json?include=relations |
-                    jq '.issue.relations[] | select(.relation_type | contains("copied_to")) | .issue_id')
-    [ -z "$related_issue" ] && echo "Could not find original issue." && return 1
-    echo "Original issue: $related_issue"
+redmine_url="${redmine_endpoint}/issues/${issue}"
+debug "Considering Redmine issue: $redmine_url - is it in the Backport tracker?"
 
-    pr=$(curl --silent ${redmine_endpoint}/issues/${related_issue}.json |
-         jq '.issue.custom_fields[] | select(.id | contains(21)) | .value' |
-         tr -d '\\"')
-    [ -z "$pr" ] && echo "Could not find PR." && return 1
-    echo "Original PR: $pr"
+tracker=$(curl --silent "${redmine_url}.json" | jq -r '.issue.tracker.name')
+if [ "$tracker" = "Backport" ]; then
+    debug "Yes, $redmine_url is a Backport issue"
+else
+    error "Issue $redmine_url is not a Backport"
+    info "(This script only works with Backport tracker issues.)"
+    exit 1
+fi
 
-    number=$(curl --silent https://api.github.com/repos/ceph/ceph/pulls/${pr}?access_token=${github_token} | jq .commits)
-    [ -z "$number" ] && echo "Could not determine the number of commits." && return 1
-    echo "Found $number commit(s)"
+debug "Looking up release/milestone of $redmine_url"
+milestone=$(curl --silent "${redmine_url}.json" | jq -r '.issue.custom_fields[0].value')
+if [ "$milestone" ] ; then
+    debug "Release/milestone: $milestone"
+else
+    error "could not obtain release/milestone from ${redmine_url}"
+    exit 1
+fi
 
-    git fetch $ceph_repo
-    echo "fetch latest $milestone branch."
+# milestone numbers can be obtained manually with:
+#   curl --verbose -X GET https://api.github.com/repos/ceph/ceph/milestones
+milestone_number=$(curl -s -X GET 'https://api.github.com/repos/ceph/ceph/milestones?access_token='$github_token | jq --arg milestone $milestone '.[] | select(.title==$milestone) | .number')
+if test -n "$milestone_number" ; then
+    target_branch="$milestone"
+else
+    error "Unsupported milestone"
+    info "Valid values are $(curl -s -X GET 'https://api.github.com/repos/ceph/ceph/milestones?access_token='$github_token | jq '.[].title')"
+    info "(This probably means the Release field of ${redmine_url} is populated with"
+    info "an unexpected value - i.e. it does not match any of the GitHub milestones.)"
+    exit 1
+fi
+info "Milestone/release is $milestone"
+debug "Milestone number is $milestone_number"
 
-    git checkout $ceph_repo/$milestone -b wip-$issue-$milestone
+local_branch=wip-${issue}-${target_branch}
 
-    git fetch $ceph_repo pull/$pr/head:pr-$pr
+if [ $(curl --silent ${redmine_url}.json | jq -r .issue.tracker.name) != "Backport" ]
+then
+    error "${redmine_endpoint}/issues/${issue} is not in the Backport tracker"
+    exit 1
+fi
 
-    git cherry-pick -x pr-$pr~$number..pr-$pr
-    echo "cherry picked pr-$pr into wip-$issue-$milestone"
+if [[ $* == *--prepare* ]]; then
+    debug "'--prepare' found, will only prepare the backport"
+    prepare  # does not return
+fi
 
-    exit 0
-}
+debug "Pushing local branch $local_branch to remote $github_repo"
+git push -u $github_repo $local_branch
 
-if [[ $* == *--prepare* ]]
-then
-    echo "'--prepare' found, will only prepare the backport"
-    prepare
+debug "Generating backport PR description"
+populate_original_issue
+populate_original_pr
+desc="backport tracker: ${redmine_url}"
+if [ "$original_pr" -o "$original_issue" ] ; then
+    desc="${desc}\n\n---\n"
+    [ "$original_pr"    ] && desc="${desc}\nbackport of ${github_endpoint}/pull/${original_pr}"
+    [ "$original_issue" ] && desc="${desc}\nparent tracker: ${redmine_endpoint}/issues/${original_issue}"
 fi
+desc="${desc}\n\nthis backport was staged using ${github_endpoint}/blob/master/src/script/ceph-backport.sh"
 
-git push -u $github_repo wip-$issue-$milestone
+debug "Generating backport PR title"
+title="${milestone}: $(curl --silent https://api.github.com/repos/ceph/ceph/pulls/${original_pr} | jq -r '.title')"
+if [[ $title =~ \" ]] ; then
+    title=$(echo $title | sed -e 's/"/\\"/g')
+fi
 
-number=$(curl --silent --data-binary '{"title":"'"$title"'","head":"'$github_user':wip-'$issue-$milestone'","base":"'$target_branch'","body":"'$redmine_endpoint'/issues/'$issue'"}' 'https://api.github.com/repos/ceph/ceph/pulls?access_token='$github_token | jq .number)
-echo "Opened pull request $number"
+debug "Opening backport PR"
+number=$(curl --silent --data-binary '{"title":"'"$title"'","head":"'$github_user':'$local_branch'","base":"'$target_branch'","body":"'"${desc}"'"}' 'https://api.github.com/repos/ceph/ceph/pulls?access_token='$github_token | jq -r .number)
+component=${COMPONENT:-core}
+info "Opened backport PR ${github_endpoint}/pull/$number"
+debug "Setting ${component} label"
+curl --silent --data-binary '{"milestone":'$milestone_number',"assignee":"'$github_user'","labels":["'$component'"]}' 'https://api.github.com/repos/ceph/ceph/issues/'$number'?access_token='$github_token >/dev/null
+info "Set ${component} label in PR"
+pgrep firefox >/dev/null && firefox ${github_endpoint}/pull/$number
 
-component=${COMPONENT:-core}; curl --silent --data-binary '{"milestone":'$milestone_number',"assignee":"'$github_user'","labels":["bug fix","'$component'"]}' 'https://api.github.com/repos/ceph/ceph/issues/'$number'?access_token='$github_token
-firefox https://github.com/ceph/ceph/pull/$number
+debug "Updating backport tracker issue in Redmine"
 redmine_status=2 # In Progress
-curl --verbose -X PUT --header 'Content-type: application/json' --data-binary '{"issue":{"description":"https://github.com/ceph/ceph/pull/'$number'","status_id":'$redmine_status',"assigned_to_id":'$redmine_user_id'}}' $redmine_endpoint'/issues/'$issue.json?key=$redmine_key
-echo "Staged ${redmine_endpoint}/${issue}"
-
-firefox ${redmine_endpoint}/issues/${issue}
+curl -X PUT --header 'Content-type: application/json' --data-binary '{"issue":{"description":"https://github.com/ceph/ceph/pull/'$number'","status_id":'$redmine_status',"assigned_to_id":'$redmine_user_id'}}' ${redmine_url}'.json?key='$redmine_key
+info "${redmine_url} updated"
+pgrep firefox >/dev/null && firefox ${redmine_url}