pulp_upload.sh: include the flavor in repository and distribution names

Repository names were branch/os/version-scoped and distribution names
derived from them, so default and debug builds of the same branch/sha1
shared repositories and produced identically named distributions: the
stale-distribution delete would clobber one flavor's distribution with
the other's, and both flavors' packages mixed in one repository even
though their base_paths are flavor-specific.

Add FLAVOR to REPO_NAME, which flows into the per-arch repository names
and the dist-<repo>-<short_sha1> distribution names, and update the
existence check in the ceph-dev-pipeline Jenkinsfile to match.

Distributions created under the old naming may still own a base_path
(base_paths are unique in pulp), so also delete any differently named
distribution occupying the base_path before creating the new one.
Existing repositories with unflavored names are left in place and can
be cleaned up manually.

Signed-off-by: David Galloway <david.galloway@ibm.com>

diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile
index cd8d404dee0bdbccecf9d56c17c062412ea16deb..8fdf92bd4b00d3cb7fcfbf86fa2764dcff7ec773 100644 (file)
--- a/ceph-dev-pipeline/build/Jenkinsfile
+++ b/ceph-dev-pipeline/build/Jenkinsfile
@@ -274,15 +274,13 @@ def doCheckForPulpPackagesStage() {
   def os = get_os_info(env.DIST)
 
   // pulp_upload.sh reuses branch-scoped repositories; only its per-sha1
-  // distribution (dist-ceph-<branch>-<os>-<version>-<arch>-<short_sha1>)
+  // distribution (dist-ceph-<branch>-<os>-<version>-<flavor>-<arch>-<short_sha1>)
   // proves this sha1 was uploaded. The name must mirror pulp_upload.sh:
   // ubuntu uses the codename, rpm arches use aarch64 (not arm64), and rpm
-  // lookups use --distribution while deb uses --name. Distribution names
-  // don't include the flavor (a pulp_upload.sh limitation), so this check
-  // can't tell default and debug builds of the same sha1 apart.
+  // lookups use --distribution while deb uses --name.
   def os_version = (os.name == "ubuntu") ? os.version_name : os.version
   def dist_arch = (os.pkg_type == "rpm" && env.ARCH == "arm64") ? "aarch64" : env.ARCH
-  def dist_name = "dist-ceph-${env.BRANCH}-${os.name}-${os_version}-${dist_arch}-${env.SHA1[-8..-1]}"
+  def dist_name = "dist-ceph-${env.BRANCH}-${os.name}-${os_version}-${env.FLAVOR}-${dist_arch}-${env.SHA1[-8..-1]}"
   def lookup_flag = (os.pkg_type == "rpm") ? "--distribution" : "--name"
   def pulp_dist_rc = sh(
     script: """#!/bin/bash -ex

diff --git a/scripts/pulp_upload.sh b/scripts/pulp_upload.sh
index 38c39d8fcd85403f6c4b74cb03b2deb39243c0ca..00c1c36274ba711db5a4de829f89b9be29ae61f7 100755 (executable)
--- a/scripts/pulp_upload.sh
+++ b/scripts/pulp_upload.sh
@@ -319,7 +319,7 @@ publish_pulp_distribution() {
     local repo_endpoint="$2"
     local repo_arch="$3"
     local package_version="$4"
-    local final_version pub_href dist_name lookup_flag
+    local final_version pub_href dist_name lookup_flag stale_dist
 
     if [ "$OS_PKG_TYPE" == "rpm" ]; then
         pub_href=$(
@@ -357,6 +357,21 @@ publish_pulp_distribution() {
         fi
     fi
 
+    # A distribution created before FLAVOR was part of the naming scheme may
+    # still own this base_path under a different name; base_paths are unique
+    # in pulp, so it has to be removed before the new distribution is created.
+    stale_dist=$(pulp "${OS_PKG_TYPE}" distribution list \
+        --base-path "${repo_endpoint}" 2> /dev/null \
+        | jq -r '.[0].name // empty' || true)
+    if [ -n "$stale_dist" ] && [ "$stale_dist" != "$dist_name" ]; then
+        log "Distribution ${stale_dist} owns base_path ${repo_endpoint}; deleting"
+        if ! pulp "${OS_PKG_TYPE}" distribution destroy \
+                "${lookup_flag}" "${stale_dist}"; then
+            log "ERROR: Failed to delete ${OS_PKG_TYPE} distribution ${stale_dist}"
+            return
+        fi
+    fi
+
     log "Creating distribution ${dist_name} " \
         "with base_path=${repo_endpoint}"
     if ! pulp "${OS_PKG_TYPE}" distribution create \
@@ -387,7 +402,10 @@ log "Uploading artifacts to Pulp repository ..."
 _OS_VERSION=$(resolve_os_version_for_repo)
 REPO_VERSIONS_TO_RETAIN=$(get_repo_versions_to_retain)
 
-REPO_NAME="${PULP_PROJECT}-${BRANCH}-${OS_NAME}-${_OS_VERSION}"
+# FLAVOR is part of the repository (and therefore distribution) name so that
+# default and debug builds of the same branch/sha1 don't share repositories
+# or clobber each other's distributions.
+REPO_NAME="${PULP_PROJECT}-${BRANCH}-${OS_NAME}-${_OS_VERSION}-${FLAVOR}"
 REPO_ENDPOINT="repos/${PULP_PROJECT}/${BRANCH}/${SHA1}/${OS_NAME}"
 REPO_ENDPOINT="${REPO_ENDPOINT}/${_OS_VERSION}/flavors/${FLAVOR}"