podman auth: share one authfile via same-shell exports

This reverts the two previous attempts at fixing the Docker Hub
rate-limit failures -- commit aae7f2fd ("bwc: pin podman auth to a
persistent authfile") and PR #2656 (merge f1996164, the
ceph-dev-pipeline equivalent) -- and replaces both. They were built on
wrong assumptions:

- aae7f2fd blamed logind tearing down the XDG runtime dir between
  steps. An auditd watch on /run/user/<uid>/containers/ disproved
  that: nothing external ever touches auth.json there; every write is
  podman login's own atomic tmp+rename. Linger is enabled and the
  agent runs as a systemd service with no login sessions, so the
  directory is stable across the whole job.
- PR #2656 added --authfile to the logins plus a Groovy
  env.REGISTRY_AUTH_FILE assignment. Tracker 77920 recurred on
  2026-07-22 with it deployed: login wrote docker.io credentials to
  $HOME/.config/containers/auth.json, and the
  FROM docker.io/ubuntu:22.04 pull inside podman build (spawned by
  build-with-container.py) still ran anonymous and hit
  toomanyrequests.

A debug run on a builder pinned the real mechanism. With the variable
present in podman build's environment, the base image pull uses it:

  $ REGISTRY_AUTH_FILE=$HOME/.config/containers/auth.json \
      podman build --pull --log-level=debug ... 2>&1 | grep -i cred
  "Found credentials for docker.io/library/ubuntu ... in file
   /home/jenkins-build/.config/containers/auth.json"

So the pull authenticates whenever REGISTRY_AUTH_FILE actually reaches
the process, which means on 2026-07-22 it did not. That dictates the
shape of this change:

- No --authfile on logins: pointing only the login at a file steers
  credentials somewhere later pulls never look, which is worse than
  the podman default.
- No Groovy env threading: it demonstrably failed to deliver the
  variable to the bwc subprocess, and it silently depends on stage
  ordering and when{} guards.
- Instead, every sh block that invokes podman or
  build-with-container.py exports REGISTRY_AUTH_FILE itself. A
  same-shell export reaches podman through ordinary process
  inheritance and cannot be lost. podman login honors the variable
  for writes, so login and every later read use the same persistent
  file by construction. The export line is repeated verbatim in each
  block on purpose: each block is self-sufficient and the pattern is
  greppable.

Fixes: https://tracker.ceph.com/issues/77920
Signed-off-by: David Galloway <david.galloway@ibm.com>

diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile
index 7744b89d65e4169e41bdf9b7d2b8cd6e074ff4f3..d2f96577347e5ee98f45e090d850dc5aae8fe1a8 100644 (file)
--- a/ceph-dev-pipeline/build/Jenkinsfile
+++ b/ceph-dev-pipeline/build/Jenkinsfile
@@ -346,26 +346,20 @@ def doArtifactsChecksStage() {
 // Stage 6 (builder container): log into container registries and pull/build/push the ceph-build container image for this matrix cell.
 def doBuilderContainerStage() {
   env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build"
-  // Pin podman auth to a persistent authfile. The agent runs as a systemd
-  // service with no login session and no XDG_RUNTIME_DIR, so podman derives
-  // the default auth.json location from ambient node state at each
-  // invocation, landing in runtime tmpfs the job doesn't control.
-  // Credentials written by podman login here have been observed unavailable
-  // to podman build in the same job minutes later, so the base image pull
-  // goes anonymous and hits Docker Hub's rate limit (toomanyrequests,
-  // https://tracker.ceph.com/issues/77920). Set via env so every subsequent
-  // sh step (pull/build/push here, and build-with-container.py in the build
-  // stage) resolves the same file.
-  env.REGISTRY_AUTH_FILE = "${env.HOME}/.config/containers/auth.json"
+  // Every podman-running sh block exports REGISTRY_AUTH_FILE itself so login
+  // and all later pulls (incl. podman build inside build-with-container.py)
+  // share one persistent authfile: https://tracker.ceph.com/issues/77920
   sh '''#!/bin/bash
     set -ex
+    export REGISTRY_AUTH_FILE="${HOME}/.config/containers/auth.json"
     mkdir -p "${REGISTRY_AUTH_FILE%/*}"
-    podman login --authfile ${REGISTRY_AUTH_FILE} -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION}
-    podman login --authfile ${REGISTRY_AUTH_FILE} -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io
+    podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION}
+    podman login -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io
   '''
   def ceph_builder_tag_short = "${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}"
   def ceph_builder_tag = "${env.SHA1[0..6]}.${ceph_builder_tag_short}"
   sh """#!/bin/bash -ex
+    export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json"
     podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} || \
     podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} || \
     true
@@ -373,6 +367,7 @@ def doBuilderContainerStage() {
   def withCrimson = !(env.BRANCH in ['tentacle', 'squid', 'reef'])
   sh """#!/bin/bash
     set -ex
+    export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json"
     echo > .env
     echo "WITH_CRIMSON=${withCrimson}" >> .env
     cd dist/ceph
@@ -380,6 +375,7 @@ def doBuilderContainerStage() {
     podman tag ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short}
   """
   sh """#!/bin/bash -ex
+    export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json"
     podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short}
     podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag}
   """
@@ -473,18 +469,21 @@ def doBuildStage() {
     echo "CEPH_EXTRA_CMAKE_ARGS=${ceph_extra_cmake_args}" >> .env
   """
   sh """#!/bin/bash -ex
+    export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json"
     cd dist/ceph
     ln ../ceph-${env.VERSION}.tar.bz2 .
     ${bwc_command}
   """
   if ( os.pkg_type == "deb" ) {
     sh """#!/bin/bash -ex
+      export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json"
       cd dist/ceph
       ${bwc_command_base} -e custom -- "dpkg-deb --fsys-tarfile /ceph/debs/*/pool/main/c/ceph/cephadm_${env.VERSION}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm"
       ln ./cephadm ../../
     """
   } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) {
     sh """#!/bin/bash -ex
+      export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json"
       cd dist/ceph
       ${bwc_command_base} -e custom -- "rpm2cpio /ceph/rpmbuild/RPMS/noarch/cephadm-*.rpm | cpio -i --to-stdout *sbin/cephadm > cephadm"
       ln ./cephadm ../../

diff --git a/scripts/bwc.sh b/scripts/bwc.sh
index 184a3e30df8bbc9d630328b69fda200204528489..54c667cb579f8c5f26b3661b877d2a7de51007c0 100644 (file)
--- a/scripts/bwc.sh
+++ b/scripts/bwc.sh
@@ -83,10 +83,11 @@ bwc_login() {
     if [ -z "${DOCKER_HUB_USERNAME}" ] || [ -z "${DOCKER_HUB_PASSWORD}" ]; then
         return 0
     fi
+    # Same-shell export so login and all later podman calls share one
+    # persistent authfile: https://tracker.ceph.com/issues/77920
     export REGISTRY_AUTH_FILE="${HOME}/.config/containers/auth.json"
     mkdir -p "${REGISTRY_AUTH_FILE%/*}"
-    podman login --authfile "${REGISTRY_AUTH_FILE}" \
-        -u "${DOCKER_HUB_USERNAME}" -p "${DOCKER_HUB_PASSWORD}" docker.io
+    podman login -u "${DOCKER_HUB_USERNAME}" -p "${DOCKER_HUB_PASSWORD}" docker.io
 }
 
 # bwc_arch - Print the architecture of the current host in the style