From ebb51dd71e81e8012fa7381bdadcde2944e707d7 Mon Sep 17 00:00:00 2001 From: David Galloway Date: Fri, 31 Jul 2026 16:17:33 -0400 Subject: [PATCH] podman auth: share one authfile via same-shell exports This reverts the two previous attempts at fixing the Docker Hub rate-limit failures -- commit aae7f2fd ("bwc: pin podman auth to a persistent authfile") and PR #2656 (merge f1996164, the ceph-dev-pipeline equivalent) -- and replaces both. They were built on wrong assumptions: - aae7f2fd blamed logind tearing down the XDG runtime dir between steps. An auditd watch on /run/user//containers/ disproved that: nothing external ever touches auth.json there; every write is podman login's own atomic tmp+rename. Linger is enabled and the agent runs as a systemd service with no login sessions, so the directory is stable across the whole job. - PR #2656 added --authfile to the logins plus a Groovy env.REGISTRY_AUTH_FILE assignment. Tracker 77920 recurred on 2026-07-22 with it deployed: login wrote docker.io credentials to $HOME/.config/containers/auth.json, and the FROM docker.io/ubuntu:22.04 pull inside podman build (spawned by build-with-container.py) still ran anonymous and hit toomanyrequests. A debug run on a builder pinned the real mechanism. With the variable present in podman build's environment, the base image pull uses it: $ REGISTRY_AUTH_FILE=$HOME/.config/containers/auth.json \ podman build --pull --log-level=debug ... 2>&1 | grep -i cred "Found credentials for docker.io/library/ubuntu ... in file /home/jenkins-build/.config/containers/auth.json" So the pull authenticates whenever REGISTRY_AUTH_FILE actually reaches the process, which means on 2026-07-22 it did not. That dictates the shape of this change: - No --authfile on logins: pointing only the login at a file steers credentials somewhere later pulls never look, which is worse than the podman default. - No Groovy env threading: it demonstrably failed to deliver the variable to the bwc subprocess, and it silently depends on stage ordering and when{} guards. - Instead, every sh block that invokes podman or build-with-container.py exports REGISTRY_AUTH_FILE itself. A same-shell export reaches podman through ordinary process inheritance and cannot be lost. podman login honors the variable for writes, so login and every later read use the same persistent file by construction. The export line is repeated verbatim in each block on purpose: each block is self-sufficient and the pattern is greppable. Fixes: https://tracker.ceph.com/issues/77920 Signed-off-by: David Galloway --- ceph-dev-pipeline/build/Jenkinsfile | 25 ++++++++++++------------- scripts/bwc.sh | 5 +++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile index 7744b89d6..d2f965773 100644 --- a/ceph-dev-pipeline/build/Jenkinsfile +++ b/ceph-dev-pipeline/build/Jenkinsfile @@ -346,26 +346,20 @@ def doArtifactsChecksStage() { // Stage 6 (builder container): log into container registries and pull/build/push the ceph-build container image for this matrix cell. def doBuilderContainerStage() { env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build" - // Pin podman auth to a persistent authfile. The agent runs as a systemd - // service with no login session and no XDG_RUNTIME_DIR, so podman derives - // the default auth.json location from ambient node state at each - // invocation, landing in runtime tmpfs the job doesn't control. - // Credentials written by podman login here have been observed unavailable - // to podman build in the same job minutes later, so the base image pull - // goes anonymous and hits Docker Hub's rate limit (toomanyrequests, - // https://tracker.ceph.com/issues/77920). Set via env so every subsequent - // sh step (pull/build/push here, and build-with-container.py in the build - // stage) resolves the same file. - env.REGISTRY_AUTH_FILE = "${env.HOME}/.config/containers/auth.json" + // Every podman-running sh block exports REGISTRY_AUTH_FILE itself so login + // and all later pulls (incl. podman build inside build-with-container.py) + // share one persistent authfile: https://tracker.ceph.com/issues/77920 sh '''#!/bin/bash set -ex + export REGISTRY_AUTH_FILE="${HOME}/.config/containers/auth.json" mkdir -p "${REGISTRY_AUTH_FILE%/*}" - podman login --authfile ${REGISTRY_AUTH_FILE} -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION} - podman login --authfile ${REGISTRY_AUTH_FILE} -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io + podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION} + podman login -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io ''' def ceph_builder_tag_short = "${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}" def ceph_builder_tag = "${env.SHA1[0..6]}.${ceph_builder_tag_short}" sh """#!/bin/bash -ex + export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json" podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} || \ podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} || \ true @@ -373,6 +367,7 @@ def doBuilderContainerStage() { def withCrimson = !(env.BRANCH in ['tentacle', 'squid', 'reef']) sh """#!/bin/bash set -ex + export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json" echo > .env echo "WITH_CRIMSON=${withCrimson}" >> .env cd dist/ceph @@ -380,6 +375,7 @@ def doBuilderContainerStage() { podman tag ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} """ sh """#!/bin/bash -ex + export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json" podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} """ @@ -473,18 +469,21 @@ def doBuildStage() { echo "CEPH_EXTRA_CMAKE_ARGS=${ceph_extra_cmake_args}" >> .env """ sh """#!/bin/bash -ex + export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json" cd dist/ceph ln ../ceph-${env.VERSION}.tar.bz2 . ${bwc_command} """ if ( os.pkg_type == "deb" ) { sh """#!/bin/bash -ex + export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json" cd dist/ceph ${bwc_command_base} -e custom -- "dpkg-deb --fsys-tarfile /ceph/debs/*/pool/main/c/ceph/cephadm_${env.VERSION}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm" ln ./cephadm ../../ """ } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) { sh """#!/bin/bash -ex + export REGISTRY_AUTH_FILE="\${HOME}/.config/containers/auth.json" cd dist/ceph ${bwc_command_base} -e custom -- "rpm2cpio /ceph/rpmbuild/RPMS/noarch/cephadm-*.rpm | cpio -i --to-stdout *sbin/cephadm > cephadm" ln ./cephadm ../../ diff --git a/scripts/bwc.sh b/scripts/bwc.sh index 184a3e30d..54c667cb5 100644 --- a/scripts/bwc.sh +++ b/scripts/bwc.sh @@ -83,10 +83,11 @@ bwc_login() { if [ -z "${DOCKER_HUB_USERNAME}" ] || [ -z "${DOCKER_HUB_PASSWORD}" ]; then return 0 fi + # Same-shell export so login and all later podman calls share one + # persistent authfile: https://tracker.ceph.com/issues/77920 export REGISTRY_AUTH_FILE="${HOME}/.config/containers/auth.json" mkdir -p "${REGISTRY_AUTH_FILE%/*}" - podman login --authfile "${REGISTRY_AUTH_FILE}" \ - -u "${DOCKER_HUB_USERNAME}" -p "${DOCKER_HUB_PASSWORD}" docker.io + podman login -u "${DOCKER_HUB_USERNAME}" -p "${DOCKER_HUB_PASSWORD}" docker.io } # bwc_arch - Print the architecture of the current host in the style -- 2.47.3