From 49e8a0729c4a57cbaec0078a91d4f2fb7d9167df Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Fri, 10 Jul 2026 11:35:21 +0800 Subject: [PATCH] vstart.sh: retry ceph-osd --mkfs on transient monitor auth failure "ceph-osd --mkfs" authenticates to the monitor as the freshly-created osd.N entity. Right after "ceph osd new", the monitor can briefly reject that auth with EACCES: mon: _auth_bad_method auth_method 2 r (13) Permission denied osd: handle_auth_bad_method server allowed_methods [2] but i only support [2] failed to fetch mon config (--no-mon-config to skip) A bad-method reply is terminal, so mkfs aborts and the cluster never becomes healthy. It is intermittent and was seen with a single monitor, so it is not cross-monitor propagation lag. Same symptom as ceph commit 7afd38f84689, which teuthology already works around; vstart.sh has no guard. Add a retry() helper and retry the mkfs. The failure is in MonClient bootstrap, before any objectstore write, so each attempt starts clean. Output goes to the per-osd log rather than through tee, so mkfs's own exit status is checked and a real failure still aborts vstart. Signed-off-by: Kefu Chai --- src/vstart.sh | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/vstart.sh b/src/vstart.sh index c64c9cf99837..6a2eb1c48cf9 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -37,6 +37,29 @@ prun() { PATH=$CEPH_BIN:$PATH "$@" } +# retry [args...] +# Run the command until it succeeds or attempts have been made +# (use 0 for unlimited). Returns the command's last exit status. +retry() { + local max=$1 delay=$2 what=$3 + shift 3 + local n=0 rc=0 + while true; do + rc=0 + "$@" || rc=$? + if [ "$rc" -eq 0 ]; then + return 0 + fi + n=$((n + 1)) + if [ "$max" -ne 0 ] && [ "$n" -ge "$max" ]; then + echo "$what failed after $n attempt(s) (rc=$rc)" >&2 + return "$rc" + fi + echo "$what failed (rc=$rc), retry $n in ${delay}s" >&2 + sleep "$delay" + done +} + if [ -n "$VSTART_DEST" ]; then SRC_PATH=`dirname $0` @@ -1330,8 +1353,14 @@ EOF echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd$osd/new.json ceph_adm osd new $uuid -i $CEPH_DEV_DIR/osd$osd/new.json rm $CEPH_DEV_DIR/osd$osd/new.json - prun $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args \ - 2>&1 | tee $CEPH_OUT_DIR/osd-mkfs.$osd.log + # ceph-osd --mkfs authenticates to the monitor as the just-created + # osd.$osd entity, which the monitor can briefly reject with EACCES + # right after "osd new" (handle_auth_bad_method / failed to fetch + # mon config). Transient; retry past it. + retry 10 2 "ceph-osd --mkfs for osd.$osd" \ + prun $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS \ + --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args \ + > $CEPH_OUT_DIR/osd-mkfs.$osd.log 2>&1 || exit $? local key_fn=$CEPH_DEV_DIR/osd$osd/keyring cat > $key_fn<