From 2ade9390c63b0175531e92cf07c048b109f71648 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sat, 7 Jan 2023 15:49:00 -0500
Subject: [PATCH] tools/cephfs/first-damage: repair specific first=CEPH_NOSNAP
 damage

first==CEPH_NOSNAP is never valid. Apparently somehow the MDS may set
this in some code path yet to be discovered, even without snapshots.

Note, we probably should not necessarily automate this fix via the MDS
because we do not know if this dentry (visible in HEAD) should also be
visible before NEXT_SNAP.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/workunits/fs/damage/test-first-damage.sh | 16 ++++++++++++----
 src/tools/cephfs/first-damage.py            | 17 +++++++++++++++--
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/qa/workunits/fs/damage/test-first-damage.sh b/qa/workunits/fs/damage/test-first-damage.sh
index ed1fc060367..c1ae71ed013 100755
--- a/qa/workunits/fs/damage/test-first-damage.sh
+++ b/qa/workunits/fs/damage/test-first-damage.sh
@@ -49,14 +49,19 @@ function damage {
 
   # nuke snap 1 version of "a"
   rados --pool="$METADATA_POOL" getomapval "$IS" a_$(printf %x $((LS-4))) "$T"
-  printf '\xff\xff\xff\xf0' | dd of="$T" count=4 bs=1
+  printf '\xff\xff\xff\xf0' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
   rados --pool="$METADATA_POOL" setomapval "$IS" a_$(printf %x $((LS-4))) --input-file="$T"
 
   # nuke snap 4 version of "a"
   rados --pool="$METADATA_POOL" getomapval "$IS" a_$(printf %x $((LS-1))) "$T"
-  printf '\xff\xff\xff\xff' | dd of="$T" count=4 bs=1
+  printf '\xff\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
   rados --pool="$METADATA_POOL" setomapval "$IS" a_$(printf %x $((LS-1))) --input-file="$T"
 
+  # screw up HEAD
+  rados --pool="$METADATA_POOL" getomapval "$IS" a_head "$T"
+  printf '\xfe\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
+  rados --pool="$METADATA_POOL" setomapval "$IS" a_head --input-file="$T"
+
   rm -f "$T"
 }
 
@@ -66,13 +71,15 @@ function recover {
   sleep 5
   cephfs-journal-tool --rank="$FS":0 event recover_dentries summary
   cephfs-journal-tool --rank="$FS":0 journal reset
-  python3 $FIRST_DAMAGE --memo /tmp/memo1 "$METADATA_POOL"
-  python3 $FIRST_DAMAGE --memo /tmp/memo2 --remove "$METADATA_POOL"
+  python3 $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL"
+  python3 $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap  "$METADATA_POOL"
+  python3 $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL"
   ceph fs set "$FS" joinable true
 }
 
 function check {
   stat dir || exit 1
+  stat dir/a || exit 1
   for i in `seq 1 5`; do
     stat dir/.snap/$i || exit 2
   done
@@ -94,6 +101,7 @@ function check {
 
 function cleanup {
   rmdir dir/.snap/*
+  find dir
   rm -rf dir
 }
 
diff --git a/src/tools/cephfs/first-damage.py b/src/tools/cephfs/first-damage.py
index bc683473bcc..0446aacf177 100644
--- a/src/tools/cephfs/first-damage.py
+++ b/src/tools/cephfs/first-damage.py
@@ -57,6 +57,9 @@ REMOVE = False
 POOL = None
 NEXT_SNAP = None
 CONF = os.environ['CEPH_CONF']
+REPAIR_NOSNAP = None
+
+CEPH_NOSNAP = 0xfffffffe # int32 -2
 
 DIR_PATTERN = re.compile(r'[0-9a-fA-F]{8,}\.[0-9a-fA-F]+')
 
@@ -76,11 +79,19 @@ def traverse(MEMO, ioctx):
             it = ioctx.get_omap_vals(rctx, None, None, 100000)[0]
             ioctx.operate_read_op(rctx, o.key)
             for (dnk, val) in it:
-                log.debug('\t%s', dnk)
+                log.debug('\t%s: val size %d', dnk, len(val))
                 (first,) = struct.unpack('<I', val[:4])
                 if first > NEXT_SNAP:
                     log.warning(f"found {o.key}:{dnk} first (0x{first:x}) > NEXT_SNAP (0x{NEXT_SNAP:x})")
-                    if REMOVE:
+                    if REPAIR_NOSNAP and dnk.endswith("_head") and first == CEPH_NOSNAP:
+                        log.warning(f"repairing first==CEPH_NOSNAP damage, setting to NEXT_SNAP (0x{NEXT_SNAP:x})")
+                        first = NEXT_SNAP
+                        nval = bytearray(val)
+                        struct.pack_into("<I", nval, 0, NEXT_SNAP)
+                        with rados.WriteOpCtx() as wctx:
+                            ioctx.set_omap(wctx, (dnk,), (bytes(nval),))
+                            ioctx.operate_write_op(wctx, o.key)
+                    elif REMOVE:
                         log.warning(f"removing {o.key}:{dnk}")
                         with rados.WriteOpCtx() as wctx:
                             ioctx.remove_omap_keys(wctx, [dnk])
@@ -95,6 +106,7 @@ if __name__ == '__main__':
     P.add_argument('--memo', action='store', help='db for traversed dirs', default=outpath+'.memo')
     P.add_argument('--next-snap', action='store', help='force next-snap (dev)', type=int)
     P.add_argument('--remove', action='store_true', help='remove bad dentries', default=False)
+    P.add_argument('--repair-nosnap', action='store_true', help='repair first=CEPH_NOSNAP damage', default=False)
     P.add_argument('pool', action='store', help='metadata pool', type=str)
     NS = P.parse_args()
 
@@ -105,6 +117,7 @@ if __name__ == '__main__':
     POOL = NS.pool
     NEXT_SNAP = NS.next_snap
     CONF = NS.conf
+    REPAIR_NOSNAP = NS.repair_nosnap
 
     log.info("running as pid %d", os.getpid())
 
-- 
2.39.5