tools/cephfs/first-damage: repair specific first=CEPH_NOSNAP damage

first==CEPH_NOSNAP is never valid. Apparently somehow the MDS may set
this in some code path yet to be discovered, even without snapshots.

Note, we probably should not necessarily automate this fix via the MDS
because we do not know if this dentry (visible in HEAD) should also be
visible before NEXT_SNAP.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/qa/workunits/fs/damage/test-first-damage.sh b/qa/workunits/fs/damage/test-first-damage.sh
index ed1fc0603677fa41b23b2c92b57a62f63752d2b7..c1ae71ed013bf3c1c1e2e6eeea9fbe51e247de68 100755 (executable)
--- a/qa/workunits/fs/damage/test-first-damage.sh
+++ b/qa/workunits/fs/damage/test-first-damage.sh
@@ -49,14 +49,19 @@ function damage {
 
   # nuke snap 1 version of "a"
   rados --pool="$METADATA_POOL" getomapval "$IS" a_$(printf %x $((LS-4))) "$T"
-  printf '\xff\xff\xff\xf0' | dd of="$T" count=4 bs=1
+  printf '\xff\xff\xff\xf0' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
   rados --pool="$METADATA_POOL" setomapval "$IS" a_$(printf %x $((LS-4))) --input-file="$T"
 
   # nuke snap 4 version of "a"
   rados --pool="$METADATA_POOL" getomapval "$IS" a_$(printf %x $((LS-1))) "$T"
-  printf '\xff\xff\xff\xff' | dd of="$T" count=4 bs=1
+  printf '\xff\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
   rados --pool="$METADATA_POOL" setomapval "$IS" a_$(printf %x $((LS-1))) --input-file="$T"
 
+  # screw up HEAD
+  rados --pool="$METADATA_POOL" getomapval "$IS" a_head "$T"
+  printf '\xfe\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
+  rados --pool="$METADATA_POOL" setomapval "$IS" a_head --input-file="$T"
+
   rm -f "$T"
 }
 
@@ -66,13 +71,15 @@ function recover {
   sleep 5
   cephfs-journal-tool --rank="$FS":0 event recover_dentries summary
   cephfs-journal-tool --rank="$FS":0 journal reset
-  python3 $FIRST_DAMAGE --memo /tmp/memo1 "$METADATA_POOL"
-  python3 $FIRST_DAMAGE --memo /tmp/memo2 --remove "$METADATA_POOL"
+  python3 $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL"
+  python3 $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap  "$METADATA_POOL"
+  python3 $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL"
   ceph fs set "$FS" joinable true
 }
 
 function check {
   stat dir || exit 1
+  stat dir/a || exit 1
   for i in `seq 1 5`; do
     stat dir/.snap/$i || exit 2
   done
@@ -94,6 +101,7 @@ function check {
 
 function cleanup {
   rmdir dir/.snap/*
+  find dir
   rm -rf dir
 }
 

diff --git a/src/tools/cephfs/first-damage.py b/src/tools/cephfs/first-damage.py
index bc683473bcc3ff5ec41cef4e42dadce7a533b63a..0446aacf177b80bec474770a3715c5df172662e4 100644 (file)
--- a/src/tools/cephfs/first-damage.py
+++ b/src/tools/cephfs/first-damage.py
@@ -57,6 +57,9 @@ REMOVE = False
 POOL = None
 NEXT_SNAP = None
 CONF = os.environ['CEPH_CONF']
+REPAIR_NOSNAP = None
+
+CEPH_NOSNAP = 0xfffffffe # int32 -2
 
 DIR_PATTERN = re.compile(r'[0-9a-fA-F]{8,}\.[0-9a-fA-F]+')
 
@@ -76,11 +79,19 @@ def traverse(MEMO, ioctx):
             it = ioctx.get_omap_vals(rctx, None, None, 100000)[0]
             ioctx.operate_read_op(rctx, o.key)
             for (dnk, val) in it:
-                log.debug('\t%s', dnk)
+                log.debug('\t%s: val size %d', dnk, len(val))
                 (first,) = struct.unpack('<I', val[:4])
                 if first > NEXT_SNAP:
                     log.warning(f"found {o.key}:{dnk} first (0x{first:x}) > NEXT_SNAP (0x{NEXT_SNAP:x})")
-                    if REMOVE:
+                    if REPAIR_NOSNAP and dnk.endswith("_head") and first == CEPH_NOSNAP:
+                        log.warning(f"repairing first==CEPH_NOSNAP damage, setting to NEXT_SNAP (0x{NEXT_SNAP:x})")
+                        first = NEXT_SNAP
+                        nval = bytearray(val)
+                        struct.pack_into("<I", nval, 0, NEXT_SNAP)
+                        with rados.WriteOpCtx() as wctx:
+                            ioctx.set_omap(wctx, (dnk,), (bytes(nval),))
+                            ioctx.operate_write_op(wctx, o.key)
+                    elif REMOVE:
                         log.warning(f"removing {o.key}:{dnk}")
                         with rados.WriteOpCtx() as wctx:
                             ioctx.remove_omap_keys(wctx, [dnk])
@@ -95,6 +106,7 @@ if __name__ == '__main__':
     P.add_argument('--memo', action='store', help='db for traversed dirs', default=outpath+'.memo')
     P.add_argument('--next-snap', action='store', help='force next-snap (dev)', type=int)
     P.add_argument('--remove', action='store_true', help='remove bad dentries', default=False)
+    P.add_argument('--repair-nosnap', action='store_true', help='repair first=CEPH_NOSNAP damage', default=False)
     P.add_argument('pool', action='store', help='metadata pool', type=str)
     NS = P.parse_args()
 
@@ -105,6 +117,7 @@ if __name__ == '__main__':
     POOL = NS.pool
     NEXT_SNAP = NS.next_snap
     CONF = NS.conf
+    REPAIR_NOSNAP = NS.repair_nosnap
 
     log.info("running as pid %d", os.getpid())