test: ceph_filestore_dump.sh test improvements

Add some usage error tests
Don't use the same var in second for loop
Add xattr/omap to rep pool and xattr to ec pool
Add list, get-bytes and set-bytes testing
Add list-attrs and get-attr

Signed-off-by: David Zafman <david.zafman@inktank.com>

diff --git a/src/test/ceph_filestore_dump.sh b/src/test/ceph_filestore_dump.sh
index 66eaffe5a9b96caba6582b4768d31632b0ca6338..36ffe068a9af35629272800d0c2e0aaeef84eb46 100755 (executable)
--- a/src/test/ceph_filestore_dump.sh
+++ b/src/test/ceph_filestore_dump.sh
@@ -17,9 +17,10 @@ NUM_OBJECTS=40
 ERRORS=0
 TESTDIR="/tmp/test.$$"
 DATADIR="/tmp/data.$$"
+JSONOBJ="/tmp/json.$$"
 
 echo -n "vstarting...."
-OSD=4 ./vstart.sh -l -n -d -o "mon_pg_warn_min_per_osd=1" > /dev/null 2>&1
+OSD=4 ./vstart.sh -l -n -d > /dev/null 2>&1
 echo DONE
 
 wait_for_health
@@ -42,11 +43,33 @@ for i in `seq 1 $NUM_OBJECTS`
 do
   NAME="${REP_NAME}$i"
   rm -f $DATADIR/$NAME
-  for i in `seq 1 10000`
+  for j in `seq 1 10000`
   do
     echo "This is the replicated data for $NAME" >> $DATADIR/$NAME
   done
   ./rados -p $REP_POOL put $NAME $DATADIR/$NAME  2> /dev/null
+  ACNT=`expr $i - 1`
+  for k in `seq 0 $ACNT`
+  do
+    if [ $k = "0" ];
+    then
+      continue
+    fi
+    ./rados -p $REP_POOL setxattr $NAME key${i}-${k} val${i}-${k}
+  done
+  # Create omap header in all objects but REPobject1
+  if [ $i != "1" ];
+  then
+    ./rados -p $REP_POOL setomapheader $NAME hdr${i}
+  fi
+  for k in `seq 0 $ACNT`
+  do
+    if [ $k = "0" ];
+    then
+      continue
+    fi
+    ./rados -p $REP_POOL setomapval $NAME okey${i}-${k} oval${i}-${k}
+  done
 done
 
 echo "Creating $NUM_OBJECTS objects in erausre coded pool"
@@ -54,11 +77,21 @@ for i in `seq 1 $NUM_OBJECTS`
 do
   NAME="${EC_NAME}$i"
   rm -f $DATADIR/$NAME
-  for i in `seq 1 10000`
+  for j in `seq 1 10000`
   do
     echo "This is the erasure coded data for $NAME" >> $DATADIR/$NAME
   done
   ./rados -p $EC_POOL put $NAME $DATADIR/$NAME  2> /dev/null
+  ACNT=`expr $i - 1`
+  for k in `seq 0 $ACNT`
+  do
+    if [ $k = "0" ];
+    then
+      continue
+    fi
+    ./rados -p $EC_POOL setxattr $NAME key${i}-${k} val${i}-${k}
+  done
+  # Omap isn't supported in EC pools
 done
 
 ./stop.sh > /dev/null 2>&1
@@ -73,6 +106,213 @@ OBJREPPGS=`ls dev/*/current/${REPID}.*_head/${REP_NAME}* | awk -F / '{ print $4}
 OBJECPGS=`ls dev/*/current/${ECID}.*_head/${EC_NAME}* | awk -F / '{ print $4}' | sed 's/_head//' | sort -u`
 #echo OBJECT EC PGS: $OBJECPGS
 
+ONEPG=`echo $ALLREPPGS | awk '{ print $1 }'`
+#echo $ONEPG
+ONEOSD=`ls -d dev/*/current/${ONEPG}_head | awk -F / '{ print $2 }' | head -1`
+#echo $ONEOSD
+
+# On export can't use stdout to a terminal
+./ceph_filestore_dump --filestore-path dev/$ONEOSD --journal-path dev/$ONEOSD.journal --type export --pgid $ONEPG > /dev/tty 2> /tmp/tmp.$$
+if [ $? = "0" ];
+then
+  echo Should have failed, but got exit 0
+  ERRORS=`expr $ERRORS + 1`
+fi
+if head -1 /tmp/tmp.$$ | grep -- "stdout is a tty and no --file option specified" > /dev/null
+then
+  echo Correctly failed with message \"`head -1 /tmp/tmp.$$`\"
+else
+  echo Bad message to stderr \"`head -1 /tmp/tmp.$$`\"
+  ERRORS=`expr $ERRORS + 1`
+fi
+
+# On import can't specify a PG
+touch /tmp/foo.$$
+./ceph_filestore_dump --filestore-path dev/$ONEOSD --journal-path dev/$ONEOSD.journal --type import --pgid $ONEPG --file /tmp/foo.$$ 2> /tmp/tmp.$$
+if [ $? = "0" ];
+then
+  echo Should have failed, but got exit 0
+  ERRORS=`expr $ERRORS + 1`
+fi
+if head -1 /tmp/tmp.$$ | grep -- "--pgid option invalid with import" > /dev/null
+then
+  echo Correctly failed with message \"`head -1 /tmp/tmp.$$`\"
+else
+  echo Bad message to stderr \"`head -1 /tmp/tmp.$$`\"
+  ERRORS=`expr $ERRORS + 1`
+fi
+rm -f /tmp/foo.$$
+
+# On import input file not found
+./ceph_filestore_dump --filestore-path dev/$ONEOSD --journal-path dev/$ONEOSD.journal --type import --file /tmp/foo.$$ 2> /tmp/tmp.$$
+if [ $? = "0" ];
+then
+  echo Should have failed, but got exit 0
+  ERRORS=`expr $ERRORS + 1`
+fi
+if head -1 /tmp/tmp.$$ | grep -- "open: No such file or directory" > /dev/null
+then
+  echo Correctly failed with message \"`head -1 /tmp/tmp.$$`\"
+else
+  echo Bad message to stderr \"`head -1 /tmp/tmp.$$`\"
+  ERRORS=`expr $ERRORS + 1`
+fi
+
+# On import can't use stdin from a terminal
+./ceph_filestore_dump --filestore-path dev/$ONEOSD --journal-path dev/$ONEOSD.journal --type import --pgid $ONEPG < /dev/tty 2> /tmp/tmp.$$
+if [ $? = "0" ];
+then
+  echo Should have failed, but got exit 0
+  ERRORS=`expr $ERRORS + 1`
+fi
+if head -1 /tmp/tmp.$$ | grep -- "stdin is a tty and no --file option specified" > /dev/null
+then
+  echo Correctly failed with message \"`head -1 /tmp/tmp.$$`\"
+else
+  echo Bad message to stderr \"`head -1 /tmp/tmp.$$`\"
+  ERRORS=`expr $ERRORS + 1`
+fi
+
+rm -f /tmp/tmp.$$
+
+# Test --type list and generate json for all objects
+echo "Testing --type list by generating json for all objects"
+for pg in $OBJREPPGS $OBJECPGS
+do
+  OSDS=`ls -d dev/*/current/${pg}_head | awk -F / '{ print $2 }'`
+  for osd in $OSDS
+  do
+    ./ceph_filestore_dump --filestore-path dev/$osd --journal-path dev/$osd.journal --type list --pgid $pg >> /tmp/tmp.$$
+    if [ "$?" != "0" ];
+    then
+      echo "Bad exit status $? from --type list request"
+      ERRORS=`expr $ERRORS + 1`
+    fi
+  done
+done
+
+sort -u /tmp/tmp.$$ > $JSONOBJ
+rm -f /tmp/tmp.$$
+
+# Test get-bytes
+echo "Testing get-bytes and set-bytes"
+for file in ${DATADIR}/${REP_NAME}*
+do
+  rm -f /tmp/tmp.$$
+  BASENAME=`basename $file`
+  JSON=`grep \"$BASENAME\" $JSONOBJ`
+  for pg in $OBJREPPGS
+  do
+    OSDS=`ls -d dev/*/current/${pg}_head | awk -F / '{ print $2 }'`
+    for osd in $OSDS
+    do
+      if [ -e dev/$osd/current/${pg}_head/${BASENAME}_* ];
+      then
+        rm -f /tmp/getbytes.$$
+        ./ceph_filestore_dump --filestore-path dev/$osd --journal-path dev/$osd.journal  --pgid $pg "$JSON" get-bytes /tmp/getbytes.$$
+        if [ $? != "0" ];
+        then
+          echo "Bad exit status $?"
+          ERRORS=`expr $ERRORS + 1`
+          continue
+        fi
+        diff -q $file /tmp/getbytes.$$
+        if [ $? != "0" ];
+        then
+          echo "Data from get-bytes differ"
+          echo "Got:"
+          cat /tmp/getbyte.$$
+          echo "Expected:"
+          cat $file
+          ERRORS=`expr $ERRORS + 1`
+        fi
+        echo "put-bytes going into $file" > /tmp/setbytes.$$
+        ./ceph_filestore_dump --filestore-path dev/$osd --journal-path dev/$osd.journal  --pgid $pg "$JSON" set-bytes - < /tmp/setbytes.$$
+        if [ $? != "0" ];
+        then
+          echo "Bad exit status $? from set-bytes"
+          ERRORS=`expr $ERRORS + 1`
+        fi
+        ./ceph_filestore_dump --filestore-path dev/$osd --journal-path dev/$osd.journal  --pgid $pg "$JSON" get-bytes - > /tmp/testbytes.$$
+        if [ $? != "0" ];
+        then
+          echo "Bad exit status $? from get-bytes"
+          ERRORS=`expr $ERRORS + 1`
+        fi
+        diff -q /tmp/setbytes.$$ /tmp/testbytes.$$
+        if [ $? != "0" ];
+        then
+          echo "Data after set-bytes differ"
+          echo "Got:"
+          cat /tmp/testbyte.$$
+          echo "Expected:"
+          cat /tmp/setbytes.$$
+          ERRORS=`expr $ERRORS + 1`
+        fi
+        ./ceph_filestore_dump --filestore-path dev/$osd --journal-path dev/$osd.journal  --pgid $pg "$JSON" set-bytes < $file
+        if [ $? != "0" ];
+        then
+          echo "Bad exit status $? from set-bytes to restore object"
+          ERRORS=`expr $ERRORS + 1`
+        fi
+      fi
+    done
+  done
+done
+
+rm -f /tmp/getbytes.$$ /tmp/testbytes.$$ /tmp/setbytes.$$
+
+# Testing attrs
+echo "Testing list-attrs get-attr"
+for file in ${DATADIR}/${REP_NAME}*
+do
+  rm -f /tmp/tmp.$$
+  BASENAME=`basename $file`
+  JSON=`grep \"$BASENAME\" $JSONOBJ`
+  for pg in $OBJREPPGS
+  do
+    OSDS=`ls -d dev/*/current/${pg}_head | awk -F / '{ print $2 }'`
+    for osd in $OSDS
+    do
+      if [ -e dev/$osd/current/${pg}_head/${BASENAME}_* ];
+      then
+        ./ceph_filestore_dump --filestore-path dev/$osd --journal-path dev/$osd.journal  --pgid $pg "$JSON" list-attrs > /tmp/attrs.$$
+        if [ $? != "0" ];
+        then
+          echo "Bad exit status $?"
+          ERRORS=`expr $ERRORS + 1`
+          continue
+        fi
+        for key in `cat /tmp/attrs.$$`
+        do
+          ./ceph_filestore_dump --filestore-path dev/$osd --journal-path dev/$osd.journal  --pgid $pg "$JSON" get-attr $key > /tmp/val.$$
+          if [ $? != "0" ];
+          then
+            echo "Bad exit status $?"
+            ERRORS=`expr $ERRORS + 1`
+            continue
+          fi
+          if [ "$key" = "_" -o "$key" = "snapset" ];
+          then
+            continue
+          fi
+          OBJNUM=`echo $BASENAME | sed "s/$REP_NAME//"`
+          echo -n $key | sed 's/_key//' > /tmp/checkval.$$
+          cat /tmp/val.$$ | sed 's/val//' > /tmp/testval.$$
+          diff -q /tmp/testval.$$ /tmp/checkval.$$
+          if [ "$?" != "0" ];
+          then
+            echo Got `cat /tmp/val.$$` instead of val`cat /tmp/check.$$`
+            ERRORS=`expr $ERRORS + 1`
+          fi
+        done
+      fi
+    done
+  done
+done
+
+rm -rf /tmp/testval.$$ /tmp/checkval.$$ /tmp/val.$$ /tmp/attrs.$$
+
 echo Checking pg info
 for pg in $ALLREPPGS $ALLECPGS
 do
@@ -210,7 +450,7 @@ then
     done
   done
 
-  OSD=4 ./vstart.sh -l -d -o "mon_pg_warn_min_per_osd=1" > /dev/null 2>&1
+  OSD=4 ./vstart.sh -l -d > /dev/null 2>&1
   wait_for_health
 
   echo "Checking erasure coded import data"
@@ -232,7 +472,7 @@ else
   echo "SKIPPING CHECKING IMPORT DATA DUE TO PREVIOUS FAILURES"
 fi
 
-rm -rf $DATADIR
+rm -rf $DATADIR $JSONOBJ
 
 if [ $ERRORS = "0" ];
 then