From 69fa515c95fbc7ceba313c64751ca60d54977f6e Mon Sep 17 00:00:00 2001
From: David Zafman <dzafman@redhat.com>
Date: Tue, 9 Apr 2019 21:34:59 -0700
Subject: [PATCH] test: Make most tests use default objectstore bluestore

Change run_osd() to default objectstore bluestore
Use run_osd_filestore() to use the non-default objectstore
Fix inject_eio to handle any objectstore if config prefixed with type

Remaining tests using filestore:
	osd-pool-create.sh TEST_pool_create_rep_expected_num_objects
		Test filestore directory creation
	qa/standalone/osd/osd-dup.sh TEST_filestore_to_bluestore
		Obvious
	qa/standalone/osd/osd-rep-recov-eio.sh TEST_rep_read_unfound
		Requires data digest in object info
	qa/standalone/scrub/osd-scrub-repair.sh multiple tests
		Erasure code pools append mode for filestore is tested
	qa/standalone/special/ceph_objectstore_tool.py
		Test code verifies COT by directly examining filestore contents

Fixes: https://tracker.ceph.com/issues/39162

Signed-off-by: David Zafman <dzafman@redhat.com>
---
 qa/standalone/ceph-helpers.sh                 | 12 ++---
 .../erasure-code/test-erasure-eio.sh          |  1 -
 qa/standalone/mon/mon-osdmap-prune.sh         |  5 ---
 qa/standalone/mon/osd-pool-create.sh          |  5 +--
 qa/standalone/osd/ec-error-rollforward.sh     |  2 +-
 qa/standalone/osd/osd-bluefs-volume-ops.sh    | 32 ++++++-------
 qa/standalone/osd/osd-dup.sh                  |  8 ++--
 qa/standalone/osd/osd-rep-recov-eio.sh        |  7 +--
 qa/standalone/scrub/osd-scrub-repair.sh       | 45 +++++++++----------
 qa/standalone/scrub/osd-scrub-snaps.sh        | 18 ++++----
 10 files changed, 62 insertions(+), 73 deletions(-)

diff --git a/qa/standalone/ceph-helpers.sh b/qa/standalone/ceph-helpers.sh
index 152396f6c02..001a701bde8 100755
--- a/qa/standalone/ceph-helpers.sh
+++ b/qa/standalone/ceph-helpers.sh
@@ -655,7 +655,7 @@ EOF
 
 }
 
-function run_osd_bluestore() {
+function run_osd_filestore() {
     local dir=$1
     shift
     local id=$1
@@ -688,7 +688,7 @@ function run_osd_bluestore() {
     echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $osd_data/new.json
     ceph osd new $uuid -i $osd_data/new.json
     rm $osd_data/new.json
-    ceph-osd -i $id $ceph_args --mkfs --key $OSD_SECRET --osd-uuid $uuid --osd-objectstore=bluestore
+    ceph-osd -i $id $ceph_args --mkfs --key $OSD_SECRET --osd-uuid $uuid --osd-objectstore=filestore
 
     local key_fn=$osd_data/keyring
     cat > $key_fn<<EOF
@@ -1157,13 +1157,8 @@ function _objectstore_tool_nodown() {
     shift
     local osd_data=$dir/$id
 
-    local journal_args
-    if [ "$objectstore_type" == "filestore" ]; then
-	journal_args=" --journal-path $osd_data/journal"
-    fi
     ceph-objectstore-tool \
         --data-path $osd_data \
-        $journal_args \
         "$@" || return 1
 }
 
@@ -2111,7 +2106,8 @@ function inject_eio() {
     if [ "$pooltype" != "ec" ]; then
         shard_id=""
     fi
-    set_config osd $osd_id filestore_debug_inject_read_err true || return 1
+    type=$(cat $dir/$osd_id/type)
+    set_config osd $osd_id ${type}_debug_inject_read_err true || return 1
     local loop=0
     while ( CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.$osd_id) \
              inject${which}err $poolname $objname $shard_id | grep -q Invalid ); do
diff --git a/qa/standalone/erasure-code/test-erasure-eio.sh b/qa/standalone/erasure-code/test-erasure-eio.sh
index af8b29b9c50..ce234d4adbd 100755
--- a/qa/standalone/erasure-code/test-erasure-eio.sh
+++ b/qa/standalone/erasure-code/test-erasure-eio.sh
@@ -26,7 +26,6 @@ function run() {
     export CEPH_ARGS
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
     CEPH_ARGS+="--mon-host=$CEPH_MON "
-    CEPH_ARGS+="--osd-objectstore=filestore "
 
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for func in $funcs ; do
diff --git a/qa/standalone/mon/mon-osdmap-prune.sh b/qa/standalone/mon/mon-osdmap-prune.sh
index e3928087aac..f8f7876bb76 100755
--- a/qa/standalone/mon/mon-osdmap-prune.sh
+++ b/qa/standalone/mon/mon-osdmap-prune.sh
@@ -4,11 +4,6 @@ source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
 
 base_test=$CEPH_ROOT/qa/workunits/mon/test_mon_osdmap_prune.sh
 
-# We are going to open and close a lot of files, and generate a lot of maps
-# that the osds will need to process. If we don't increase the fd ulimit, we
-# risk having the osds asserting when handling filestore transactions.
-ulimit -n 4096
-
 function run() {
 
   local dir=$1
diff --git a/qa/standalone/mon/osd-pool-create.sh b/qa/standalone/mon/osd-pool-create.sh
index 992cbf583aa..68a35942858 100755
--- a/qa/standalone/mon/osd-pool-create.sh
+++ b/qa/standalone/mon/osd-pool-create.sh
@@ -212,12 +212,11 @@ function TEST_pool_create_rep_expected_num_objects() {
     local dir=$1
     setup $dir || return 1
 
-    # disable pg dir merge
-    CEPH_ARGS+="--osd-objectstore=filestore"
     export CEPH_ARGS
     run_mon $dir a || return 1
     run_mgr $dir x || return 1
-    run_osd $dir 0 || return 1
+    # disable pg dir merge
+    run_osd_filestore $dir 0 || return 1
 
     ceph osd pool create rep_expected_num_objects 64 64 replicated  replicated_rule 100000 || return 1
     # wait for pg dir creating
diff --git a/qa/standalone/osd/ec-error-rollforward.sh b/qa/standalone/osd/ec-error-rollforward.sh
index cfbf2871927..05188ad5096 100755
--- a/qa/standalone/osd/ec-error-rollforward.sh
+++ b/qa/standalone/osd/ec-error-rollforward.sh
@@ -10,7 +10,7 @@ function run() {
     export CEPH_MON="127.0.0.1:7132" # git grep '\<7132\>' : there must be only one
     export CEPH_ARGS
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
-    CEPH_ARGS+="--mon-host=$CEPH_MON --osd-objectstore filestore"
+    CEPH_ARGS+="--mon-host=$CEPH_MON "
     export margin=10
     export objects=200
     export poolname=test
diff --git a/qa/standalone/osd/osd-bluefs-volume-ops.sh b/qa/standalone/osd/osd-bluefs-volume-ops.sh
index 550708963bc..5258cfc6d5b 100755
--- a/qa/standalone/osd/osd-bluefs-volume-ops.sh
+++ b/qa/standalone/osd/osd-bluefs-volume-ops.sh
@@ -38,13 +38,13 @@ function TEST_bluestore() {
 
     run_mon $dir a || return 1
     run_mgr $dir x || return 1
-    run_osd_bluestore $dir 0 || return 1
+    run_osd $dir 0 || return 1
     osd_pid0=$(cat $dir/osd.0.pid)
-    run_osd_bluestore $dir 1 || return 1
+    run_osd $dir 1 || return 1
     osd_pid1=$(cat $dir/osd.1.pid)
-    run_osd_bluestore $dir 2 || return 1
+    run_osd $dir 2 || return 1
     osd_pid2=$(cat $dir/osd.2.pid)
-    run_osd_bluestore $dir 3 || return 1
+    run_osd $dir 3 || return 1
     osd_pid3=$(cat $dir/osd.3.pid)
 
     sleep 5
@@ -140,13 +140,13 @@ function TEST_bluestore() {
 
     ceph-bluestore-tool --path $dir/3 fsck || return 1
 
-    run_osd_bluestore $dir 0 || return 1
+    run_osd $dir 0 || return 1
     osd_pid0=$(cat $dir/osd.0.pid)
-    run_osd_bluestore $dir 1 || return 1
+    run_osd $dir 1 || return 1
     osd_pid1=$(cat $dir/osd.1.pid)
-    run_osd_bluestore $dir 2 || return 1
+    run_osd $dir 2 || return 1
     osd_pid2=$(cat $dir/osd.2.pid)
-    run_osd_bluestore $dir 3 || return 1
+    run_osd $dir 3 || return 1
     osd_pid3=$(cat $dir/osd.3.pid)
 
     wait_for_clean || return 1
@@ -218,13 +218,13 @@ function TEST_bluestore() {
 
     ceph-bluestore-tool --path $dir/3 fsck || return 1
 
-    run_osd_bluestore $dir 0 || return 1
+    run_osd $dir 0 || return 1
     osd_pid0=$(cat $dir/osd.0.pid)
-    run_osd_bluestore $dir 1 || return 1
+    run_osd $dir 1 || return 1
     osd_pid1=$(cat $dir/osd.1.pid)
-    run_osd_bluestore $dir 2 || return 1
+    run_osd $dir 2 || return 1
     osd_pid2=$(cat $dir/osd.2.pid)
-    run_osd_bluestore $dir 3 || return 1
+    run_osd $dir 3 || return 1
     osd_pid3=$(cat $dir/osd.3.pid)
 
     # write some objects
@@ -324,13 +324,13 @@ function TEST_bluestore() {
 
     ceph-bluestore-tool --path $dir/3 fsck || return 1
 
-    run_osd_bluestore $dir 0 || return 1
+    run_osd $dir 0 || return 1
     osd_pid0=$(cat $dir/osd.0.pid)
-    run_osd_bluestore $dir 1 || return 1
+    run_osd $dir 1 || return 1
     osd_pid1=$(cat $dir/osd.1.pid)
-    run_osd_bluestore $dir 2 || return 1
+    run_osd $dir 2 || return 1
     osd_pid2=$(cat $dir/osd.2.pid)
-    run_osd_bluestore $dir 3 || return 1
+    run_osd $dir 3 || return 1
     osd_pid3=$(cat $dir/osd.3.pid)
 
     # write some objects
diff --git a/qa/standalone/osd/osd-dup.sh b/qa/standalone/osd/osd-dup.sh
index 324840b04b4..26f583b9990 100755
--- a/qa/standalone/osd/osd-dup.sh
+++ b/qa/standalone/osd/osd-dup.sh
@@ -33,10 +33,10 @@ function TEST_filestore_to_bluestore() {
 
     run_mon $dir a || return 1
     run_mgr $dir x || return 1
-    run_osd $dir 0 || return 1
+    run_osd_filestore $dir 0 || return 1
     osd_pid=$(cat $dir/osd.0.pid)
-    run_osd $dir 1 || return 1
-    run_osd $dir 2 || return 1
+    run_osd_filestore $dir 1 || return 1
+    run_osd_filestore $dir 2 || return 1
 
     sleep 5
 
@@ -61,7 +61,7 @@ function TEST_filestore_to_bluestore() {
 			  --op dup || return 1
     CEPH_ARGS=$O
 
-    run_osd_bluestore $dir 0 || return 1
+    run_osd $dir 0 || return 1
 
     while ! ceph osd stat | grep '3 up' ; do sleep 1 ; done
     ceph osd metadata 0 | grep bluestore || return 1
diff --git a/qa/standalone/osd/osd-rep-recov-eio.sh b/qa/standalone/osd/osd-rep-recov-eio.sh
index 6b501bc875c..1ec9869b868 100755
--- a/qa/standalone/osd/osd-rep-recov-eio.sh
+++ b/qa/standalone/osd/osd-rep-recov-eio.sh
@@ -27,7 +27,7 @@ function run() {
     export CEPH_ARGS
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
     CEPH_ARGS+="--mon-host=$CEPH_MON "
-    CEPH_ARGS+="--osd-objectstore=filestore "
+
 
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for func in $funcs ; do
@@ -44,9 +44,10 @@ function run() {
 function setup_osds() {
     local count=$1
     shift
+    local type=$1
 
     for id in $(seq 0 $(expr $count - 1)) ; do
-        run_osd $dir $id || return 1
+        run_osd${type} $dir $id || return 1
     done
     wait_for_clean || return 1
 }
@@ -310,7 +311,7 @@ function TEST_rep_read_unfound() {
     local dir=$1
     local objname=myobject
 
-    setup_osds 3 || return 1
+    setup_osds 3 _filestore || return 1
 
     ceph osd pool delete foo foo --yes-i-really-really-mean-it || return 1
     local poolname=test-pool
diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh
index b62e2c086c8..5d5477b8ec6 100755
--- a/qa/standalone/scrub/osd-scrub-repair.sh
+++ b/qa/standalone/scrub/osd-scrub-repair.sh
@@ -56,7 +56,6 @@ function run() {
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
     CEPH_ARGS+="--mon-host=$CEPH_MON "
     CEPH_ARGS+="--osd-skip-data-digest=false "
-    CEPH_ARGS+="--osd-objectstore=filestore "
 
     export -n CEPH_CLI_TEST_DUP_COMMAND
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
@@ -225,9 +224,9 @@ function auto_repair_erasure_coded() {
             --osd-scrub-interval-randomize-ratio=0"
     for id in $(seq 0 2) ; do
 	if [ "$allow_overwrites" = "true" ]; then
-            run_osd_bluestore $dir $id $ceph_osd_args || return 1
-	else
             run_osd $dir $id $ceph_osd_args || return 1
+	else
+            run_osd_filestore $dir $id $ceph_osd_args || return 1
 	fi
     done
     create_rbd_pool || return 1
@@ -280,7 +279,7 @@ function TEST_auto_repair_bluestore_basic() {
 	    --osd_deep_scrub_randomize_ratio=0 \
             --osd-scrub-interval-randomize-ratio=0"
     for id in $(seq 0 2) ; do
-        run_osd_bluestore $dir $id $ceph_osd_args || return 1
+        run_osd $dir $id $ceph_osd_args || return 1
     done
 
     create_pool $poolname 1 1 || return 1
@@ -329,7 +328,7 @@ function TEST_auto_repair_bluestore_scrub() {
 	    --osd_deep_scrub_randomize_ratio=0 \
             --osd-scrub-interval-randomize-ratio=0"
     for id in $(seq 0 2) ; do
-        run_osd_bluestore $dir $id $ceph_osd_args || return 1
+        run_osd $dir $id $ceph_osd_args || return 1
     done
 
     create_pool $poolname 1 1 || return 1
@@ -384,7 +383,7 @@ function TEST_auto_repair_bluestore_failed() {
 	    --osd_deep_scrub_randomize_ratio=0 \
             --osd-scrub-interval-randomize-ratio=0"
     for id in $(seq 0 2) ; do
-        run_osd_bluestore $dir $id $ceph_osd_args || return 1
+        run_osd $dir $id $ceph_osd_args || return 1
     done
 
     create_pool $poolname 1 1 || return 1
@@ -453,7 +452,7 @@ function TEST_auto_repair_bluestore_failed_norecov() {
 	    --osd_deep_scrub_randomize_ratio=0 \
             --osd-scrub-interval-randomize-ratio=0"
     for id in $(seq 0 2) ; do
-        run_osd_bluestore $dir $id $ceph_osd_args || return 1
+        run_osd $dir $id $ceph_osd_args || return 1
     done
 
     create_pool $poolname 1 1 || return 1
@@ -510,7 +509,7 @@ function TEST_repair_stats() {
     local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
             --osd-scrub-interval-randomize-ratio=0"
     for id in $(seq 0 $(expr $OSDS - 1)) ; do
-        run_osd_bluestore $dir $id $ceph_osd_args || return 1
+        run_osd $dir $id $ceph_osd_args || return 1
     done
 
     create_pool $poolname 1 1 || return 1
@@ -539,8 +538,8 @@ function TEST_repair_stats() {
       OSD=$(expr $i % 2)
       _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
     done
-    run_osd_bluestore $dir $primary $ceph_osd_args || return 1
-    run_osd_bluestore $dir $other $ceph_osd_args || return 1
+    run_osd $dir $primary $ceph_osd_args || return 1
+    run_osd $dir $other $ceph_osd_args || return 1
     wait_for_clean || return 1
 
     repair $pgid
@@ -584,7 +583,7 @@ function TEST_repair_stats_ec() {
     local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
             --osd-scrub-interval-randomize-ratio=0"
     for id in $(seq 0 $(expr $OSDS - 1)) ; do
-        run_osd_bluestore $dir $id $ceph_osd_args || return 1
+        run_osd $dir $id $ceph_osd_args || return 1
     done
 
     # Create an EC pool
@@ -612,8 +611,8 @@ function TEST_repair_stats_ec() {
       OSD=$(expr $i % 2)
       _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
     done
-    run_osd_bluestore $dir $primary $ceph_osd_args || return 1
-    run_osd_bluestore $dir $other $ceph_osd_args || return 1
+    run_osd $dir $primary $ceph_osd_args || return 1
+    run_osd $dir $other $ceph_osd_args || return 1
     wait_for_clean || return 1
 
     repair $pgid
@@ -655,9 +654,9 @@ function corrupt_and_repair_jerasure() {
     run_mgr $dir x || return 1
     for id in $(seq 0 3) ; do
 	if [ "$allow_overwrites" = "true" ]; then
-            run_osd_bluestore $dir $id || return 1
-	else
             run_osd $dir $id || return 1
+	else
+            run_osd_filestore $dir $id || return 1
 	fi
     done
     create_rbd_pool || return 1
@@ -689,9 +688,9 @@ function corrupt_and_repair_lrc() {
     run_mgr $dir x || return 1
     for id in $(seq 0 9) ; do
 	if [ "$allow_overwrites" = "true" ]; then
-            run_osd_bluestore $dir $id || return 1
-	else
             run_osd $dir $id || return 1
+	else
+            run_osd_filestore $dir $id || return 1
 	fi
     done
     create_rbd_pool || return 1
@@ -724,9 +723,9 @@ function unfound_erasure_coded() {
     run_mgr $dir x || return 1
     for id in $(seq 0 3) ; do
 	if [ "$allow_overwrites" = "true" ]; then
-            run_osd_bluestore $dir $id || return 1
-	else
             run_osd $dir $id || return 1
+	else
+            run_osd_filestore $dir $id || return 1
 	fi
     done
 
@@ -794,9 +793,9 @@ function list_missing_erasure_coded() {
     run_mgr $dir x || return 1
     for id in $(seq 0 2) ; do
 	if [ "$allow_overwrites" = "true" ]; then
-            run_osd_bluestore $dir $id || return 1
-	else
             run_osd $dir $id || return 1
+	else
+            run_osd_filestore $dir $id || return 1
 	fi
     done
     create_rbd_pool || return 1
@@ -3405,9 +3404,9 @@ function corrupt_scrub_erasure() {
     run_mgr $dir x || return 1
     for id in $(seq 0 2) ; do
 	if [ "$allow_overwrites" = "true" ]; then
-            run_osd_bluestore $dir $id || return 1
-	else
             run_osd $dir $id || return 1
+	else
+            run_osd_filestore $dir $id || return 1
 	fi
     done
     create_rbd_pool || return 1
diff --git a/qa/standalone/scrub/osd-scrub-snaps.sh b/qa/standalone/scrub/osd-scrub-snaps.sh
index 4ed94fea691..e09a551f54e 100755
--- a/qa/standalone/scrub/osd-scrub-snaps.sh
+++ b/qa/standalone/scrub/osd-scrub-snaps.sh
@@ -30,7 +30,7 @@ function run() {
     export CEPH_MON="127.0.0.1:7121" # git grep '\<7121\>' : there must be only one
     export CEPH_ARGS
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
-    CEPH_ARGS+="--mon-host=$CEPH_MON --osd-objectstore=filestore"
+    CEPH_ARGS+="--mon-host=$CEPH_MON "
 
     export -n CEPH_CLI_TEST_DUP_COMMAND
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
@@ -100,12 +100,12 @@ function create_scenario() {
     JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":1)"
     OBJ5SAVE="$JSON"
     # Starts with a snapmap
-    ceph-osdomap-tool --no-mon-config --omap-path $dir/${osd}/current/omap --command dump-raw-keys > $dir/drk.log
-    grep "_USER_[0-9]*_USER_,MAP_.*[.]1[.]obj5[.][.]" $dir/drk.log || return 1
+    ceph-kvstore-tool bluestore-kv $dir/${osd} list 2> /dev/null > $dir/drk.log
+    grep "^M.*MAP_.*[.]1[.]obj5[.][.]$" $dir/drk.log || return 1
     ceph-objectstore-tool --data-path $dir/${osd} --rmtype nosnapmap "$JSON" remove || return 1
     # Check that snapmap is stil there
-    ceph-osdomap-tool --no-mon-config --omap-path $dir/${osd}/current/omap --command dump-raw-keys > $dir/drk.log
-    grep "_USER_[0-9]*_USER_,MAP_.*[.]1[.]obj5[.][.]" $dir/drk.log || return 1
+    ceph-kvstore-tool bluestore-kv $dir/${osd} list 2> /dev/null > $dir/drk.log
+    grep "^M.*MAP_.*[.]1[.]obj5[.][.]$" $dir/drk.log || return 1
     rm -f $dir/drk.log
 
     JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":4)"
@@ -120,13 +120,13 @@ function create_scenario() {
     ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove || return 1
 
     # Starts with a snapmap
-    ceph-osdomap-tool --no-mon-config --omap-path $dir/${osd}/current/omap --command dump-raw-keys > $dir/drk.log
-    grep "_USER_[0-9]*_USER_,MAP_.*[.]7[.]obj16[.][.]" $dir/drk.log || return 1
+    ceph-kvstore-tool bluestore-kv $dir/${osd} list 2> /dev/null > $dir/drk.log
+    grep "^M.*MAP_.*[.]7[.]obj16[.][.]$" $dir/drk.log || return 1
     JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj16 | grep \"snapid\":7)"
     ceph-objectstore-tool --data-path $dir/${osd} --rmtype snapmap "$JSON" remove || return 1
     # Check that snapmap is now removed
-    ceph-osdomap-tool --no-mon-config --omap-path $dir/${osd}/current/omap --command dump-raw-keys > $dir/drk.log
-    ! grep "_USER_[0-9]*_USER_,MAP_.*[.]7[.]obj16[.][.]" $dir/drk.log || return 1
+    ceph-kvstore-tool bluestore-kv $dir/${osd} list 2> /dev/null > $dir/drk.log
+    ! grep "^M.*MAP_.*[.]7[.]obj16[.][.]$" $dir/drk.log || return 1
     rm -f $dir/drk.log
 
     JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj2)"
-- 
2.39.5