From de490a15adc4c0d262e1450f6f8dc447108b3921 Mon Sep 17 00:00:00 2001
From: David Zafman <dzafman@redhat.com>
Date: Tue, 2 Apr 2019 20:21:35 -0700
Subject: [PATCH] test: Divergent testing of _merge_object_divergent_entries()
 cases

Case 1: A more recent update exists
Case 2: The first entry in the divergent sequence is a create
Case 3  NOT TESTED - Ohject currently missing
Case 4: We can rollback all of the entries
Case 5: We cannot rollback at least 1 of the entries

Support starting OSDs even when "noup" is set (don't wait for up).
Move create_ec_pool() to ceph-helpers.sh

Fixes: https://tracker.ceph.com/issues/39162

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 7e778980016fcac80ca30df081d85df415865139)
---
 qa/standalone/ceph-helpers.sh           |  30 ++
 qa/standalone/osd/divergent-priors.sh   | 630 ++++++++++++++++++++++++
 qa/standalone/scrub/osd-scrub-repair.sh |  18 -
 3 files changed, 660 insertions(+), 18 deletions(-)
 create mode 100755 qa/standalone/osd/divergent-priors.sh

diff --git a/qa/standalone/ceph-helpers.sh b/qa/standalone/ceph-helpers.sh
index 7ffd88a6676da..9ab9ad1b851a6 100755
--- a/qa/standalone/ceph-helpers.sh
+++ b/qa/standalone/ceph-helpers.sh
@@ -673,6 +673,10 @@ EOF
     echo start osd.$id
     ceph-osd -i $id $ceph_args &
 
+    # If noup is set, then can't wait for this osd
+    if ceph osd dump --format=json | jq '.flags_set[]' | grep -q '"noup"' ; then
+      return 0
+    fi
     wait_for_osd up $id || return 1
 
 }
@@ -722,6 +726,10 @@ EOF
     echo start osd.$id
     ceph-osd -i $id $ceph_args &
 
+    # If noup is set, then can't wait for this osd
+    if ceph osd dump --format=json | jq '.flags_set[]' | grep -q '"noup"' ; then
+      return 0
+    fi
     wait_for_osd up $id || return 1
 
 
@@ -859,6 +867,10 @@ function activate_osd() {
 
     [ "$id" = "$(cat $osd_data/whoami)" ] || return 1
 
+    # If noup is set, then can't wait for this osd
+    if ceph osd dump --format=json | jq '.flags_set[]' | grep -q '"noup"' ; then
+      return 0
+    fi
     wait_for_osd up $id || return 1
 }
 
@@ -2150,6 +2162,24 @@ function multidiff() {
     fi
 }
 
+function create_ec_pool() {
+    local pool_name=$1
+    shift
+    local allow_overwrites=$1
+    shift
+
+    ceph osd erasure-code-profile set myprofile crush-failure-domain=osd "$@" || return 1
+
+    create_pool "$poolname" 1 1 erasure myprofile || return 1
+
+    if [ "$allow_overwrites" = "true" ]; then
+        ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
+    fi
+
+    wait_for_clean || return 1
+    return 0
+}
+
 # Local Variables:
 # compile-command: "cd ../../src ; make -j4 && ../qa/standalone/ceph-helpers.sh TESTS # test_get_config"
 # End:
diff --git a/qa/standalone/osd/divergent-priors.sh b/qa/standalone/osd/divergent-priors.sh
new file mode 100755
index 0000000000000..608a524b179fd
--- /dev/null
+++ b/qa/standalone/osd/divergent-priors.sh
@@ -0,0 +1,630 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2019 Red Hat <contact@redhat.com>
+#
+# Author: David Zafman <dzafman@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+
+source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
+
+function run() {
+    local dir=$1
+    shift
+
+    # This should multiple of 6
+    export loglen=12
+    export divisor=3
+    export trim=$(expr $loglen / 2)
+    export DIVERGENT_WRITE=$(expr $trim / $divisor)
+    export DIVERGENT_REMOVE=$(expr $trim / $divisor)
+    export DIVERGENT_CREATE=$(expr $trim / $divisor)
+    export poolname=test
+    export testobjects=100
+    # Fix port????
+    export CEPH_MON="127.0.0.1:7115" # git grep '\<7115\>' : there must be only one
+    export CEPH_ARGS
+    CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
+    CEPH_ARGS+="--mon-host=$CEPH_MON "
+    # so we will not force auth_log_shard to be acting_primary
+    CEPH_ARGS+="--osd_force_auth_primary_missing_objects=1000000 "
+    CEPH_ARGS+="--osd_debug_pg_log_writeout=true "
+    CEPH_ARGS+="--osd_min_pg_log_entries=$loglen --osd_max_pg_log_entries=$loglen --osd_pg_log_trim_min=$trim "
+
+    local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
+    for func in $funcs ; do
+        setup $dir || return 1
+        $func $dir || return 1
+        teardown $dir || return 1
+    done
+}
+
+
+# Special case divergence test
+#	Test handling of divergent entries with prior_version
+#	prior to log_tail
+# 	based on qa/tasks/divergent_prior.py
+function TEST_divergent() {
+    local dir=$1
+
+    # something that is always there
+    local dummyfile='/etc/fstab'
+    local dummyfile2='/etc/resolv.conf'
+
+    local num_osds=3
+    local osds="$(seq 0 $(expr $num_osds - 1))"
+    run_mon $dir a || return 1
+    run_mgr $dir x || return 1
+    for i in $osds
+    do
+      run_osd $dir $i || return 1
+    done
+
+    ceph osd set noout
+    ceph osd set noin
+    ceph osd set nodown
+    create_pool $poolname 1 1
+    ceph osd pool set $poolname size 3
+    ceph osd pool set $poolname min_size 2
+
+    flush_pg_stats || return 1
+    wait_for_clean || return 1
+
+    # determine primary
+    local divergent="$(ceph pg dump pgs --format=json | jq '.pg_stats[0].up_primary')"
+    echo "primary and soon to be divergent is $divergent"
+    ceph pg dump pgs
+    local non_divergent=""
+    for i in $osds
+    do
+      if [ "$i" = "$divergent" ]; then
+	  continue
+      fi
+      non_divergent="$non_divergent $i"
+    done
+
+    echo "writing initial objects"
+    # write a bunch of objects
+    for i in $(seq 1 $testobjects)
+    do
+      rados -p $poolname put existing_$i $dummyfile
+    done
+
+    WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    local pgid=$(get_pg $poolname existing_1)
+
+    # blackhole non_divergent
+    echo "blackholing osds $non_divergent"
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${i}) config set objectstore_blackhole 1
+    done
+
+    local case5=$testobjects
+    local case3=$(expr $testobjects - 1)
+    # Write some soon to be divergent
+    echo 'writing divergent object'
+    rados -p $poolname put existing_$case5 $dummyfile &
+    echo 'create missing divergent object'
+    inject_eio rep data $poolname existing_$case3 $dir 0 || return 1
+    rados -p $poolname get existing_$case3 $dir/existing &
+    sleep 10
+    killall -9 rados
+
+    # kill all the osds but leave divergent in
+    echo 'killing all the osds'
+    ceph pg dump pgs
+    kill_daemons $dir KILL osd || return 1
+    for i in $osds
+    do
+      ceph osd down osd.$i
+    done
+    for i in $non_divergent
+    do
+      ceph osd out osd.$i
+    done
+
+    # bring up non-divergent
+    echo "bringing up non_divergent $non_divergent"
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      activate_osd $dir $i || return 1
+    done
+    for i in $non_divergent
+    do
+      ceph osd in osd.$i
+    done
+
+    WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    # write 1 non-divergent object (ensure that old divergent one is divergent)
+    objname="existing_$(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE)"
+    echo "writing non-divergent object $objname"
+    ceph pg dump pgs
+    rados -p $poolname put $objname $dummyfile2
+
+    # ensure no recovery of up osds first
+    echo 'delay recovery'
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${i}) set_recovery_delay 100000
+    done
+
+    # bring in our divergent friend
+    echo "revive divergent $divergent"
+    ceph pg dump pgs
+    ceph osd set noup
+    activate_osd $dir $divergent
+    sleep 5
+
+    echo 'delay recovery divergent'
+    ceph pg dump pgs
+    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${divergent}) set_recovery_delay 100000
+
+    ceph osd unset noup
+
+    wait_for_osd up 0
+    wait_for_osd up 1
+    wait_for_osd up 2
+
+    ceph pg dump pgs
+    echo 'wait for peering'
+    ceph pg dump pgs
+    rados -p $poolname put foo $dummyfile
+
+    echo "killing divergent $divergent"
+    ceph pg dump pgs
+    kill_daemons $dir KILL osd.$divergent
+    #_objectstore_tool_nodown $dir $divergent --op log --pgid $pgid
+    echo "reviving divergent $divergent"
+    ceph pg dump pgs
+    activate_osd $dir $divergent
+
+    sleep 20
+
+    echo "allowing recovery"
+    ceph pg dump pgs
+    # Set osd_recovery_delay_start back to 0 and kick the queue
+    for i in $osds
+    do
+	 ceph tell osd.$i debug kick_recovery_wq 0
+    done
+
+    echo 'reading divergent objects'
+    ceph pg dump pgs
+    for i in $(seq 1 $(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE))
+    do
+      rados -p $poolname get existing_$i $dir/existing || return 1
+    done
+    rm -f $dir/existing
+
+    grep _merge_object_divergent_entries $(find $dir -name '*osd*log')
+    # Check for _merge_object_divergent_entries for case #5
+    if ! grep -q "_merge_object_divergent_entries.*cannot roll back, removing and adding to missing" $(find $dir -name '*osd*log')
+    then
+	    echo failure
+	    return 1
+    fi
+    echo "success"
+
+    delete_pool $poolname
+    kill_daemons $dir || return 1
+}
+
+function TEST_divergent_ec() {
+    local dir=$1
+
+    # something that is always there
+    local dummyfile='/etc/fstab'
+    local dummyfile2='/etc/resolv.conf'
+
+    local num_osds=3
+    local osds="$(seq 0 $(expr $num_osds - 1))"
+    run_mon $dir a || return 1
+    run_mgr $dir x || return 1
+    for i in $osds
+    do
+      run_osd $dir $i || return 1
+    done
+
+    ceph osd set noout
+    ceph osd set noin
+    ceph osd set nodown
+    create_ec_pool $poolname true k=2 m=1 || return 1
+
+    flush_pg_stats || return 1
+    wait_for_clean || return 1
+
+    # determine primary
+    local divergent="$(ceph pg dump pgs --format=json | jq '.pg_stats[0].up_primary')"
+    echo "primary and soon to be divergent is $divergent"
+    ceph pg dump pgs
+    local non_divergent=""
+    for i in $osds
+    do
+      if [ "$i" = "$divergent" ]; then
+	  continue
+      fi
+      non_divergent="$non_divergent $i"
+    done
+
+    echo "writing initial objects"
+    # write a bunch of objects
+    for i in $(seq 1 $testobjects)
+    do
+      rados -p $poolname put existing_$i $dummyfile
+    done
+
+    WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    local pgid=$(get_pg $poolname existing_1)
+
+    # blackhole non_divergent
+    echo "blackholing osds $non_divergent"
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${i}) config set objectstore_blackhole 1
+    done
+
+    # Write some soon to be divergent
+    echo 'writing divergent object'
+    rados -p $poolname put existing_$testobjects $dummyfile2 &
+    sleep 1
+    rados -p $poolname put existing_$testobjects $dummyfile &
+    rados -p $poolname mksnap snap1
+    rados -p $poolname put existing_$(expr $testobjects - 1) $dummyfile &
+    sleep 10
+    killall -9 rados
+
+    # kill all the osds but leave divergent in
+    echo 'killing all the osds'
+    ceph pg dump pgs
+    kill_daemons $dir KILL osd || return 1
+    for i in $osds
+    do
+      ceph osd down osd.$i
+    done
+    for i in $non_divergent
+    do
+      ceph osd out osd.$i
+    done
+
+    # bring up non-divergent
+    echo "bringing up non_divergent $non_divergent"
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      activate_osd $dir $i || return 1
+    done
+    for i in $non_divergent
+    do
+      ceph osd in osd.$i
+    done
+
+    sleep 5
+    #WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    # write 1 non-divergent object (ensure that old divergent one is divergent)
+    objname="existing_$(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE)"
+    echo "writing non-divergent object $objname"
+    ceph pg dump pgs
+    rados -p $poolname put $objname $dummyfile2
+
+    WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    # Dump logs
+    for i in $non_divergent
+    do
+      kill_daemons $dir KILL osd.$i || return 1
+      _objectstore_tool_nodown $dir $i --op log --pgid $pgid
+      activate_osd $dir $i || return 1
+    done
+    _objectstore_tool_nodown $dir $divergent --op log --pgid $pgid
+
+    WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    # ensure no recovery of up osds first
+    echo 'delay recovery'
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${i}) set_recovery_delay 100000
+    done
+
+    # bring in our divergent friend
+    echo "revive divergent $divergent"
+    ceph pg dump pgs
+    ceph osd set noup
+    activate_osd $dir $divergent
+    sleep 5
+
+    echo 'delay recovery divergent'
+    ceph pg dump pgs
+    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${divergent}) set_recovery_delay 100000
+
+    ceph osd unset noup
+
+    wait_for_osd up 0
+    wait_for_osd up 1
+    wait_for_osd up 2
+
+    ceph pg dump pgs
+    echo 'wait for peering'
+    ceph pg dump pgs
+    rados -p $poolname put foo $dummyfile
+
+    echo "killing divergent $divergent"
+    ceph pg dump pgs
+    kill_daemons $dir KILL osd.$divergent
+    #_objectstore_tool_nodown $dir $divergent --op log --pgid $pgid
+    echo "reviving divergent $divergent"
+    ceph pg dump pgs
+    activate_osd $dir $divergent
+
+    sleep 20
+
+    echo "allowing recovery"
+    ceph pg dump pgs
+    # Set osd_recovery_delay_start back to 0 and kick the queue
+    for i in $osds
+    do
+	 ceph tell osd.$i debug kick_recovery_wq 0
+    done
+
+    echo 'reading divergent objects'
+    ceph pg dump pgs
+    for i in $(seq 1 $(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE))
+    do
+      rados -p $poolname get existing_$i $dir/existing || return 1
+    done
+    rm -f $dir/existing
+
+    grep _merge_object_divergent_entries $(find $dir -name '*osd*log')
+    # Check for _merge_object_divergent_entries for case #3
+    # XXX: Not reproducing this case
+#    if ! grep -q "_merge_object_divergent_entries.* missing, .* adjusting" $(find $dir -name '*osd*log')
+#    then
+#	echo failure
+#	return 1
+#    fi
+    # Check for _merge_object_divergent_entries for case #4
+    if ! grep -q "_merge_object_divergent_entries.*rolled back" $(find $dir -name '*osd*log')
+    then
+	echo failure
+	return 1
+    fi
+    echo "success"
+
+    delete_pool $poolname
+    kill_daemons $dir || return 1
+}
+
+# Special case divergence test with ceph-objectstore-tool export/remove/import
+# 	Test handling of divergent entries with prior_version
+# 	prior to log_tail and a ceph-objectstore-tool export/import
+# 	based on qa/tasks/divergent_prior2.py
+function TEST_divergent_2() {
+    local dir=$1
+
+    # something that is always there
+    local dummyfile='/etc/fstab'
+    local dummyfile2='/etc/resolv.conf'
+
+    local num_osds=3
+    local osds="$(seq 0 $(expr $num_osds - 1))"
+    run_mon $dir a || return 1
+    run_mgr $dir x || return 1
+    for i in $osds
+    do
+      run_osd $dir $i || return 1
+    done
+
+    ceph osd set noout
+    ceph osd set noin
+    ceph osd set nodown
+    create_pool $poolname 1 1
+    ceph osd pool set $poolname size 3
+    ceph osd pool set $poolname min_size 2
+
+    flush_pg_stats || return 1
+    wait_for_clean || return 1
+
+    # determine primary
+    local divergent="$(ceph pg dump pgs --format=json | jq '.pg_stats[0].up_primary')"
+    echo "primary and soon to be divergent is $divergent"
+    ceph pg dump pgs
+    local non_divergent=""
+    for i in $osds
+    do
+      if [ "$i" = "$divergent" ]; then
+	  continue
+      fi
+      non_divergent="$non_divergent $i"
+    done
+
+    echo "writing initial objects"
+    # write a bunch of objects
+    for i in $(seq 1 $testobjects)
+    do
+      rados -p $poolname put existing_$i $dummyfile
+    done
+
+    WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    local pgid=$(get_pg $poolname existing_1)
+
+    # blackhole non_divergent
+    echo "blackholing osds $non_divergent"
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${i}) config set objectstore_blackhole 1
+    done
+
+    # Do some creates to hit case 2
+    echo 'create new divergent objects'
+    for i in $(seq 1 $DIVERGENT_CREATE)
+    do
+      rados -p $poolname create newobject_$i &
+    done
+    # Write some soon to be divergent
+    echo 'writing divergent objects'
+    for i in $(seq 1 $DIVERGENT_WRITE)
+    do
+      rados -p $poolname put existing_$i $dummyfile2 &
+    done
+    # Remove some soon to be divergent
+    echo 'remove divergent objects'
+    for i in $(seq 1 $DIVERGENT_REMOVE)
+    do
+      rmi=$(expr $i + $DIVERGENT_WRITE)
+      rados -p $poolname rm existing_$rmi &
+    done
+    sleep 10
+    killall -9 rados
+
+    # kill all the osds but leave divergent in
+    echo 'killing all the osds'
+    ceph pg dump pgs
+    kill_daemons $dir KILL osd || return 1
+    for i in $osds
+    do
+      ceph osd down osd.$i
+    done
+    for i in $non_divergent
+    do
+      ceph osd out osd.$i
+    done
+
+    # bring up non-divergent
+    echo "bringing up non_divergent $non_divergent"
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      activate_osd $dir $i || return 1
+    done
+    for i in $non_divergent
+    do
+      ceph osd in osd.$i
+    done
+
+    WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    # write 1 non-divergent object (ensure that old divergent one is divergent)
+    objname="existing_$(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE)"
+    echo "writing non-divergent object $objname"
+    ceph pg dump pgs
+    rados -p $poolname put $objname $dummyfile2
+
+    WAIT_FOR_CLEAN_TIMEOUT=20 wait_for_clean
+
+    # ensure no recovery of up osds first
+    echo 'delay recovery'
+    ceph pg dump pgs
+    for i in $non_divergent
+    do
+      CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${i}) set_recovery_delay 100000
+    done
+
+    # bring in our divergent friend
+    echo "revive divergent $divergent"
+    ceph pg dump pgs
+    ceph osd set noup
+    activate_osd $dir $divergent
+    sleep 5
+
+    echo 'delay recovery divergent'
+    ceph pg dump pgs
+    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${divergent}) set_recovery_delay 100000
+
+    ceph osd unset noup
+
+    wait_for_osd up 0
+    wait_for_osd up 1
+    wait_for_osd up 2
+
+    ceph pg dump pgs
+    echo 'wait for peering'
+    ceph pg dump pgs
+    rados -p $poolname put foo $dummyfile
+
+    # At this point the divergent_priors should have been detected
+
+    echo "killing divergent $divergent"
+    ceph pg dump pgs
+    kill_daemons $dir KILL osd.$divergent
+
+    # export a pg
+    expfile=$dir/exp.$$.out
+    _objectstore_tool_nodown $dir $divergent --op export-remove --pgid $pgid --file $expfile
+    _objectstore_tool_nodown $dir $divergent --op import --file $expfile
+
+    echo "reviving divergent $divergent"
+    ceph pg dump pgs
+    activate_osd $dir $divergent
+    wait_for_osd up $divergent
+
+    sleep 20
+    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${divergent}) dump_ops_in_flight
+
+    echo "allowing recovery"
+    ceph pg dump pgs
+    # Set osd_recovery_delay_start back to 0 and kick the queue
+    for i in $osds
+    do
+	 ceph tell osd.$i debug kick_recovery_wq 0
+    done
+
+    echo 'reading divergent objects'
+    ceph pg dump pgs
+    for i in $(seq 1 $(expr $DIVERGENT_WRITE + $DIVERGENT_REMOVE))
+    do
+      rados -p $poolname get existing_$i $dir/existing || return 1
+    done
+    for i in $(seq 1 $DIVERGENT_CREATE)
+    do
+      rados -p $poolname get newobject_$i $dir/existing
+    done
+    rm -f $dir/existing
+
+    grep _merge_object_divergent_entries $(find $dir -name '*osd*log')
+    # Check for _merge_object_divergent_entries for case #1
+    if ! grep -q "_merge_object_divergent_entries: more recent entry found:" $(find $dir -name '*osd*log')
+    then
+	    echo failure
+	    return 1
+    fi
+    # Check for _merge_object_divergent_entries for case #2
+    if ! grep -q "_merge_object_divergent_entries.*prior_version or op type indicates creation" $(find $dir -name '*osd*log')
+    then
+	    echo failure
+	    return 1
+    fi
+    echo "success"
+
+    rm $dir/$expfile
+
+    delete_pool $poolname
+    kill_daemons $dir || return 1
+}
+
+
+main divergent-priors "$@"
+
+# Local Variables:
+# compile-command: "make -j4 && ../qa/run-standalone.sh divergent-priors.sh"
+# End:
diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh
index 5d5477b8ec6f1..0582fa1417be5 100755
--- a/qa/standalone/scrub/osd-scrub-repair.sh
+++ b/qa/standalone/scrub/osd-scrub-repair.sh
@@ -190,24 +190,6 @@ function corrupt_and_repair_erasure_coded() {
 
 }
 
-function create_ec_pool() {
-    local pool_name=$1
-    shift
-    local allow_overwrites=$1
-    shift
-
-    ceph osd erasure-code-profile set myprofile crush-failure-domain=osd "$@" || return 1
-
-    create_pool "$poolname" 1 1 erasure myprofile || return 1
-
-    if [ "$allow_overwrites" = "true" ]; then
-        ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
-    fi
-
-    wait_for_clean || return 1
-    return 0
-}
-
 function auto_repair_erasure_coded() {
     local dir=$1
     local allow_overwrites=$2
-- 
2.39.5