From 41c903a4bf294b6f123484451ff36c449ca51a39 Mon Sep 17 00:00:00 2001 From: Sridhar Seshasayee Date: Mon, 15 May 2023 12:46:54 +0530 Subject: [PATCH] osd/scheduler: Reset ephemeral changes to mClock built-in profile This is a follow-up to PR: https://github.com/ceph/ceph/pull/48703. This commit also considers changes made ephemerally using either the 'daemon' or the 'tell' interfaces to override the built-in mClock QoS parameters. In such a scenario, the ephemeral changes are removed using the rm_val() method exposed by the config subsytem and logging this information. Other changes: 1. Add a standalone test to exercise the fix. 2. Add documentation note on the outcome of the attempt to modify built-in profile defaults. Fixes: https://tracker.ceph.com/issues/61155 Signed-off-by: Sridhar Seshasayee (cherry picked from commit 414ac7dd2ce48d2a31f798e8df09bd3c975e95b1) --- doc/rados/configuration/mclock-config-ref.rst | 5 ++ qa/standalone/misc/mclock-config.sh | 79 +++++++++++++++++++ src/osd/scheduler/mClockScheduler.cc | 6 ++ 3 files changed, 90 insertions(+) diff --git a/doc/rados/configuration/mclock-config-ref.rst b/doc/rados/configuration/mclock-config-ref.rst index cb607a326f19e..e49391698507d 100644 --- a/doc/rados/configuration/mclock-config-ref.rst +++ b/doc/rados/configuration/mclock-config-ref.rst @@ -177,6 +177,11 @@ in order to ensure mClock scheduler is able to provide predictable QoS. mClock Config Options --------------------- +.. important:: These defaults cannot be changed using any of the config + subsytem commands like *config set* or via the *config daemon* or *config + tell* interfaces. Although the above command(s) report success, the mclock + QoS parameters are reverted to their respective built-in profile defaults. + When a built-in profile is enabled, the mClock scheduler calculates the low level mclock parameters [*reservation*, *weight*, *limit*] based on the profile enabled for each client type. The mclock parameters are calculated based on diff --git a/qa/standalone/misc/mclock-config.sh b/qa/standalone/misc/mclock-config.sh index bc5473114a959..59f002584d9da 100755 --- a/qa/standalone/misc/mclock-config.sh +++ b/qa/standalone/misc/mclock-config.sh @@ -380,6 +380,85 @@ function TEST_profile_disallow_builtin_params_modify() { teardown $dir || return 1 } +function TEST_profile_disallow_builtin_params_override() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + + run_osd $dir 0 --osd_op_queue=mclock_scheduler || return 1 + + # Verify that the default mclock profile is set on the OSD + local def_mclock_profile=$(ceph config get osd.0 osd_mclock_profile) + test "$def_mclock_profile" = "balanced" || return 1 + + # Verify the running mClock profile + local cur_mclock_profile=$(CEPH_ARGS='' ceph --format=json daemon \ + $(get_asok_path osd.0) config get osd_mclock_profile |\ + jq .osd_mclock_profile) + cur_mclock_profile=$(eval echo $cur_mclock_profile) + test $cur_mclock_profile = "high_recovery_ops" || return 1 + + declare -a options=("osd_mclock_scheduler_background_recovery_res" + "osd_mclock_scheduler_client_res") + + local retries=10 + local errors=0 + for opt in "${options[@]}" + do + # Override a mclock config param and confirm that no change occurred + local opt_val_orig=$(CEPH_ARGS='' ceph --format=json daemon \ + $(get_asok_path osd.0) config get $opt | jq .$opt | bc) + local opt_val_new=$(echo "$opt_val_orig + 0.1" | bc -l) + ceph tell osd.0 config set $opt $opt_val_new || return 1 + + # Check configuration values + for count in $(seq 0 $(expr $retries - 1)) + do + errors=0 + sleep 2 # Allow time for changes to take effect + + echo "Check configuration values - Attempt#: $count" + # Check configuration value on Mon store (or the default) for the osd + local res=$(ceph config get osd.0 $opt) || return 1 + echo "Mon db (or default): osd.0 $opt = $res" + if (( $(echo "$res == $opt_val_new" | bc -l) )); then + errors=$(expr $errors + 1) + fi + + # Check running configuration value using "config show" cmd + res=$(ceph config show osd.0 | grep $opt |\ + awk '{ print $2 }' | bc ) || return 1 + echo "Running config: osd.0 $opt = $res" + if (( $(echo "$res == $opt_val_new" | bc -l) || \ + $(echo "$res != $opt_val_orig" | bc -l) )); then + errors=$(expr $errors + 1) + fi + + # Check value in the in-memory 'values' map is unmodified + res=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path \ + osd.0) config get $opt | jq .$opt | bc) + echo "Values map: osd.0 $opt = $res" + if (( $(echo "$res == $opt_val_new" | bc -l) || \ + $(echo "$res != $opt_val_orig" | bc -l) )); then + errors=$(expr $errors + 1) + fi + + # Check if we succeeded or exhausted retry count + if [ $errors -eq 0 ] + then + break + elif [ $count -eq $(expr $retries - 1) ] + then + return 1 + fi + done + done + + teardown $dir || return 1 +} + main mclock-config "$@" # Local Variables: diff --git a/src/osd/scheduler/mClockScheduler.cc b/src/osd/scheduler/mClockScheduler.cc index 4f56a8e48177a..4b30766796c6d 100644 --- a/src/osd/scheduler/mClockScheduler.cc +++ b/src/osd/scheduler/mClockScheduler.cc @@ -539,6 +539,12 @@ void mClockScheduler::handle_conf_change( } } } + // Alternatively, the QoS parameter, if set ephemerally for this OSD via + // the 'daemon' or 'tell' interfaces must be removed. + if (!cct->_conf.rm_val(*key)) { + dout(10) << __func__ << " Restored " << *key << " to default" << dendl; + cct->_conf.apply_changes(nullptr); + } } } -- 2.39.5