From 2e5fba072d92dfda232b12aba84c199a08234b07 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 27 May 2017 00:46:10 +0300 Subject: [PATCH] crush: update choose_args on adjust_item_weight The osd_crush_update_weight_set (true by default) can be used to disable the update of the weights. Signed-off-by: Loic Dachary --- src/common/config_opts.h | 1 + src/crush/CrushWrapper.cc | 26 ++++++- src/crush/CrushWrapper.h | 3 +- ...crush-choose-args-expected-one-more-0.txt} | 24 ++++-- .../crush-choose-args-expected-one-more-3.txt | 75 ++++++++++++++++++ src/test/crush/crush-choose-args.sh | 78 +++++++++++++++++-- 6 files changed, 190 insertions(+), 17 deletions(-) rename src/test/crush/{crush-choose-args-expected-one-more.txt => crush-choose-args-expected-one-more-0.txt} (77%) create mode 100644 src/test/crush/crush-choose-args-expected-one-more-3.txt diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 122e84401fe8a..6acd4f7cb7f4b 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -679,6 +679,7 @@ OPTION(osd_client_message_size_cap, OPT_U64, 500*1024L*1024L) // client data all OPTION(osd_client_message_cap, OPT_U64, 100) // num client messages allowed in-memory OPTION(osd_pg_bits, OPT_INT, 6) // bits per osd OPTION(osd_pgp_bits, OPT_INT, 6) // bits per osd +OPTION(osd_crush_update_weight_set, OPT_BOOL, true) // update weight set while updating weights OPTION(osd_crush_chooseleaf_type, OPT_INT, 1) // 1 = host OPTION(osd_pool_use_gmt_hitset, OPT_BOOL, true) // try to use gmt for hitset archive names if all osds in cluster support it. OPTION(osd_crush_update_on_start, OPT_BOOL, true) diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc index 1a62590430c63..1e84904ba4f40 100644 --- a/src/crush/CrushWrapper.cc +++ b/src/crush/CrushWrapper.cc @@ -975,7 +975,7 @@ int CrushWrapper::adjust_item_weight(CephContext *cct, int id, int weight) continue; for (unsigned i = 0; i < b->size; i++) { if (b->items[i] == id) { - int diff = crush_bucket_adjust_item_weight(crush, b, id, weight); + int diff = bucket_adjust_item_weight(cct, b, id, weight); ldout(cct, 5) << "adjust_item_weight " << id << " diff " << diff << " in bucket " << bidx << dendl; adjust_item_weight(cct, -1 - bidx, b->weight); changed++; @@ -999,7 +999,7 @@ int CrushWrapper::adjust_item_weight_in_loc(CephContext *cct, int id, int weight crush_bucket *b = get_bucket(bid); for (unsigned int i = 0; i < b->size; i++) { if (b->items[i] == id) { - int diff = crush_bucket_adjust_item_weight(crush, b, id, weight); + int diff = bucket_adjust_item_weight(cct, b, id, weight); ldout(cct, 5) << "adjust_item_weight_in_loc " << id << " diff " << diff << " in bucket " << bid << dendl; adjust_item_weight(cct, bid, b->weight); changed++; @@ -1027,7 +1027,7 @@ int CrushWrapper::adjust_subtree_weight(CephContext *cct, int id, int weight) for (unsigned i=0; isize; ++i) { int n = b->items[i]; if (n >= 0) { - crush_bucket_adjust_item_weight(crush, b, n, weight); + bucket_adjust_item_weight(cct, b, n, weight); ++changed; ++local_changed; } else { @@ -1345,6 +1345,26 @@ int CrushWrapper::remove_rule(int ruleno) return 0; } +int CrushWrapper::bucket_adjust_item_weight(CephContext *cct, crush_bucket *bucket, int item, int weight) +{ + if (cct->_conf->osd_crush_update_weight_set) { + unsigned position; + for (position = 0; position < bucket->size; position++) + if (bucket->items[position] == item) + break; + assert(position != bucket->size); + for (auto w : choose_args) { + crush_choose_arg_map arg_map = w.second; + crush_choose_arg *arg = &arg_map.args[-1-bucket->id]; + for (__u32 j = 0; j < arg->weight_set_size; j++) { + crush_weight_set *weight_set = &arg->weight_set[j]; + weight_set->weights[position] = weight; + } + } + } + return crush_bucket_adjust_item_weight(crush, bucket, item, weight); +} + int CrushWrapper::bucket_add_item(crush_bucket *bucket, int item, int weight) { __u32 new_size = bucket->size + 1; diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h index bdb0f5e85773d..41e4de3a881aa 100644 --- a/src/crush/CrushWrapper.h +++ b/src/crush/CrushWrapper.h @@ -1044,7 +1044,7 @@ private: if (!IS_ERR(parent_bucket)) { // zero out the bucket weight - crush_bucket_adjust_item_weight(crush, parent_bucket, item, 0); + bucket_adjust_item_weight(cct, parent_bucket, item, 0); adjust_item_weight(cct, parent_bucket->id, parent_bucket->weight); // remove the bucket from the parent @@ -1143,6 +1143,7 @@ public: int bucket_add_item(crush_bucket *bucket, int item, int weight); int bucket_remove_item(struct crush_bucket *bucket, int item); + int bucket_adjust_item_weight(CephContext *cct, struct crush_bucket *bucket, int item, int weight); void finalize() { assert(crush); diff --git a/src/test/crush/crush-choose-args-expected-one-more.txt b/src/test/crush/crush-choose-args-expected-one-more-0.txt similarity index 77% rename from src/test/crush/crush-choose-args-expected-one-more.txt rename to src/test/crush/crush-choose-args-expected-one-more-0.txt index daff5611185d5..726a90736258f 100644 --- a/src/test/crush/crush-choose-args-expected-one-more.txt +++ b/src/test/crush/crush-choose-args-expected-one-more-0.txt @@ -25,20 +25,20 @@ type 9 region type 10 root # buckets -host fold { +host HOST { id -2 # do not change unnecessarily - # weight 0.356 + # weight 6.000 alg straw2 hash 0 # rjenkins1 - item osd.0 weight 0.178 - item osd.1 weight 0.178 + item osd.0 weight 3.000 + item osd.1 weight 3.000 } root default { id -1 # do not change unnecessarily - # weight 0.356 + # weight 6.000 alg straw2 hash 0 # rjenkins1 - item fold weight 0.356 + item HOST weight 6.000 } # rules @@ -54,11 +54,19 @@ rule replicated_ruleset { # choose_args choose_args 0 { + { + bucket_id -1 + weight_set [ + [ 6.000 ] + [ 7.000 ] + ] + ids [ -10 ] + } { bucket_id -2 weight_set [ - [ 1.000 0.000 ] - [ 1.000 0.000 ] + [ 2.000 0.000 ] + [ 2.000 0.000 ] ] ids [ -20 1 ] } diff --git a/src/test/crush/crush-choose-args-expected-one-more-3.txt b/src/test/crush/crush-choose-args-expected-one-more-3.txt new file mode 100644 index 0000000000000..75424a3d52a54 --- /dev/null +++ b/src/test/crush/crush-choose-args-expected-one-more-3.txt @@ -0,0 +1,75 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 +tunable allowed_bucket_algs 54 + +# devices +device 0 osd.0 +device 1 osd.1 + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host HOST { + id -2 # do not change unnecessarily + # weight 6.000 + alg straw2 + hash 0 # rjenkins1 + item osd.0 weight 3.000 + item osd.1 weight 3.000 +} +root default { + id -1 # do not change unnecessarily + # weight 6.000 + alg straw2 + hash 0 # rjenkins1 + item HOST weight 6.000 +} + +# rules +rule replicated_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take default + step choose firstn 0 type osd + step emit +} + +# choose_args +choose_args 0 { + { + bucket_id -1 + weight_set [ + [ 6.000 ] + [ 6.000 ] + ] + ids [ -10 ] + } + { + bucket_id -2 + weight_set [ + [ 2.000 3.000 ] + [ 2.000 3.000 ] + ] + ids [ -20 1 ] + } +} + +# end crush map diff --git a/src/test/crush/crush-choose-args.sh b/src/test/crush/crush-choose-args.sh index 3ea22b15b2e0d..c0aefb23834fc 100755 --- a/src/test/crush/crush-choose-args.sh +++ b/src/test/crush/crush-choose-args.sh @@ -26,6 +26,8 @@ function run() { export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--crush-location=root=default,host=HOST " + CEPH_ARGS+="--osd-crush-initial-weight=3 " local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do @@ -35,7 +37,10 @@ function run() { done } -function TEST_choose_args() { +function TEST_choose_args_update() { + # + # adding a weighted OSD updates the weight up to the top + # local dir=$1 run_mon $dir a || return 1 @@ -43,16 +48,23 @@ function TEST_choose_args() { ceph osd getcrushmap > $dir/map || return 1 crushtool -d $dir/map -o $dir/map.txt || return 1 - cat $dir/map.txt sed -i -e '/end crush map/d' $dir/map.txt cat >> $dir/map.txt < $dir/map-one-more || return 1 crushtool -d $dir/map-one-more -o $dir/map-one-more.txt || return 1 cat $dir/map-one-more.txt - diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more.txt || return 1 + diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more-3.txt || return 1 destroy_osd $dir 1 || return 1 ceph osd getcrushmap > $dir/map-one-less || return 1 @@ -75,6 +87,62 @@ EOF diff -u $dir/map-one-less.txt $dir/map.txt || return 1 } +function TEST_no_update_weight_set() { + # + # adding a zero weight OSD does not update the weight set at all + # + local dir=$1 + + ORIG_CEPH_ARGS="$CEPH_ARGS" + CEPH_ARGS+="--osd-crush-update-weight-set=false " + + run_mon $dir a || return 1 + run_osd $dir 0 || return 1 + + ceph osd getcrushmap > $dir/map || return 1 + crushtool -d $dir/map -o $dir/map.txt || return 1 + sed -i -e '/end crush map/d' $dir/map.txt + cat >> $dir/map.txt < $dir/map-one-more || return 1 + crushtool -d $dir/map-one-more -o $dir/map-one-more.txt || return 1 + cat $dir/map-one-more.txt + diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more-0.txt || return 1 + + destroy_osd $dir 1 || return 1 + ceph osd getcrushmap > $dir/map-one-less || return 1 + crushtool -d $dir/map-one-less -o $dir/map-one-less.txt || return 1 + diff -u $dir/map-one-less.txt $dir/map.txt || return 1 + + CEPH_ARGS="$ORIG_CEPH_ARGS" +} + main crush-choose-args "$@" # Local Variables: -- 2.47.3