From: Loic Dachary Date: Fri, 26 May 2017 21:46:10 +0000 (+0300) Subject: crush: update choose_args on adjust_item_weight X-Git-Tag: v12.1.0~332^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2e5fba072d92dfda232b12aba84c199a08234b07;p=ceph-ci.git crush: update choose_args on adjust_item_weight The osd_crush_update_weight_set (true by default) can be used to disable the update of the weights. Signed-off-by: Loic Dachary --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 122e84401fe..6acd4f7cb7f 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -679,6 +679,7 @@ OPTION(osd_client_message_size_cap, OPT_U64, 500*1024L*1024L) // client data all OPTION(osd_client_message_cap, OPT_U64, 100) // num client messages allowed in-memory OPTION(osd_pg_bits, OPT_INT, 6) // bits per osd OPTION(osd_pgp_bits, OPT_INT, 6) // bits per osd +OPTION(osd_crush_update_weight_set, OPT_BOOL, true) // update weight set while updating weights OPTION(osd_crush_chooseleaf_type, OPT_INT, 1) // 1 = host OPTION(osd_pool_use_gmt_hitset, OPT_BOOL, true) // try to use gmt for hitset archive names if all osds in cluster support it. OPTION(osd_crush_update_on_start, OPT_BOOL, true) diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc index 1a62590430c..1e84904ba4f 100644 --- a/src/crush/CrushWrapper.cc +++ b/src/crush/CrushWrapper.cc @@ -975,7 +975,7 @@ int CrushWrapper::adjust_item_weight(CephContext *cct, int id, int weight) continue; for (unsigned i = 0; i < b->size; i++) { if (b->items[i] == id) { - int diff = crush_bucket_adjust_item_weight(crush, b, id, weight); + int diff = bucket_adjust_item_weight(cct, b, id, weight); ldout(cct, 5) << "adjust_item_weight " << id << " diff " << diff << " in bucket " << bidx << dendl; adjust_item_weight(cct, -1 - bidx, b->weight); changed++; @@ -999,7 +999,7 @@ int CrushWrapper::adjust_item_weight_in_loc(CephContext *cct, int id, int weight crush_bucket *b = get_bucket(bid); for (unsigned int i = 0; i < b->size; i++) { if (b->items[i] == id) { - int diff = crush_bucket_adjust_item_weight(crush, b, id, weight); + int diff = bucket_adjust_item_weight(cct, b, id, weight); ldout(cct, 5) << "adjust_item_weight_in_loc " << id << " diff " << diff << " in bucket " << bid << dendl; adjust_item_weight(cct, bid, b->weight); changed++; @@ -1027,7 +1027,7 @@ int CrushWrapper::adjust_subtree_weight(CephContext *cct, int id, int weight) for (unsigned i=0; isize; ++i) { int n = b->items[i]; if (n >= 0) { - crush_bucket_adjust_item_weight(crush, b, n, weight); + bucket_adjust_item_weight(cct, b, n, weight); ++changed; ++local_changed; } else { @@ -1345,6 +1345,26 @@ int CrushWrapper::remove_rule(int ruleno) return 0; } +int CrushWrapper::bucket_adjust_item_weight(CephContext *cct, crush_bucket *bucket, int item, int weight) +{ + if (cct->_conf->osd_crush_update_weight_set) { + unsigned position; + for (position = 0; position < bucket->size; position++) + if (bucket->items[position] == item) + break; + assert(position != bucket->size); + for (auto w : choose_args) { + crush_choose_arg_map arg_map = w.second; + crush_choose_arg *arg = &arg_map.args[-1-bucket->id]; + for (__u32 j = 0; j < arg->weight_set_size; j++) { + crush_weight_set *weight_set = &arg->weight_set[j]; + weight_set->weights[position] = weight; + } + } + } + return crush_bucket_adjust_item_weight(crush, bucket, item, weight); +} + int CrushWrapper::bucket_add_item(crush_bucket *bucket, int item, int weight) { __u32 new_size = bucket->size + 1; diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h index bdb0f5e8577..41e4de3a881 100644 --- a/src/crush/CrushWrapper.h +++ b/src/crush/CrushWrapper.h @@ -1044,7 +1044,7 @@ private: if (!IS_ERR(parent_bucket)) { // zero out the bucket weight - crush_bucket_adjust_item_weight(crush, parent_bucket, item, 0); + bucket_adjust_item_weight(cct, parent_bucket, item, 0); adjust_item_weight(cct, parent_bucket->id, parent_bucket->weight); // remove the bucket from the parent @@ -1143,6 +1143,7 @@ public: int bucket_add_item(crush_bucket *bucket, int item, int weight); int bucket_remove_item(struct crush_bucket *bucket, int item); + int bucket_adjust_item_weight(CephContext *cct, struct crush_bucket *bucket, int item, int weight); void finalize() { assert(crush); diff --git a/src/test/crush/crush-choose-args-expected-one-more-0.txt b/src/test/crush/crush-choose-args-expected-one-more-0.txt new file mode 100644 index 00000000000..726a9073625 --- /dev/null +++ b/src/test/crush/crush-choose-args-expected-one-more-0.txt @@ -0,0 +1,75 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 +tunable allowed_bucket_algs 54 + +# devices +device 0 osd.0 +device 1 osd.1 + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host HOST { + id -2 # do not change unnecessarily + # weight 6.000 + alg straw2 + hash 0 # rjenkins1 + item osd.0 weight 3.000 + item osd.1 weight 3.000 +} +root default { + id -1 # do not change unnecessarily + # weight 6.000 + alg straw2 + hash 0 # rjenkins1 + item HOST weight 6.000 +} + +# rules +rule replicated_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take default + step choose firstn 0 type osd + step emit +} + +# choose_args +choose_args 0 { + { + bucket_id -1 + weight_set [ + [ 6.000 ] + [ 7.000 ] + ] + ids [ -10 ] + } + { + bucket_id -2 + weight_set [ + [ 2.000 0.000 ] + [ 2.000 0.000 ] + ] + ids [ -20 1 ] + } +} + +# end crush map diff --git a/src/test/crush/crush-choose-args-expected-one-more-3.txt b/src/test/crush/crush-choose-args-expected-one-more-3.txt new file mode 100644 index 00000000000..75424a3d52a --- /dev/null +++ b/src/test/crush/crush-choose-args-expected-one-more-3.txt @@ -0,0 +1,75 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 +tunable allowed_bucket_algs 54 + +# devices +device 0 osd.0 +device 1 osd.1 + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host HOST { + id -2 # do not change unnecessarily + # weight 6.000 + alg straw2 + hash 0 # rjenkins1 + item osd.0 weight 3.000 + item osd.1 weight 3.000 +} +root default { + id -1 # do not change unnecessarily + # weight 6.000 + alg straw2 + hash 0 # rjenkins1 + item HOST weight 6.000 +} + +# rules +rule replicated_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take default + step choose firstn 0 type osd + step emit +} + +# choose_args +choose_args 0 { + { + bucket_id -1 + weight_set [ + [ 6.000 ] + [ 6.000 ] + ] + ids [ -10 ] + } + { + bucket_id -2 + weight_set [ + [ 2.000 3.000 ] + [ 2.000 3.000 ] + ] + ids [ -20 1 ] + } +} + +# end crush map diff --git a/src/test/crush/crush-choose-args-expected-one-more.txt b/src/test/crush/crush-choose-args-expected-one-more.txt deleted file mode 100644 index daff5611185..00000000000 --- a/src/test/crush/crush-choose-args-expected-one-more.txt +++ /dev/null @@ -1,67 +0,0 @@ -# begin crush map -tunable choose_local_tries 0 -tunable choose_local_fallback_tries 0 -tunable choose_total_tries 50 -tunable chooseleaf_descend_once 1 -tunable chooseleaf_vary_r 1 -tunable straw_calc_version 1 -tunable allowed_bucket_algs 54 - -# devices -device 0 osd.0 -device 1 osd.1 - -# types -type 0 osd -type 1 host -type 2 chassis -type 3 rack -type 4 row -type 5 pdu -type 6 pod -type 7 room -type 8 datacenter -type 9 region -type 10 root - -# buckets -host fold { - id -2 # do not change unnecessarily - # weight 0.356 - alg straw2 - hash 0 # rjenkins1 - item osd.0 weight 0.178 - item osd.1 weight 0.178 -} -root default { - id -1 # do not change unnecessarily - # weight 0.356 - alg straw2 - hash 0 # rjenkins1 - item fold weight 0.356 -} - -# rules -rule replicated_ruleset { - ruleset 0 - type replicated - min_size 1 - max_size 10 - step take default - step choose firstn 0 type osd - step emit -} - -# choose_args -choose_args 0 { - { - bucket_id -2 - weight_set [ - [ 1.000 0.000 ] - [ 1.000 0.000 ] - ] - ids [ -20 1 ] - } -} - -# end crush map diff --git a/src/test/crush/crush-choose-args.sh b/src/test/crush/crush-choose-args.sh index 3ea22b15b2e..c0aefb23834 100755 --- a/src/test/crush/crush-choose-args.sh +++ b/src/test/crush/crush-choose-args.sh @@ -26,6 +26,8 @@ function run() { export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--crush-location=root=default,host=HOST " + CEPH_ARGS+="--osd-crush-initial-weight=3 " local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do @@ -35,7 +37,10 @@ function run() { done } -function TEST_choose_args() { +function TEST_choose_args_update() { + # + # adding a weighted OSD updates the weight up to the top + # local dir=$1 run_mon $dir a || return 1 @@ -43,16 +48,23 @@ function TEST_choose_args() { ceph osd getcrushmap > $dir/map || return 1 crushtool -d $dir/map -o $dir/map.txt || return 1 - cat $dir/map.txt sed -i -e '/end crush map/d' $dir/map.txt cat >> $dir/map.txt < $dir/map-one-more || return 1 crushtool -d $dir/map-one-more -o $dir/map-one-more.txt || return 1 cat $dir/map-one-more.txt - diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more.txt || return 1 + diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more-3.txt || return 1 destroy_osd $dir 1 || return 1 ceph osd getcrushmap > $dir/map-one-less || return 1 @@ -75,6 +87,62 @@ EOF diff -u $dir/map-one-less.txt $dir/map.txt || return 1 } +function TEST_no_update_weight_set() { + # + # adding a zero weight OSD does not update the weight set at all + # + local dir=$1 + + ORIG_CEPH_ARGS="$CEPH_ARGS" + CEPH_ARGS+="--osd-crush-update-weight-set=false " + + run_mon $dir a || return 1 + run_osd $dir 0 || return 1 + + ceph osd getcrushmap > $dir/map || return 1 + crushtool -d $dir/map -o $dir/map.txt || return 1 + sed -i -e '/end crush map/d' $dir/map.txt + cat >> $dir/map.txt < $dir/map-one-more || return 1 + crushtool -d $dir/map-one-more -o $dir/map-one-more.txt || return 1 + cat $dir/map-one-more.txt + diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more-0.txt || return 1 + + destroy_osd $dir 1 || return 1 + ceph osd getcrushmap > $dir/map-one-less || return 1 + crushtool -d $dir/map-one-less -o $dir/map-one-less.txt || return 1 + diff -u $dir/map-one-less.txt $dir/map.txt || return 1 + + CEPH_ARGS="$ORIG_CEPH_ARGS" +} + main crush-choose-args "$@" # Local Variables: