From 3b8a276c437cfd599c55a935d141375afda676ff Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sun, 23 Jul 2017 00:10:56 -0400 Subject: [PATCH] mgr/PyOSDMap: get_crush, find_takes, get_take_weight_osd_map These let us identify distinct CRUSH hierarchies that rules distribute data over, and create relative weight maps for the OSDs they map to. Signed-off-by: Sage Weil --- src/mgr/PyOSDMap.cc | 212 ++++++++++++++++++++++++++++++++++- src/pybind/mgr/mgr_module.py | 46 +++++++- 2 files changed, 250 insertions(+), 8 deletions(-) diff --git a/src/mgr/PyOSDMap.cc b/src/mgr/PyOSDMap.cc index 9b1bdc4e3b79d..69bcea42a2ade 100644 --- a/src/mgr/PyOSDMap.cc +++ b/src/mgr/PyOSDMap.cc @@ -6,6 +6,7 @@ #include "osd/OSDMap.h" #include "common/errno.h" #include "common/version.h" +#include "include/stringify.h" #include "PyOSDMap.h" #include "PyFormatter.h" @@ -22,6 +23,12 @@ static PyObject *osdmap_get_epoch(PyObject *self, PyObject *obj) return PyInt_FromLong(osdmap->get_epoch()); } +static PyObject *osdmap_get_crush_version(PyObject *self, PyObject *obj) +{ + OSDMap *osdmap = static_cast(PyCapsule_GetPointer(obj, nullptr)); + return PyInt_FromLong(osdmap->get_crush_version()); +} + static PyObject *osdmap_dump(PyObject *self, PyObject *obj) { OSDMap *osdmap = static_cast(PyCapsule_GetPointer(obj, nullptr)); @@ -35,21 +42,84 @@ static void delete_osdmap_incremental(PyObject *object) { OSDMap::Incremental *inc = static_cast( PyCapsule_GetPointer(object, nullptr)); - derr << __func__ << " " << inc << dendl; + dout(10) << __func__ << " " << inc << dendl; delete inc; } static PyObject *osdmap_new_incremental(PyObject *self, PyObject *obj) { OSDMap *osdmap = static_cast(PyCapsule_GetPointer(obj, nullptr)); - - // Construct a capsule containing an OSDMap. OSDMap::Incremental *inc = new OSDMap::Incremental; inc->fsid = osdmap->get_fsid(); inc->epoch = osdmap->get_epoch() + 1; + // always include latest crush map here... this is okay since we never + // actually use this map in the real world (and even if we did it would + // be a no-op). + osdmap->crush->encode(inc->crush, CEPH_FEATURES_ALL); + dout(10) << __func__ << " " << inc << dendl; return PyCapsule_New(inc, nullptr, &delete_osdmap_incremental); } +static void delete_osdmap(PyObject *object) +{ + OSDMap *osdmap = static_cast(PyCapsule_GetPointer(object, nullptr)); + assert(osdmap); + dout(10) << __func__ << " " << osdmap << dendl; + delete osdmap; +} + +static PyObject *osdmap_apply_incremental(PyObject *self, PyObject *args) +{ + PyObject *mapobj, *incobj; + if (!PyArg_ParseTuple(args, "OO:apply_incremental", + &mapobj, &incobj)) { + return nullptr; + } + OSDMap *osdmap = static_cast(PyCapsule_GetPointer(mapobj, nullptr)); + OSDMap::Incremental *inc = static_cast( + PyCapsule_GetPointer(incobj, nullptr)); + if (!osdmap || !inc) { + return nullptr; + } + + bufferlist bl; + osdmap->encode(bl, CEPH_FEATURES_ALL|CEPH_FEATURE_RESERVED); + OSDMap *next = new OSDMap; + next->decode(bl); + next->apply_incremental(*inc); + dout(10) << __func__ << " map " << osdmap << " inc " << inc + << " next " << next << dendl; + return PyCapsule_New(next, nullptr, &delete_osdmap); +} + +static PyObject *osdmap_get_crush(PyObject *self, PyObject *obj) +{ + OSDMap *osdmap = static_cast(PyCapsule_GetPointer(obj, nullptr)); + + // Construct a capsule containing a the CrushWrapper. + return PyCapsule_New(osdmap->crush.get(), nullptr, nullptr); +} + +static PyObject *osdmap_get_pools_by_take(PyObject *self, PyObject *args) +{ + PyObject *mapobj; + int take; + if (!PyArg_ParseTuple(args, "Oi:get_pools_by_take", + &mapobj, &take)) { + return nullptr; + } + OSDMap *osdmap = static_cast(PyCapsule_GetPointer(mapobj, nullptr)); + PyFormatter f; + f.open_array_section("pools"); + for (auto& p : osdmap->get_pools()) { + if (osdmap->crush->rule_has_take(p.second.crush_rule, take)) { + f.dump_int("pool", p.first); + } + } + f.close_section(); + return f.get(); +} + static PyObject *osdmap_calc_pg_upmaps(PyObject *self, PyObject *args) { PyObject *mapobj, *incobj, *pool_list; @@ -82,9 +152,15 @@ static PyObject *osdmap_calc_pg_upmaps(PyObject *self, PyObject *args) PyMethodDef OSDMapMethods[] = { {"get_epoch", osdmap_get_epoch, METH_O, "Get OSDMap epoch"}, + {"get_crush_version", osdmap_get_crush_version, METH_O, "Get CRUSH version"}, {"dump", osdmap_dump, METH_O, "Dump OSDMap::Incremental"}, {"new_incremental", osdmap_new_incremental, METH_O, "Create OSDMap::Incremental"}, + {"apply_incremental", osdmap_apply_incremental, METH_VARARGS, + "Apply OSDMap::Incremental and return the resulting OSDMap"}, + {"get_crush", osdmap_get_crush, METH_O, "Get CrushWrapper"}, + {"get_pools_by_take", osdmap_get_pools_by_take, METH_VARARGS, + "Get pools that have CRUSH rules that TAKE the given root"}, {"calc_pg_upmaps", osdmap_calc_pg_upmaps, METH_VARARGS, "Calculate new pg-upmap values"}, {NULL, NULL, 0, NULL} @@ -108,18 +184,146 @@ static PyObject *osdmap_inc_dump(PyObject *self, PyObject *obj) return f.get(); } +static int get_int_float_map(PyObject *obj, map *out) +{ + PyObject *ls = PyDict_Items(obj); + for (int j = 0; j < PyList_Size(ls); ++j) { + PyObject *pair = PyList_GET_ITEM(ls, j); + if (!PyTuple_Check(pair)) { + derr << __func__ << " item " << j << " not a tuple" << dendl; + return -1; + } + int k; + double v; + if (!PyArg_ParseTuple(pair, "id:pair", &k, &v)) { + derr << __func__ << " item " << j << " not a size 2 tuple" << dendl; + return -1; + } + (*out)[k] = v; + } + return 0; +} + +static PyObject *osdmap_inc_set_osd_reweights(PyObject *self, PyObject *args) +{ + PyObject *incobj, *weightobj; + if (!PyArg_ParseTuple(args, "OO:set_osd_reweights", + &incobj, &weightobj)) { + return nullptr; + } + OSDMap::Incremental *inc = static_cast( + PyCapsule_GetPointer(incobj, nullptr)); + map wm; + if (get_int_float_map(weightobj, &wm) < 0) { + return nullptr; + } + + for (auto i : wm) { + inc->new_weight[i.first] = std::max(0.0, std::min(1.0, i.second)) * 0x10000; + } + Py_RETURN_NONE; +} + +static PyObject *osdmap_inc_set_compat_weight_set_weights( + PyObject *self, PyObject *args) +{ + PyObject *incobj, *weightobj; + if (!PyArg_ParseTuple(args, "OO:set_compat_weight_set_weights", + &incobj, &weightobj)) { + return nullptr; + } + OSDMap::Incremental *inc = static_cast( + PyCapsule_GetPointer(incobj, nullptr)); + map wm; + if (get_int_float_map(weightobj, &wm) < 0) { + return nullptr; + } + + CrushWrapper crush; + assert(inc->crush.length()); // see new_incremental + auto p = inc->crush.begin(); + ::decode(crush, p); + crush.create_choose_args(CrushWrapper::DEFAULT_CHOOSE_ARGS, 1); + for (auto i : wm) { + crush.choose_args_adjust_item_weightf( + g_ceph_context, + crush.choose_args_get(CrushWrapper::DEFAULT_CHOOSE_ARGS), + i.first, + { i.second }, + nullptr); + } + inc->crush.clear(); + crush.encode(inc->crush, CEPH_FEATURES_ALL); + Py_RETURN_NONE; +} + + + PyMethodDef OSDMapIncrementalMethods[] = { {"get_epoch", osdmap_inc_get_epoch, METH_O, "Get OSDMap::Incremental epoch"}, {"dump", osdmap_inc_dump, METH_O, "Dump OSDMap::Incremental"}, + {"set_osd_reweights", osdmap_inc_set_osd_reweights, METH_VARARGS, + "Set osd reweight values"}, + {"set_crush_compat_weight_set_weights", + osdmap_inc_set_compat_weight_set_weights, METH_VARARGS, + "Set weight values in the pending CRUSH compat weight-set"}, {NULL, NULL, 0, NULL} }; // ---------- +static PyObject *crush_dump(PyObject *self, PyObject *obj) +{ + CrushWrapper *crush = static_cast( + PyCapsule_GetPointer(obj, nullptr)); + PyFormatter f; + crush->dump(&f); + return f.get(); +} + +static PyObject *crush_find_takes(PyObject *self, PyObject *obj) +{ + CrushWrapper *crush = static_cast( + PyCapsule_GetPointer(obj, nullptr)); + set takes; + crush->find_takes(&takes); + PyFormatter f; + f.open_array_section("takes"); + for (auto root : takes) { + f.dump_int("root", root); + } + f.close_section(); + return f.get(); +} + +static PyObject *crush_get_take_weight_osd_map(PyObject *self, PyObject *args) +{ + PyObject *obj; + int root; + if (!PyArg_ParseTuple(args, "Oi:get_take_weight_osd_map", + &obj, &root)) { + return nullptr; + } + CrushWrapper *crush = static_cast( + PyCapsule_GetPointer(obj, nullptr)); + map wmap; + crush->get_take_weight_osd_map(root, &wmap); + PyFormatter f; + f.open_object_section("weights"); + for (auto& p : wmap) { + string n = stringify(p.first); // ick + f.dump_float(n.c_str(), p.second); + } + f.close_section(); + return f.get(); +} PyMethodDef CRUSHMapMethods[] = { -// {"get_epoch", osdmap_get_epoch, METH_O, "Get OSDMap epoch"}, + {"dump", crush_dump, METH_O, "Dump map"}, + {"find_takes", crush_find_takes, METH_O, "Find distinct TAKE roots"}, + {"get_take_weight_osd_map", crush_get_take_weight_osd_map, METH_VARARGS, + "Get OSD weight map for a given TAKE root node"}, {NULL, NULL, 0, NULL} }; diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py index 0236e92d76c7b..9f5781fbc5027 100644 --- a/src/pybind/mgr/mgr_module.py +++ b/src/pybind/mgr/mgr_module.py @@ -40,13 +40,26 @@ class OSDMap(object): def get_epoch(self): return ceph_osdmap.get_epoch(self._handle) + def get_crush_version(self): + return ceph_osdmap.get_crush_version(self._handle) + def dump(self): return ceph_osdmap.dump(self._handle) def new_incremental(self): return OSDMapIncremental(ceph_osdmap.new_incremental(self._handle)) - def calc_pg_upmaps(self, inc, max_deviation=.01, max_iterations=10, pools=[]): + def apply_incremental(self, inc): + return OSDMap(ceph_osdmap.apply_incremental(self._handle, inc._handle)) + + def get_crush(self): + return CRUSHMap(ceph_osdmap.get_crush(self._handle), self) + + def get_pools_by_take(self, take): + return ceph_osdmap.get_pools_by_take(self._handle, take).get('pools', []) + + def calc_pg_upmaps(self, inc, + max_deviation=.01, max_iterations=10, pools=[]): return ceph_osdmap.calc_pg_upmaps( self._handle, inc._handle, @@ -63,12 +76,37 @@ class OSDMapIncremental(object): def dump(self): return ceph_osdmap_incremental.dump(self._handle) + def set_osd_reweights(self, weightmap): + """ + weightmap is a dict, int to float. e.g. { 0: .9, 1: 1.0, 3: .997 } + """ + return ceph_osdmap_incremental.set_osd_reweights(self._handle, weightmap) + + def set_crush_compat_weight_set_weights(self, weightmap): + """ + weightmap is a dict, int to float. devices only. e.g., + { 0: 3.4, 1: 3.3, 2: 3.334 } + """ + return ceph_osdmap_incremental.set_crush_compat_weight_set_weights( + self._handle, weightmap) + + + class CRUSHMap(object): - def __init__(self, handle): + def __init__(self, handle, parent_osdmap): self._handle = handle + # keep ref to parent osdmap since handle lifecycle is owned by it + self._parent_osdmap = parent_osdmap + + def dump(self): + return ceph_crushmap.dump(self._handle) + + def find_takes(self): + return ceph_crushmap.find_takes(self._handle).get('takes',[]) -# def get_epoch(self): -# return ceph_crushmap.get_epoch(self._handle) + def get_take_weight_osd_map(self, root): + uglymap = ceph_crushmap.get_take_weight_osd_map(self._handle, root) + return { int(k): v for k, v in uglymap.get('weights', {}).iteritems() } class MgrModule(object): -- 2.39.5