From 5bdf539474657dcfdbfc3242f4dd955eb5f9f51f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 18 Jan 2017 17:45:55 -0600 Subject: [PATCH] osd/OSDMap: add OSDMapMapping class This is a precalculated table of all pg mappings. Signed-off-by: Sage Weil --- src/CMakeLists.txt | 1 + src/osd/OSDMap.h | 4 +- src/osd/OSDMapMapping.cc | 103 +++++++++++++++++++++++++++++++ src/osd/OSDMapMapping.h | 121 +++++++++++++++++++++++++++++++++++++ src/test/osd/TestOSDMap.cc | 33 +++++++--- 5 files changed, 252 insertions(+), 10 deletions(-) create mode 100644 src/osd/OSDMapMapping.cc create mode 100644 src/osd/OSDMapMapping.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4f6325745768..19c97bf98026 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -460,6 +460,7 @@ set(libcommon_files msg/msg_types.cc common/hobject.cc osd/OSDMap.cc + osd/OSDMapMapping.cc common/histogram.cc osd/osd_types.cc osd/OpRequest.cc diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 7522d7f62f5b..e01127dbb953 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -36,9 +36,11 @@ #include "include/memory.h" using namespace std; -//forward declaration +// forward declaration class CephContext; class CrushWrapper; + + /* * we track up to two intervals during which the osd was alive and * healthy. the most recent is [up_from,up_thru), where up_thru is diff --git a/src/osd/OSDMapMapping.cc b/src/osd/OSDMapMapping.cc new file mode 100644 index 000000000000..7c022419d1fd --- /dev/null +++ b/src/osd/OSDMapMapping.cc @@ -0,0 +1,103 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "OSDMapMapping.h" +#include "OSDMap.h" + +// ensure that we have a PoolMappings for each pool and that +// the dimensions (pg_num and size) match up. +void OSDMapMapping::_init_mappings(const OSDMap& osdmap) +{ + auto q = pools.begin(); + for (auto& p : osdmap.get_pools()) { + // drop unneeded pools + while (q != pools.end() && q->first < p.first) { + q = pools.erase(q); + } + if (q != pools.end() && q->first == p.first) { + if (q->second.pg_num != p.second.get_pg_num() || + q->second.size != p.second.get_size()) { + // pg_num changed + q = pools.erase(q); + } else { + // keep it + ++q; + continue; + } + } + pools.emplace(p.first, PoolMapping(p.second.get_size(), + p.second.get_pg_num())); + } + pools.erase(q, pools.end()); + assert(pools.size() == osdmap.get_pools().size()); +} + +void OSDMapMapping::update(const OSDMap& osdmap) +{ + _init_mappings(osdmap); + for (auto& p : osdmap.get_pools()) { + _update_range(osdmap, p.first, 0, p.second.get_pg_num()); + } + _build_rmap(osdmap); + //_dump(); // for debugging +} + +void OSDMapMapping::_build_rmap(const OSDMap& osdmap) +{ + acting_rmap.resize(osdmap.get_max_osd()); + up_rmap.resize(osdmap.get_max_osd()); + for (auto& v : acting_rmap) { + v.resize(0); + } + for (auto& v : up_rmap) { + v.resize(0); + } + for (auto& p : pools) { + pg_t pgid(0, p.first); + for (unsigned ps = 0; ps < p.second.pg_num; ++ps) { + pgid.set_ps(ps); + int32_t *row = &p.second.table[p.second.row_size() * ps]; + for (int i = 0; i < row[2]; ++i) { + if (row[4 + i] != CRUSH_ITEM_NONE) { + acting_rmap[row[4 + i]].push_back(pgid); + } + } + for (int i = 0; i < row[3]; ++i) { + up_rmap[row[4 + p.second.size + i]].push_back(pgid); + } + } + } +} + +void OSDMapMapping::_dump() +{ + for (auto& p : pools) { + cout << "pool " << p.first << std::endl; + for (unsigned i = 0; i < p.second.table.size(); ++i) { + cout << " " << p.second.table[i]; + if (i % p.second.row_size() == p.second.row_size() - 1) + cout << std::endl; + } + } +} + +void OSDMapMapping::_update_range( + const OSDMap& osdmap, + int64_t pool, + unsigned pg_begin, + unsigned pg_end) +{ + auto i = pools.find(pool); + assert(i != pools.end()); + assert(pg_begin <= pg_end); + assert(pg_end <= i->second.pg_num); + for (unsigned ps = pg_begin; ps < pg_end; ++ps) { + vector up, acting; + int up_primary, acting_primary; + osdmap.pg_to_up_acting_osds( + pg_t(ps, pool), + &up, &up_primary, &acting, &acting_primary); + i->second.set(ps, std::move(up), up_primary, + std::move(acting), acting_primary); + } +} diff --git a/src/osd/OSDMapMapping.h b/src/osd/OSDMapMapping.h new file mode 100644 index 000000000000..ff8ed9c63465 --- /dev/null +++ b/src/osd/OSDMapMapping.h @@ -0,0 +1,121 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + + +#ifndef CEPH_OSDMAPMAPPING_H +#define CEPH_OSDMAPMAPPING_H + +#include +#include + +#include "osd/osd_types.h" + +class OSDMap; + +/// a precalculated mapping of every PG for a given OSDMap +class OSDMapMapping { + struct PoolMapping { + unsigned size = 0; + unsigned pg_num = 0; + std::vector table; + + size_t row_size() const { + return + 1 + // acting_primary + 1 + // up_primary + 1 + // num acting + 1 + // num up + size + // acting + size; // up + } + + PoolMapping(int s, int p) + : size(s), + pg_num(p), + table(pg_num * row_size()) { + } + + void get(size_t ps, + std::vector *up, + int *up_primary, + std::vector *acting, + int *acting_primary) { + int32_t *row = &table[row_size() * ps]; + if (acting_primary) { + *acting_primary = row[0]; + } + if (up_primary) { + *up_primary = row[1]; + } + if (acting) { + acting->resize(row[2]); + for (int i = 0; i < row[2]; ++i) { + (*acting)[i] = row[4 + i]; + } + } + if (up) { + up->resize(row[3]); + for (int i = 0; i < row[3]; ++i) { + (*up)[i] = row[4 + size + i]; + } + } + } + + void set(size_t ps, + const std::vector& up, + int up_primary, + const std::vector& acting, + int acting_primary) { + int32_t *row = &table[row_size() * ps]; + row[0] = acting_primary; + row[1] = up_primary; + row[2] = acting.size(); + row[3] = up.size(); + for (int i = 0; i < row[2]; ++i) { + row[4 + i] = acting[i]; + } + for (int i = 0; i < row[3]; ++i) { + row[4 + size + i] = up[i]; + } + } + }; + + std::map pools; + std::vector> acting_rmap; // osd -> pg + std::vector> up_rmap; // osd -> pg + + void _init_mappings(const OSDMap& osdmap); + void _update_range( + const OSDMap& map, + int64_t pool, + unsigned pg_begin, unsigned pg_end); + + void _build_rmap(const OSDMap& osdmap); + + void _dump(); + +public: + void get(pg_t pgid, + std::vector *up, + int *up_primary, + std::vector *acting, + int *acting_primary) { + auto p = pools.find(pgid.pool()); + assert(p != pools.end()); + p->second.get(pgid.ps(), up, up_primary, acting, acting_primary); + } + + const std::vector& get_osd_acting_pgs(unsigned osd) { + assert(osd < acting_rmap.size()); + return acting_rmap[osd]; + } + const std::vector& get_osd_up_pgs(unsigned osd) { + assert(osd < up_rmap.size()); + return up_rmap[osd]; + } + + void update(const OSDMap& map); +}; + + +#endif diff --git a/src/test/osd/TestOSDMap.cc b/src/test/osd/TestOSDMap.cc index 6c660e8f9f3f..29360ae578b7 100644 --- a/src/test/osd/TestOSDMap.cc +++ b/src/test/osd/TestOSDMap.cc @@ -1,6 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- #include "gtest/gtest.h" #include "osd/OSDMap.h" +#include "osd/OSDMapMapping.h" #include "global/global_context.h" #include "global/global_init.h" @@ -28,6 +29,8 @@ class OSDMapTest : public testing::Test { const static int num_osds = 6; public: OSDMap osdmap; + OSDMapMapping mapping; + OSDMapTest() {} void set_up_map() { @@ -76,17 +79,29 @@ public: vector *any, vector *first, vector *primary) { + mapping.update(osdmap); for (int i=0; i o; - int p; + vector up, acting; + int up_primary, acting_primary; pg_t pgid(i, pool); - osdmap.pg_to_acting_osds(pgid, &o, &p); - for (unsigned j=0; j= 0) - (*primary)[p]++; + osdmap.pg_to_up_acting_osds(pgid, + &up, &up_primary, &acting, &acting_primary); + for (unsigned j=0; j= 0) + (*primary)[acting_primary]++; + + // compare to precalc mapping + vector up2, acting2; + int up_primary2, acting_primary2; + pgid = osdmap.raw_pg_to_pg(pgid); + mapping.get(pgid, &up2, &up_primary2, &acting2, &acting_primary2); + ASSERT_EQ(up, up2); + ASSERT_EQ(up_primary, up_primary2); + ASSERT_EQ(acting, acting2); + ASSERT_EQ(acting_primary, acting_primary2); } } }; -- 2.47.3