From 7b56e70385c9be19fdf97b74c0cde1bc36078c22 Mon Sep 17 00:00:00 2001
From: Kefu Chai <kchai@redhat.com>
Date: Wed, 23 Jan 2019 17:58:20 +0800
Subject: [PATCH] crimson/osd: load_pgs()

Signed-off-by: Kefu Chai <kchai@redhat.com>
---
 src/crimson/osd/CMakeLists.txt |   4 +-
 src/crimson/osd/osd.cc         |  55 +++++++++++++++++
 src/crimson/osd/osd.h          |   7 +++
 src/crimson/osd/pg.cc          |   6 ++
 src/crimson/osd/pg.h           |  21 +++++++
 src/crimson/osd/pg_meta.cc     | 104 +++++++++++++++++++++++++++++++++
 src/crimson/osd/pg_meta.h      |  22 +++++++
 7 files changed, 218 insertions(+), 1 deletion(-)
 create mode 100644 src/crimson/osd/pg.cc
 create mode 100644 src/crimson/osd/pg.h
 create mode 100644 src/crimson/osd/pg_meta.cc
 create mode 100644 src/crimson/osd/pg_meta.h

diff --git a/src/crimson/osd/CMakeLists.txt b/src/crimson/osd/CMakeLists.txt
index 6bf330eebaa3d..624414701538f 100644
--- a/src/crimson/osd/CMakeLists.txt
+++ b/src/crimson/osd/CMakeLists.txt
@@ -2,6 +2,8 @@ add_executable(crimson-osd
   chained_dispatchers.cc
   main.cc
   osd.cc
-  osd_meta.cc)
+  osd_meta.cc
+  pg.cc
+  pg_meta.cc)
 target_link_libraries(crimson-osd
   crimson-common crimson-os crimson)
diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index d5619f87a519e..71eee16173c3f 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -10,6 +10,8 @@
 #include "crimson/os/cyan_store.h"
 #include "crimson/os/Transaction.h"
 #include "crimson/osd/osd_meta.h"
+#include "crimson/osd/pg.h"
+#include "crimson/osd/pg_meta.h"
 
 namespace {
   seastar::logger& logger() {
@@ -122,6 +124,8 @@ seastar::future<> OSD::start()
     return get_map(superblock.current_epoch);
   }).then([this](seastar::lw_shared_ptr<OSDMap> map) {
     osdmap = std::move(map);
+    return load_pgs();
+  }).then([this] {
     return client_msgr->start(&dispatchers);
   }).then([this] {
     return monc.start();
@@ -204,6 +208,57 @@ seastar::future<> OSD::stop()
   });
 }
 
+seastar::future<> OSD::load_pgs()
+{
+  return seastar::parallel_for_each(store->list_collections(),
+    [this](auto coll) {
+      spg_t pgid;
+      if (coll.is_pg(&pgid)) {
+        return load_pg(pgid).then([pgid, this](auto&& pg) {
+          logger().info("load_pgs: loaded {}", pgid);
+          pgs.emplace(pgid, std::move(pg));
+          return seastar::now();
+        });
+      } else if (coll.is_temp(&pgid)) {
+        // TODO: remove the collection
+        return seastar::now();
+      } else {
+        logger().warn("ignoring unrecognized collection: {}", coll);
+        return seastar::now();
+      }
+    });
+}
+
+seastar::future<Ref<PG>> OSD::load_pg(spg_t pgid)
+{
+  using ec_profile_t = map<string,string>;
+  return PGMeta{store.get(), pgid}.get_epoch().then([this](epoch_t e) {
+    return get_map(e);
+  }).then([pgid, this] (auto&& create_map) {
+    if (create_map->have_pg_pool(pgid.pool())) {
+      pg_pool_t pi = *create_map->get_pg_pool(pgid.pool());
+      string name = create_map->get_pool_name(pgid.pool());
+      ec_profile_t ec_profile;
+      if (pi.is_erasure()) {
+        ec_profile = create_map->get_erasure_code_profile(pi.erasure_code_profile);
+      }
+      return seastar::make_ready_future<pg_pool_t,
+                                        string,
+                                        ec_profile_t>(std::move(pi),
+                                                      std::move(name),
+                                                      std::move(ec_profile));
+    } else {
+      // pool was deleted; grab final pg_pool_t off disk.
+      return meta_coll->load_final_pool_info(pgid.pool());
+    }
+  }).then([this](pg_pool_t&& pool, string&& name, ec_profile_t&& ec_profile) {
+    Ref<PG> pg{new PG{std::move(pool),
+                      std::move(name),
+                      std::move(ec_profile)}};
+    return seastar::make_ready_future<Ref<PG>>(std::move(pg));
+  });
+}
+
 seastar::future<> OSD::ms_dispatch(ceph::net::ConnectionRef conn, MessageRef m)
 {
   logger().info("ms_dispatch {}", *m);
diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index 0498c97d23b76..8c4b1790044c8 100644
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -16,6 +16,7 @@
 class MOSDMap;
 class OSDMap;
 class OSDMeta;
+class PG;
 
 namespace ceph::net {
   class Messenger;
@@ -27,6 +28,8 @@ namespace ceph::os {
   class Transaction;
 }
 
+template<typename T> using Ref = boost::intrusive_ptr<T>;
+
 class OSD : public ceph::net::Dispatcher {
   seastar::gate gate;
   seastar::timer<seastar::lowres_clock> beacon_timer;
@@ -46,6 +49,7 @@ class OSD : public ceph::net::Dispatcher {
   std::unique_ptr<ceph::os::CyanStore> store;
   std::unique_ptr<OSDMeta> meta_coll;
 
+  std::unordered_map<spg_t, Ref<PG>> pgs;
   OSDState state;
 
   /// _first_ epoch we were marked up (after this process started)
@@ -81,6 +85,9 @@ private:
   seastar::future<> _preboot(version_t newest_osdmap, version_t oldest_osdmap);
   seastar::future<> _send_boot();
 
+  seastar::future<Ref<PG>> load_pg(spg_t pgid);
+  seastar::future<> load_pgs();
+
   seastar::future<seastar::lw_shared_ptr<OSDMap>> get_map(epoch_t e);
   seastar::future<bufferlist> load_map_bl(epoch_t e);
   void store_map_bl(ceph::os::Transaction& t,
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
new file mode 100644
index 0000000000000..bc7b8c2d8a7a2
--- /dev/null
+++ b/src/crimson/osd/pg.cc
@@ -0,0 +1,6 @@
+#include "pg.h"
+
+PG::PG(pg_pool_t&& pool, std::string&& name, ec_profile_t&& ec_profile)
+{
+  // TODO
+}
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
new file mode 100644
index 0000000000000..4bb672f4b08af
--- /dev/null
+++ b/src/crimson/osd/pg.h
@@ -0,0 +1,21 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <boost/intrusive_ptr.hpp>
+#include <boost/smart_ptr/intrusive_ref_counter.hpp>
+#include <seastar/core/future.hh>
+
+#include "osd/osd_types.h"
+
+template<typename T> using Ref = boost::intrusive_ptr<T>;
+
+class PG : public boost::intrusive_ref_counter<
+  PG,
+  boost::thread_unsafe_counter>
+{
+  using ec_profile_t = std::map<std::string,std::string>;
+public:
+  PG(pg_pool_t&& pool, std::string&& name, ec_profile_t&& ec_profile);
+};
diff --git a/src/crimson/osd/pg_meta.cc b/src/crimson/osd/pg_meta.cc
new file mode 100644
index 0000000000000..2098e50a2e3e6
--- /dev/null
+++ b/src/crimson/osd/pg_meta.cc
@@ -0,0 +1,104 @@
+#include "pg_meta.h"
+
+#include <string_view>
+
+#include "crimson/os/cyan_collection.h"
+#include "crimson/os/cyan_store.h"
+
+// prefix pgmeta_oid keys with _ so that PGLog::read_log_and_missing() can
+// easily skip them
+
+static const string_view infover_key = "_infover"sv;
+static const string_view info_key = "_info"sv;
+static const string_view biginfo_key = "_biginfo"sv;
+static const string_view epoch_key = "_epoch"sv;
+static const string_view fastinfo_key = "_fastinfo"sv;
+
+using ceph::os::CyanStore;
+
+PGMeta::PGMeta(CyanStore* store, spg_t pgid)
+  : store{store},
+    pgid{pgid}
+{}
+
+namespace {
+  template<typename T>
+  std::optional<T> find_value(const CyanStore::omap_values_t& values,
+                              string_view key)
+  {
+    auto found = values.find(key);
+    if (found == values.end()) {
+      return {};
+    }
+    auto p = found->second.cbegin();
+    T value;
+    decode(value, p);
+    return std::make_optional(std::move(value));
+  }
+}
+seastar::future<epoch_t> PGMeta::get_epoch()
+{
+  auto ch = store->open_collection(coll_t{pgid});
+  return store->omap_get_values(ch,
+                                pgid.make_pgmeta_oid(),
+                                {string{infover_key},
+                                 string{epoch_key}}).then(
+    [](auto&& values) {
+      {
+        // sanity check
+        auto infover = find_value<__u8>(values, infover_key);
+        assert(infover);
+        if (*infover < 10) {
+          throw std::runtime_error("incompatible pg meta");
+        }
+      }
+      {
+        auto epoch = find_value<epoch_t>(values, epoch_key);
+        assert(epoch);
+        return seastar::make_ready_future<epoch_t>(*epoch);
+      }
+    });
+}
+
+seastar::future<pg_info_t, PastIntervals> PGMeta::load()
+{
+  auto ch = store->open_collection(coll_t{pgid});
+  return store->omap_get_values(ch,
+                                pgid.make_pgmeta_oid(),
+                                {string{infover_key},
+                                 string{info_key},
+                                 string{biginfo_key},
+                                 string{fastinfo_key}}).then(
+    [this](auto&& values) {
+      {
+        // sanity check
+        auto infover = find_value<__u8>(values, infover_key);
+        assert(infover);
+        if (infover < 10) {
+          throw std::runtime_error("incompatible pg meta");
+        }
+      }
+      pg_info_t info;
+      {
+        auto found = find_value<pg_info_t>(values, info_key);
+        assert(found);
+        info = *std::move(found);
+      }
+      PastIntervals past_intervals;
+      {
+        using biginfo_t = std::pair<PastIntervals, decltype(info.purged_snaps)>;
+        auto big_info = find_value<biginfo_t>(values, biginfo_key);
+        assert(big_info);
+        past_intervals = std::move(big_info->first);
+        info.purged_snaps = std::move(big_info->second);
+      }
+      {
+        auto fast_info = find_value<pg_fast_info_t>(values, fastinfo_key);
+        assert(fast_info);
+        fast_info->try_apply_to(&info);
+      }
+      return seastar::make_ready_future<pg_info_t, PastIntervals>(
+        std::move(info),
+        std::move(past_intervals));
+    });
+}
diff --git a/src/crimson/osd/pg_meta.h b/src/crimson/osd/pg_meta.h
new file mode 100644
index 0000000000000..10f2234a7922f
--- /dev/null
+++ b/src/crimson/osd/pg_meta.h
@@ -0,0 +1,22 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <seastar/core/future.hh>
+#include "osd/osd_types.h"
+
+namespace ceph::os {
+  class CyanStore;
+}
+
+/// PG related metadata
+class PGMeta
+{
+  ceph::os::CyanStore* store;
+  const spg_t pgid;
+public:
+  PGMeta(ceph::os::CyanStore *store, spg_t pgid);
+  seastar::future<epoch_t> get_epoch();
+  seastar::future<pg_info_t, PastIntervals> load();
+};
-- 
2.39.5