crimson/os: make load_pgs() load pg on proper core

Signed-off-by: chunmei-liu <chunmei.liu@intel.com>

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index dd5b54084116a86b795c8b86d96b0f9dbdebca4d..a458d321b86990b928ce8beea47d2a19c1c72a6a 100644 (file)
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -262,7 +262,7 @@ seastar::future<CollectionRef> AlienStore::open_collection(const coll_t& cid)
   });
 }
 
-seastar::future<std::vector<coll_t>> AlienStore::list_collections()
+seastar::future<std::vector<coll_core_t>> AlienStore::list_collections()
 {
   logger().debug("{}", __func__);
   assert(tp);
@@ -270,9 +270,14 @@ seastar::future<std::vector<coll_t>> AlienStore::list_collections()
   return do_with_op_gate(std::vector<coll_t>{}, [this] (auto &ls) {
     return tp->submit([this, &ls] {
       return store->list_collections(ls);
-    }).then([&ls] (int r) {
+    }).then([&ls] (int r) -> seastar::future<std::vector<coll_core_t>> {
       assert(r == 0);
-      return seastar::make_ready_future<std::vector<coll_t>>(std::move(ls));
+      std::vector<coll_core_t> ret;
+      ret.resize(ls.size());
+      std::transform(
+        ls.begin(), ls.end(), ret.begin(),
+        [](auto p) { return std::make_pair(p, NULL_CORE); });
+      return seastar::make_ready_future<std::vector<coll_core_t>>(std::move(ret));
     });
   });
 }

diff --git a/src/crimson/os/alienstore/alien_store.h b/src/crimson/os/alienstore/alien_store.h
index 1b6a4276479f1a2a9d9bd4d10e3a4e9656a2515d..f299ffda811c20d4474080f04351d537f8200f89 100644 (file)
--- a/src/crimson/os/alienstore/alien_store.h
+++ b/src/crimson/os/alienstore/alien_store.h
@@ -19,6 +19,7 @@ class Transaction;
 }
 
 namespace crimson::os {
+using coll_core_t = FuturizedStore::coll_core_t;
 class AlienStore final : public FuturizedStore {
 public:
   AlienStore(const std::string& type,
@@ -69,7 +70,7 @@ public:
 
   seastar::future<CollectionRef> create_new_collection(const coll_t& cid) final;
   seastar::future<CollectionRef> open_collection(const coll_t& cid) final;
-  seastar::future<std::vector<coll_t>> list_collections() final;
+  seastar::future<std::vector<coll_core_t>> list_collections() final;
 
   seastar::future<> do_transaction_no_callbacks(
     CollectionRef c,

diff --git a/src/crimson/os/cyanstore/cyan_store.cc b/src/crimson/os/cyanstore/cyan_store.cc
index 5336f1eb971a4ddac432a06290a47aa6a522ea2c..daeb1e567670a4ca2242c835839ff81e818127b0 100644 (file)
--- a/src/crimson/os/cyanstore/cyan_store.cc
+++ b/src/crimson/os/cyanstore/cyan_store.cc
@@ -123,17 +123,17 @@ seastar::future<> CyanStore::ShardStores::mkfs()
   return crimson::write_file(std::move(bl), fn);
 }
 
-seastar::future<std::vector<coll_t>>
+seastar::future<std::vector<coll_core_t>>
 CyanStore::list_collections()
 {
-  return seastar::do_with(std::vector<coll_t>{}, [this](auto &collections) {
+  return seastar::do_with(std::vector<coll_core_t>{}, [this](auto &collections) {
     return shard_stores.map([](auto &local_store) {
       return local_store.list_collections();
-    }).then([&collections](std::vector<std::vector<coll_t>> results) {
+    }).then([&collections](std::vector<std::vector<coll_core_t>> results) {
       for (auto& colls : results) {
         collections.insert(collections.end(), colls.begin(), colls.end());
       }
-      return seastar::make_ready_future<std::vector<coll_t>>(
+      return seastar::make_ready_future<std::vector<coll_core_t>>(
         std::move(collections));
     });
   });
@@ -233,14 +233,14 @@ CyanStore::ShardStores::open_collection(const coll_t& cid)
   return seastar::make_ready_future<CollectionRef>(_get_collection(cid));
 }
 
-seastar::future<std::vector<coll_t>>
+seastar::future<std::vector<coll_core_t>>
 CyanStore::ShardStores::list_collections()
 {
-  std::vector<coll_t> collections;
+  std::vector<coll_core_t> collections;
   for (auto& coll : coll_map) {
-    collections.push_back(coll.first);
+    collections.push_back(std::make_pair(coll.first, seastar::this_shard_id()));
   }
-  return seastar::make_ready_future<std::vector<coll_t>>(std::move(collections));
+  return seastar::make_ready_future<std::vector<coll_core_t>>(std::move(collections));
 }
 
 CyanStore::read_errorator::future<ceph::bufferlist>

diff --git a/src/crimson/os/cyanstore/cyan_store.h b/src/crimson/os/cyanstore/cyan_store.h
index 738157e8a85eb78518487b4104aaa5f6a0f759b5..715f8ae8caf0e15ae582ac4d09249bdde8a2697c 100644 (file)
--- a/src/crimson/os/cyanstore/cyan_store.h
+++ b/src/crimson/os/cyanstore/cyan_store.h
@@ -25,7 +25,7 @@ class Transaction;
 }
 
 namespace crimson::os {
-
+using coll_core_t = FuturizedStore::coll_core_t;
 class CyanStore final : public FuturizedStore {
   class ShardStores {
   public:
@@ -92,7 +92,7 @@ class CyanStore final : public FuturizedStore {
 
     seastar::future<CollectionRef> open_collection(const coll_t& cid);
 
-    seastar::future<std::vector<coll_t>> list_collections();
+    seastar::future<std::vector<coll_core_t>> list_collections();
 
     seastar::future<> do_transaction_no_callbacks(
       CollectionRef ch,
@@ -195,7 +195,7 @@ public:
     });
   }
 
-  seastar::future<std::vector<coll_t>> list_collections() final;
+  seastar::future<std::vector<coll_core_t>> list_collections() final;
 
 // public interfaces called by each shard osd
 public:

diff --git a/src/crimson/os/futurized_store.h b/src/crimson/os/futurized_store.h
index acde6447ed628aacb8d86887ed33fc3a451c6e5e..2fae5520275b6c3c181d04a62c68e407643ebeab 100644 (file)
--- a/src/crimson/os/futurized_store.h
+++ b/src/crimson/os/futurized_store.h
@@ -12,6 +12,7 @@
 
 #include "os/Transaction.h"
 #include "crimson/common/smp_helpers.h"
+#include "crimson/common/smp_helpers.h"
 #include "crimson/osd/exceptions.h"
 #include "include/buffer_fwd.h"
 #include "include/uuid.h"
@@ -106,7 +107,8 @@ public:
 
   virtual seastar::future<CollectionRef> create_new_collection(const coll_t& cid) = 0;
   virtual seastar::future<CollectionRef> open_collection(const coll_t& cid) = 0;
-  virtual seastar::future<std::vector<coll_t>> list_collections() = 0;
+  using coll_core_t = std::pair<coll_t, core_id_t>;
+  virtual seastar::future<std::vector<coll_core_t>> list_collections() = 0;
 
 protected:
   virtual seastar::future<> do_transaction_no_callbacks(
@@ -298,7 +300,7 @@ public:
   seastar::future<CollectionRef> open_collection(const coll_t &cid) final {
     return proxy(&T::open_collection, cid);
   }
-  seastar::future<std::vector<coll_t>> list_collections() final {
+  seastar::future<std::vector<coll_core_t>> list_collections() final {
     return proxy(&T::list_collections);
   }
   seastar::future<> do_transaction_no_callbacks(

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 27c1cc515497a7139886568fe1ace04e8badf9c0..86917c0ebc6046991297e3f584eb7c4b2e54a0dd 100644 (file)
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -574,9 +574,10 @@ seastar::future<CollectionRef> SeaStore::open_collection(const coll_t& cid)
 {
   LOG_PREFIX(SeaStore::open_collection);
   DEBUG("{}", cid);
-  return list_collections().then([cid, this] (auto colls) {
-    if (auto found = std::find(colls.begin(), colls.end(), cid);
-       found != colls.end()) {
+  return list_collections().then([cid, this] (auto colls_cores) {
+    if (auto found = std::find(colls_cores.begin(), colls_cores.end(),
+        std::make_pair(cid, NULL_CORE));
+      found != colls_cores.end()) {
       return seastar::make_ready_future<CollectionRef>(_get_collection(cid));
     } else {
       return seastar::make_ready_future<CollectionRef>();
@@ -584,10 +585,10 @@ seastar::future<CollectionRef> SeaStore::open_collection(const coll_t& cid)
   });
 }
 
-seastar::future<std::vector<coll_t>> SeaStore::list_collections()
+seastar::future<std::vector<coll_core_t>> SeaStore::list_collections()
 {
   return seastar::do_with(
-    std::vector<coll_t>(),
+    std::vector<coll_core_t>(),
     [this](auto &ret) {
       return repeat_eagain([this, &ret] {
         return transaction_manager->with_transaction_intr(
@@ -602,11 +603,11 @@ seastar::future<std::vector<coll_t>> SeaStore::list_collections()
             ret.resize(colls.size());
             std::transform(
               colls.begin(), colls.end(), ret.begin(),
-              [](auto p) { return p.first; });
+              [](auto p) { return std::make_pair(p.first, NULL_CORE); });
           });
         });
       }).safe_then([&ret] {
-        return seastar::make_ready_future<std::vector<coll_t>>(ret);
+        return seastar::make_ready_future<std::vector<coll_core_t>>(ret);
       });
     }
   ).handle_error(

diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index 9b8833e73a570dada6319316f36d838c5dce9da4..36b5368345011118f89e39dbe27acb2c5791269c 100644 (file)
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -55,6 +55,7 @@ struct col_obj_ranges_t {
   ghobject_t obj_end;
 };
 
+using coll_core_t = FuturizedStore::coll_core_t;
 class SeaStore final : public FuturizedStore {
 public:
   class MDStore {
@@ -145,7 +146,7 @@ public:
 
   seastar::future<CollectionRef> create_new_collection(const coll_t& cid) final;
   seastar::future<CollectionRef> open_collection(const coll_t& cid) final;
-  seastar::future<std::vector<coll_t>> list_collections() final;
+  seastar::future<std::vector<coll_core_t>> list_collections() final;
 
   seastar::future<> do_transaction_no_callbacks(
     CollectionRef ch,

diff --git a/src/crimson/osd/pg_map.h b/src/crimson/osd/pg_map.h
index ed753749a55e883f0e563a17b0fa1a63a6666e9c..3af746e60529c29c3772eafc332482c3b678d0cd 100644 (file)
--- a/src/crimson/osd/pg_map.h
+++ b/src/crimson/osd/pg_map.h
@@ -33,19 +33,27 @@ public:
   }
 
   /// Returns mapping for pgid, creates new one if it doesn't already exist
-  core_id_t maybe_create_pg(spg_t pgid) {
-    auto [insert_iter, inserted] = pg_to_core.emplace(pgid, NULL_CORE);
+  core_id_t maybe_create_pg(spg_t pgid, core_id_t core = NULL_CORE) {
+    auto [insert_iter, inserted] = pg_to_core.emplace(pgid, core);
     if (!inserted) {
       ceph_assert_always(insert_iter->second != NULL_CORE);
+      if (core != NULL_CORE) {
+       ceph_assert_always(insert_iter->second == core);
+      }
       return insert_iter->second;
     } else {
       ceph_assert_always(core_to_num_pgs.size() > 0);
-      auto core_iter = std::min_element(
-       core_to_num_pgs.begin(),
-       core_to_num_pgs.end(),
-       [](const auto &left, const auto &right) {
-         return left.second < right.second;
-       });
+      std::map<core_id_t, unsigned>::iterator core_iter;
+      if (core == NULL_CORE) {
+        core_iter = std::min_element(
+          core_to_num_pgs.begin(),
+          core_to_num_pgs.end(),
+          [](const auto &left, const auto &right) {
+            return left.second < right.second;
+        });
+      } else {
+       core_iter = core_to_num_pgs.find(core);
+      }
       ceph_assert_always(core_to_num_pgs.end() != core_iter);
       insert_iter->second = core_iter->first;
       core_iter->second++;

diff --git a/src/crimson/osd/pg_shard_manager.cc b/src/crimson/osd/pg_shard_manager.cc
index afe9d0a5014626534b19507d327ad5e037ad60b2..c81ea8a01f41c39271353b8f3e58913ca2d762ad 100644 (file)
--- a/src/crimson/osd/pg_shard_manager.cc
+++ b/src/crimson/osd/pg_shard_manager.cc
@@ -49,15 +49,16 @@ seastar::future<> PGShardManager::load_pgs()
 {
   ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
   return get_local_state().store.list_collections(
-  ).then([this](auto colls) {
+  ).then([this](auto colls_cores) {
     return seastar::parallel_for_each(
-      colls,
-      [this](auto coll) {
+      colls_cores,
+      [this](auto coll_core) {
+        auto[coll, shard_core] = coll_core;
        spg_t pgid;
        if (coll.is_pg(&pgid)) {
          auto core = get_osd_singleton_state(
          ).pg_to_shard_mapping.maybe_create_pg(
-           pgid);
+           pgid, shard_core);
          return with_remote_shard_state(
            core,
            [pgid](

diff --git a/src/test/crimson/seastore/test_seastore.cc b/src/test/crimson/seastore/test_seastore.cc
index a2e450de4ae24b1f6f8058e2abfaedea8cd3a0f6..fc49edc54ab7acb9468f7a1951bd10ad756e9a43 100644 (file)
--- a/src/test/crimson/seastore/test_seastore.cc
+++ b/src/test/crimson/seastore/test_seastore.cc
@@ -605,10 +605,15 @@ TEST_F(seastore_test_t, collection_create_list_remove)
        t.create_collection(test_coll, 4);
        do_transaction(std::move(t));
       }
-      auto collections = seastore->list_collections().get0();
-      EXPECT_EQ(collections.size(), 2);
-      EXPECT_TRUE(contains(collections, coll_name));
-      EXPECT_TRUE(contains(collections,  test_coll));
+      auto colls_cores = seastore->list_collections().get0();
+      std::vector<coll_t> colls;
+      colls.resize(colls_cores.size());
+      std::transform(
+        colls_cores.begin(), colls_cores.end(), colls.begin(),
+        [](auto p) { return p.first; });
+      EXPECT_EQ(colls.size(), 2);
+      EXPECT_TRUE(contains(colls, coll_name));
+      EXPECT_TRUE(contains(colls,  test_coll));
     }
 
     {
@@ -617,9 +622,14 @@ TEST_F(seastore_test_t, collection_create_list_remove)
        t.remove_collection(test_coll);
        do_transaction(std::move(t));
       }
-      auto collections = seastore->list_collections().get0();
-      EXPECT_EQ(collections.size(), 1);
-      EXPECT_TRUE(contains(collections, coll_name));
+      auto colls_cores = seastore->list_collections().get0();
+      std::vector<coll_t> colls;
+      colls.resize(colls_cores.size());
+      std::transform(
+        colls_cores.begin(), colls_cores.end(), colls.begin(),
+        [](auto p) { return p.first; });
+      EXPECT_EQ(colls.size(), 1);
+      EXPECT_TRUE(contains(colls, coll_name));
     }
   });
 }