From 1bd011a101904b9bfd76a61882f16c60321058c1 Mon Sep 17 00:00:00 2001
From: Samuel Just <sam.just@inktank.com>
Date: Mon, 29 Apr 2013 09:07:19 -0700
Subject: [PATCH] PG,OSD: _remove_pg must remove pg keys

Instead of doing this in OSD::_remove_pg, pass a transaction
to on_removal and do it in PG.

Signed-off-by: Samuel Just <sam.just@inktank.com>
---
 src/osd/OSD.cc          | 15 +++++++--------
 src/osd/PG.cc           | 15 +++++++++++++++
 src/osd/PG.h            |  8 +++++++-
 src/osd/ReplicatedPG.cc |  4 +++-
 src/osd/ReplicatedPG.h  |  2 +-
 5 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index f5571c6f5dbab..85edf66f3b458 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -5834,6 +5834,13 @@ void OSD::_remove_pg(PG *pg)
 {
   vector<coll_t> removals;
   ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
+
+  // on_removal, which calls remove_watchers_and_notifies, and the erasure from
+  // the pg_map must be done together without unlocking the pg lock,
+  // to avoid racing with watcher cleanup in ms_handle_reset
+  // and handle_notify_timeout
+  pg->on_removal(rmt);
+
   coll_t to_remove = get_next_removal_coll(pg->info.pgid);
   removals.push_back(to_remove);
   rmt->collection_rename(coll_t(pg->info.pgid), to_remove);
@@ -5842,8 +5849,6 @@ void OSD::_remove_pg(PG *pg)
     removals.push_back(to_remove);
     rmt->collection_rename(pg->get_temp_coll(), to_remove);
   }
-  rmt->remove(coll_t::META_COLL, pg->log_oid);
-  rmt->remove(coll_t::META_COLL, pg->biginfo_oid);
 
   store->queue_transaction(
     pg->osr.get(), rmt,
@@ -5852,12 +5857,6 @@ void OSD::_remove_pg(PG *pg)
     new ContainerContext<
       SequencerRef>(pg->osr));
 
-  // on_removal, which calls remove_watchers_and_notifies, and the erasure from 
-  // the pg_map must be done together without unlocking the pg lock,
-  // to avoid racing with watcher cleanup in ms_handle_reset
-  // and handle_notify_timeout
-  pg->on_removal();
-
   DeletingStateRef deleting = service.deleting_pgs.lookup_or_create(pg->info.pgid);
   for (vector<coll_t>::iterator i = removals.begin();
        i != removals.end();
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index c1155c33bbe1b..70584e4459123 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2629,6 +2629,21 @@ void PG::write_info(ObjectStore::Transaction& t)
   dirty_big_info = false;
 }
 
+void PG::clear_info_log(
+  pg_t pgid,
+  const hobject_t &infos_oid,
+  const hobject_t &log_oid,
+  ObjectStore::Transaction *t) {
+
+  set<string> keys_to_remove;
+  keys_to_remove.insert(get_epoch_key(pgid));
+  keys_to_remove.insert(get_biginfo_key(pgid));
+  keys_to_remove.insert(get_info_key(pgid));
+
+  t->remove(coll_t::META_COLL, log_oid);
+  t->omap_rmkeys(coll_t::META_COLL, infos_oid, keys_to_remove);
+}
+
 epoch_t PG::peek_map_epoch(ObjectStore *store, coll_t coll, hobject_t &infos_oid, bufferlist *bl)
 {
   assert(bl);
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 48636476812be..bdd276d0bc7d5 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1854,7 +1854,13 @@ public:
 private:
   void write_info(ObjectStore::Transaction& t);
   void write_log(ObjectStore::Transaction& t);
+
 public:
+  static void clear_info_log(
+    pg_t pgid,
+    const hobject_t &infos_oid,
+    const hobject_t &log_oid,
+    ObjectStore::Transaction *t);
 
   static int _write_info(ObjectStore::Transaction& t, epoch_t epoch,
     pg_info_t &info, coll_t coll,
@@ -1969,7 +1975,7 @@ public:
   void handle_loaded(RecoveryCtx *rctx);
   void handle_query_state(Formatter *f);
 
-  virtual void on_removal() = 0;
+  virtual void on_removal(ObjectStore::Transaction *t) = 0;
 
 
   // abstract bits
diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index b08fffbb235cc..fb24375dca6bb 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -6241,10 +6241,12 @@ void ReplicatedPG::apply_and_flush_repops(bool requeue)
   waiting_for_ack.clear();
 }
 
-void ReplicatedPG::on_removal()
+void ReplicatedPG::on_removal(ObjectStore::Transaction *t)
 {
   dout(10) << "on_removal" << dendl;
 
+  clear_info_log(info.pgid, osd->infos_oid, log_oid, t);
+
   on_shutdown();
 }
 
diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 2ef2627ba7079..644a277f0dc9c 100644
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -1004,7 +1004,7 @@ public:
   void on_change();
   void on_activate();
   void on_flushed();
-  void on_removal();
+  void on_removal(ObjectStore::Transaction *t);
   void on_shutdown();
 };
 
-- 
2.39.5