From d5fc95dc6ba1177e3604c80d4f6acaf831542c5e Mon Sep 17 00:00:00 2001
From: Igor Fedotov <ifedotov@suse.com>
Date: Fri, 28 Aug 2020 22:10:56 +0300
Subject: [PATCH] osd/pg: use next when calling collection_list for pg removal

Signed-off-by: Igor Fedotov <ifedotov@suse.com>
(cherry picked from commit 7f047005fc72e1f37a45cde2d742bb2eb1e62881)

 Conflicts:
	src/crimson/osd/pg.cc
	src/crimson/osd/pg.h
	src/osd/PG.cc
	src/osd/PG.h
	src/osd/PeeringState.cc
	src/osd/PeeringState.h
 Legacy design for Deleting peering stat.
---
 src/osd/PG.cc | 39 ++++++++++++++++++++++++++++++++++-----
 src/osd/PG.h  |  5 ++++-
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 287b576c545..fda8d569259 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -7094,7 +7094,8 @@ struct C_DeleteMore : public Context {
   }
 };
 
-void PG::_delete_some(ObjectStore::Transaction *t)
+ghobject_t PG::_delete_some(ObjectStore::Transaction *t,
+  ghobject_t _next)
 {
   dout(10) << __func__ << dendl;
 
@@ -7121,25 +7122,45 @@ void PG::_delete_some(ObjectStore::Transaction *t)
       osd->sleep_timer.add_event_at(delete_schedule_time,
 	                                        delete_requeue_callback);
       dout(20) << __func__ << " Delete scheduled at " << delete_schedule_time << dendl;
-      return;
+      return _next;
     }
   }
 
   delete_needs_sleep = true;
 
+  ghobject_t next;
+
   vector<ghobject_t> olist;
   int max = std::min(osd->store->get_ideal_list_max(),
 		     (int)cct->_conf->osd_target_transaction_size);
-  ghobject_t next;
+
   osd->store->collection_list(
     ch,
-    next,
+    _next,
     ghobject_t::get_max(),
     max,
     &olist,
     &next);
   dout(20) << __func__ << " " << olist << dendl;
 
+  // make sure we've removed everything
+  // by one more listing from the beginning
+  if (_next != ghobject_t() && olist.empty()) {
+    next = ghobject_t();
+    osd->store->collection_list(
+      ch,
+      next,
+      ghobject_t::get_max(),
+      max,
+      &olist,
+      &next);
+    if (!olist.empty()) {
+      dout(0) << __func__ << " additional unexpected onode list"
+              <<" (new onodes has appeared since PG removal started"
+              << olist << dendl;
+    }
+  }
+
   OSDriver::OSTransaction _t(osdriver.get_transaction(t));
   int64_t num = 0;
   for (auto& oid : olist) {
@@ -7199,6 +7220,7 @@ void PG::_delete_some(ObjectStore::Transaction *t)
       osd->logger->dec(l_osd_pg_removing);
     }
   }
+  return next;
 }
 
 // Compute pending backfill data
@@ -9241,7 +9263,10 @@ PG::RecoveryState::Deleting::Deleting(my_context ctx)
   : my_base(ctx),
     NamedState(context< RecoveryMachine >().pg, "Started/ToDelete/Deleting")
 {
+  start = ceph::mono_clock::now();
+  
   context< RecoveryMachine >().log_enter(state_name);
+  
   PG *pg = context< RecoveryMachine >().pg;
   pg->deleting = true;
   ObjectStore::Transaction* t = context<RecoveryMachine>().get_cur_transaction();
@@ -9253,7 +9278,8 @@ boost::statechart::result PG::RecoveryState::Deleting::react(
   const DeleteSome& evt)
 {
   PG *pg = context< RecoveryMachine >().pg;
-  pg->_delete_some(context<RecoveryMachine>().get_cur_transaction());
+  next = pg->_delete_some(context<RecoveryMachine>().get_cur_transaction(),
+    next);
   return discard_event();
 }
 
@@ -9263,6 +9289,9 @@ void PG::RecoveryState::Deleting::exit()
   PG *pg = context< RecoveryMachine >().pg;
   pg->deleting = false;
   pg->osd->local_reserver.cancel_reservation(pg->info.pgid);
+  ldout(pg->cct, 20) << "Deleting::" << __func__ << this <<" finished in "
+                     << ceph::mono_clock::now() - start
+                     << dendl;
 }
 
 /*--------GetInfo---------*/
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 1ebf0c8f875..7655493bd13 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -499,7 +499,8 @@ public:
 
   virtual void on_removal(ObjectStore::Transaction *t) = 0;
 
-  void _delete_some(ObjectStore::Transaction *t);
+  ghobject_t _delete_some(ObjectStore::Transaction *t,
+    ghobject_t _next);
 
   virtual void set_dynamic_perf_stats_queries(
     const std::list<OSDPerfMetricQuery> &queries) {
@@ -2744,6 +2745,8 @@ protected:
 	boost::statechart::custom_reaction< DeleteSome >,
 	boost::statechart::transition<DeleteInterrupted, WaitDeleteReserved>
 	> reactions;
+      ghobject_t next;
+      ceph::mono_clock::time_point start;	
       explicit Deleting(my_context ctx);
       boost::statechart::result react(const DeleteSome &evt);
       void exit();
-- 
2.47.3