From e5c336851c367b60cfd93572813dc401697704a5 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Tue, 16 Jan 2018 16:42:28 -0600
Subject: [PATCH] osd: move recovery contexts to normal wq

We have a specific PGRecoveryContext type/event--even though we are just
calling a GenContext--so that we can distinguish the event type properly.

Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/osd/OSD.cc          | 16 ++++++++++++++++
 src/osd/OSD.h           |  1 +
 src/osd/OpQueueItem.cc  | 10 +++++++++-
 src/osd/OpQueueItem.h   | 20 ++++++++++++++++++++
 src/osd/PrimaryLogPG.cc |  2 +-
 5 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index e882f342c3906..6d529e8cab8b8 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1691,6 +1691,22 @@ void OSDService::enqueue_front(OpQueueItem&& qi)
   osd->op_shardedwq.queue_front(std::move(qi));
 }
 
+void OSDService::queue_recovery_context(
+  PG *pg,
+  GenContext<ThreadPool::TPHandle&> *c)
+{
+  epoch_t e = get_osdmap()->get_epoch();
+  enqueue_back(
+    OpQueueItem(
+      unique_ptr<OpQueueItem::OpQueueable>(
+	new PGRecoveryContext(pg->get_pgid(), c, e)),
+      cct->_conf->osd_recovery_cost,
+      cct->_conf->osd_recovery_priority,
+      ceph_clock_now(),
+      0,
+      e));
+}
+
 void OSDService::queue_for_snap_trim(PG *pg)
 {
   dout(10) << "queueing " << *pg << " for snaptrim" << dendl;
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index da17b9e85a1c4..f9be95ceb098e 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -796,6 +796,7 @@ public:
   SafeTimer scrub_sleep_timer;
 
   AsyncReserver<spg_t> snap_reserver;
+  void queue_recovery_context(PG *pg, GenContext<ThreadPool::TPHandle&> *c);
   void queue_for_snap_trim(PG *pg);
   void queue_for_scrub(PG *pg, bool with_high_priority);
   void queue_for_pg_delete(spg_t pgid, epoch_t e);
diff --git a/src/osd/OpQueueItem.cc b/src/osd/OpQueueItem.cc
index 5e2bda20b8709..6a0560d7c934f 100644
--- a/src/osd/OpQueueItem.cc
+++ b/src/osd/OpQueueItem.cc
@@ -55,6 +55,15 @@ void PGRecovery::run(OSD *osd,
   pg->unlock();
 }
 
+void PGRecoveryContext::run(
+  OSD *osd,
+  PGRef& pg,
+  ThreadPool::TPHandle &handle)
+{
+  c.release()->complete(handle);
+  pg->unlock();
+}
+
 void PGDelete::run(
   OSD *osd,
   PGRef& pg,
@@ -62,4 +71,3 @@ void PGDelete::run(
 {
   osd->dequeue_delete(pg.get(), epoch_queued, handle);
 }
-
diff --git a/src/osd/OpQueueItem.h b/src/osd/OpQueueItem.h
index fe3f66296adfc..b55114ae32ed7 100644
--- a/src/osd/OpQueueItem.h
+++ b/src/osd/OpQueueItem.h
@@ -298,6 +298,26 @@ public:
     OSD *osd, PGRef& pg, ThreadPool::TPHandle &handle) override final;
 };
 
+class PGRecoveryContext : public PGOpQueueable {
+  unique_ptr<GenContext<ThreadPool::TPHandle&>> c;
+  epoch_t epoch;
+public:
+  PGRecoveryContext(spg_t pgid,
+		    GenContext<ThreadPool::TPHandle&> *c, epoch_t epoch)
+    : PGOpQueueable(pgid),
+      c(c), epoch(epoch) {}
+  op_type_t get_op_type() const override final {
+    return op_type_t::bg_recovery;
+  }
+  ostream &print(ostream &rhs) const override final {
+    return rhs << "PGRecoveryContext(pgid=" << get_pgid()
+	       << " c=" << c.get() << " epoch=" << epoch
+	       << ")";
+  }
+  void run(
+    OSD *osd, PGRef& pg, ThreadPool::TPHandle &handle) override final;
+};
+
 class PGDelete : public PGOpQueueable {
   epoch_t epoch_queued;
 public:
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 9e711a571f677..9198238d38439 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -485,7 +485,7 @@ void PrimaryLogPG::begin_peer_recover(
 void PrimaryLogPG::schedule_recovery_work(
   GenContext<ThreadPool::TPHandle&> *c)
 {
-  osd->recovery_gen_wq.queue(c);
+  osd->queue_recovery_context(this, c);
 }
 
 void PrimaryLogPG::send_message_osd_cluster(
-- 
2.39.5