From 3266786527a9e4e05922abdedc640facc8260dee Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Mon, 13 Jan 2020 07:11:51 -0600
Subject: [PATCH] osd/PG: clean up fastinfo key when last_update does not
 increase

If the last_update field goes backwards, we write a full info instead of
the fastinfo key. However, a stale fastinfo key may still be present, and
it may have a last_update that looks valid to a restarting OSD.

Fixes: https://tracker.ceph.com/issues/43580
Reported-by: song <jinglesong0814@163.com>
Signed-off-by: Sage Weil <sage@redhat.com>
---
 src/crimson/osd/pg.cc              | 5 +++++
 src/osd/PG.cc                      | 4 ++++
 src/osd/osd_types.cc               | 5 +++++
 src/osd/osd_types.h                | 1 +
 src/tools/ceph_objectstore_tool.cc | 7 ++++++-
 5 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 3955dd7765d29..6b7556e960ff7 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -243,10 +243,12 @@ void PG::prepare_write(pg_info_t &info,
 		       ceph::os::Transaction &t)
 {
   std::map<string,bufferlist> km;
+  std::string key_to_remove;
   if (dirty_big_info || dirty_info) {
     int ret = prepare_info_keymap(
       shard_services.get_cct(),
       &km,
+      &key_to_remove,
       get_osdmap_epoch(),
       info,
       last_written_info,
@@ -264,6 +266,9 @@ void PG::prepare_write(pg_info_t &info,
   if (!km.empty()) {
     t.omap_setkeys(coll, pgmeta_oid, km);
   }
+  if (!key_to_remove.empty()) {
+    t.omap_rmkey(coll, pgmeta_oid, key_to_remove);
+  }
 }
 
 void PG::log_state_enter(const char *state) {
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index db29e0056ace2..2019ebe1bd4f0 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -949,10 +949,12 @@ void PG::prepare_write(
   info.stats.stats.add(unstable_stats);
   unstable_stats.clear();
   map<string,bufferlist> km;
+  string key_to_remove;
   if (dirty_big_info || dirty_info) {
     int ret = prepare_info_keymap(
       cct,
       &km,
+      &key_to_remove,
       get_osdmap_epoch(),
       info,
       last_written_info,
@@ -968,6 +970,8 @@ void PG::prepare_write(
     t, &km, coll, pgmeta_oid, pool.info.require_rollback());
   if (!km.empty())
     t.omap_setkeys(coll, pgmeta_oid, km);
+  if (!key_to_remove.empty())
+    t.omap_rmkey(coll, pgmeta_oid, key_to_remove);
 }
 
 #pragma GCC diagnostic ignored "-Wpragmas"
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index a4b322d20170a..901480b849f3c 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -6840,6 +6840,7 @@ void OSDOp::clear_data(vector<OSDOp>& ops)
 int prepare_info_keymap(
   CephContext* cct,
   map<string,bufferlist> *km,
+  string *key_to_remove,
   epoch_t epoch,
   pg_info_t &info,
   pg_info_t &last_written_info,
@@ -6885,6 +6886,10 @@ int prepare_info_keymap(
       }
       *_dout << dendl;
     }
+  } else if (info.last_update <= last_written_info.last_update) {
+    // clean up any potentially stale fastinfo key resulting from last_update
+    // not moving forwards (e.g., a backwards jump during peering)
+    *key_to_remove = fastinfo_key;
   }
 
   last_written_info = info;
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 9fa3fd8758d36..be90f083ee254 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -6229,6 +6229,7 @@ static const __u8 pg_compat_struct_v = 10;
 int prepare_info_keymap(
   CephContext* cct,
   map<string,bufferlist> *km,
+  string *key_to_remove,
   epoch_t epoch,
   pg_info_t &info,
   pg_info_t &last_written_info,
diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index 77c929feb510e..b534e0ab317b0 100644
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -577,16 +577,21 @@ int write_info(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info,
   coll_t coll(info.pgid);
   ghobject_t pgmeta_oid(info.pgid.make_pgmeta_oid());
   map<string,bufferlist> km;
+  string key_to_remove;
   pg_info_t last_written_info;
   int ret = prepare_info_keymap(
     g_ceph_context,
-    &km, epoch,
+    &km, &key_to_remove,
+    epoch,
     info,
     last_written_info,
     past_intervals,
     true, true, false);
   if (ret) cerr << "Failed to write info" << std::endl;
   t.omap_setkeys(coll, pgmeta_oid, km);
+  if (!key_to_remove.empty()) {
+    t.omap_rmkey(coll, pgmeta_oid, key_to_remove);
+  }
   return ret;
 }
 
-- 
2.47.3