OSD: use per-pg temp collections, bug #2255

Signed-off-by: Samuel Just <samuel.just@dreamhost.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 76421959c186e4289553f7b4cc5da30d4fcf54da..7f158ed926123dacc0a436de66d6602d3934cc21 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -121,7 +121,6 @@ static ostream& _prefix(std::ostream* _dout, int whoami, OSDMapRef osdmap) {
 }
 
 const coll_t coll_t::META_COLL("meta");
-const coll_t coll_t::TEMP_COLL("temp");
 
 static CompatSet get_osd_compat_set() {
   CompatSet::FeatureSet ceph_osd_feature_compat;
@@ -346,7 +345,6 @@ int OSD::mkfs(const std::string &dev, const std::string &jdev, uuid_d fsid, int
       ObjectStore::Transaction t;
       t.create_collection(coll_t::META_COLL);
       t.write(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, bl.length(), bl);
-      t.create_collection(coll_t::TEMP_COLL);
       ret = store->apply_transaction(t);
       if (ret) {
        derr << "OSD::mkfs: error while writing OSD_SUPERBLOCK_POBJECT: "
@@ -681,16 +679,6 @@ int OSD::init()
 
   bind_epoch = osdmap->get_epoch();
 
-  clear_temp();
-
-  // make sure (newish) temp dir exists
-  if (!store->collection_exists(coll_t::TEMP_COLL)) {
-    dout(10) << "creating temp pg dir" << dendl;
-    ObjectStore::Transaction t;
-    t.create_collection(coll_t::TEMP_COLL);
-    store->apply_transaction(t);
-  }
-
   // load up pgs (as they previously existed)
   load_pgs();
 
@@ -1026,12 +1014,12 @@ int OSD::read_superblock()
 
 
 
-void OSD::clear_temp()
+void OSD::clear_temp(coll_t tmp)
 {
   dout(10) << "clear_temp" << dendl;
 
   vector<hobject_t> objects;
-  store->collection_list(coll_t::TEMP_COLL, objects);
+  store->collection_list(tmp, objects);
 
   dout(10) << objects.size() << " objects" << dendl;
   if (objects.empty())
@@ -1042,7 +1030,8 @@ void OSD::clear_temp()
   for (vector<hobject_t>::iterator p = objects.begin();
        p != objects.end();
        p++)
-    t->collection_remove(coll_t::TEMP_COLL, *p);
+    t->collection_remove(tmp, *p);
+  t->remove_collection(tmp);
   int r = store->queue_transaction(NULL, t);
   assert(r == 0);
   store->sync_and_flush();
@@ -1205,6 +1194,8 @@ void OSD::load_pgs()
     pg_t pgid;
     snapid_t snap;
     if (!it->is_pg(pgid, snap)) {
+      if (it->is_temp(pgid))
+       clear_temp(*it);
       dout(10) << "load_pgs skipping non-pg " << *it << dendl;
       continue;
     }
@@ -4833,6 +4824,9 @@ void OSD::_remove_pg(PG *pg)
     assert(tr == 0);
   }
 
+  if (store->collection_exists(coll_t::make_temp_coll(pg->get_pgid()))) {
+    clear_temp(coll_t::make_temp_coll(pg->get_pgid()));
+  }
 
   // on_removal, which calls remove_watchers_and_notifies, and the erasure from 
   // the pg_map must be done together without unlocking the pg lock,

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 83f015fd274281209e8f7c46e859777350295367..562aad5f77e4cb913c80ded77ae8c064646227fe 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -518,7 +518,7 @@ protected:
   
   friend class C_OSD_GetVersion;
 
-  void clear_temp();
+  void clear_temp(coll_t tmp);
 
   // -- alive --
   epoch_t up_thru_wanted;

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 0b380c49040e32eb743a07dcd2af2da82def52f5..69f0ed2a757259a4e59f6a2eca7c652a66b3de68 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -574,7 +574,8 @@ void ReplicatedPG::calc_trim_to()
 }
 
 ReplicatedPG::ReplicatedPG(OSD *o, PGPool *_pool, pg_t p, const hobject_t& oid, const hobject_t& ioid) : 
-  PG(o, _pool, p, oid, ioid), snap_trimmer_machine(this)
+  PG(o, _pool, p, oid, ioid), temp_created(false),
+  temp_coll(coll_t::make_temp_coll(p)), snap_trimmer_machine(this)
 { 
   snap_trimmer_machine.initiate();
 }
@@ -3079,6 +3080,15 @@ void ReplicatedPG::do_osd_op_effects(OpContext *ctx)
   }
 }
 
+coll_t ReplicatedPG::get_temp_coll(ObjectStore::Transaction *t)
+{
+  if (temp_created)
+    return temp_coll;
+  if (!osd->store->collection_exists(temp_coll))
+      t->create_collection(temp_coll);
+  temp_created = true;
+  return temp_coll;
+}
 
 int ReplicatedPG::prepare_transaction(OpContext *ctx)
 {
@@ -4644,9 +4654,9 @@ void ReplicatedPG::submit_push_data(
   ObjectStore::Transaction *t)
 {
   if (first) {
-    t->remove(coll_t::TEMP_COLL, recovery_info.soid);
-    t->touch(coll_t::TEMP_COLL, recovery_info.soid);
-    t->omap_setheader(coll_t::TEMP_COLL, recovery_info.soid, omap_header);
+    t->remove(get_temp_coll(t), recovery_info.soid);
+    t->touch(get_temp_coll(t), recovery_info.soid);
+    t->omap_setheader(get_temp_coll(t), recovery_info.soid, omap_header);
   }
   uint64_t off = 0;
   for (interval_set<uint64_t>::const_iterator p = intervals_included.begin();
@@ -4654,14 +4664,14 @@ void ReplicatedPG::submit_push_data(
        ++p) {
     bufferlist bit;
     bit.substr_of(data_included, off, p.get_len());
-    t->write(coll_t::TEMP_COLL, recovery_info.soid,
+    t->write(get_temp_coll(t), recovery_info.soid,
             p.get_start(), p.get_len(), bit);
     off += p.get_len();
   }
 
-  t->omap_setkeys(coll_t::TEMP_COLL, recovery_info.soid,
+  t->omap_setkeys(get_temp_coll(t), recovery_info.soid,
                  omap_entries);
-  t->setattrs(coll_t::TEMP_COLL, recovery_info.soid,
+  t->setattrs(get_temp_coll(t), recovery_info.soid,
              attrs);
 }
 
@@ -4669,7 +4679,7 @@ void ReplicatedPG::submit_push_complete(ObjectRecoveryInfo &recovery_info,
                                        ObjectStore::Transaction *t)
 {
   remove_object_with_snap_hardlinks(*t, recovery_info.soid);
-  t->collection_move(coll, coll_t::TEMP_COLL, recovery_info.soid);
+  t->collection_move(coll, get_temp_coll(t), recovery_info.soid);
   for (map<hobject_t, interval_set<uint64_t> >::const_iterator p =
         recovery_info.clone_subset.begin();
        p != recovery_info.clone_subset.end();

diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 0bda36404b85d8dfd3381eb1bd5f382e9bc3ee14..f09d5d05a4ce9fc05e6da7e88edd43b8fffed5dd 100644 (file)
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -808,6 +808,9 @@ public:
   int do_osd_ops(OpContext *ctx, vector<OSDOp>& ops);
   void do_osd_op_effects(OpContext *ctx);
 private:
+  bool temp_created;
+  coll_t temp_coll;
+  coll_t get_temp_coll(ObjectStore::Transaction *t);
   struct NotTrimming;
   struct SnapTrim : boost::statechart::event< SnapTrim > {
     SnapTrim() : boost::statechart::event < SnapTrim >() {}

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index d520c0c96c24b3a86fa842bd9b833a1d435fd0e8..66e0890435653ece6c926c3a6601e83a3679e58e 100644 (file)
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -250,6 +250,19 @@ ostream& operator<<(ostream& out, const pg_t &pg)
 
 // -- coll_t --
 
+bool coll_t::is_temp(pg_t& pgid) const
+{
+  const char *cstr(str.c_str());
+  if (!pgid.parse(cstr))
+    return false;
+  const char *tmp_start = strchr(cstr, '_');
+  if (!tmp_start)
+    return false;
+  if (strncmp(tmp_start, "_TEMP", 4) == 0)
+    return true;
+  return false;
+}
+
 bool coll_t::is_pg(pg_t& pgid, snapid_t& snap) const
 {
   const char *cstr(str.c_str());

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 0177da87e01469d39b0c95b0bcaea1b51484c74f..66a7aff5a09798c2713040bab778d4d3fe22fd4a 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -314,7 +314,6 @@ namespace __gnu_cxx {
 class coll_t {
 public:
   const static coll_t META_COLL;
-  const static coll_t TEMP_COLL;
 
   coll_t()
     : str("meta")
@@ -328,6 +327,10 @@ public:
     : str(pg_and_snap_to_str(pgid, snap))
   { }
 
+  static coll_t make_temp_coll(pg_t pgid) {
+    return coll_t(pg_to_tmp_str(pgid));
+  }
+
   const std::string& to_str() const {
     return str;
   }
@@ -341,6 +344,7 @@ public:
   }
 
   bool is_pg(pg_t& pgid, snapid_t& snap) const;
+  bool is_temp(pg_t& pgid) const;
   void encode(bufferlist& bl) const;
   void decode(bufferlist::iterator& bl);
   inline bool operator==(const coll_t& rhs) const {
@@ -359,6 +363,11 @@ private:
     oss << p << "_" << s;
     return oss.str();
   }
+  static std::string pg_to_tmp_str(pg_t p) {
+    std::ostringstream oss;
+    oss << p << "_TEMP";
+    return oss.str();
+  }
 
   std::string str;
 };