ObjctStore: More accurate disk space usage calculation when considering 'whiteout...

This also adds cache protection against potential overflow with removed objects

Signed-off-by: Igor Fedotov <ifedotov@mirantis.com>

diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index fba6b76fcf18e7d42c054add3c5a06f276a98b18..db31f3c5d377e7ffb548537bff4df4b9c63086c0 100644 (file)
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -2388,6 +2388,12 @@ public:
   virtual void set_fsid(uuid_d u) = 0;
   virtual uuid_d get_fsid() = 0;
 
+  /**
+  * Estimates additional disk space used by the specified amount of objects and caused by file allocation granularity and metadata store
+  * - num objects - total (including witeouts) object count to measure used space for.
+  */
+  virtual uint64_t estimate_objects_overhead(uint64_t num_objects) = 0;
+
   // DEBUG
   virtual void inject_data_error(const ghobject_t &oid) {}
   virtual void inject_mdata_error(const ghobject_t &oid) {}

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index b1d0fcb19f842cc2d352a5e3d41fd67737f01bdc..ab17c21887c131a4db8e5f48d9169ada3ef7e4fd 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -799,6 +799,10 @@ public:
     return fsid;
   }
 
+  uint64_t estimate_objects_overhead(uint64_t num_objects) override { 
+    return num_objects * 300; //assuming per-object overhead is 300 bytes
+  }
+
   objectstore_perf_stat_t get_cur_stats() override {
     return objectstore_perf_stat_t();
   }

diff --git a/src/os/filestore/FileStore.cc b/src/os/filestore/FileStore.cc
index 0696afc5518a68dbe3a6f2d4f343e456ff1a7efe..dbd9ca2d5dee90ecae124bf080b738e64638826d 100644 (file)
--- a/src/os/filestore/FileStore.cc
+++ b/src/os/filestore/FileStore.cc
@@ -5685,6 +5685,12 @@ void FileStore::set_xattr_limits_via_conf()
     m_filestore_max_inline_xattrs = fs_xattrs;
 }
 
+uint64_t FileStore::estimate_objects_overhead(uint64_t num_objects)
+{
+  uint64_t res = num_objects * blk_size / 2; //assumes that each object uses ( in average ) additional 1/2 block due to FS allocation granularity.
+  return res;
+}
+
 // -- FSSuperblock --
 
 void FSSuperblock::encode(bufferlist &bl) const

diff --git a/src/os/filestore/FileStore.h b/src/os/filestore/FileStore.h
index d81f8b0691b02917216157902eb63f04eb7baab8..008c430dae68e4c7b405e54297ad00c5a6f415b3 100644 (file)
--- a/src/os/filestore/FileStore.h
+++ b/src/os/filestore/FileStore.h
@@ -588,6 +588,8 @@ public:
     fsid = u;
   }
   uuid_d get_fsid() { return fsid; }
+  
+  uint64_t estimate_objects_overhead(uint64_t num_objects);
 
   // DEBUG read error injection, an object is removed from both on delete()
   Mutex read_error_lock;

diff --git a/src/os/kstore/KStore.h b/src/os/kstore/KStore.h
index 4b0066331deeac3ee3dc17dce121fc2acadea7fd..3202bdf42bec216755f0a2f68a669b376494e47f 100644 (file)
--- a/src/os/kstore/KStore.h
+++ b/src/os/kstore/KStore.h
@@ -521,6 +521,10 @@ public:
     return fsid;
   }
 
+  uint64_t estimate_objects_overhead(uint64_t num_objects) override {
+    return num_objects * 300; //assuming per-object overhead is 300 bytes
+  }
+
   objectstore_perf_stat_t get_cur_stats() {
     return objectstore_perf_stat_t();
   }

diff --git a/src/os/memstore/MemStore.h b/src/os/memstore/MemStore.h
index 2d809f3384f954ce926f6df97ee31663672f59de..cdf90f79418d2c330613d07a3e955c9b8f3d0feb 100644 (file)
--- a/src/os/memstore/MemStore.h
+++ b/src/os/memstore/MemStore.h
@@ -485,6 +485,10 @@ public:
   void set_fsid(uuid_d u);
   uuid_d get_fsid();
 
+  uint64_t estimate_objects_overhead(uint64_t num_objects) override {
+    return 0; //do not care
+  }
+
   objectstore_perf_stat_t get_cur_stats();
 
   int queue_transactions(

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index d2fd6e9c6874e400870df5aa149de12ad6c08e89..e29fbc3b0c895047a183b5331ad457c748153291 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -12211,6 +12211,8 @@ bool ReplicatedPG::agent_choose_mode(bool restart, OpRequestRef op)
   uint64_t num_user_bytes = info.stats.stats.sum.num_bytes;
   uint64_t unflushable_bytes = info.stats.stats.sum.num_bytes_hit_set_archive;
   num_user_bytes -= unflushable_bytes;
+  uint64_t num_overhead_bytes = osd->store->estimate_objects_overhead(num_user_objects);
+  num_user_bytes += num_overhead_bytes;
 
   // also reduce the num_dirty by num_objects_omap
   int64_t num_dirty = info.stats.stats.sum.num_objects_dirty;
@@ -12233,6 +12235,7 @@ bool ReplicatedPG::agent_choose_mode(bool restart, OpRequestRef op)
           << " num_dirty: " << num_dirty
           << " num_user_objects: " << num_user_objects
           << " num_user_bytes: " << num_user_bytes
+          << " num_overhead_bytes: " << num_overhead_bytes
           << " pool.info.target_max_bytes: " << pool.info.target_max_bytes
           << " pool.info.target_max_objects: " << pool.info.target_max_objects
           << dendl;