From f6afe92e9103f95ea63c4e041cc951661a4f3b02 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Mon, 21 Mar 2016 15:51:24 +0300 Subject: [PATCH] ObjctStore: More accurate disk space usage calculation when considering 'whiteout'/empty/small objects. This also adds cache protection against potential overflow with removed objects Signed-off-by: Igor Fedotov --- src/os/ObjectStore.h | 6 ++++++ src/os/bluestore/BlueStore.h | 4 ++++ src/os/filestore/FileStore.cc | 6 ++++++ src/os/filestore/FileStore.h | 2 ++ src/os/kstore/KStore.h | 4 ++++ src/os/memstore/MemStore.h | 4 ++++ src/osd/ReplicatedPG.cc | 3 +++ 7 files changed, 29 insertions(+) diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h index fba6b76fcf18..db31f3c5d377 100644 --- a/src/os/ObjectStore.h +++ b/src/os/ObjectStore.h @@ -2388,6 +2388,12 @@ public: virtual void set_fsid(uuid_d u) = 0; virtual uuid_d get_fsid() = 0; + /** + * Estimates additional disk space used by the specified amount of objects and caused by file allocation granularity and metadata store + * - num objects - total (including witeouts) object count to measure used space for. + */ + virtual uint64_t estimate_objects_overhead(uint64_t num_objects) = 0; + // DEBUG virtual void inject_data_error(const ghobject_t &oid) {} virtual void inject_mdata_error(const ghobject_t &oid) {} diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index b1d0fcb19f84..ab17c21887c1 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -799,6 +799,10 @@ public: return fsid; } + uint64_t estimate_objects_overhead(uint64_t num_objects) override { + return num_objects * 300; //assuming per-object overhead is 300 bytes + } + objectstore_perf_stat_t get_cur_stats() override { return objectstore_perf_stat_t(); } diff --git a/src/os/filestore/FileStore.cc b/src/os/filestore/FileStore.cc index 0696afc5518a..dbd9ca2d5dee 100644 --- a/src/os/filestore/FileStore.cc +++ b/src/os/filestore/FileStore.cc @@ -5685,6 +5685,12 @@ void FileStore::set_xattr_limits_via_conf() m_filestore_max_inline_xattrs = fs_xattrs; } +uint64_t FileStore::estimate_objects_overhead(uint64_t num_objects) +{ + uint64_t res = num_objects * blk_size / 2; //assumes that each object uses ( in average ) additional 1/2 block due to FS allocation granularity. + return res; +} + // -- FSSuperblock -- void FSSuperblock::encode(bufferlist &bl) const diff --git a/src/os/filestore/FileStore.h b/src/os/filestore/FileStore.h index d81f8b0691b0..008c430dae68 100644 --- a/src/os/filestore/FileStore.h +++ b/src/os/filestore/FileStore.h @@ -588,6 +588,8 @@ public: fsid = u; } uuid_d get_fsid() { return fsid; } + + uint64_t estimate_objects_overhead(uint64_t num_objects); // DEBUG read error injection, an object is removed from both on delete() Mutex read_error_lock; diff --git a/src/os/kstore/KStore.h b/src/os/kstore/KStore.h index 4b0066331dee..3202bdf42bec 100644 --- a/src/os/kstore/KStore.h +++ b/src/os/kstore/KStore.h @@ -521,6 +521,10 @@ public: return fsid; } + uint64_t estimate_objects_overhead(uint64_t num_objects) override { + return num_objects * 300; //assuming per-object overhead is 300 bytes + } + objectstore_perf_stat_t get_cur_stats() { return objectstore_perf_stat_t(); } diff --git a/src/os/memstore/MemStore.h b/src/os/memstore/MemStore.h index 2d809f3384f9..cdf90f79418d 100644 --- a/src/os/memstore/MemStore.h +++ b/src/os/memstore/MemStore.h @@ -485,6 +485,10 @@ public: void set_fsid(uuid_d u); uuid_d get_fsid(); + uint64_t estimate_objects_overhead(uint64_t num_objects) override { + return 0; //do not care + } + objectstore_perf_stat_t get_cur_stats(); int queue_transactions( diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index d2fd6e9c6874..e29fbc3b0c89 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -12211,6 +12211,8 @@ bool ReplicatedPG::agent_choose_mode(bool restart, OpRequestRef op) uint64_t num_user_bytes = info.stats.stats.sum.num_bytes; uint64_t unflushable_bytes = info.stats.stats.sum.num_bytes_hit_set_archive; num_user_bytes -= unflushable_bytes; + uint64_t num_overhead_bytes = osd->store->estimate_objects_overhead(num_user_objects); + num_user_bytes += num_overhead_bytes; // also reduce the num_dirty by num_objects_omap int64_t num_dirty = info.stats.stats.sum.num_objects_dirty; @@ -12233,6 +12235,7 @@ bool ReplicatedPG::agent_choose_mode(bool restart, OpRequestRef op) << " num_dirty: " << num_dirty << " num_user_objects: " << num_user_objects << " num_user_bytes: " << num_user_bytes + << " num_overhead_bytes: " << num_overhead_bytes << " pool.info.target_max_bytes: " << pool.info.target_max_bytes << " pool.info.target_max_objects: " << pool.info.target_max_objects << dendl; -- 2.47.3