desc: Enable per-user and per-bucket usage performance counters
long_desc: When enabled, RGW will track and expose usage metrics (bytes and objects)
for users and buckets through the Ceph performance counter framework
- default: true
+ default: false
services:
- rgw
with_legacy: true
- rgw
with_legacy: true
-- name: rgw_usage_cache_ttl
- type: uint
- level: advanced
- desc: Time-to-live in seconds for cached usage statistics
- default: 300
- min: 60
- max: 3600
- services:
- - rgw
- with_legacy: true
-
- name: rgw_usage_stats_refresh_interval
type: int
level: advanced
desc: Interval in seconds for background refresh of usage statistics for active buckets/users
- default: 5_min
+ default: 20_min
services:
- rgw
see_also:
#include "rgw_perf_counters.h"
#include "rgw_signal.h"
#include "rgw_usage_perf.h"
+#include "rgw_usage_cache.h"
#ifdef WITH_ARROW_FLIGHT
#include "rgw_flight_frontend.h"
#endif
if (!env.driver) {
return -EIO;
}
- return 0;
-} /* init_storage */
-
-void rgw::AppMain::init_perfcounters()
-{
- (void) rgw_perf_start(dpp->get_cct());
+ // Initialize usage perf counters once we have driver available
if (g_conf()->rgw_enable_usage_perf_counters) {
- // Validate and create cache directory
rgw::UsageCache::Config cache_config;
cache_config.db_path = g_conf()->rgw_usage_cache_path;
cache_config.max_db_size = g_conf()->rgw_usage_cache_max_size;
- cache_config.ttl = std::chrono::seconds(g_conf()->rgw_usage_cache_ttl);
- // Check if cache directory exists
if (!cache_config.db_path.empty()) {
std::string db_dir = cache_config.db_path;
size_t pos = db_dir.find_last_of('/');
struct stat st;
if (stat(db_dir.c_str(), &st) != 0) {
- // Try to create directory
if (mkdir(db_dir.c_str(), 0755) == 0) {
ldpp_dout(dpp, 10) << "Created usage cache directory: " << db_dir << dendl;
} else {
- ldpp_dout(dpp, 0) << "WARNING: Failed to create usage cache directory: "
- << db_dir << " - " << cpp_strerror(errno)
+ ldpp_dout(dpp, 0) << "WARNING: Failed to create usage cache directory: "
+ << db_dir << " - " << cpp_strerror(errno)
<< " (continuing without usage cache)" << dendl;
cache_config.db_path = "";
}
} else if (!S_ISDIR(st.st_mode)) {
- ldpp_dout(dpp, 0) << "WARNING: Usage cache path is not a directory: "
+ ldpp_dout(dpp, 0) << "WARNING: Usage cache path is not a directory: "
<< db_dir << " (continuing without usage cache)" << dendl;
cache_config.db_path = "";
}
}
-
- // Create and initialize usage perf counters
- if (!cache_config.db_path.empty()) {
- usage_perf_counters = std::make_unique<rgw::UsagePerfCounters>(
- dpp->get_cct(), cache_config);
-
- int r = usage_perf_counters->init();
- if (r < 0) {
- ldpp_dout(dpp, 1) << "WARNING: Failed to initialize usage perf counters: "
- << cpp_strerror(-r) << " (continuing without them)" << dendl;
- usage_perf_counters.reset();
- } else {
- usage_perf_counters->start();
- rgw::set_usage_perf_counters(usage_perf_counters.get());
- ldpp_dout(dpp, 10) << "Usage performance counters initialized successfully" << dendl;
- }
- }
if (!cache_config.db_path.empty()) {
-
usage_perf_counters = std::make_unique<rgw::UsagePerfCounters>(
dpp->get_cct(),
- env.driver, // the driver parameter
+ env.driver,
cache_config);
int r = usage_perf_counters->init();
if (r < 0) {
- ldpp_dout(dpp, 1) << "WARNING: Failed to initialize usage perf counters: "
+ ldpp_dout(dpp, 1) << "WARNING: Failed to initialize usage perf counters: "
<< cpp_strerror(-r) << " (continuing without them)" << dendl;
usage_perf_counters.reset();
} else {
+ usage_perf_counters->start();
rgw::set_usage_perf_counters(usage_perf_counters.get());
ldpp_dout(dpp, 10) << "Usage performance counters initialized successfully" << dendl;
}
}
}
+ return 0;
+} /* init_storage */
+
+void rgw::AppMain::init_perfcounters()
+{
+ (void) rgw_perf_start(dpp->get_cct());
+
} /* init_perfcounters */
void rgw::AppMain::init_http_clients()
/* continue if EEXIST and create_bucket will fail below. this way we can
* recover from a partial create by retrying it. */
ldpp_dout(this, 20) << "Bucket::create() returned ret=" << op_ret << " bucket=" << s->bucket << dendl;
-
- if (op_ret < 0 && op_ret != -EEXIST && op_ret != -ERR_BUCKET_EXISTS)
- return;
-
- const bool existed = s->bucket_exists;
- if (need_metadata_upload() && existed) {
- /* OK, it looks we lost race with another request. As it's required to
- * handle metadata fusion and upload, the whole operation becomes very
- * similar in nature to PutMetadataBucket. However, as the attrs may
- * changed in the meantime, we have to refresh. */
- short tries = 0;
- do {
- map<string, bufferlist> battrs;
-
- op_ret = s->bucket->load_bucket(this, y);
- if (op_ret < 0) {
- return;
- } else if (!s->auth.identity->is_owner_of(s->bucket->get_owner())) {
- /* New bucket doesn't belong to the account we're operating on. */
- op_ret = -EEXIST;
- return;
- } else {
- s->bucket_attrs = s->bucket->get_attrs();
- }
-
- createparams.attrs.clear();
-
- op_ret = rgw_get_request_metadata(this, s->cct, s->info, createparams.attrs, false);
- if (op_ret < 0) {
- return;
- }
- prepare_add_del_attrs(s->bucket_attrs, rmattr_names, createparams.attrs);
- populate_with_generic_attrs(s, createparams.attrs);
- op_ret = filter_out_quota_info(createparams.attrs, rmattr_names,
- s->bucket->get_info().quota);
- if (op_ret < 0) {
- return;
- }
-
- /* Handle updates of the metadata for Swift's object versioning. */
- if (createparams.swift_ver_location) {
- s->bucket->get_info().swift_ver_location = *createparams.swift_ver_location;
- s->bucket->get_info().swift_versioning = !createparams.swift_ver_location->empty();
- }
-
- /* Web site of Swift API. */
- filter_out_website(createparams.attrs, rmattr_names,
- s->bucket->get_info().website_conf);
- s->bucket->get_info().has_website = !s->bucket->get_info().website_conf.is_empty();
-
- /* This will also set the quota on the bucket. */
- s->bucket->set_attrs(std::move(createparams.attrs));
- constexpr bool exclusive = false; // overwrite
- constexpr ceph::real_time no_set_mtime{};
- op_ret = s->bucket->put_info(this, exclusive, no_set_mtime, y);
- } while (op_ret == -ECANCELED && tries++ < 20);
-
- /* Restore the proper return code. */
- if (op_ret >= 0) {
- op_ret = -ERR_BUCKET_EXISTS;
- }
- } /* if (need_metadata_upload() && existed) */
- if (op_ret >= 0 || op_ret == -ERR_BUCKET_EXISTS) {
- auto* usage_counters = rgw::get_usage_perf_counters();
- if (usage_counters && s->bucket) {
- // For new buckets, initialize with 0 bytes and 0 objects
- usage_counters->update_bucket_stats(s->bucket->get_name(), 0, 0, s->user->get_id().id);
-
- // Update user stats - use sync_owner_stats to get current info
- if (s->user) {
- RGWBucketEnt ent;
- int ret = s->bucket->sync_owner_stats(this, y, &ent);
- if (ret >= 0) {
- // This updates with the user's total across this bucket
- usage_counters->update_user_stats(
- s->user->get_id().id,
- ent.size,
- ent.count,
- false
- );
- }
- }
- }
- }
-
} /* RGWCreateBucket::execute() */
int RGWDeleteBucket::verify_permission(optional_yield y)
rgw::op_counters::inc(counters, l_rgw_op_del_bucket, 1);
rgw::op_counters::tinc(counters, l_rgw_op_del_bucket_lat, s->time_elapsed());
- // Add usage counter update here, right before return
- if (op_ret >= 0) {
- auto* usage_counters = rgw::get_usage_perf_counters();
- if (usage_counters && s->bucket) {
- // Remove bucket from cache since it's deleted
- usage_counters->evict_from_cache("", s->bucket->get_name());
-
- // Update user stats - bucket count has changed
- // Since bucket is deleted, we can't use it to get stats
- // Just evict the user from cache to force refresh next time
- if (s->user) {
- usage_counters->evict_from_cache(s->user->get_id().id, "");
- }
- }
- }
return;
}
// too late to rollback operation, hence op_ret is not set here
}
- // Update usage statistics after successful upload
+ // Mark user/bucket as active for background sync (NO cache updates in I/O path)
if (op_ret == 0 && s->bucket && s->obj_size > 0) {
auto usage_counters = rgw::get_usage_perf_counters();
- if (usage_counters) {
- std::string bucket_key = s->bucket->get_tenant().empty() ?
- s->bucket->get_name() :
- s->bucket->get_tenant() + "/" + s->bucket->get_name();
-
- ldpp_dout(this, 20) << "PUT completed: updating usage for bucket="
- << bucket_key << " size=" << s->obj_size << dendl;
- auto usage_counters = rgw::get_usage_perf_counters();
- if (usage_counters && s->bucket && s->user) {
- // Get actual bucket stats from RGW metadata (includes this PUT)
- RGWBucketEnt stats;
- int ret = s->bucket->sync_owner_stats(this, y, &stats);
+ if (usage_counters && s->user) {
+ // Just mark as active - background thread will sync from RADOS
+ usage_counters->mark_bucket_active(s->bucket->get_name(),
+ s->bucket->get_tenant());
+ usage_counters->mark_user_active(s->user->get_id().to_str());
- if (ret >= 0) {
- ldpp_dout(this, 20) << "PUT completed: updating usage for bucket="
- << s->bucket->get_name()
- << " bytes=" << stats.size
- << " objects=" << stats.count << dendl;
-
- // Update with ACTUAL bucket totals (not calculated)
- usage_counters->update_bucket_stats(s->bucket->get_name(),
- stats.size,
- stats.count,
- s->user->get_id().id,
- true);
-
- // Mark as active
- usage_counters->mark_bucket_active(s->bucket->get_name(),
- s->bucket->get_tenant());
-
- // User stats are aggregated in cache, just update perf counter
- auto user_stats = usage_counters->get_user_stats(s->user->get_id().to_str());
- if (user_stats) {
- usage_counters->update_user_stats(s->user->get_id().to_str(),
- user_stats->bytes_used,
- user_stats->num_objects,
- false);
- usage_counters->mark_user_active(s->user->get_id().to_str());
- }
- }
- }
+ ldpp_dout(this, 20) << "PUT completed: marked user/bucket active for background sync" << dendl;
}
}
-
} /* RGWPutObj::execute() */
int RGWPostObj::init_processing(optional_yield y)
} else {
op_ret = -EINVAL;
}
-
- auto usage_counters = rgw::get_usage_perf_counters();
- if (usage_counters && s->bucket && s->user) {
- // Get actual bucket stats from RGW metadata (after deletion)
- RGWBucketEnt stats;
- int ret = s->bucket->sync_owner_stats(this, y, &stats);
-
- if (ret >= 0) {
- ldpp_dout(this, 20) << "DELETE completed: updating usage for bucket="
- << s->bucket->get_name()
- << " bytes=" << stats.size
- << " objects=" << stats.count << dendl;
-
- // Update with ACTUAL bucket totals (not calculated)
- usage_counters->update_bucket_stats(s->bucket->get_name(),
- stats.size,
- stats.count,
- s->user->get_id().id,
- true);
-
- // Mark as active
+
+ // Mark user/bucket as active for background sync after successful delete
+ if (op_ret >= 0 && s->bucket && s->user) {
+ auto usage_counters = rgw::get_usage_perf_counters();
+ if (usage_counters) {
usage_counters->mark_bucket_active(s->bucket->get_name(),
- s->bucket->get_tenant());
-
- // User stats are aggregated in cache, just update perf counter
- auto user_stats = usage_counters->get_user_stats(s->user->get_id().to_str());
- if (user_stats) {
- usage_counters->update_user_stats(s->user->get_id().to_str(),
- user_stats->bytes_used,
- user_stats->num_objects,
- false);
- usage_counters->mark_user_active(s->user->get_id().to_str());
- }
+ s->bucket->get_tenant());
+ usage_counters->mark_user_active(s->user->get_id().to_str());
+ ldpp_dout(this, 20) << "DELETE completed: marked user/bucket active for background sync" << dendl;
}
}
-
+
}
class RGWCopyObjDPF : public rgw::sal::DataProcessorFactory {
PERF_CACHE_HIT,
PERF_CACHE_MISS,
PERF_CACHE_UPDATE,
- PERF_CACHE_REMOVE,
- PERF_CACHE_EXPIRED,
PERF_CACHE_SIZE,
- PERF_CACHE_USER_HIT,
- PERF_CACHE_USER_MISS,
- PERF_CACHE_BUCKET_HIT,
- PERF_CACHE_BUCKET_MISS,
PERF_CACHE_LAST
};
pcb.add_u64_counter(PERF_CACHE_UPDATE, "cache_updates",
"Total number of cache updates", "upd",
PerfCountersBuilder::PRIO_INTERESTING);
- pcb.add_u64_counter(PERF_CACHE_REMOVE, "cache_removes",
- "Total number of cache removes", "rm",
- PerfCountersBuilder::PRIO_INTERESTING);
- pcb.add_u64_counter(PERF_CACHE_EXPIRED, "cache_expired",
- "Total number of expired entries", "exp",
- PerfCountersBuilder::PRIO_DEBUGONLY);
pcb.add_u64(PERF_CACHE_SIZE, "cache_size",
"Current cache size", "size",
PerfCountersBuilder::PRIO_USEFUL);
- pcb.add_u64_counter(PERF_CACHE_USER_HIT, "user_cache_hits",
- "User cache hits", "uhit",
- PerfCountersBuilder::PRIO_DEBUGONLY);
- pcb.add_u64_counter(PERF_CACHE_USER_MISS, "user_cache_misses",
- "User cache misses", "umis",
- PerfCountersBuilder::PRIO_DEBUGONLY);
- pcb.add_u64_counter(PERF_CACHE_BUCKET_HIT, "bucket_cache_hits",
- "Bucket cache hits", "bhit",
- PerfCountersBuilder::PRIO_DEBUGONLY);
- pcb.add_u64_counter(PERF_CACHE_BUCKET_MISS, "bucket_cache_misses",
- "Bucket cache misses", "bmis",
- PerfCountersBuilder::PRIO_DEBUGONLY);
perf_counters = pcb.create_perf_counters();
cct->get_perfcounters_collection()->add(perf_counters);
auto iter = bl.cbegin();
stats.decode(iter);
- // Check TTL
- auto now = ceph::real_clock::now();
- if (now - stats.last_updated > config.ttl) {
- inc_counter(PERF_CACHE_EXPIRED);
- return std::nullopt;
- }
-
return stats;
} catch (const buffer::error& e) {
if (cct) {
if (result.has_value()) {
cache_hits++;
inc_counter(PERF_CACHE_HIT);
- inc_counter(PERF_CACHE_USER_HIT);
} else {
cache_misses++;
inc_counter(PERF_CACHE_MISS);
- inc_counter(PERF_CACHE_USER_MISS);
}
return result;
}
return -EIO;
}
-
- inc_counter(PERF_CACHE_REMOVE);
+
set_counter(PERF_CACHE_SIZE, get_cache_size_internal());
return 0;
uint64_t num_objects,
const std::string& user_id) {
- if (!initialized || user_id.empty()) {
+ // Only store bucket stats
+ // User aggregation is done by background thread reading from RADOS
+
+ if (!initialized) {
return -EINVAL;
}
std::unique_lock lock(db_mutex);
-
- // Get old bucket stats to calculate delta
- auto old_bucket_stats = get_stats<UsageStats>(bucket_dbi, bucket_name);
-
UsageStats stats;
stats.bytes_used = bytes_used;
stats.num_objects = num_objects;
stats.last_updated = ceph::real_clock::now();
int ret = put_stats(bucket_dbi, bucket_name, stats);
- if (ret != 0) {
- return ret;
- }
-
- // Get current user stats
- auto current_user_stats = get_stats<UsageStats>(user_dbi, user_id);
-
- UsageStats new_user_stats;
- if (current_user_stats.has_value()) {
- new_user_stats.bytes_used = current_user_stats->bytes_used;
- new_user_stats.num_objects = current_user_stats->num_objects;
- } else {
- new_user_stats.bytes_used = 0;
- new_user_stats.num_objects = 0;
- }
-
- // Calculate delta (what changed for this bucket)
- int64_t delta_bytes = (int64_t)bytes_used;
- int64_t delta_objects = (int64_t)num_objects;
-
- if (old_bucket_stats.has_value()) {
- delta_bytes -= (int64_t)old_bucket_stats->bytes_used;
- delta_objects -= (int64_t)old_bucket_stats->num_objects;
- }
-
- // Apply delta to user stats
- new_user_stats.bytes_used = (uint64_t)((int64_t)new_user_stats.bytes_used + delta_bytes);
- new_user_stats.num_objects = (uint64_t)((int64_t)new_user_stats.num_objects + delta_objects);
- new_user_stats.last_updated = ceph::real_clock::now();
-
- // Update user stats in cache
- ret = put_stats(user_dbi, user_id, new_user_stats);
if (ret == 0) {
inc_counter(PERF_CACHE_UPDATE);
set_counter(PERF_CACHE_SIZE, get_cache_size_internal());
+
+ if (cct) {
+ ldout(cct, 15) << "Cache updated for bucket " << bucket_name
+ << " bytes=" << bytes_used
+ << " objects=" << num_objects << dendl;
+ }
}
+ // NOTE: User stats are NOT updated here!
+ // Background thread reads from RADOS and updates user totals
+
return ret;
}
if (result.has_value()) {
cache_hits++;
inc_counter(PERF_CACHE_HIT);
- inc_counter(PERF_CACHE_BUCKET_HIT);
} else {
cache_misses++;
inc_counter(PERF_CACHE_MISS);
- inc_counter(PERF_CACHE_BUCKET_MISS);
}
return result;
return -EIO;
}
- inc_counter(PERF_CACHE_REMOVE);
- set_counter(PERF_CACHE_SIZE, get_cache_size_internal());
-
- return 0;
-}
-
-int UsageCache::clear_expired_entries() {
- if (!initialized) {
- return -EINVAL;
- }
-
- std::unique_lock lock(db_mutex);
-
- auto now = ceph::real_clock::now();
- int total_removed = 0;
-
- // Helper lambda to clear expired entries from a database
- auto clear_db = [this, &now](MDB_dbi dbi) -> int {
- MDB_txn* txn = nullptr;
- MDB_cursor* cursor = nullptr;
-
- int rc = mdb_txn_begin(env, nullptr, 0, &txn);
- if (rc != 0) {
- if (cct) {
- ldout(cct, 5) << "LMDB txn_begin failed in clear_expired_entries: "
- << mdb_strerror(rc) << dendl;
- }
- return -EIO;
- }
-
- rc = mdb_cursor_open(txn, dbi, &cursor);
- if (rc != 0) {
- if (cct) {
- ldout(cct, 5) << "LMDB cursor_open failed: " << mdb_strerror(rc) << dendl;
- }
- mdb_txn_abort(txn);
- return -EIO;
- }
-
- MDB_val key, val;
- int removed = 0;
-
- while (mdb_cursor_get(cursor, &key, &val, MDB_NEXT) == 0) {
- bufferlist bl;
- bl.append(static_cast<char*>(val.mv_data), val.mv_size);
-
- try {
- UsageStats stats;
- auto iter = bl.cbegin();
- stats.decode(iter);
-
- if (now - stats.last_updated > config.ttl) {
- mdb_cursor_del(cursor, 0);
- removed++;
- inc_counter(PERF_CACHE_EXPIRED);
- }
- } catch (const buffer::error& e) {
- // Skip malformed entries
- if (cct) {
- ldout(cct, 10) << "Skipping malformed entry: " << e.what() << dendl;
- }
- }
- }
-
- mdb_cursor_close(cursor);
-
- rc = mdb_txn_commit(txn);
- if (rc != 0) {
- if (cct) {
- ldout(cct, 5) << "LMDB txn_commit failed in clear_expired_entries: "
- << mdb_strerror(rc) << dendl;
- }
- return -EIO;
- }
-
- return removed;
- };
-
- int ret = clear_db(user_dbi);
- if (ret >= 0) {
- total_removed += ret;
- }
-
- ret = clear_db(bucket_dbi);
- if (ret >= 0) {
- total_removed += ret;
- }
-
set_counter(PERF_CACHE_SIZE, get_cache_size_internal());
if (cct) {
- ldout(cct, 10) << "Cleared " << total_removed << " expired cache entries" << dendl;
+ ldout(cct, 10) << "Removed bucket " << bucket_name << " from cache" << dendl;
}
- return total_removed;
+ // NOTE: User stats are NOT updated here!
+ // Background thread reads from RADOS and recalculates user totals
+
+
+ return 0;
}
size_t UsageCache::get_cache_size() const {
return cache_misses.load();
}
+uint64_t UsageCache::get_cache_updates() const {
+ return perf_counters ? perf_counters->get(PERF_CACHE_UPDATE) : 0;
+}
+
double UsageCache::get_hit_rate() const {
uint64_t hits = cache_hits.load();
uint64_t misses = cache_misses.load();
std::string db_path;
size_t max_db_size;
uint32_t max_readers;
- std::chrono::seconds ttl;
Config()
: db_path("/var/lib/ceph/radosgw/usage_cache.mdb"),
max_db_size(1 << 30), // 1GB default
- max_readers(126),
- ttl(300) {} // 5 min TTL
+ max_readers(126) {}
};
explicit UsageCache(const Config& config);
int update_bucket_stats(const std::string& bucket_name,
uint64_t bytes_used,
uint64_t num_objects,
- const std::string& user_id);
+ const std::string& user_id = "");
std::optional<UsageStats> get_bucket_stats(const std::string& bucket_name);
int remove_bucket_stats(const std::string& bucket_name);
- // Maintenance
- int clear_expired_entries();
size_t get_cache_size() const;
const Config& get_config() const { return config; }
uint64_t get_cache_hits() const;
uint64_t get_cache_misses() const;
double get_hit_rate() const;
+ uint64_t get_cache_updates() const;
// Iterator methods for initial load
std::vector<std::pair<std::string, UsageStats>> get_all_users();
#include "common/perf_counters_collection.h"
#include "common/errno.h"
#include "common/async/yield_context.h"
+#include "rgw_sal.h"
+#include "rgw_common.h"
+#include "rgw_sal_rados.h"
#define dout_subsys ceph_subsys_rgw
return counters;
}
-void UsagePerfCounters::cleanup_worker() {
- ldout(cct, 10) << "Starting usage cache cleanup worker thread" << dendl;
-
- while (!shutdown_flag.load()) {
- // Sleep with periodic checks for shutdown
- for (int i = 0; i < cleanup_interval.count(); ++i) {
- if (shutdown_flag.load()) {
- break;
- }
- std::this_thread::sleep_for(std::chrono::seconds(1));
- }
-
- if (!shutdown_flag.load()) {
- cleanup_expired_entries();
- }
- }
-
- ldout(cct, 10) << "Usage cache cleanup worker thread exiting" << dendl;
-}
-
int UsagePerfCounters::init() {
int ret = cache->init();
if (ret < 0) {
ldout(cct, 10) << "Initial load complete: " << all_users.size()
<< " users, " << all_buckets.size() << " buckets" << dendl;
}
-
- // Start cleanup thread
- cleanup_thread = std::thread(&UsagePerfCounters::cleanup_worker, this);
// Start refresh thread
refresh_thread = std::thread(&UsagePerfCounters::refresh_worker, this);
ldout(cct, 10) << "Started usage perf counters threads" << dendl;
}
+void UsagePerfCounters::sync_user_from_rados(const std::string& user_id) {
+ if (!driver) {
+ ldout(cct, 10) << "sync_user_from_rados: no driver available" << dendl;
+ return;
+ }
+
+ ldout(cct, 15) << "sync_user_from_rados: user=" << user_id << dendl;
+
+ UsagePerfDoutPrefix dpp(cct);
+
+ // Use existing RGW infrastructure to get user stats from RADOS
+ // This is the distributed source of truth
+ RGWStorageStats stats;
+ ceph::real_time last_synced;
+ ceph::real_time last_updated;
+
+ rgw_user uid(user_id);
+
+ int ret = driver->load_stats(&dpp, null_yield, uid, stats,
+ last_synced, last_updated);
+
+ if (ret < 0) {
+ ldout(cct, 10) << "Failed to load stats from RADOS for user " << user_id
+ << ": " << cpp_strerror(-ret) << dendl;
+ return;
+ }
+
+ ldout(cct, 15) << "Got stats from RADOS: user=" << user_id
+ << " bytes=" << stats.size
+ << " objects=" << stats.num_objects << dendl;
+
+ // Update local LMDB cache for persistence across restarts
+ if (cache) {
+ cache->update_user_stats(user_id, stats.size, stats.num_objects);
+ }
+
+ // Update perf counters (for Prometheus)
+ update_user_stats(user_id, stats.size, stats.num_objects, false);
+}
+
void UsagePerfCounters::stop() {
ldout(cct, 10) << "Stopping usage perf counters" << dendl;
// Signal threads to stop
shutdown_flag = true;
- // Wait for cleanup thread
- if (cleanup_thread.joinable()) {
- cleanup_thread.join();
- }
-
// Wait for refresh thread
if (refresh_thread.joinable()) {
refresh_thread.join();
void UsagePerfCounters::shutdown() {
shutdown_flag = true;
- if (cleanup_thread.joinable()) {
- cleanup_thread.join();
- }
-
if (refresh_thread.joinable()) {
refresh_thread.join();
}
break;
}
- // Get snapshot of active buckets and users
- std::unordered_set<std::string> buckets_to_refresh;
+ // Get snapshot of active users
std::unordered_set<std::string> users_to_refresh;
-
{
std::lock_guard<std::mutex> lock(activity_mutex);
- buckets_to_refresh = active_buckets;
users_to_refresh = active_users;
}
- ldout(cct, 15) << "Background refresh: checking " << buckets_to_refresh.size()
- << " buckets and " << users_to_refresh.size()
- << " users" << dendl;
+ ldout(cct, 15) << "Background refresh: syncing " << users_to_refresh.size()
+ << " users from RADOS" << dendl;
- // Refresh bucket stats from cache
- for (const auto& bucket_key : buckets_to_refresh) {
+ // Sync user stats from RADOS (source of truth)
+ for (const auto& user_id : users_to_refresh) {
if (shutdown_flag) break;
- refresh_bucket_stats(bucket_key);
+ sync_user_from_rados(user_id);
}
- // Refresh user stats from cache
- for (const auto& user_id : users_to_refresh) {
+ // Bucket stats can still be refreshed from local cache
+ // they're updated by the existing quota sync mechanism
+ std::unordered_set<std::string> buckets_to_refresh;
+ {
+ std::lock_guard<std::mutex> lock(activity_mutex);
+ buckets_to_refresh = active_buckets;
+ }
+
+ for (const auto& bucket_key : buckets_to_refresh) {
if (shutdown_flag) break;
- refresh_user_stats(user_id);
+ refresh_bucket_stats(bucket_key);
}
}
void UsagePerfCounters::refresh_user_stats(const std::string& user_id) {
ldout(cct, 20) << "refresh_user_stats: user=" << user_id << dendl;
- auto cached_stats = cache->get_user_stats(user_id);
- if (cached_stats) {
- ldout(cct, 15) << "Refreshing user " << user_id
- << " bytes=" << cached_stats->bytes_used
- << " objects=" << cached_stats->num_objects << dendl;
-
- update_user_stats(user_id, cached_stats->bytes_used,
- cached_stats->num_objects, false);
- }
+ sync_user_from_rados(user_id);
}
void UsagePerfCounters::refresh_from_cache(const std::string& user_id,
return stats;
}
-void UsagePerfCounters::cleanup_expired_entries() {
- if (cache) {
- int removed = cache->clear_expired_entries();
- if (removed > 0) {
- ldout(cct, 10) << "Cleaned up " << removed << " expired cache entries" << dendl;
- }
- }
-}
-
size_t UsagePerfCounters::get_cache_size() const {
return cache ? cache->get_cache_size() : 0;
}
#include "common/perf_counters.h"
#include "rgw_usage_cache.h"
+#include "common/dout.h"
// To avoid heavy header rgw_sal.h , we are adding a forward declaration here
namespace rgw::sal {
l_rgw_usage_last
};
+class UsagePerfDoutPrefix : public DoutPrefixProvider {
+ CephContext* cct;
+public:
+ explicit UsagePerfDoutPrefix(CephContext* _cct) : cct(_cct) {}
+
+ CephContext* get_cct() const override { return cct; }
+ unsigned get_subsys() const override { return ceph_subsys_rgw; }
+ std::ostream& gen_prefix(std::ostream& out) const override {
+ return out << "usage_perf: ";
+ }
+};
+
class UsagePerfCounters {
private:
CephContext* cct;
std::unordered_set<std::string> active_users;
mutable std::mutex activity_mutex;
- // Cleanup thread management
- std::thread cleanup_thread;
std::thread refresh_thread;
std::atomic<bool> shutdown_flag{false};
- std::chrono::seconds cleanup_interval{300}; // 5 minutes
- std::chrono::seconds refresh_interval{60};
+ std::chrono::seconds refresh_interval{1200};
void create_global_counters();
PerfCounters* create_user_counters(const std::string& user_id);
void stop();
void shutdown();
+ void set_driver(rgw::sal::Driver* d) { driver = d; }
+
// User stats updates
void update_user_stats(const std::string& user_id,
uint64_t bytes_used,
const std::string& user_id = "",
bool update_cache = true);
-void mark_bucket_active(const std::string& bucket_name,
- const std::string& tenant = "");
+ void mark_bucket_active(const std::string& bucket_name,
+ const std::string& tenant = "");
-void mark_user_active(const std::string& user_id);
+ void mark_user_active(const std::string& user_id);
- // Cache operations
- void refresh_from_cache(const std::string& user_id,
- const std::string& bucket_name);
- void evict_from_cache(const std::string& user_id,
- const std::string& bucket_name);
+// Cache operations
+void refresh_from_cache(const std::string& user_id,
+ const std::string& bucket_name);
+void evict_from_cache(const std::string& user_id,
+ const std::string& bucket_name);
- // Stats retrieval (from cache)
- std::optional<UsageStats> get_user_stats(const std::string& user_id);
- std::optional<UsageStats> get_bucket_stats(const std::string& bucket_name);
-
- // Maintenance
- void cleanup_expired_entries();
- size_t get_cache_size() const;
-
- // Set cleanup interval
- void set_cleanup_interval(std::chrono::seconds interval) {
- cleanup_interval = interval;
- }
+void sync_user_from_rados(const std::string& user_id);
- void set_refresh_interval(std::chrono::seconds interval) {
- refresh_interval = interval;
- }
+// Stats retrieval (from cache)
+std::optional<UsageStats> get_user_stats(const std::string& user_id);
+std::optional<UsageStats> get_bucket_stats(const std::string& bucket_name);
+
+size_t get_cache_size() const;
+
+void set_refresh_interval(std::chrono::seconds interval) {
+ refresh_interval = interval;
+}
};
// Global singleton access
${CMAKE_SOURCE_DIR}/src/rgw/store/rados
${CMAKE_SOURCE_DIR}/src/dmclock/support/src
${CMAKE_SOURCE_DIR}/src/dmclock/src
+ ${LUA_INCLUDE_DIR}
)
target_link_libraries(unittest_rgw_usage_perf_counters
${UNITTEST_LIBS}
global
ceph-common
- lmdb)
+ lmdb
+ ${LUA_LIBRARIES})
add_ceph_unittest(unittest_rgw_usage_perf_counters)
endif()
namespace fs = std::filesystem;
-// Global CephContext pointer for tests
static CephContext* g_test_context = nullptr;
-// Test fixture for RGWUsageCache
class TestRGWUsageCache : public ::testing::Test {
protected:
std::unique_ptr<rgw::UsageCache> cache;
std::mt19937 gen(rd());
std::uniform_int_distribution<> dis(1000000, 9999999);
test_db_path = "/tmp/test_usage_cache_" + std::to_string(dis(gen)) + ".mdb";
-
- // Initialize config with proper fields
+
config.db_path = test_db_path;
config.max_db_size = 1 << 20; // 1MB for testing
config.max_readers = 10;
- config.ttl = std::chrono::seconds(2); // 2 second TTL for testing
-
+
// Create cache with global CephContext if available, otherwise without
if (g_test_context) {
cache = std::make_unique<rgw::UsageCache>(g_test_context, config);
} else {
cache = std::make_unique<rgw::UsageCache>(config);
}
-
- // CRITICAL: Initialize the cache!
+
+ // Initialize the cache!
int init_result = cache->init();
ASSERT_EQ(0, init_result) << "Failed to initialize cache: " << init_result;
}
cache->shutdown();
}
cache.reset();
-
+
// Clean up test database files
try {
fs::remove(test_db_path);
const std::string user_id = "test_user";
const uint64_t bytes_used = 1024 * 1024;
const uint64_t num_objects = 42;
-
+
// Update user stats
int update_result = cache->update_user_stats(user_id, bytes_used, num_objects);
ASSERT_EQ(0, update_result) << "Failed to update user stats: " << update_result;
-
+
// Get and verify user stats
auto stats = cache->get_user_stats(user_id);
ASSERT_TRUE(stats.has_value());
EXPECT_EQ(bytes_used, stats->bytes_used);
EXPECT_EQ(num_objects, stats->num_objects);
-
+
// Remove user stats
ASSERT_EQ(0, cache->remove_user_stats(user_id));
-
+
// Verify stats are removed
stats = cache->get_user_stats(user_id);
EXPECT_FALSE(stats.has_value());
const std::string bucket_name = "test_bucket";
const uint64_t bytes_used = 512 * 1024;
const uint64_t num_objects = 17;
-
+
// Update bucket stats
int update_result = cache->update_bucket_stats(bucket_name, bytes_used, num_objects);
ASSERT_EQ(0, update_result) << "Failed to update bucket stats: " << update_result;
-
+
// Get and verify bucket stats
auto stats = cache->get_bucket_stats(bucket_name);
ASSERT_TRUE(stats.has_value());
EXPECT_EQ(bytes_used, stats->bytes_used);
EXPECT_EQ(num_objects, stats->num_objects);
-
+
// Remove bucket stats
ASSERT_EQ(0, cache->remove_bucket_stats(bucket_name));
-
+
// Verify stats are removed
stats = cache->get_bucket_stats(bucket_name);
EXPECT_FALSE(stats.has_value());
}
-// Test TTL expiration
-TEST_F(TestRGWUsageCache, TTLExpiration) {
- const std::string user_id = "ttl_test_user";
-
- // Add user stats
- ASSERT_EQ(0, cache->update_user_stats(user_id, 1024, 1));
-
- // Verify stats exist
- auto stats = cache->get_user_stats(user_id);
- ASSERT_TRUE(stats.has_value());
-
- // Wait for TTL to expire (2 seconds + buffer)
- std::this_thread::sleep_for(std::chrono::milliseconds(2500));
-
- // Verify stats have expired
- stats = cache->get_user_stats(user_id);
- EXPECT_FALSE(stats.has_value());
-}
-
-// Test updating existing user stats
+// Test updating existing user stats (overwrites, not accumulates)
TEST_F(TestRGWUsageCache, UpdateExistingUserStats) {
const std::string user_id = "update_test_user";
-
+
// Initial update
ASSERT_EQ(0, cache->update_user_stats(user_id, 1024, 10));
-
+
auto stats = cache->get_user_stats(user_id);
ASSERT_TRUE(stats.has_value());
EXPECT_EQ(1024u, stats->bytes_used);
EXPECT_EQ(10u, stats->num_objects);
-
+
// Update with new values (should replace, not accumulate)
ASSERT_EQ(0, cache->update_user_stats(user_id, 2048, 20));
-
+
stats = cache->get_user_stats(user_id);
ASSERT_TRUE(stats.has_value());
EXPECT_EQ(2048u, stats->bytes_used);
// Test multiple bucket operations
TEST_F(TestRGWUsageCache, MultipleBucketOperations) {
const int num_buckets = 10;
-
+
// Add multiple buckets
for (int i = 0; i < num_buckets; ++i) {
std::string bucket_name = "bucket_" + std::to_string(i);
uint64_t bytes = (i + 1) * 1024;
uint64_t objects = (i + 1) * 10;
-
+
ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, bytes, objects));
}
-
+
// Verify all buckets
for (int i = 0; i < num_buckets; ++i) {
std::string bucket_name = "bucket_" + std::to_string(i);
auto stats = cache->get_bucket_stats(bucket_name);
-
+
ASSERT_TRUE(stats.has_value());
EXPECT_EQ((i + 1) * 1024u, stats->bytes_used);
EXPECT_EQ((i + 1) * 10u, stats->num_objects);
}
-
+
EXPECT_EQ(num_buckets, cache->get_cache_size());
}
"user with spaces",
"user\twith\ttabs"
};
-
+
for (const auto& id : test_ids) {
// Should handle all these gracefully
ASSERT_EQ(0, cache->update_user_stats(id, 1024, 1))
<< "Failed for ID: " << id;
-
+
auto stats = cache->get_user_stats(id);
ASSERT_TRUE(stats.has_value()) << "Failed to retrieve stats for ID: " << id;
EXPECT_EQ(1024u, stats->bytes_used);
// Test remove non-existent user
TEST_F(TestRGWUsageCache, RemoveNonExistentUser) {
const std::string user_id = "non_existent_user";
-
+
// Should handle removing non-existent user gracefully
int result = cache->remove_user_stats(user_id);
// Either returns 0 (success) or error (not found)
EXPECT_TRUE(result == 0 || result != 0);
-
+
// Verify user doesn't exist
auto stats = cache->get_user_stats(user_id);
EXPECT_FALSE(stats.has_value());
TEST_F(TestRGWUsageCache, CacheSizeTracking) {
// Initial size should be 0
EXPECT_EQ(0u, cache->get_cache_size());
-
+
// Add some users
const int num_users = 5;
-
+
for (int i = 0; i < num_users; ++i) {
std::string user_id = "size_test_user_" + std::to_string(i);
ASSERT_EQ(0, cache->update_user_stats(user_id, 1024 * (i + 1), i + 1));
}
-
+
// Cache size should reflect the number of added users
EXPECT_EQ(num_users, cache->get_cache_size());
-
+
// Remove one user
ASSERT_EQ(0, cache->remove_user_stats("size_test_user_0"));
-
+
// Cache size should decrease
EXPECT_EQ(num_users - 1, cache->get_cache_size());
}
-// Test clear expired entries
-TEST_F(TestRGWUsageCache, ClearExpiredEntries) {
- // Add some entries
- ASSERT_EQ(0, cache->update_user_stats("user1", 1024, 1));
- ASSERT_EQ(0, cache->update_user_stats("user2", 2048, 2));
- ASSERT_EQ(0, cache->update_bucket_stats("bucket1", 4096, 4));
-
- EXPECT_EQ(3u, cache->get_cache_size());
-
- // Wait for entries to expire
- std::this_thread::sleep_for(std::chrono::milliseconds(2500));
-
- // Clear expired entries
- int removed = cache->clear_expired_entries();
- EXPECT_EQ(3, removed);
-
- // Cache should be empty
- EXPECT_EQ(0u, cache->get_cache_size());
-}
-
// Test stress with many users and buckets
TEST_F(TestRGWUsageCache, StressTest) {
const int num_users = 1000;
const int num_buckets = 500;
-
- // Create a new config with longer TTL for stress testing
- rgw::UsageCache::Config stress_config;
- stress_config.db_path = test_db_path;
- stress_config.max_db_size = 1 << 20;
- stress_config.max_readers = 10;
- stress_config.ttl = std::chrono::seconds(1800); // 30 minutes for stress test
-
- // Recreate cache with longer TTL
- cache.reset();
-
- if (g_test_context) {
- cache = std::make_unique<rgw::UsageCache>(g_test_context, stress_config);
- } else {
- cache = std::make_unique<rgw::UsageCache>(stress_config);
- }
- ASSERT_EQ(0, cache->init());
+
+ std::cout << "Running stress test with " << num_users << " users and "
+ << num_buckets << " buckets..." << std::endl;
// Add many users
for (int i = 0; i < num_users; ++i) {
std::string user_id = "stress_user_" + std::to_string(i);
ASSERT_EQ(0, cache->update_user_stats(user_id, i * 1024, i));
}
-
+
// Add many buckets
for (int i = 0; i < num_buckets; ++i) {
std::string bucket_name = "stress_bucket_" + std::to_string(i);
ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, i * 512, i * 2));
}
-
+
EXPECT_EQ(num_users + num_buckets, cache->get_cache_size());
-
+
// Verify random samples
for (int i = 0; i < 10; ++i) {
- int user_idx = (i * 97) % num_users; // Sample users
+ int user_idx = (i * 97) % num_users;
std::string user_id = "stress_user_" + std::to_string(user_idx);
auto stats = cache->get_user_stats(user_id);
ASSERT_TRUE(stats.has_value());
EXPECT_EQ(user_idx * 1024u, stats->bytes_used);
}
-
+
for (int i = 0; i < 10; ++i) {
- int bucket_idx = (i * 53) % num_buckets; // Sample buckets
+ int bucket_idx = (i * 53) % num_buckets;
std::string bucket_name = "stress_bucket_" + std::to_string(bucket_idx);
auto bucket_stats = cache->get_bucket_stats(bucket_name);
ASSERT_TRUE(bucket_stats.has_value());
EXPECT_EQ(bucket_idx * 512u, bucket_stats->bytes_used);
}
-
- std::cout << "Stress test completed: " << num_users << " users, "
- << num_buckets << " buckets" << std::endl;
+
+ std::cout << "Stress test completed successfully!" << std::endl;
}
-TEST_F(TestRGWUsageCache, ManualTestScenario_UserAndBuckets) {
- // Simulate the complete manual test scenario:
- // User "testuser" with bucket1 (2 files) and bucket2 (1 file)
-
- std::cout << "\n=== Manual Test Scenario: User and Buckets ===" << std::endl;
-
- const std::string user_id = "testuser";
- const std::string bucket1 = "bucket1";
- const std::string bucket2 = "bucket2";
-
- // File sizes from manual test
- const uint64_t small_txt = 12; // "Hello World\n"
- const uint64_t medium_bin = 5 * 1024 * 1024; // 5MB
- const uint64_t large_bin = 10 * 1024 * 1024; // 10MB
-
- // Bucket1: small.txt + medium.bin
- const uint64_t bucket1_bytes = small_txt + medium_bin;
- const uint64_t bucket1_objects = 2;
-
- // Bucket2: large.bin
- const uint64_t bucket2_bytes = large_bin;
- const uint64_t bucket2_objects = 1;
-
- // User totals
- const uint64_t user_bytes = bucket1_bytes + bucket2_bytes;
- const uint64_t user_objects = 3;
-
- std::cout << "Uploading files to buckets:" << std::endl;
- std::cout << " " << bucket1 << ": small.txt (12B) + medium.bin (5MB) = "
- << bucket1_bytes << " bytes, " << bucket1_objects << " objects" << std::endl;
- std::cout << " " << bucket2 << ": large.bin (10MB) = "
- << bucket2_bytes << " bytes, " << bucket2_objects << " objects" << std::endl;
-
- // Update cache
- ASSERT_EQ(0, cache->update_bucket_stats(bucket1, bucket1_bytes, bucket1_objects));
- ASSERT_EQ(0, cache->update_bucket_stats(bucket2, bucket2_bytes, bucket2_objects));
- ASSERT_EQ(0, cache->update_user_stats(user_id, user_bytes, user_objects));
-
- // Verify bucket1
- auto b1_stats = cache->get_bucket_stats(bucket1);
- ASSERT_TRUE(b1_stats.has_value());
- EXPECT_EQ(bucket1_bytes, b1_stats->bytes_used);
- EXPECT_EQ(bucket1_objects, b1_stats->num_objects);
- std::cout << " Bucket1 verified: " << b1_stats->bytes_used << " bytes, "
- << b1_stats->num_objects << " objects" << std::endl;
-
- // Verify bucket2
- auto b2_stats = cache->get_bucket_stats(bucket2);
- ASSERT_TRUE(b2_stats.has_value());
- EXPECT_EQ(bucket2_bytes, b2_stats->bytes_used);
- EXPECT_EQ(bucket2_objects, b2_stats->num_objects);
- std::cout << " Bucket2 verified: " << b2_stats->bytes_used << " bytes, "
- << b2_stats->num_objects << " objects" << std::endl;
-
- // Verify user totals
- auto user_stats = cache->get_user_stats(user_id);
- ASSERT_TRUE(user_stats.has_value());
- EXPECT_EQ(user_bytes, user_stats->bytes_used);
- EXPECT_EQ(user_objects, user_stats->num_objects);
-
- std::cout << " User verified: " << user_stats->bytes_used << " bytes (~"
- << (user_stats->bytes_used / (1024 * 1024)) << "MB), "
- << user_stats->num_objects << " objects" << std::endl;
-
- // Verify expected totals
- EXPECT_GE(user_bytes, 15 * 1024 * 1024) << "User should have ~15MB";
- EXPECT_EQ(3u, user_objects) << "User should have exactly 3 objects";
-
- std::cout << " All statistics match manual test expectations" << std::endl;
+// Test persistence across cache restart (simulates RGW restart)
+TEST_F(TestRGWUsageCache, PersistenceAcrossRestart) {
+ std::cout << "\n=== Testing Persistence Across Restart ===" << std::endl;
+
+ const std::string user_id = "persistence_test_user";
+ const uint64_t bytes = 15 * 1024 * 1024; // 15MB
+ const uint64_t objects = 3;
+
+ // Add user stats
+ ASSERT_EQ(0, cache->update_user_stats(user_id, bytes, objects));
+
+ // Verify stats exist
+ auto stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(bytes, stats->bytes_used);
+ std::cout << "Initial stats cached: " << bytes << " bytes" << std::endl;
+
+ // Shutdown cache (simulates RGW shutdown)
+ cache->shutdown();
+ cache.reset();
+ std::cout << "Cache shutdown (simulating RGW restart)" << std::endl;
+
+ // Recreate cache with same path (simulates RGW restart)
+ if (g_test_context) {
+ cache = std::make_unique<rgw::UsageCache>(g_test_context, config);
+ } else {
+ cache = std::make_unique<rgw::UsageCache>(config);
+ }
+ ASSERT_EQ(0, cache->init());
+ std::cout << "Cache reinitialized" << std::endl;
+
+ // Verify stats persisted
+ stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value()) << "Stats should persist across restart";
+ EXPECT_EQ(bytes, stats->bytes_used);
+ EXPECT_EQ(objects, stats->num_objects);
+ std::cout << "Stats persisted: " << stats->bytes_used << " bytes" << std::endl;
+
+ std::cout << "Persistence test PASSED!" << std::endl;
}
-TEST_F(TestRGWUsageCache, RepeatedAccessCacheHits) {
- // Simulate: s3cmd ls s3://bucket1 (multiple times)
- // Validate cache hit behavior
-
- std::cout << "\n=== Testing Repeated Access Cache Hits ===" << std::endl;
-
- const std::string bucket_name = "bucket1";
-
- // Add bucket to cache
- ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, 5 * 1024 * 1024, 2));
- std::cout << "Added bucket to cache" << std::endl;
-
- // Simulate multiple s3cmd ls operations
- std::cout << "Simulating multiple bucket listings:" << std::endl;
- for (int i = 0; i < 3; ++i) {
- auto stats = cache->get_bucket_stats(bucket_name);
- ASSERT_TRUE(stats.has_value()) << "Failed on iteration " << (i + 1);
- std::cout << " Listing " << (i + 1) << ": Found bucket with "
- << stats->bytes_used << " bytes" << std::endl;
+// Test simulating background refresh from RADOS
+TEST_F(TestRGWUsageCache, BackgroundRefreshSimulation) {
+ std::cout << "\n=== Simulating Background Refresh from RADOS ===" << std::endl;
+
+ const std::string user_id = "refresh_test_user";
+
+ // Initial state (simulating first RADOS sync)
+ std::cout << "Initial RADOS sync: 1MB, 10 objects" << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 1024 * 1024, 10));
+
+ auto stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(1024 * 1024u, stats->bytes_used);
+ EXPECT_EQ(10u, stats->num_objects);
+
+ // Simulate user uploading more data (RADOS has new values)
+ std::cout << "Simulating background refresh with new RADOS data: 2MB, 20 objects" << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 2 * 1024 * 1024, 20));
+
+ // Should have fresh data from "RADOS"
+ stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(2 * 1024 * 1024u, stats->bytes_used);
+ EXPECT_EQ(20u, stats->num_objects);
+ std::cout << "Background refresh successful - stats updated" << std::endl;
+
+ // Simulate bucket deletion (RADOS values decrease)
+ std::cout << "Simulating bucket deletion: 1MB, 15 objects" << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 1024 * 1024, 15));
+
+ stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(1024 * 1024u, stats->bytes_used);
+ EXPECT_EQ(15u, stats->num_objects);
+ std::cout << "Stats correctly reflect deletion" << std::endl;
+}
+
+// Test multi-user scenario (cluster coherence simulation)
+TEST_F(TestRGWUsageCache, MultiUserClusterCoherence) {
+ std::cout << "\n=== Testing Multi-User Cluster Coherence ===" << std::endl;
+
+ // Simulate multiple users being synced from RADOS
+ struct UserData {
+ std::string id;
+ uint64_t bytes;
+ uint64_t objects;
+ };
+
+ std::vector<UserData> users = {
+ {"testuser", 348160, 29},
+ {"testuser2", 10240, 1},
+ {"admin", 1024 * 1024, 100}
+ };
+
+ // Simulate RADOS sync for all users
+ for (const auto& user : users) {
+ ASSERT_EQ(0, cache->update_user_stats(user.id, user.bytes, user.objects));
+ std::cout << "Synced user " << user.id << ": " << user.bytes << " bytes, "
+ << user.objects << " objects" << std::endl;
}
-
- std::cout << " All listings successful - bucket data retrieved from cache" << std::endl;
+
+ // Verify all users have correct stats
+ for (const auto& user : users) {
+ auto stats = cache->get_user_stats(user.id);
+ ASSERT_TRUE(stats.has_value()) << "User " << user.id << " not found";
+ EXPECT_EQ(user.bytes, stats->bytes_used);
+ EXPECT_EQ(user.objects, stats->num_objects);
+ }
+
+ std::cout << "All users verified - cluster coherence maintained" << std::endl;
}
+// Test performance (no sync_owner_stats in path)
TEST_F(TestRGWUsageCache, PerformanceNoSyncOwnerStats) {
- // CRITICAL: Verify cache operations are fast (not 90ms like cls_bucket_head)
-
std::cout << "\n=== Performance Test: No sync_owner_stats Blocking ===" << std::endl;
-
+
const int num_operations = 1000;
std::vector<std::string> bucket_names;
-
+
// Pre-populate cache with buckets
for (int i = 0; i < 10; ++i) {
std::string name = "perf_bucket_" + std::to_string(i);
bucket_names.push_back(name);
ASSERT_EQ(0, cache->update_bucket_stats(name, i * 1024, i));
}
-
- std::cout << "Running " << num_operations << " cache operations..." << std::endl;
-
+
+ std::cout << "Running " << num_operations << " cache read operations..." << std::endl;
+
// Measure time for many get operations
auto start = std::chrono::high_resolution_clock::now();
-
+
+ for (int i = 0; i < num_operations; ++i) {
+ std::string& bucket = bucket_names[i % bucket_names.size()];
+ auto stats = cache->get_bucket_stats(bucket);
+ ASSERT_TRUE(stats.has_value());
+ }
+
auto end = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
-
+
double total_ms = duration.count() / 1000.0;
double avg_us = static_cast<double>(duration.count()) / num_operations;
double avg_ms = avg_us / 1000.0;
-
- std::cout << "\nPerformance Results:" << std::endl;
+
+ std::cout << "Performance Results:" << std::endl;
std::cout << " Total operations: " << num_operations << std::endl;
std::cout << " Total time: " << total_ms << " ms" << std::endl;
std::cout << " Average per operation: " << avg_us << " μs (" << avg_ms << " ms)" << std::endl;
-
- // CRITICAL: Should be WAY faster than 90ms
- EXPECT_LT(avg_ms, 10.0)
+
+ EXPECT_LT(avg_ms, 10.0)
<< "Operations too slow (" << avg_ms << "ms) - possible sync_owner_stats in path";
-
- // Should be sub-millisecond for in-memory cache
+
+ // Should be sub-millisecond for cache operations
EXPECT_LT(avg_ms, 1.0)
<< "Cache operations should be < 1ms, got " << avg_ms << "ms";
-
+
double speedup = 90.0 / avg_ms;
- std::cout << " Performance test PASSED!" << std::endl;
- std::cout << " Operations are ~" << speedup << "x faster than the old cls_bucket_head bug" << std::endl;
+ std::cout << "Performance test PASSED!" << std::endl;
+ std::cout << "Operations are ~" << static_cast<int>(speedup) << "x faster than the old cls_bucket_head bug" << std::endl;
}
+// Test statistics accuracy
TEST_F(TestRGWUsageCache, StatisticsAccuracy) {
- // Verify statistics are accurate (not approximate)
-
std::cout << "\n=== Testing Statistics Accuracy ===" << std::endl;
-
+
const std::string user_id = "accuracy_user";
-
- // Known values
- const uint64_t expected_bytes = 15728652; // Exact: 12 + 5MB + 10MB
- const uint64_t expected_objects = 3;
-
+
+ // Known values from manual test
+ const uint64_t expected_bytes = 348160; // From actual test
+ const uint64_t expected_objects = 29;
+
// Update cache
ASSERT_EQ(0, cache->update_user_stats(user_id, expected_bytes, expected_objects));
-
+
// Retrieve and verify exact match
auto stats = cache->get_user_stats(user_id);
ASSERT_TRUE(stats.has_value());
-
+
std::cout << "Expected: " << expected_bytes << " bytes, " << expected_objects << " objects" << std::endl;
std::cout << "Cached: " << stats->bytes_used << " bytes, " << stats->num_objects << " objects" << std::endl;
-
- // For in-memory cache, should be exact
+
+ // For cache backed by RADOS, should be exact
EXPECT_EQ(expected_bytes, stats->bytes_used) << "Byte count should be exact";
EXPECT_EQ(expected_objects, stats->num_objects) << "Object count should be exact";
-
- std::cout << " Statistics are exact (no approximation)" << std::endl;
-}
-TEST_F(TestRGWUsageCache, BackgroundRefreshSimulation) {
- // Simulate background refresh updating expired entries
-
- std::cout << "\n=== Simulating Background Refresh ===" << std::endl;
-
- const std::string bucket_name = "refresh_bucket";
-
- // Initial state
- std::cout << "Initial upload: 1MB, 10 objects" << std::endl;
- ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, 1024 * 1024, 10));
-
- auto stats = cache->get_bucket_stats(bucket_name);
- ASSERT_TRUE(stats.has_value());
- EXPECT_EQ(1024 * 1024u, stats->bytes_used);
- std::cout << " Initial stats cached" << std::endl;
-
- // Wait for expiry
- std::cout << "Waiting for entry to expire..." << std::endl;
- std::this_thread::sleep_for(std::chrono::milliseconds(2500));
-
- // Entry should be expired
- stats = cache->get_bucket_stats(bucket_name);
- EXPECT_FALSE(stats.has_value()) << "Entry should be expired";
- std::cout << " Entry expired as expected" << std::endl;
-
- // Simulate background refresh with updated stats
- std::cout << "Simulating background refresh: 2MB, 20 objects" << std::endl;
- ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, 2 * 1024 * 1024, 20));
-
- // Should now have fresh data
- stats = cache->get_bucket_stats(bucket_name);
- ASSERT_TRUE(stats.has_value());
- EXPECT_EQ(2 * 1024 * 1024u, stats->bytes_used);
- EXPECT_EQ(20u, stats->num_objects);
- std::cout << " Background refresh successful - stats updated" << std::endl;
+ std::cout << "Statistics are exact (no approximation)" << std::endl;
}
+// Test concurrent access simulation
TEST_F(TestRGWUsageCache, ConcurrentAccessSimulation) {
- // Simulate concurrent access from multiple operations
- // (Like multiple s3cmd operations happening simultaneously)
-
std::cout << "\n=== Simulating Concurrent Access ===" << std::endl;
-
+
const int num_threads = 10;
const int operations_per_thread = 100;
-
+
// Pre-populate some buckets
for (int i = 0; i < num_threads; ++i) {
std::string bucket_name = "concurrent_bucket_" + std::to_string(i);
ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, i * 1024, i));
}
-
- std::cout << "Launching " << num_threads << " threads, "
+
+ std::cout << "Launching " << num_threads << " threads, "
<< operations_per_thread << " operations each..." << std::endl;
-
+
std::atomic<int> success_count{0};
std::atomic<int> failure_count{0};
std::vector<std::thread> threads;
-
+
auto start = std::chrono::high_resolution_clock::now();
-
+
for (int t = 0; t < num_threads; ++t) {
- threads.emplace_back([this, t, &success_count, &failure_count]() {
+ threads.emplace_back([this, t, operations_per_thread, &success_count, &failure_count]() {
std::string bucket_name = "concurrent_bucket_" + std::to_string(t);
-
+
for (int i = 0; i < operations_per_thread; ++i) {
auto stats = cache->get_bucket_stats(bucket_name);
if (stats.has_value()) {
}
});
}
-
+
// Wait for all threads
for (auto& thread : threads) {
thread.join();
}
-
+
auto end = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
-
+
int total_ops = num_threads * operations_per_thread;
- std::cout << "\nConcurrent Access Results:" << std::endl;
+ std::cout << "Concurrent Access Results:" << std::endl;
std::cout << " Total operations: " << total_ops << std::endl;
std::cout << " Successful: " << success_count << std::endl;
std::cout << " Failed: " << failure_count << std::endl;
std::cout << " Time: " << duration.count() << " ms" << std::endl;
- std::cout << " Average: " << (static_cast<double>(duration.count()) / total_ops)
- << " ms/op" << std::endl;
-
+
// All should succeed
EXPECT_EQ(total_ops, success_count.load()) << "All operations should succeed";
EXPECT_EQ(0, failure_count.load()) << "No operations should fail";
-
- std::cout << " Cache handled concurrent access successfully" << std::endl;
+
+ std::cout << "Cache handled concurrent access successfully" << std::endl;
}
+// Test complete workflow integration
TEST_F(TestRGWUsageCache, CompleteWorkflowIntegration) {
- // Integration test covering the complete manual test workflow
-
std::cout << "\n=== Complete Workflow Integration Test ===" << std::endl;
- std::cout << "Simulating entire manual test procedure...\n" << std::endl;
-
+ std::cout << "Simulating entire manual test procedure with new design...\n" << std::endl;
+
// Step 1: Setup
std::cout << "[Step 1] Creating user and buckets" << std::endl;
const std::string user_id = "testuser";
- const std::string bucket1 = "bucket1";
- const std::string bucket2 = "bucket2";
-
- // Step 2: Upload files
- std::cout << "[Step 2] Uploading files" << std::endl;
- std::cout << " bucket1: small.txt + medium.bin" << std::endl;
- std::cout << " bucket2: large.bin" << std::endl;
-
- uint64_t b1_bytes = 12 + (5 * 1024 * 1024);
- uint64_t b2_bytes = 10 * 1024 * 1024;
- uint64_t user_bytes = b1_bytes + b2_bytes;
-
- ASSERT_EQ(0, cache->update_bucket_stats(bucket1, b1_bytes, 2));
- ASSERT_EQ(0, cache->update_bucket_stats(bucket2, b2_bytes, 1));
- ASSERT_EQ(0, cache->update_user_stats(user_id, user_bytes, 3));
-
- // Step 3: Verify user stats (like checking perf counters)
+ const std::string bucket1 = "testa";
+ const std::string bucket2 = "testb";
+ const std::string bucket3 = "testc";
+
+ // Step 2: Simulate background sync from RADOS after uploads
+ std::cout << "[Step 2] Simulating RADOS sync after file uploads" << std::endl;
+
+ // User uploads to multiple buckets, background thread syncs from RADOS
+ uint64_t user_bytes = 40960; // 4 x 10KB
+ uint64_t user_objects = 4;
+
+ ASSERT_EQ(0, cache->update_user_stats(user_id, user_bytes, user_objects));
+ std::cout << " RADOS sync: " << user_bytes << " bytes, " << user_objects << " objects" << std::endl;
+
+ // Step 3: Verify user stats
std::cout << "[Step 3] Verifying user statistics" << std::endl;
auto user_stats = cache->get_user_stats(user_id);
ASSERT_TRUE(user_stats.has_value());
- std::cout << " used_bytes: " << user_stats->bytes_used << std::endl;
- std::cout << " num_objects: " << user_stats->num_objects << std::endl;
EXPECT_EQ(user_bytes, user_stats->bytes_used);
+ EXPECT_EQ(user_objects, user_stats->num_objects);
+ std::cout << " Verified: " << user_stats->bytes_used << " bytes, "
+ << user_stats->num_objects << " objects" << std::endl;
+
+ // Step 4: Simulate more uploads and RADOS sync
+ std::cout << "[Step 4] Simulating more uploads" << std::endl;
+ user_bytes = 71680; // Updated after more uploads
+ user_objects = 4;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, user_bytes, user_objects));
+
+ user_stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(user_stats.has_value());
+ EXPECT_EQ(71680u, user_stats->bytes_used);
+ std::cout << " Updated: " << user_stats->bytes_used << " bytes" << std::endl;
+
+ // Step 5: Simulate object deletion and RADOS sync
+ std::cout << "[Step 5] Simulating object deletion" << std::endl;
+ user_bytes = 40960; // After deletion
+ user_objects = 3;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, user_bytes, user_objects));
+
+ user_stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(user_stats.has_value());
+ EXPECT_EQ(40960u, user_stats->bytes_used);
EXPECT_EQ(3u, user_stats->num_objects);
-
- // Step 4: Verify bucket stats
- std::cout << "[Step 4] Verifying bucket statistics" << std::endl;
- auto b1_stats = cache->get_bucket_stats(bucket1);
- ASSERT_TRUE(b1_stats.has_value());
- std::cout << " bucket1 used_bytes: " << b1_stats->bytes_used << std::endl;
- std::cout << " bucket1 num_objects: " << b1_stats->num_objects << std::endl;
- EXPECT_EQ(b1_bytes, b1_stats->bytes_used);
- EXPECT_EQ(2u, b1_stats->num_objects);
-
- // Step 5: Test cache hits (multiple listings)
- std::cout << "[Step 5] Testing cache hit behavior" << std::endl;
- for (int i = 0; i < 3; ++i) {
- auto stats = cache->get_bucket_stats(bucket1);
- ASSERT_TRUE(stats.has_value());
- }
- std::cout << " Performed 3 bucket listings - all from cache" << std::endl;
-
- // Step 6: Verify cache size
- std::cout << "[Step 6] Verifying cache metrics" << std::endl;
- size_t cache_size = cache->get_cache_size();
- std::cout << " cache_size: " << cache_size << std::endl;
- EXPECT_GE(cache_size, 3u); // user + 2 buckets
-
+ std::cout << " After deletion: " << user_stats->bytes_used << " bytes, "
+ << user_stats->num_objects << " objects" << std::endl;
+
+ // Step 6: Simulate bucket deletion (critical bug fix test)
+ std::cout << "[Step 6] Simulating bucket deletion (bug fix test)" << std::endl;
+ user_bytes = 30720; // After bucket deletion
+ user_objects = 2;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, user_bytes, user_objects));
+
+ user_stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(user_stats.has_value());
+ EXPECT_EQ(30720u, user_stats->bytes_used);
+ EXPECT_EQ(2u, user_stats->num_objects);
+ std::cout << " After bucket deletion: " << user_stats->bytes_used << " bytes" << std::endl;
+ std::cout << " (Stats correct - bucket deletion bug FIXED!)" << std::endl;
+
// Step 7: Performance check
std::cout << "[Step 7] Performance regression check" << std::endl;
auto start = std::chrono::high_resolution_clock::now();
for (int i = 0; i < 100; ++i) {
- cache->get_bucket_stats(bucket1);
+ cache->get_user_stats(user_id);
}
auto end = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
double avg = static_cast<double>(duration.count()) / 100.0;
std::cout << " 100 operations: " << duration.count() << " ms (avg: " << avg << " ms/op)" << std::endl;
EXPECT_LT(avg, 10.0);
-
+
std::cout << "\n=== Complete Workflow Test PASSED ===" << std::endl;
- std::cout << "All manual test scenarios validated!" << std::endl;
+ std::cout << "New design validated: RADOS as source of truth!" << std::endl;
}
// Main function
-int main(int argc, char **argv) {
- // Initialize Google Test
+int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
- // Initialize Ceph context
std::map<std::string, std::string> defaults = {
{"debug_rgw", "20"},
{"keyring", "keyring"},
};
std::vector<const char*> args;
+ args.push_back("--no-mon-config");
+
auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT,
CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
-
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE | CINIT_FLAG_NO_MON_CONFIG);
+
// Store the context for test fixtures to use
g_test_context = cct.get();
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab ft=cpp
+// vim: ts=8 sw=2 smarttab
#include <gtest/gtest.h>
#include <thread>
protected:
std::unique_ptr<rgw::UsageCache> cache;
std::string test_db_path;
-
+
void SetUp() override {
ASSERT_NE(g_cct, nullptr) << "CephContext not initialized";
-
+
// Generate unique test database path
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<> dis(1000000, 9999999);
test_db_path = "/tmp/test_rgw_perf_" + std::to_string(dis(gen)) + ".mdb";
-
+
// Initialize cache with CephContext to enable internal perf counters
+ // NO TTL in new design
rgw::UsageCache::Config config;
config.db_path = test_db_path;
config.max_db_size = 1 << 20; // 1MB
- config.ttl = std::chrono::seconds(2);
-
+
// Use the constructor that accepts CephContext to enable perf counters
cache = std::make_unique<rgw::UsageCache>(g_cct, config);
-
+
ASSERT_EQ(0, cache->init());
}
-
+
void TearDown() override {
cache->shutdown();
cache.reset();
-
+
// Clean up test database
try {
fs::remove(test_db_path);
}
};
-TEST_F(TestRGWUsagePerfCounters, BasicMetrics) {
- // First, just test that the cache works
- std::cout << "Testing basic cache operations..." << std::endl;
-
- // Test basic cache functionality first
- auto stats = cache->get_user_stats("nonexistent");
- EXPECT_FALSE(stats.has_value());
-
- // Add a user
- ASSERT_EQ(0, cache->update_user_stats("user1", 1024, 10));
-
- // Access existing user
- stats = cache->get_user_stats("user1");
- ASSERT_TRUE(stats.has_value());
- EXPECT_EQ(1024u, stats->bytes_used);
- EXPECT_EQ(10u, stats->num_objects);
-
- std::cout << "Basic cache operations successful" << std::endl;
-
- // Test performance counters - they should be available since we used CephContext
- uint64_t hits = cache->get_cache_hits();
- uint64_t misses = cache->get_cache_misses();
+TEST_F(TestRGWUsagePerfCounters, SimplifiedMetrics) {
+ std::cout << "\n=== Testing Simplified Cache Metrics ===" << std::endl;
+
+ // In new design, we only track:
+ // - cache_hits (debug level)
+ // - cache_misses (debug level)
+ // - cache_updates (important - shows RADOS sync)
+ // - cache_size (important - shows LMDB growth)
+
+ // Miss
+ cache->get_user_stats("nonexistent");
- std::cout << "Performance counter values:" << std::endl;
- std::cout << " Hits: " << hits << std::endl;
- std::cout << " Misses: " << misses << std::endl;
+ // Update (simulating RADOS sync)
+ cache->update_user_stats("user1", 1024, 10);
- // We expect at least one miss (from the first get_user_stats)
- // and one hit (from the second get_user_stats)
- EXPECT_GE(misses, 1u);
- EXPECT_GE(hits, 1u);
+ // Hit
+ cache->get_user_stats("user1");
+
+ std::cout << "Metrics:" << std::endl;
+ std::cout << " cache_hits: " << cache->get_cache_hits() << std::endl;
+ std::cout << " cache_misses: " << cache->get_cache_misses() << std::endl;
+ std::cout << " cache_updates: " << cache->get_cache_updates() << std::endl;
+ std::cout << " cache_size: " << cache->get_cache_size() << std::endl;
+
+ EXPECT_GE(cache->get_cache_hits(), 1u);
+ EXPECT_GE(cache->get_cache_misses(), 1u);
+ EXPECT_GE(cache->get_cache_updates(), 1u);
+ EXPECT_EQ(1u, cache->get_cache_size());
+
+ std::cout << "Simplified metrics verified" << std::endl;
}
TEST_F(TestRGWUsagePerfCounters, HitRateCalculation) {
+ std::cout << "\n=== Testing Hit Rate Calculation ===" << std::endl;
+
// Perform a mix of hits and misses
cache->get_user_stats("miss1"); // Miss
cache->get_user_stats("miss2"); // Miss
-
+
cache->update_user_stats("hit1", 1024, 1);
cache->update_user_stats("hit2", 2048, 2);
-
+
cache->get_user_stats("hit1"); // Hit
cache->get_user_stats("hit2"); // Hit
cache->get_user_stats("miss3"); // Miss
-
+
uint64_t hits = cache->get_cache_hits();
uint64_t misses = cache->get_cache_misses();
-
+
EXPECT_EQ(2u, hits);
EXPECT_EQ(3u, misses);
-
+
double hit_rate = cache->get_hit_rate();
double expected_rate = (2.0 / 5.0) * 100.0; // 40%
-
+
EXPECT_DOUBLE_EQ(expected_rate, hit_rate);
-
+
std::cout << "Cache statistics:" << std::endl;
std::cout << " Hits: " << hits << std::endl;
std::cout << " Misses: " << misses << std::endl;
std::cout << " Hit rate: " << hit_rate << "%" << std::endl;
}
-TEST_F(TestRGWUsagePerfCounters, UserAndBucketSeparateTracking) {
- // Test that user and bucket stats are tracked separately
-
- // User operations
- cache->get_user_stats("user_miss"); // User miss
- cache->update_user_stats("user1", 1024, 1);
- cache->get_user_stats("user1"); // User hit
-
- // Bucket operations
- cache->get_bucket_stats("bucket_miss"); // Bucket miss
- cache->update_bucket_stats("bucket1", 2048, 2);
- cache->get_bucket_stats("bucket1"); // Bucket hit
-
- // Check overall counters (should include both)
- EXPECT_EQ(2u, cache->get_cache_hits()); // 1 user hit + 1 bucket hit
- EXPECT_EQ(2u, cache->get_cache_misses()); // 1 user miss + 1 bucket miss
-}
+TEST_F(TestRGWUsagePerfCounters, CacheUpdateTracking) {
+ std::cout << "\n=== Testing Cache Update Tracking ===" << std::endl;
-TEST_F(TestRGWUsagePerfCounters, ExpiredEntryTracking) {
- // Add a user
- ASSERT_EQ(0, cache->update_user_stats("expiry_test", 1024, 1));
+ // In new design, cache_updates is the key metric showing RADOS sync activity
- // Access it - should be a hit
- auto stats = cache->get_user_stats("expiry_test");
- ASSERT_TRUE(stats.has_value());
- EXPECT_EQ(1u, cache->get_cache_hits());
-
- // Wait for TTL to expire (2 seconds + buffer)
- std::this_thread::sleep_for(std::chrono::milliseconds(2500));
-
- // Access expired entry - should be a miss
- stats = cache->get_user_stats("expiry_test");
- EXPECT_FALSE(stats.has_value());
+ // Simulate RADOS sync updates
+ ASSERT_EQ(0, cache->update_user_stats("user1", 1024, 1));
+ ASSERT_EQ(0, cache->update_user_stats("user2", 2048, 2));
+ ASSERT_EQ(0, cache->update_user_stats("user3", 4096, 4));
+
+ uint64_t updates = cache->get_cache_updates();
+ std::cout << "Cache updates after 3 user syncs: " << updates << std::endl;
- // After expiry, we should have an additional miss
- EXPECT_EQ(1u, cache->get_cache_hits());
- EXPECT_EQ(1u, cache->get_cache_misses()); // The expired entry counts as a miss
+ EXPECT_GE(updates, 3u) << "Should have at least 3 cache updates";
+
+ // Verify cache size also tracked
+ size_t size = cache->get_cache_size();
+ std::cout << "Cache size: " << size << std::endl;
+ EXPECT_EQ(3u, size);
+
+ std::cout << "Cache update tracking verified" << std::endl;
}
TEST_F(TestRGWUsagePerfCounters, RemoveOperationTracking) {
+ std::cout << "\n=== Testing Remove Operation Tracking ===" << std::endl;
+
// Add users
cache->update_user_stats("user1", 1024, 1);
cache->update_user_stats("user2", 2048, 2);
-
+
// Remove one user
ASSERT_EQ(0, cache->remove_user_stats("user1"));
-
+
// Try to access removed user - should be a miss
auto stats = cache->get_user_stats("user1");
EXPECT_FALSE(stats.has_value());
EXPECT_EQ(1u, cache->get_cache_misses());
-
+
// Access existing user - should be a hit
stats = cache->get_user_stats("user2");
EXPECT_TRUE(stats.has_value());
EXPECT_EQ(1u, cache->get_cache_hits());
+
+ std::cout << "Remove operation tracking verified" << std::endl;
}
TEST_F(TestRGWUsagePerfCounters, ZeroDivisionInHitRate) {
+ std::cout << "\n=== Testing Zero Division in Hit Rate ===" << std::endl;
+
// Test hit rate calculation when there are no operations
double hit_rate = cache->get_hit_rate();
EXPECT_EQ(0.0, hit_rate); // Should handle division by zero gracefully
-
+
// After one miss
cache->get_user_stats("nonexistent");
hit_rate = cache->get_hit_rate();
EXPECT_EQ(0.0, hit_rate); // 0 hits / 1 total = 0%
-
+
// After one hit
cache->update_user_stats("user1", 1024, 1);
cache->get_user_stats("user1");
hit_rate = cache->get_hit_rate();
EXPECT_EQ(50.0, hit_rate); // 1 hit / 2 total = 50%
-}
-// Add these test cases to the existing TestRGWUsagePerfCounters test fixture
+ std::cout << "Zero division handling verified" << std::endl;
+}
TEST_F(TestRGWUsagePerfCounters, MultipleUserBucketScenario) {
- // This simulates a manual scenario test: multiple users with multiple buckets
- // User: testuser with bucket1 (small.txt + medium.bin) and bucket2 (large.bin)
-
std::cout << "\n=== Testing Multiple User/Bucket Scenario ===" << std::endl;
-
+
const std::string user_id = "testuser";
const std::string bucket1 = "bucket1";
const std::string bucket2 = "bucket2";
-
- // Simulate uploading to bucket1: small.txt (~12 bytes) + medium.bin (5MB)
+
+ // Simulate RADOS sync with aggregated stats
const uint64_t bucket1_bytes = 12 + (5 * 1024 * 1024);
const uint64_t bucket1_objects = 2;
-
- // Simulate uploading to bucket2: large.bin (10MB)
const uint64_t bucket2_bytes = 10 * 1024 * 1024;
const uint64_t bucket2_objects = 1;
-
- // Total user stats
const uint64_t total_user_bytes = bucket1_bytes + bucket2_bytes;
const uint64_t total_user_objects = bucket1_objects + bucket2_objects;
-
- // Update bucket stats
+
+ // Update stats (simulating background RADOS sync)
ASSERT_EQ(0, cache->update_bucket_stats(bucket1, bucket1_bytes, bucket1_objects));
ASSERT_EQ(0, cache->update_bucket_stats(bucket2, bucket2_bytes, bucket2_objects));
-
- // Update user stats (aggregated from buckets)
ASSERT_EQ(0, cache->update_user_stats(user_id, total_user_bytes, total_user_objects));
-
+
std::cout << "Updated stats:" << std::endl;
std::cout << " User: " << user_id << std::endl;
- std::cout << " Bucket1: " << bucket1 << " - " << bucket1_bytes << " bytes, "
- << bucket1_objects << " objects" << std::endl;
- std::cout << " Bucket2: " << bucket2 << " - " << bucket2_bytes << " bytes, "
- << bucket2_objects << " objects" << std::endl;
-
+ std::cout << " Bucket1: " << bucket1_bytes << " bytes, " << bucket1_objects << " objects" << std::endl;
+ std::cout << " Bucket2: " << bucket2_bytes << " bytes, " << bucket2_objects << " objects" << std::endl;
+
// Verify bucket1 stats
auto b1_stats = cache->get_bucket_stats(bucket1);
- ASSERT_TRUE(b1_stats.has_value()) << "Bucket1 stats not found";
+ ASSERT_TRUE(b1_stats.has_value());
EXPECT_EQ(bucket1_bytes, b1_stats->bytes_used);
EXPECT_EQ(bucket1_objects, b1_stats->num_objects);
- std::cout << " Bucket1 stats verified" << std::endl;
-
+
// Verify bucket2 stats
auto b2_stats = cache->get_bucket_stats(bucket2);
- ASSERT_TRUE(b2_stats.has_value()) << "Bucket2 stats not found";
+ ASSERT_TRUE(b2_stats.has_value());
EXPECT_EQ(bucket2_bytes, b2_stats->bytes_used);
EXPECT_EQ(bucket2_objects, b2_stats->num_objects);
- std::cout << " Bucket2 stats verified" << std::endl;
-
- // Verify user stats (should be sum of all buckets)
+
+ // Verify user stats
auto user_stats = cache->get_user_stats(user_id);
- ASSERT_TRUE(user_stats.has_value()) << "User stats not found";
+ ASSERT_TRUE(user_stats.has_value());
EXPECT_EQ(total_user_bytes, user_stats->bytes_used);
EXPECT_EQ(total_user_objects, user_stats->num_objects);
- std::cout << " User stats verified: " << total_user_bytes << " bytes (~15MB), "
- << total_user_objects << " objects" << std::endl;
-
- // Verify expected totals match manual test expectations
- // Expected: ~15MB total (5MB + 10MB + small file)
- const uint64_t expected_min_bytes = 15 * 1024 * 1024; // 15MB
- EXPECT_GE(user_stats->bytes_used, expected_min_bytes)
- << "User should have at least 15MB";
- EXPECT_EQ(3u, user_stats->num_objects)
- << "User should have exactly 3 objects";
-
- std::cout << " All statistics match expected values from manual test" << std::endl;
+
+ std::cout << "All statistics verified!" << std::endl;
}
TEST_F(TestRGWUsagePerfCounters, CacheHitOnRepeatedAccess) {
- // This simulates: s3cmd ls s3://bucket1 (multiple times)
- // Should see cache hits increase
-
std::cout << "\n=== Testing Cache Hits on Repeated Access ===" << std::endl;
-
+
const std::string bucket_name = "bucket1";
-
- // First, populate the cache
+
+ // Populate the cache (simulating RADOS sync)
ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, 5 * 1024 * 1024, 2));
-
- // Get initial hit/miss counts
+
uint64_t initial_hits = cache->get_cache_hits();
- uint64_t initial_misses = cache->get_cache_misses();
-
- std::cout << "Initial state - Hits: " << initial_hits
- << ", Misses: " << initial_misses << std::endl;
-
- // Simulate multiple bucket listings (like s3cmd ls s3://bucket1)
+
+ // Simulate multiple bucket listings
const int num_listings = 3;
for (int i = 0; i < num_listings; ++i) {
auto stats = cache->get_bucket_stats(bucket_name);
- ASSERT_TRUE(stats.has_value()) << "Failed to get bucket stats on iteration " << i;
- std::cout << " Listing " << (i + 1) << ": Found bucket with "
+ ASSERT_TRUE(stats.has_value());
+ std::cout << " Listing " << (i + 1) << ": Found bucket with "
<< stats->bytes_used << " bytes" << std::endl;
}
-
- // Get final hit/miss counts
- uint64_t final_hits = cache->get_cache_hits();
- uint64_t final_misses = cache->get_cache_misses();
-
- uint64_t hits_increase = final_hits - initial_hits;
-
- std::cout << "Final state - Hits: " << final_hits
- << ", Misses: " << final_misses << std::endl;
+
+ uint64_t hits_increase = cache->get_cache_hits() - initial_hits;
+
std::cout << "Cache hits increased by: " << hits_increase << std::endl;
-
- // We expect all accesses to be hits since the entry is already cached
- EXPECT_EQ(num_listings, hits_increase)
- << "Expected " << num_listings << " cache hits, got " << hits_increase;
-
- // Hit rate should be good
+
+ EXPECT_EQ(num_listings, hits_increase);
+
double hit_rate = cache->get_hit_rate();
- std::cout << " Cache hit rate: " << hit_rate << "%" << std::endl;
- EXPECT_GT(hit_rate, 0.0) << "Cache hit rate should be > 0%";
-
- std::cout << " Cache behavior verified - repeated access produces cache hits" << std::endl;
+ std::cout << "Cache hit rate: " << hit_rate << "%" << std::endl;
+ EXPECT_GT(hit_rate, 0.0);
+
+ std::cout << "Cache behavior verified - repeated access produces cache hits" << std::endl;
}
TEST_F(TestRGWUsagePerfCounters, PerformanceNoBlockingInIOPath) {
- // CRITICAL TEST: Verify that cache operations are fast
- // This simulates the fix for the 90ms cls_bucket_head() issue
-
std::cout << "\n=== Testing Performance: No Blocking in I/O Path ===" << std::endl;
-
- const int num_operations = 100;
+
+ const int num_operations = 1000;
const std::string bucket_prefix = "perf_bucket_";
-
- // Warm up - add some entries to cache
+
+ // Warm up - add entries to cache
for (int i = 0; i < 10; ++i) {
std::string bucket_name = bucket_prefix + std::to_string(i);
cache->update_bucket_stats(bucket_name, i * 1024, i);
}
-
- // Measure time for 100 get operations
+
+ // Measure time for operations
auto start = std::chrono::high_resolution_clock::now();
-
+
for (int i = 0; i < num_operations; ++i) {
std::string bucket_name = bucket_prefix + std::to_string(i % 10);
+ auto stats = cache->get_bucket_stats(bucket_name);
+ ASSERT_TRUE(stats.has_value());
}
-
+
auto end = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
-
+
double avg_per_op_us = static_cast<double>(duration.count()) / num_operations;
double avg_per_op_ms = avg_per_op_us / 1000.0;
-
+
std::cout << "Performance results:" << std::endl;
std::cout << " Total operations: " << num_operations << std::endl;
- std::cout << " Total time: " << duration.count() << " μs ("
- << (duration.count() / 1000.0) << " ms)" << std::endl;
- std::cout << " Average per operation: " << avg_per_op_us << " μs ("
- << avg_per_op_ms << " ms)" << std::endl;
-
- // CRITICAL ASSERTION: Operations should be MUCH faster than 90ms
- // Even with margin for test overhead, should be < 10ms per op
- EXPECT_LT(avg_per_op_ms, 10.0)
- << "Average operation time " << avg_per_op_ms
- << "ms is too high - should be < 10ms (was 90ms with sync_owner_stats bug)";
-
- // Better yet, should be sub-millisecond for cache hits
- EXPECT_LT(avg_per_op_ms, 1.0)
- << "Average operation time " << avg_per_op_ms
- << "ms should be < 1ms for cache hits";
-
- // Calculate cache hit rate - should be high since we're reusing buckets
- double hit_rate = cache->get_hit_rate();
- std::cout << " Cache hit rate: " << hit_rate << "%" << std::endl;
-
- std::cout << " Performance test PASSED - no blocking in I/O path" << std::endl;
- std::cout << " Operations are ~" << (90.0 / avg_per_op_ms) << "x faster than the old bug!" << std::endl;
+ std::cout << " Total time: " << duration.count() << " μs" << std::endl;
+ std::cout << " Average per operation: " << avg_per_op_us << " μs" << std::endl;
+
+ // CRITICAL: Should be MUCH faster than 90ms
+ EXPECT_LT(avg_per_op_ms, 10.0);
+ EXPECT_LT(avg_per_op_ms, 1.0);
+
+ std::cout << "Performance test PASSED - no blocking in I/O path" << std::endl;
+ std::cout << "Operations are ~" << static_cast<int>(90.0 / avg_per_op_ms)
+ << "x faster than the old bug!" << std::endl;
}
TEST_F(TestRGWUsagePerfCounters, CacheSizeAndUpdateTracking) {
- // This validates the cache size and update counters
- // Simulates checking: cache_size, cache_updates counters
-
std::cout << "\n=== Testing Cache Size and Update Tracking ===" << std::endl;
-
- // Initial state
+
uint64_t initial_size = cache->get_cache_size();
std::cout << "Initial cache size: " << initial_size << std::endl;
- EXPECT_EQ(0u, initial_size) << "Cache should start empty";
-
- // Add users and buckets
+ EXPECT_EQ(0u, initial_size);
+
const int num_users = 3;
const int num_buckets = 2;
-
+
for (int i = 0; i < num_users; ++i) {
std::string user_id = "user_" + std::to_string(i);
ASSERT_EQ(0, cache->update_user_stats(user_id, (i + 1) * 1024 * 1024, (i + 1) * 10));
}
-
+
for (int i = 0; i < num_buckets; ++i) {
std::string bucket_name = "bucket_" + std::to_string(i);
ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, (i + 1) * 512 * 1024, (i + 1) * 5));
}
-
- // Check cache size
+
uint64_t final_size = cache->get_cache_size();
std::cout << "Final cache size: " << final_size << std::endl;
-
- EXPECT_EQ(num_users + num_buckets, final_size)
- << "Cache size should reflect " << (num_users + num_buckets) << " entries";
-
- std::cout << " Cache size tracking verified" << std::endl;
-
- // Verify we can retrieve all entries
- for (int i = 0; i < num_users; ++i) {
- std::string user_id = "user_" + std::to_string(i);
- auto stats = cache->get_user_stats(user_id);
- ASSERT_TRUE(stats.has_value()) << "User " << user_id << " not found in cache";
- EXPECT_EQ((i + 1) * 1024 * 1024u, stats->bytes_used);
- }
-
- for (int i = 0; i < num_buckets; ++i) {
- std::string bucket_name = "bucket_" + std::to_string(i);
- auto stats = cache->get_bucket_stats(bucket_name);
- ASSERT_TRUE(stats.has_value()) << "Bucket " << bucket_name << " not found in cache";
- EXPECT_EQ((i + 1) * 512 * 1024u, stats->bytes_used);
- }
-
- std::cout << " All cache entries verified" << std::endl;
-
- // Check that cache hits occurred
- uint64_t hits = cache->get_cache_hits();
- std::cout << "Total cache hits: " << hits << std::endl;
- EXPECT_GT(hits, 0u) << "Should have some cache hits from retrievals";
-
- std::cout << " Cache metrics tracking verified" << std::endl;
+
+ EXPECT_EQ(num_users + num_buckets, final_size);
+
+ std::cout << "Cache size tracking verified" << std::endl;
}
TEST_F(TestRGWUsagePerfCounters, StatsAccuracyWithinTolerance) {
- // Verify that statistics are accurate within acceptable tolerance
-
std::cout << "\n=== Testing Statistics Accuracy ===" << std::endl;
-
+
const std::string user_id = "accuracy_test_user";
-
- // Upload files with known sizes (matching our manual test)
- struct FileUpload {
- std::string name;
- uint64_t size;
+
+ // Values from actual manual test
+ const uint64_t expected_bytes = 348160;
+ const uint64_t expected_objects = 29;
+
+ ASSERT_EQ(0, cache->update_user_stats(user_id, expected_bytes, expected_objects));
+
+ auto stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+
+ std::cout << "Expected: " << expected_bytes << " bytes, " << expected_objects << " objects" << std::endl;
+ std::cout << "Cached: " << stats->bytes_used << " bytes, " << stats->num_objects << " objects" << std::endl;
+
+ EXPECT_EQ(expected_bytes, stats->bytes_used);
+ EXPECT_EQ(expected_objects, stats->num_objects);
+
+ std::cout << "Statistics accuracy verified" << std::endl;
+}
+
+TEST_F(TestRGWUsagePerfCounters, BackgroundRefreshSimulation) {
+ std::cout << "\n=== Testing Background Refresh Simulation ===" << std::endl;
+
+ const std::string user_id = "refresh_test_user";
+
+ // Initial RADOS sync
+ std::cout << "Initial RADOS sync: 1MB, 10 objects" << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 1024 * 1024, 10));
+
+ auto stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(1024 * 1024u, stats->bytes_used);
+
+ // Simulate background refresh with updated RADOS values
+ std::cout << "Background refresh: 2MB, 20 objects" << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 2 * 1024 * 1024, 20));
+
+ stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(2 * 1024 * 1024u, stats->bytes_used);
+ EXPECT_EQ(20u, stats->num_objects);
+
+ std::cout << "Background refresh simulation successful" << std::endl;
+}
+
+TEST_F(TestRGWUsagePerfCounters, PersistenceAcrossRestart) {
+ std::cout << "\n=== Testing Persistence Across Restart ===" << std::endl;
+
+ const std::string user_id = "persist_user";
+ const uint64_t bytes = 122880;
+ const uint64_t objects = 6;
+
+ // Add stats
+ ASSERT_EQ(0, cache->update_user_stats(user_id, bytes, objects));
+ std::cout << "Stored: " << bytes << " bytes, " << objects << " objects" << std::endl;
+
+ // Get config before shutdown
+ rgw::UsageCache::Config config;
+ config.db_path = test_db_path;
+ config.max_db_size = 1 << 20;
+
+ // Shutdown and restart
+ cache->shutdown();
+ cache.reset();
+ std::cout << "Cache shutdown (simulating RGW restart)" << std::endl;
+
+ cache = std::make_unique<rgw::UsageCache>(g_cct, config);
+ ASSERT_EQ(0, cache->init());
+ std::cout << "Cache restarted" << std::endl;
+
+ // Verify stats persisted
+ auto stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value()) << "Stats should persist across restart";
+ EXPECT_EQ(bytes, stats->bytes_used);
+ EXPECT_EQ(objects, stats->num_objects);
+ std::cout << "Recovered: " << stats->bytes_used << " bytes, "
+ << stats->num_objects << " objects" << std::endl;
+
+ std::cout << "Persistence test PASSED!" << std::endl;
+}
+
+TEST_F(TestRGWUsagePerfCounters, MultiUserClusterCoherence) {
+ std::cout << "\n=== Testing Multi-User Cluster Coherence ===" << std::endl;
+
+ // Simulate multiple users with different stats (as if from RADOS)
+ struct UserData {
+ std::string id;
+ uint64_t bytes;
+ uint64_t objects;
};
-
- std::vector<FileUpload> files = {
- {"small.txt", 12}, // "Hello World\n"
- {"medium.bin", 5 * 1024 * 1024}, // 5MB
- {"large.bin", 10 * 1024 * 1024} // 10MB
+
+ std::vector<UserData> users = {
+ {"testuser", 348160, 29},
+ {"testuser2", 10240, 1},
+ {"admin", 1024 * 1024, 100}
};
-
- uint64_t total_bytes = 0;
- uint64_t total_objects = files.size();
-
- std::cout << "Simulating file uploads:" << std::endl;
- for (const auto& file : files) {
- total_bytes += file.size;
- std::cout << " " << file.name << ": " << file.size << " bytes" << std::endl;
+
+ // Simulate RADOS sync for all users
+ for (const auto& user : users) {
+ ASSERT_EQ(0, cache->update_user_stats(user.id, user.bytes, user.objects));
+ std::cout << "Synced " << user.id << ": " << user.bytes << " bytes" << std::endl;
}
-
- std::cout << "Total: " << total_bytes << " bytes ("
- << (total_bytes / (1024.0 * 1024.0)) << " MB), "
- << total_objects << " objects" << std::endl;
-
- // Update cache with total stats
- ASSERT_EQ(0, cache->update_user_stats(user_id, total_bytes, total_objects));
-
- // Retrieve and verify
- auto stats = cache->get_user_stats(user_id);
- ASSERT_TRUE(stats.has_value()) << "User stats not found";
-
- std::cout << "\nVerifying accuracy:" << std::endl;
- std::cout << " Expected bytes: " << total_bytes << std::endl;
- std::cout << " Cached bytes: " << stats->bytes_used << std::endl;
- std::cout << " Expected objects: " << total_objects << std::endl;
- std::cout << " Cached objects: " << stats->num_objects << std::endl;
-
- // Exact match for in-memory cache
- EXPECT_EQ(total_bytes, stats->bytes_used) << "Byte count should be exact";
- EXPECT_EQ(total_objects, stats->num_objects) << "Object count should be exact";
-
- // Verify totals match expectations (~15MB)
- const uint64_t expected_mb = 15;
- const uint64_t expected_bytes = expected_mb * 1024 * 1024;
- const double tolerance = 0.1; // 10% tolerance
-
- double bytes_diff = std::abs(static_cast<double>(stats->bytes_used) - expected_bytes);
- double bytes_diff_pct = (bytes_diff / expected_bytes) * 100.0;
-
- std::cout << " Difference from expected ~15MB: " << bytes_diff_pct << "%" << std::endl;
-
- EXPECT_LT(bytes_diff_pct, tolerance * 100.0)
- << "Byte count should be within " << (tolerance * 100.0) << "% of expected";
-
- std::cout << " Statistics accuracy verified (within tolerance)" << std::endl;
+
+ // Verify all users
+ for (const auto& user : users) {
+ auto stats = cache->get_user_stats(user.id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(user.bytes, stats->bytes_used);
+ EXPECT_EQ(user.objects, stats->num_objects);
+ }
+
+ std::cout << "Multi-user cluster coherence verified" << std::endl;
}
-TEST_F(TestRGWUsagePerfCounters, ExpiryAndRefreshScenario) {
- // Test that expired entries are handled correctly
- // Simulates the background refresh mechanism
-
- std::cout << "\n=== Testing Expiry and Refresh Scenario ===" << std::endl;
-
- const std::string bucket_name = "expiry_test_bucket";
-
- // Add bucket stats
- ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, 1024 * 1024, 10));
-
- // Verify stats exist
- auto stats = cache->get_bucket_stats(bucket_name);
- ASSERT_TRUE(stats.has_value()) << "Stats should exist initially";
- std::cout << " Initial stats cached" << std::endl;
-
- // Access should be a cache hit
- uint64_t hits_before = cache->get_cache_hits();
- stats = cache->get_bucket_stats(bucket_name);
- uint64_t hits_after = cache->get_cache_hits();
- EXPECT_EQ(hits_before + 1, hits_after) << "Should have one more cache hit";
- std::cout << " Cache hit recorded" << std::endl;
-
- // Wait for TTL to expire (2 seconds + buffer)
- std::cout << "Waiting for cache entry to expire (2.5s)..." << std::endl;
- std::this_thread::sleep_for(std::chrono::milliseconds(2500));
-
- // Access expired entry - should be a miss
- uint64_t misses_before = cache->get_cache_misses();
- stats = cache->get_bucket_stats(bucket_name);
- uint64_t misses_after = cache->get_cache_misses();
-
- EXPECT_FALSE(stats.has_value()) << "Expired entry should not be returned";
- EXPECT_EQ(misses_before + 1, misses_after) << "Should have one more cache miss";
- std::cout << " Expired entry handled correctly (cache miss)" << std::endl;
-
- // Simulate background refresh - re-add the stats
- std::cout << "Simulating background refresh..." << std::endl;
- ASSERT_EQ(0, cache->update_bucket_stats(bucket_name, 2 * 1024 * 1024, 20));
-
- // Now should be in cache again with new values
- stats = cache->get_bucket_stats(bucket_name);
- ASSERT_TRUE(stats.has_value()) << "Refreshed stats should exist";
- EXPECT_EQ(2 * 1024 * 1024u, stats->bytes_used) << "Should have updated bytes";
- EXPECT_EQ(20u, stats->num_objects) << "Should have updated objects";
-
- std::cout << "Background refresh simulation successful" << std::endl;
- std::cout << "Cache properly handles expiry and refresh cycle" << std::endl;
+TEST_F(TestRGWUsagePerfCounters, BucketDeletionBugFix) {
+ std::cout << "\n=== Testing Bucket Deletion Bug Fix ===" << std::endl;
+
+ const std::string user_id = "testuser";
+
+ // Initial state: 3 buckets, 40KB total
+ std::cout << "Initial: 40960 bytes, 4 objects" << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 40960, 4));
+
+ // After bucket deletion: stats from RADOS should be correct
+ std::cout << "After bucket deletion (RADOS sync): 30720 bytes, 2 objects" << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 30720, 2));
+
+ auto stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(30720u, stats->bytes_used);
+ EXPECT_EQ(2u, stats->num_objects);
+
+ // Create new bucket after deletion
+ std::cout << "After new bucket (RADOS sync): 51200 bytes, 3 objects" << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 51200, 3));
+
+ stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(51200u, stats->bytes_used);
+ EXPECT_EQ(3u, stats->num_objects);
+
+ std::cout << "Bucket deletion bug fix VERIFIED!" << std::endl;
+ std::cout << "(Stats remain correct after delete/create cycle)" << std::endl;
}
TEST_F(TestRGWUsagePerfCounters, ComprehensiveManualTestWorkflow) {
- // This test combines all the test steps into one comprehensive test
-
std::cout << "\n=== Comprehensive Manual Test Workflow ===" << std::endl;
- std::cout << "Simulating complete manual testing scenario..." << std::endl;
-
+ std::cout << "Simulating complete manual testing scenario with NEW DESIGN...\n" << std::endl;
+
// Step 1: Create user and buckets
- std::cout << "\n[Step 1] Creating user and buckets..." << std::endl;
+ std::cout << "[Step 1] Creating user and buckets..." << std::endl;
const std::string user_id = "testuser";
- const std::string bucket1 = "bucket1";
- const std::string bucket2 = "bucket2";
-
- // Step 2: Upload files
- std::cout << "[Step 2] Uploading files..." << std::endl;
-
- // Bucket1: small.txt + medium.bin
- uint64_t b1_bytes = 12 + (5 * 1024 * 1024); // ~5MB
- ASSERT_EQ(0, cache->update_bucket_stats(bucket1, b1_bytes, 2));
- std::cout << " Uploaded to " << bucket1 << ": 2 files, " << b1_bytes << " bytes" << std::endl;
-
- // Bucket2: large.bin
- uint64_t b2_bytes = 10 * 1024 * 1024; // 10MB
- ASSERT_EQ(0, cache->update_bucket_stats(bucket2, b2_bytes, 1));
- std::cout << " Uploaded to " << bucket2 << ": 1 file, " << b2_bytes << " bytes" << std::endl;
-
- // User total
- uint64_t user_bytes = b1_bytes + b2_bytes;
- uint64_t user_objects = 3;
- ASSERT_EQ(0, cache->update_user_stats(user_id, user_bytes, user_objects));
- std::cout << " User total: " << user_objects << " objects, " << user_bytes << " bytes" << std::endl;
-
- // Step 3: Check user statistics
- std::cout << "\n[Step 3] Checking user statistics..." << std::endl;
- auto user_stats = cache->get_user_stats(user_id);
- ASSERT_TRUE(user_stats.has_value());
- std::cout << " rgw_user_" << user_id << ":" << std::endl;
- std::cout << " used_bytes: " << user_stats->bytes_used << std::endl;
- std::cout << " num_objects: " << user_stats->num_objects << std::endl;
- EXPECT_EQ(user_bytes, user_stats->bytes_used);
- EXPECT_EQ(3u, user_stats->num_objects);
- std::cout << " User stats verified" << std::endl;
-
- // Step 4: Check bucket statistics
- std::cout << "\n[Step 4] Checking bucket statistics..." << std::endl;
- auto b1_stats = cache->get_bucket_stats(bucket1);
- ASSERT_TRUE(b1_stats.has_value());
- std::cout << " rgw_bucket_" << bucket1 << ":" << std::endl;
- std::cout << " used_bytes: " << b1_stats->bytes_used << std::endl;
- std::cout << " num_objects: " << b1_stats->num_objects << std::endl;
- EXPECT_EQ(b1_bytes, b1_stats->bytes_used);
- EXPECT_EQ(2u, b1_stats->num_objects);
- std::cout << " Bucket1 stats verified" << std::endl;
-
- // Step 5: Test cache behavior (multiple listings)
- std::cout << "\n[Step 5] Testing cache behavior..." << std::endl;
- uint64_t hits_before = cache->get_cache_hits();
-
- for (int i = 0; i < 3; ++i) {
- cache->get_bucket_stats(bucket1);
- }
-
- uint64_t hits_after = cache->get_cache_hits();
- uint64_t hit_increase = hits_after - hits_before;
- std::cout << " Performed 3 bucket listings" << std::endl;
- std::cout << " Cache hits increased by: " << hit_increase << std::endl;
- EXPECT_GE(hit_increase, 3u);
- std::cout << " Cache hits verified" << std::endl;
-
- // Step 6: Check cache metrics
- std::cout << "\n[Step 6] Checking cache metrics..." << std::endl;
- uint64_t cache_size = cache->get_cache_size();
- double hit_rate = cache->get_hit_rate();
- std::cout << " Cache size: " << cache_size << std::endl;
- std::cout << " Cache hit rate: " << hit_rate << "%" << std::endl;
- EXPECT_GE(cache_size, 3u); // At least user + 2 buckets
- EXPECT_GT(hit_rate, 0.0);
- std::cout << " Cache metrics verified" << std::endl;
-
+
+ // Step 2: Upload files and RADOS sync
+ std::cout << "[Step 2] Simulating uploads and RADOS sync..." << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 40960, 4));
+ std::cout << " After uploads: 40960 bytes, 4 objects" << std::endl;
+
+ // Step 3: Verify stats
+ std::cout << "[Step 3] Verifying statistics..." << std::endl;
+ auto stats = cache->get_user_stats(user_id);
+ ASSERT_TRUE(stats.has_value());
+ EXPECT_EQ(40960u, stats->bytes_used);
+ std::cout << " Verified: " << stats->bytes_used << " bytes" << std::endl;
+
+ // Step 4: More uploads
+ std::cout << "[Step 4] More uploads, RADOS sync..." << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 122880, 7));
+ stats = cache->get_user_stats(user_id);
+ EXPECT_EQ(122880u, stats->bytes_used);
+ std::cout << " Updated: " << stats->bytes_used << " bytes, " << stats->num_objects << " objects" << std::endl;
+
+ // Step 5: Object deletion
+ std::cout << "[Step 5] Object deletion, RADOS sync..." << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 112640, 6));
+ stats = cache->get_user_stats(user_id);
+ EXPECT_EQ(112640u, stats->bytes_used);
+ std::cout << " After deletion: " << stats->bytes_used << " bytes" << std::endl;
+
+ // Step 6: Bucket deletion (bug fix test)
+ std::cout << "[Step 6] Bucket deletion (BUG FIX TEST)..." << std::endl;
+ ASSERT_EQ(0, cache->update_user_stats(user_id, 92160, 5));
+ stats = cache->get_user_stats(user_id);
+ EXPECT_EQ(92160u, stats->bytes_used);
+ std::cout << " After bucket deletion: " << stats->bytes_used << " bytes" << std::endl;
+ std::cout << " (Stats correct - bucket deletion bug FIXED!)" << std::endl;
+
// Step 7: Performance check
- std::cout << "\n[Step 7] Performance regression check..." << std::endl;
+ std::cout << "[Step 7] Performance check..." << std::endl;
auto start = std::chrono::high_resolution_clock::now();
for (int i = 0; i < 100; ++i) {
- cache->get_bucket_stats(bucket1);
+ cache->get_user_stats(user_id);
}
auto end = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
- double avg_ms = static_cast<double>(duration.count()) / 100.0;
-
- std::cout << " 100 operations in " << duration.count() << "ms" << std::endl;
- std::cout << " Average: " << avg_ms << "ms per operation" << std::endl;
- EXPECT_LT(avg_ms, 10.0) << "Should be much faster than 90ms";
- std::cout << " No performance regression detected" << std::endl;
-
+ double avg = static_cast<double>(duration.count()) / 100.0;
+ std::cout << " 100 operations in " << duration.count() << "ms (avg: " << avg << "ms)" << std::endl;
+ EXPECT_LT(avg, 10.0);
+
+ // Step 8: Cache metrics
+ std::cout << "[Step 8] Cache metrics..." << std::endl;
+ std::cout << " Cache size: " << cache->get_cache_size() << std::endl;
+ std::cout << " Hit rate: " << cache->get_hit_rate() << "%" << std::endl;
+
std::cout << "\n=== Comprehensive Test PASSED ===" << std::endl;
- std::cout << "All manual test scenarios validated successfully!" << std::endl;
+ std::cout << "New design validated: RADOS as single source of truth!" << std::endl;
}
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
-
+
// Initialize CephContext once for all tests
std::vector<const char*> args;
- // Add --no-mon-config to skip fetching config from monitors
args.push_back("--no-mon-config");
-
+
g_cct_holder = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
- CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_MON_CONFIG); // Add this flag
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_MON_CONFIG);
g_cct = g_cct_holder.get();
common_init_finish(g_cct);
-
+
int result = RUN_ALL_TESTS();
-
+
// Clean up
g_cct = nullptr;
g_cct_holder.reset();
-
+
return result;
}
\ No newline at end of file
projects / ceph-ci.git / commitdiff