From 7f482c05b287ffc9ee51bf445878044e0026b0f3 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Sun, 8 Nov 2015 18:11:37 -0500 Subject: [PATCH] librgw: add getattr unit tests, restructure The fids1, fids2 vectors were confusing, as fids2 mapped the objects in the union of all buckets. Now there's a matrix of , and it's trivial to scan the objects in any one bucket. Signed-off-by: Matt Benjamin --- src/rgw/librgw.cc | 2 + src/rgw/rgw_file.cc | 7 +- src/rgw/rgw_file.h | 18 +++-- src/test/librgw_file.cc | 143 ++++++++++++++++++++++++++++++++-------- 4 files changed, 136 insertions(+), 34 deletions(-) diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index 8c9cdd7b7946c..820ca67b1e1bc 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -56,6 +56,8 @@ using std::string; static std::mutex librgw_mtx; +bool global_stop = false; + RGWLib librgw; /* XXX initialize? */ class C_InitTimeout : public Context { diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc index 0cdfb5a56dcc0..bd64f116ed89b 100644 --- a/src/rgw/rgw_file.cc +++ b/src/rgw/rgw_file.cc @@ -249,7 +249,9 @@ int rgw_lookup(struct rgw_fs *rgw_fs, if (! rgw_fh) return ENOENT; } else { - RGWStatObjRequest req(cct, fs->get_user(), parent->bucket_name(), path, + std::string object_name{path}; + RGWStatObjRequest req(cct, fs->get_user(), + parent->bucket_name(), object_name, RGWStatObjRequest::FLAG_NONE); int rc = librgw.get_fe()->execute_req(&req); @@ -343,6 +345,9 @@ int rgw_getattr(struct rgw_fs *rgw_fs, #endif } else { /* object */ + const std::string bname = rgw_fh->bucket_name(); + const std::string oname = rgw_fh->object_name(); + RGWStatObjRequest req(cct, fs->get_user(), rgw_fh->bucket_name(), rgw_fh->object_name(), RGWStatObjRequest::FLAG_NONE); diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h index 2dbfe89c616a0..b1801901c4954 100644 --- a/src/rgw/rgw_file.h +++ b/src/rgw/rgw_file.h @@ -356,6 +356,16 @@ namespace rgw { } /* namespace rgw */ +static inline std::string make_uri(const std::string& bucket_name, + const std::string& object_name) { + std::string uri("/"); + uri.reserve(bucket_name.length() + object_name.length() + 2); + uri += bucket_name; + uri += "/"; + uri += object_name; + return uri; +} + /* read directory content (buckets) */ @@ -626,7 +636,7 @@ public: s->op = OP_PUT; /* XXX derp derp derp */ - string uri = "/" + bucket_name + "/" + obj_name; + std::string uri = make_uri(bucket_name, obj_name); s->relative_uri = uri; s->info.request_uri = uri; // XXX s->info.effective_uri = uri; @@ -711,7 +721,7 @@ public: s->op = OP_GET; /* XXX derp derp derp */ - string uri = "/" + bucket_name + "/" + obj_name; + std::string uri = make_uri(bucket_name, obj_name); s->relative_uri = uri; s->info.request_uri = uri; // XXX s->info.effective_uri = uri; @@ -788,7 +798,7 @@ public: s->op = OP_DELETE; /* XXX derp derp derp */ - string uri = "/" + bucket_name + "/" + obj_name; + std::string uri = make_uri(bucket_name, obj_name); s->relative_uri = uri; s->info.request_uri = uri; // XXX s->info.effective_uri = uri; @@ -863,7 +873,7 @@ public: s->op = OP_GET; /* XXX derp derp derp */ - string uri = "/" + bucket_name + "/" + obj_name; + std::string uri = make_uri(bucket_name, obj_name); s->relative_uri = uri; s->info.request_uri = uri; // XXX s->info.effective_uri = uri; diff --git a/src/test/librgw_file.cc b/src/test/librgw_file.cc index cdb7ed0ccfe37..a003fbd989f1f 100644 --- a/src/test/librgw_file.cc +++ b/src/test/librgw_file.cc @@ -32,9 +32,13 @@ namespace { string access_key(""); string secret_key(""); struct rgw_fs *fs = nullptr; - typedef std::tuple fid_type; //in c++2014 can alias... + typedef std::tuple fid_type; //in c++2014 can alias... + typedef std::tuple> bucket_type; + std::vector fids1; - std::vector fids2; + std::vector bucket_matrix; + + bool do_getattr = false; struct { int argc; @@ -55,6 +59,19 @@ TEST(LibRGW, MOUNT) { ASSERT_NE(fs, nullptr); } +TEST(LibRGW, GETATTR_ROOT) { + if (do_getattr) { + using std::get; + + if (! fs) + return; + + struct stat st; + int ret = rgw_getattr(fs, fs->root_fh, &st); + ASSERT_EQ(ret, 0); + } +} + extern "C" { extern void dump_buckets(void); } @@ -84,18 +101,12 @@ TEST(LibRGW, LIST_BUCKETS) { for (auto& fid : fids1) { std::cout << "fname: " << get<0>(fid) << " fid: " << get<1>(fid) << std::endl; + /* push row for bucket into bucket_matrix */ + bucket_matrix.push_back(bucket_type(fid, std::vector())); } ASSERT_EQ(ret, 0); } -extern "C" { - static bool r2_cb(const char* name, void *arg, uint64_t offset) { - // don't need arg--it would point to fids2 - fids2.push_back(fid_type(name, offset, nullptr)); - return true; /* XXX ? */ - } -} - TEST(LibRGW, LOOKUP_BUCKETS) { using std::get; @@ -103,8 +114,10 @@ TEST(LibRGW, LOOKUP_BUCKETS) { return; int ret = 0; - for (auto& fid : fids1) { - struct rgw_file_handle *rgw_fh; + for (auto& fid_row : bucket_matrix) { + auto& fid = get<0>(fid_row); + // auto& obj_vector = get<1>(fid_row); + struct rgw_file_handle *rgw_fh = nullptr; ret = rgw_lookup(fs, fs->root_fh, get<0>(fid).c_str(), &rgw_fh, 0 /* flags */); ASSERT_EQ(ret, 0); @@ -113,6 +126,33 @@ TEST(LibRGW, LOOKUP_BUCKETS) { } } +TEST(LibRGW, GETATTR_BUCKETS) { + if (do_getattr) { + using std::get; + + if (! fs) + return; + + int ret = 0; + struct stat st; + + for (auto& fid_row : bucket_matrix) { + auto& fid = get<0>(fid_row); + struct rgw_file_handle *rgw_fh = get<2>(fid); + ret = rgw_getattr(fs, rgw_fh, &st); + ASSERT_EQ(ret, 0); + } + } +} + +extern "C" { + static bool r2_cb(const char* name, void *arg, uint64_t offset) { + std::vector& obj_vector = *(static_cast*>(arg)); + obj_vector.push_back(fid_type(name, offset, nullptr)); + return true; /* XXX ? */ + } +} + TEST(LibRGW, LIST_OBJECTS) { /* list objects via readdir, bucketwise */ using std::get; @@ -120,31 +160,74 @@ TEST(LibRGW, LIST_OBJECTS) { if (! fs) return; - for (auto& fid : fids1) { + for (auto& fid_row : bucket_matrix) { + auto& fid = get<0>(fid_row); // bucket + std::vector& obj_vector = get<1>(fid_row); // objects in bucket + struct rgw_file_handle *bucket_fh = get<2>(fid); + ldout(g_ceph_context, 0) << __func__ << " readdir on bucket " << get<0>(fid) - << dendl; + << dendl; + bool eof = false; uint64_t offset = 0; - int ret = rgw_readdir(fs, get<2>(fid), &offset, r2_cb, &fids2, - &eof); - for (auto& fid2 : fids2) { - std::cout << "fname: " << get<0>(fid2) << " fid: " << get<1>(fid2) + int ret = rgw_readdir(fs, bucket_fh, &offset, r2_cb, &obj_vector, &eof); + for (auto& obj : obj_vector) { + std::cout << "fname: " << get<0>(obj) << " fid: " << get<1>(obj) << std::endl; } ASSERT_EQ(ret, 0); } } +extern bool global_stop; + +TEST(LibRGW, GETATTR_OBJECTS) { + if (do_getattr) { + using std::get; + struct stat st; + int ret; + + global_stop = true; + + for (auto& fid_row : bucket_matrix) { + auto& fid = get<0>(fid_row); // bucket + std::vector& obj_vector = get<1>(fid_row); // objects in bucket + struct rgw_file_handle *bucket_fh = get<2>(fid); + + for (auto& obj : obj_vector) { + struct rgw_file_handle *obj_fh = nullptr; + std::string object_name = get<0>(obj); + ret = rgw_lookup(fs, bucket_fh, get<0>(obj).c_str(), &obj_fh, + 0 /* flags */); + ASSERT_EQ(ret, 0); + get<2>(obj) = obj_fh; // stash obj_fh for cleanup + ASSERT_NE(get<2>(obj), nullptr); + ret = rgw_getattr(fs, obj_fh, &st); // now, getattr + ASSERT_EQ(ret, 0); + } + } + } +} + TEST(LibRGW, CLEANUP) { int ret = 0; using std::get; - struct rgw_file_handle *rgw_fh; + /* release file handles */ - for (auto& fids : { fids1, fids2 }) { - for (auto& fid : fids) { - rgw_fh = get<2>(fid); - if (rgw_fh) - ret = rgw_fh_rele(fs, rgw_fh, 0 /* flags */); + for (auto& fid_row : bucket_matrix) { + auto& bucket = get<0>(fid_row); // bucket + std::vector& obj_vector = get<1>(fid_row); // objects in bucket + for (auto& obj : obj_vector) { + struct rgw_file_handle *obj_fh = get<2>(obj); + if (obj_fh) { + ret = rgw_fh_rele(fs, obj_fh, 0 /* flags */); + ASSERT_EQ(ret, 0); + } + } + // now the bucket + struct rgw_file_handle *bucket_fh = get<2>(bucket); + if (bucket_fh) { + ret = rgw_fh_rele(fs, bucket_fh, 0 /* flags */); ASSERT_EQ(ret, 0); } } @@ -183,16 +266,18 @@ int main(int argc, char *argv[]) for (auto arg_iter = args.begin(); arg_iter != args.end();) { if (ceph_argparse_witharg(args, arg_iter, &val, "--access", - (char*) NULL)) { + (char*) nullptr)) { access_key = val; } else if (ceph_argparse_witharg(args, arg_iter, &val, "--secret", - (char*) NULL)) { + (char*) nullptr)) { secret_key = val; } else if (ceph_argparse_witharg(args, arg_iter, &val, "--uid", - (char*) NULL)) { + (char*) nullptr)) { uid = val; - } - else { + } else if (ceph_argparse_flag(args, arg_iter, "--getattr", + (char*) nullptr)) { + do_getattr = true; + } else { ++arg_iter; } } -- 2.39.5