From b83ac0d94585a3c0c21479e7a0fe640b475986ea Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Thu, 10 Dec 2015 18:13:27 -0500 Subject: [PATCH] librgw: incremental enumeration progress As of this change, enumeration is -almost- correct, but the rules for name, object_name(), and full_object_name() need to be checked. The rationale for storing just the last component name in dirents means that name can't ever have the form "foo/bar" (make sure it doesn't). But there may still be code which assumes name == object_name() OR assumes object_name() is hashable (it isn't, that has to be the full object name). Signed-off-by: Matt Benjamin --- src/rgw/rgw_file.cc | 2 +- src/rgw/rgw_file.h | 63 +++++++++++++++++++++++++++-------- src/test/librgw_file_nfsns.cc | 5 +++ 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc index 686c4844ab077..ba211403d2762 100644 --- a/src/rgw/rgw_file.cc +++ b/src/rgw/rgw_file.cc @@ -362,7 +362,7 @@ int rgw_lookup(struct rgw_fs *rgw_fs, struct rgw_file_handle *parent_fh, const char* path, struct rgw_file_handle **fh, uint32_t flags) { - CephContext* cct = static_cast(rgw_fs->rgw); + //CephContext* cct = static_cast(rgw_fs->rgw); RGWLibFS *fs = static_cast(rgw_fs->fs_private); RGWFileHandle* parent = get_rgwfh(parent_fh); diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h index a5195dcb3645f..d3723fa0f5a6a 100644 --- a/src/rgw/rgw_file.h +++ b/src/rgw/rgw_file.h @@ -16,6 +16,7 @@ #include #include #include +#include #include "xxhash.h" #include "include/buffer.h" #include "common/cohort_lru.h" @@ -282,8 +283,8 @@ namespace rgw { int reserve = 0; RGWFileHandle* tfh = this; while (tfh && !tfh->is_bucket()) { - segments.push_back(&parent->object_name()); - reserve += (1 + parent->object_name().length()); + segments.push_back(&tfh->object_name()); + reserve += (1 + tfh->object_name().length()); tfh = tfh->parent.get(); } bool first = true; @@ -302,10 +303,10 @@ namespace rgw { if (depth <= 1) return fh_key(fhk.fh_hk.object, name.c_str()); else { - std::vector segments = { &name, &object_name() }; - RGWFileHandle* tfh = parent.get(); + std::vector segments; + RGWFileHandle* tfh = this; while (tfh && !tfh->is_bucket()) { - segments.push_back(&parent->object_name()); + segments.push_back(&tfh->object_name()); tfh = tfh->parent.get(); } /* hash path */ @@ -644,7 +645,6 @@ public: } virtual int header_init() { - struct req_state* s = get_state(); s->info.method = "GET"; s->op = OP_GET; @@ -760,6 +760,8 @@ public: s->user = user; prefix = rgw_fh->full_object_name(); + if (prefix.length() > 0) + prefix += "/"; delimiter = '/'; return 0; @@ -782,19 +784,43 @@ public: } virtual void send_response() { - size_t size = objs.size(); - for (const auto& iter : objs) { + size_t size = objs.size();for (const auto& iter : objs) { + size_t last_del = iter.key.name.find_last_of('/'); + boost::string_ref sref; + if (last_del != string::npos) + sref = boost::string_ref{iter.key.name.substr(last_del+1)}; + else + sref = boost::string_ref{iter.key.name}; + + std::cout << "RGWListBucketRequest " + << __func__ << " " << "list uri=" << uri << " " + << " prefix=" << prefix << " " + << " obj path=" << iter.key.name + << " (" << sref << ")" << "" + << std::endl; /* call me maybe */ - this->operator()(iter.key.name, iter.key.name, + this->operator()(sref.data(), sref.data(), (ix == size-1) ? true : false); ++ix; } size += common_prefixes.size(); for (auto& iter : common_prefixes) { - std::string& pref = const_cast(iter.first); - if (pref.back() == '/') - pref.pop_back(); - this->operator()(pref, pref, (ix == size-1) ? true : false); + if (iter.first.back() == '/') + const_cast(iter.first).pop_back(); + + size_t last_del = iter.first.find_last_of('/'); + boost::string_ref sref; + if (last_del != string::npos) + sref = boost::string_ref{iter.first.substr(last_del+1)}; + else + sref = boost::string_ref{iter.first}; + + std::cout << "RGWListBucketRequest " + << __func__ << " " << "list uri=" << uri << " " + << " prefix=" << prefix << " " + << " cpref=" << sref << " (not chomped)" + << std::endl; + this->operator()(sref.data(), sref.data(), (ix == size-1) ? true : false); ++ix; } } @@ -1366,15 +1392,26 @@ public: } virtual void send_response() { + struct req_state* s = get_state(); // try objects for (const auto& iter : objs) { path = iter.key.name; + std::cout << "RGWStatLeafRequest " + << __func__ << " " << "list uri=" << s->relative_uri << " " + << " prefix=" << prefix << " " + << " obj path=" << path << "" + << std::endl; matched = true; return; } // try prefixes for (auto& iter : common_prefixes) { path = iter.first; + std::cout << "RGWStatLeafRequest " + << __func__ << " " << "list uri=" << s->relative_uri << " " + << " prefix=" << prefix << " " + << " pref path=" << path << " (not chomped)" + << std::endl; matched = true; break; } diff --git a/src/test/librgw_file_nfsns.cc b/src/test/librgw_file_nfsns.cc index 9d32214e5732c..59bbcfd96bc9c 100644 --- a/src/test/librgw_file_nfsns.cc +++ b/src/test/librgw_file_nfsns.cc @@ -131,6 +131,11 @@ TEST(LibRGW, ENUMERATE1) { // descending uint64_t offset; bool eof; // XXX + std::cout << "readdir in" + << " bucket: " << elt.rgw_fh->bucket_name() + << " object_name: " << elt.rgw_fh->object_name() + << " full name: " << elt.rgw_fh->full_object_name() + << std::endl; rc = rgw_readdir(fs, elt.fh, &offset, r1_cb, elt.fh, &eof); elt.state.readdir = true; ASSERT_EQ(rc, 0); -- 2.39.5