librgw: incremental enumeration progress

As of this change, enumeration is -almost- correct, but the
rules for name, object_name(), and full_object_name() need to be
checked.

The rationale for storing just the last component name in dirents
means that name can't ever have the form "foo/bar" (make sure it
doesn't).  But there may still be code which assumes name ==
object_name() OR assumes object_name() is hashable (it isn't, that
has to be the full object name).

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 686c4844ab07763d50a22b89708cedfcd0a74425..ba211403d2762935d40d5c4069e9d3de6f7466d4 100644 (file)
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -362,7 +362,7 @@ int rgw_lookup(struct rgw_fs *rgw_fs,
              struct rgw_file_handle *parent_fh, const char* path,
              struct rgw_file_handle **fh, uint32_t flags)
 {
-  CephContext* cct = static_cast<CephContext*>(rgw_fs->rgw);
+  //CephContext* cct = static_cast<CephContext*>(rgw_fs->rgw);
   RGWLibFS *fs = static_cast<RGWLibFS*>(rgw_fs->fs_private);
 
   RGWFileHandle* parent = get_rgwfh(parent_fh);

diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h
index a5195dcb3645f4a3d91e48c7a928904be7ece503..d3723fa0f5a6a38b77ad0d25b45873a76898b6c1 100644 (file)
--- a/src/rgw/rgw_file.h
+++ b/src/rgw/rgw_file.h
@@ -16,6 +16,7 @@
 #include <boost/range/adaptor/reversed.hpp>
 #include <boost/container/flat_map.hpp>
 #include <boost/variant.hpp>
+#include <boost/utility/string_ref.hpp>
 #include "xxhash.h"
 #include "include/buffer.h"
 #include "common/cohort_lru.h"
@@ -282,8 +283,8 @@ namespace rgw {
       int reserve = 0;
       RGWFileHandle* tfh = this;
       while (tfh && !tfh->is_bucket()) {
-       segments.push_back(&parent->object_name());
-       reserve += (1 + parent->object_name().length());
+       segments.push_back(&tfh->object_name());
+       reserve += (1 + tfh->object_name().length());
        tfh = tfh->parent.get();
       }
       bool first = true;
@@ -302,10 +303,10 @@ namespace rgw {
       if (depth <= 1)
        return fh_key(fhk.fh_hk.object, name.c_str());
       else {
-       std::vector<const std::string*> segments = { &name, &object_name() };
-       RGWFileHandle* tfh = parent.get();
+       std::vector<const std::string*> segments;
+       RGWFileHandle* tfh = this;
        while (tfh && !tfh->is_bucket()) {
-          segments.push_back(&parent->object_name());
+          segments.push_back(&tfh->object_name());
           tfh = tfh->parent.get();
        }
        /* hash path */
@@ -644,7 +645,6 @@ public:
   }
 
   virtual int header_init() {
-
     struct req_state* s = get_state();
     s->info.method = "GET";
     s->op = OP_GET;
@@ -760,6 +760,8 @@ public:
     s->user = user;
 
     prefix = rgw_fh->full_object_name();
+    if (prefix.length() > 0)
+      prefix += "/";
     delimiter = '/';
 
     return 0;
@@ -782,19 +784,43 @@ public:
   }
 
   virtual void send_response() {
-    size_t size = objs.size();
-    for (const auto& iter : objs) {
+    size_t size = objs.size();for (const auto& iter : objs) {
+      size_t last_del = iter.key.name.find_last_of('/');
+      boost::string_ref sref;
+      if (last_del != string::npos)
+       sref = boost::string_ref{iter.key.name.substr(last_del+1)};
+      else
+       sref = boost::string_ref{iter.key.name};
+       
+      std::cout << "RGWListBucketRequest "
+               << __func__ << " " << "list uri=" << uri << " "
+               << " prefix=" << prefix << " "
+               << " obj path=" << iter.key.name
+               << " (" << sref << ")" << ""
+               << std::endl;
       /* call me maybe */
-      this->operator()(iter.key.name, iter.key.name,
+      this->operator()(sref.data(), sref.data(),
                       (ix == size-1) ? true : false);
       ++ix;
     }
     size += common_prefixes.size();
     for (auto& iter : common_prefixes) {
-      std::string& pref = const_cast<std::string&>(iter.first);
-      if (pref.back() == '/')
-       pref.pop_back();
-      this->operator()(pref, pref, (ix == size-1) ? true : false);
+      if (iter.first.back() == '/')
+       const_cast<std::string&>(iter.first).pop_back();
+
+      size_t last_del = iter.first.find_last_of('/');
+      boost::string_ref sref;
+      if (last_del != string::npos)
+       sref = boost::string_ref{iter.first.substr(last_del+1)};
+      else
+       sref = boost::string_ref{iter.first};
+
+      std::cout << "RGWListBucketRequest "
+               << __func__ << " " << "list uri=" << uri << " "
+               << " prefix=" << prefix << " "
+               << " cpref=" << sref << " (not chomped)"
+               << std::endl;
+      this->operator()(sref.data(), sref.data(), (ix == size-1) ? true : false);
       ++ix;
     }
   }
@@ -1366,15 +1392,26 @@ public:
   }
 
   virtual void send_response() {
+    struct req_state* s = get_state();
     // try objects
     for (const auto& iter : objs) {
       path = iter.key.name;
+      std::cout << "RGWStatLeafRequest "
+               << __func__ << " " << "list uri=" << s->relative_uri << " "
+               << " prefix=" << prefix << " "
+               << " obj path=" << path << ""
+               << std::endl;
       matched = true;
       return;
     }
     // try prefixes
     for (auto& iter : common_prefixes) {
       path = iter.first;
+      std::cout << "RGWStatLeafRequest "
+               << __func__ << " " << "list uri=" << s->relative_uri << " "
+               << " prefix=" << prefix << " "
+               << " pref path=" << path << " (not chomped)"
+               << std::endl;
       matched = true;
       break;
     }

diff --git a/src/test/librgw_file_nfsns.cc b/src/test/librgw_file_nfsns.cc
index 9d32214e5732ccf0d5ced67528d056756145b2b8..59bbcfd96bc9c52af3a313808c7b22b6acff7fdc 100644 (file)
--- a/src/test/librgw_file_nfsns.cc
+++ b/src/test/librgw_file_nfsns.cc
@@ -131,6 +131,11 @@ TEST(LibRGW, ENUMERATE1) {
          // descending
          uint64_t offset;
          bool eof; // XXX
+         std::cout << "readdir in"
+                   << " bucket: " << elt.rgw_fh->bucket_name()
+                   << " object_name: " << elt.rgw_fh->object_name()
+                   << " full name: " << elt.rgw_fh->full_object_name()
+                   << std::endl;
          rc = rgw_readdir(fs, elt.fh, &offset, r1_cb, elt.fh, &eof);
          elt.state.readdir = true;
          ASSERT_EQ(rc, 0);