rgw_file: new fsid for fs instance

We use an in-memory fs_inst_counter for fsid currently, but it
inevitably cause nfs automount problem across a nfs-ganesha crash.
So here we relate the fsid to the s3 uid with a hash.
With this, fsid remains consistent across all fs instances and
across crashes.

We should be able to upgrade from old fsid to new fsid with
update_fh().

Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
(cherry picked from commit 178b547e83a2c0eeb024bbe4ca2ff8d8e7d50572)

diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 200a90bc4ec0cd77945d423d1ff7d19d2b3b8598..59229f64ebae5ba3c6ceb6c1029cedeba1e80263 100644 (file)
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -92,10 +92,10 @@ namespace rgw {
        auto ux_key = req.get_attr(RGW_ATTR_UNIX_KEY1);
        auto ux_attrs = req.get_attr(RGW_ATTR_UNIX1);
        if (ux_key && ux_attrs) {
-         bool old_key = rgw_fh->decode_attrs(ux_key, ux_attrs);
-         if (old_key) {
-           update_fhk(rgw_fh);
-         }
+         DecodeAttrsResult dar = rgw_fh->decode_attrs(ux_key, ux_attrs);
+         if (get<0>(dar) || get<1>(dar)) {
+           update_fh(rgw_fh);
+          }
        }
        if (! (flags & RGWFileHandle::FLAG_LOCKED)) {
          rgw_fh->mtx.unlock();
@@ -147,10 +147,10 @@ namespace rgw {
            auto ux_key = req.get_attr(RGW_ATTR_UNIX_KEY1);
            auto ux_attrs = req.get_attr(RGW_ATTR_UNIX1);
            if (ux_key && ux_attrs) {
-             bool old_key = rgw_fh->decode_attrs(ux_key, ux_attrs);
-             if (old_key) {
-               update_fhk(rgw_fh);
-             }
+              DecodeAttrsResult dar = rgw_fh->decode_attrs(ux_key, ux_attrs);
+              if (get<0>(dar) || get<1>(dar)) {
+                update_fh(rgw_fh);
+              }
            }
          }
          goto done;
@@ -181,10 +181,10 @@ namespace rgw {
            auto ux_key = req.get_attr(RGW_ATTR_UNIX_KEY1);
            auto ux_attrs = req.get_attr(RGW_ATTR_UNIX1);
            if (ux_key && ux_attrs) {
-             bool old_key = rgw_fh->decode_attrs(ux_key, ux_attrs);
-             if (old_key) {
-               update_fhk(rgw_fh);
-             }
+              DecodeAttrsResult dar = rgw_fh->decode_attrs(ux_key, ux_attrs);
+              if (get<0>(dar) || get<1>(dar)) {
+                update_fh(rgw_fh);
+              }
            }
          }
          goto done;
@@ -744,7 +744,7 @@ namespace rgw {
   } /* RGWLibFS::setattr */
 
   /* called under rgw_fh->mtx held */
-  void RGWLibFS::update_fhk(RGWFileHandle *rgw_fh)
+  void RGWLibFS::update_fh(RGWFileHandle *rgw_fh)
   {
     int rc, rc2;
     string obj_name{rgw_fh->relative_object_name()};
@@ -757,15 +757,15 @@ namespace rgw {
 
     lsubdout(get_context(), rgw, 17)
       << __func__
-      << " update old versioned fhk : " << obj_name
+      << " update old versioned fh : " << obj_name
       << dendl;
 
     RGWSetAttrsRequest req(cct, get_user(), rgw_fh->bucket_name(), obj_name);
 
     rgw_fh->encode_attrs(ux_key, ux_attrs);
 
-    /* update ux_key only */
     req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key));
+    req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs));
 
     rc = rgwlib.get_fe()->execute_req(&req);
     rc2 = req.get_ret();
@@ -773,10 +773,10 @@ namespace rgw {
     if ((rc != 0) || (rc2 != 0)) {
       lsubdout(get_context(), rgw, 17)
        << __func__
-       << " update fhk failed : " << obj_name
+       << " update fh failed : " << obj_name
        << dendl;
     }
-  } /* RGWLibFS::update_fhk */
+  } /* RGWLibFS::update_fh */
 
   void RGWLibFS::close()
   {
@@ -786,7 +786,7 @@ namespace rgw {
     {
       RGWLibFS* fs;
     public:
-      ObjUnref(RGWLibFS* fs) : fs(fs) {}
+      ObjUnref(RGWLibFS* _fs) : fs(_fs) {}
       void operator()(RGWFileHandle* fh) const {
        lsubdout(fs->get_context(), rgw, 5)
          << __func__
@@ -976,23 +976,26 @@ namespace rgw {
     rgw::encode(*this, ux_attrs1);
   } /* RGWFileHandle::encode_attrs */
 
-  bool RGWFileHandle::decode_attrs(const ceph::buffer::list* ux_key1,
-                                  const ceph::buffer::list* ux_attrs1)
+  DecodeAttrsResult RGWFileHandle::decode_attrs(const ceph::buffer::list* ux_key1,
+                                                const ceph::buffer::list* ux_attrs1)
   {
-    bool old_key = false;
+    DecodeAttrsResult dar { false, false };
     fh_key fhk;
     auto bl_iter_key1  = const_cast<buffer::list*>(ux_key1)->begin();
     rgw::decode(fhk, bl_iter_key1);
     if (fhk.version >= 2) {
       assert(this->fh.fh_hk == fhk.fh_hk);
     } else {
-      old_key = true;
+      get<0>(dar) = true;
     }
 
     auto bl_iter_unix1 = const_cast<buffer::list*>(ux_attrs1)->begin();
     rgw::decode(*this, bl_iter_unix1);
+    if (this->state.version < 2) {
+      get<1>(dar) = true;
+    }
 
-    return old_key;
+    return dar;
   } /* RGWFileHandle::decode_attrs */
 
   bool RGWFileHandle::reclaim() {
@@ -1579,8 +1582,8 @@ int rgw_statfs(struct rgw_fs *rgw_fs,
   vfs_st->f_bavail = UINT64_MAX;
   vfs_st->f_files = 1024; /* object count, do we have an est? */
   vfs_st->f_ffree = UINT64_MAX;
-  vfs_st->f_fsid[0] = fs->get_inst();
-  vfs_st->f_fsid[1] = fs->get_inst();
+  vfs_st->f_fsid[0] = fs->get_fsid();
+  vfs_st->f_fsid[1] = fs->get_fsid();
   vfs_st->f_flag = 0;
   vfs_st->f_namemax = 4096;
   return 0;

diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h
index 34f3f0bcec8616ab4e6911b1a03138eff50a492b..8bb702cb33fe15d7cd0ca35ececf7a8b99c94436 100644 (file)
--- a/src/rgw/rgw_file.h
+++ b/src/rgw/rgw_file.h
@@ -173,6 +173,8 @@ namespace rgw {
   using boost::variant;
   using boost::container::flat_map;
 
+  typedef std::tuple<bool, bool> DecodeAttrsResult;
+
   class RGWFileHandle : public cohort::lru::Object
   {
     struct rgw_file_handle fh;
@@ -204,8 +206,9 @@ namespace rgw {
       struct timespec ctime;
       struct timespec mtime;
       struct timespec atime;
+      uint32_t version;
       State() : dev(0), size(0), nlink(1), owner_uid(0), owner_gid(0),
-               ctime{0,0}, mtime{0,0}, atime{0,0} {}
+               ctime{0,0}, mtime{0,0}, atime{0,0}, version(0) {}
     } state;
 
     struct file {
@@ -257,7 +260,7 @@ namespace rgw {
     friend class RGWLibFS;
 
   private:
-    RGWFileHandle(RGWLibFS* _fs, uint32_t fs_inst)
+    RGWFileHandle(RGWLibFS* _fs)
       : fs(_fs), bucket(nullptr), parent(nullptr), variant_type{directory()},
        depth(0), flags(FLAG_ROOT)
       {
@@ -265,12 +268,15 @@ namespace rgw {
        fh.fh_type = RGW_FS_TYPE_DIRECTORY;
        variant_type = directory();
        /* stat */
-       state.dev = fs_inst;
        state.unix_mode = RGW_RWXMODE|S_IFDIR;
        /* pointer to self */
        fh.fh_private = this;
       }
 
+    uint64_t init_fsid(std::string& uid) {
+      return XXH64(uid.c_str(), uid.length(), fh_key::seed);
+    }
+
     void init_rootfs(std::string& fsid, const std::string& object_name) {
       /* fh_key */
       fh.fh_hk.bucket = XXH64(fsid.c_str(), fsid.length(), fh_key::seed);
@@ -278,12 +284,14 @@ namespace rgw {
                              fh_key::seed);
       fhk = fh.fh_hk;
       name = object_name;
+
+      state.dev = init_fsid(fsid);
     }
 
   public:
-    RGWFileHandle(RGWLibFS* fs, uint32_t fs_inst, RGWFileHandle* _parent,
+    RGWFileHandle(RGWLibFS* _fs, RGWFileHandle* _parent,
                  const fh_key& _fhk, std::string& _name, uint32_t _flags)
-      : fs(fs), bucket(nullptr), parent(_parent), name(std::move(_name)),
+      : fs(_fs), bucket(nullptr), parent(_parent), name(std::move(_name)),
        fhk(_fhk), flags(_flags) {
 
       if (parent->is_root()) {
@@ -307,8 +315,8 @@ namespace rgw {
       /* save constant fhk */
       fh.fh_hk = fhk.fh_hk; /* XXX redundant in fh_hk */
 
-      /* stat */
-      state.dev = fs_inst;
+      /* inherits parent's fsid */
+      state.dev = parent->state.dev;
 
       switch (fh.fh_type) {
       case RGW_FS_TYPE_DIRECTORY:
@@ -594,7 +602,7 @@ namespace rgw {
     }
 
     void encode(buffer::list& bl) const {
-      ENCODE_START(1, 1, bl);
+      ENCODE_START(2, 1, bl);
       ::encode(uint32_t(fh.fh_type), bl);
       ::encode(state.dev, bl);
       ::encode(state.size, bl);
@@ -605,11 +613,12 @@ namespace rgw {
       for (const auto& t : { state.ctime, state.mtime, state.atime }) {
        ::encode(real_clock::from_timespec(t), bl);
       }
+      ::encode((uint32_t)2, bl);
       ENCODE_FINISH(bl);
     }
 
     void decode(bufferlist::iterator& bl) {
-      DECODE_START(1, bl);
+      DECODE_START(2, bl);
       uint32_t fh_type;
       ::decode(fh_type, bl);
       assert(fh.fh_type == fh_type);
@@ -624,14 +633,17 @@ namespace rgw {
        ::decode(enc_time, bl);
        *t = real_clock::to_timespec(enc_time);
       }
+      if (struct_v >= 2) {
+        ::decode(state.version, bl);
+      }
       DECODE_FINISH(bl);
     }
 
     void encode_attrs(ceph::buffer::list& ux_key1,
                      ceph::buffer::list& ux_attrs1);
 
-    bool decode_attrs(const ceph::buffer::list* ux_key1,
-                     const ceph::buffer::list* ux_attrs1);
+    DecodeAttrsResult decode_attrs(const ceph::buffer::list* ux_key1,
+                                   const ceph::buffer::list* ux_attrs1);
 
     void invalidate();
 
@@ -694,7 +706,6 @@ namespace rgw {
     {
     public:
       RGWLibFS* fs;
-      uint32_t fs_inst;
       RGWFileHandle* parent;
       const fh_key& fhk;
       std::string& name;
@@ -702,20 +713,20 @@ namespace rgw {
 
       Factory() = delete;
 
-      Factory(RGWLibFS* fs, uint32_t fs_inst, RGWFileHandle* parent,
-             const fh_key& fhk, std::string& name, uint32_t flags)
-       : fs(fs), fs_inst(fs_inst), parent(parent), fhk(fhk), name(name),
-         flags(flags) {}
+      Factory(RGWLibFS* _fs, RGWFileHandle* _parent,
+             const fh_key& _fhk, std::string& _name, uint32_t _flags)
+       : fs(_fs), parent(_parent), fhk(_fhk), name(_name),
+         flags(_flags) {}
 
       void recycle (cohort::lru::Object* o) override {
        /* re-use an existing object */
        o->~Object(); // call lru::Object virtual dtor
        // placement new!
-       new (o) RGWFileHandle(fs, fs_inst, parent, fhk, name, flags);
+       new (o) RGWFileHandle(fs, parent, fhk, name, flags);
       }
 
       cohort::lru::Object* alloc() override {
-       return new RGWFileHandle(fs, fs_inst, parent, fhk, name, flags);
+       return new RGWFileHandle(fs, parent, fhk, name, flags);
       }
     }; /* Factory */
 
@@ -768,7 +779,6 @@ namespace rgw {
     static std::atomic<uint32_t> fs_inst_counter;
 
     static uint32_t write_completion_interval_s;
-    std::string fsid;
 
     using lock_guard = std::lock_guard<std::mutex>;
     using unique_lock = std::unique_lock<std::mutex>;
@@ -838,7 +848,7 @@ namespace rgw {
 
     RGWLibFS(CephContext* _cct, const char *_uid, const char *_user_id,
            const char* _key)
-      : cct(_cct), root_fh(this, new_inst()), invalidate_cb(nullptr),
+      : cct(_cct), root_fh(this), invalidate_cb(nullptr),
        invalidate_arg(nullptr), shutdown(false), refcnt(1),
        fh_cache(cct->_conf->rgw_nfs_fhcache_partitions,
                 cct->_conf->rgw_nfs_fhcache_size),
@@ -846,17 +856,15 @@ namespace rgw {
               cct->_conf->rgw_nfs_lru_lane_hiwat),
        uid(_uid), key(_user_id, _key) {
 
-      /* no bucket may be named rgw_fs_inst-(.*) */
-      fsid = RGWFileHandle::root_name + "rgw_fs_inst-" +
-       std::to_string(get_inst());
-
-      root_fh.init_rootfs(fsid /* bucket */, RGWFileHandle::root_name);
+      root_fh.init_rootfs(uid, RGWFileHandle::root_name);
 
       /* pointer to self */
       fs.fs_private = this;
 
       /* expose public root fh */
       fs.root_fh = root_fh.get_fh();
+
+      new_inst();
     }
 
     friend void intrusive_ptr_add_ref(const RGWLibFS* fs) {
@@ -1031,7 +1039,7 @@ namespace rgw {
            fh->mtx.unlock(); /* ! LOCKED */
       } else {
        /* make or re-use handle */
-       RGWFileHandle::Factory prototype(this, get_inst(), parent, fhk,
+       RGWFileHandle::Factory prototype(this, parent, fhk,
                                         obj_name, CREATE_FLAGS(flags));
        fh = static_cast<RGWFileHandle*>(
          fh_lru.insert(&prototype,
@@ -1083,8 +1091,7 @@ namespace rgw {
     int setattr(RGWFileHandle* rgw_fh, struct stat* st, uint32_t mask,
                uint32_t flags);
 
-    void update_fhk(RGWFileHandle *rgw_fh);
-
+    void update_fh(RGWFileHandle *rgw_fh);
 
     LookupFHResult stat_bucket(RGWFileHandle* parent, const char *path,
                               RGWLibFS::BucketStats& bs,
@@ -1166,7 +1173,7 @@ namespace rgw {
 
     struct rgw_fs* get_fs() { return &fs; }
 
-    uint32_t get_inst() { return root_fh.state.dev; }
+    uint64_t get_fsid() { return root_fh.state.dev; }
 
     RGWUserInfo* get_user() { return &user; }