rgw/iam: add pagination to ListRoles

rename sal::Driver::get_roles() to list_roles() and add pagination
support for the RGWListRoles op and 'radosgw-admin role list'

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index e62495c3ac94c37c2e29de2b96eaec5b8bab8d62..75e85f207497636c407b57070bdbe97f7dd705c1 100644 (file)
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -2095,10 +2095,13 @@ std::unique_ptr<RGWRole> DaosStore::get_role(std::string id) {
   return std::unique_ptr<RGWRole>(p);
 }
 
-int DaosStore::get_roles(const DoutPrefixProvider* dpp, optional_yield y,
-                         const std::string& path_prefix,
-                         const std::string& tenant,
-                         vector<std::unique_ptr<RGWRole>>& roles) {
+int DaosStore::list_roles(const DoutPrefixProvider *dpp,
+                          optional_yield y,
+                          const std::string& tenant,
+                          const std::string& path_prefix,
+                          const std::string& marker,
+                          uint32_t max_items,
+                          RoleList& listing) {
   return DAOS_NOT_IMPLEMENTED_LOG(dpp);
 }
 

diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 04263e50dd805de0d2d4338b30ebe03253189ed6..3d5446734477b2519f3d29f401565ea4b4c45509 100644 (file)
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -999,10 +999,13 @@ class DaosStore : public StoreDriver {
       std::multimap<std::string, std::string> tags = {}) override;
   virtual std::unique_ptr<RGWRole> get_role(const RGWRoleInfo& info) override;
   virtual std::unique_ptr<RGWRole> get_role(std::string id) override;
-  virtual int get_roles(const DoutPrefixProvider* dpp, optional_yield y,
-                        const std::string& path_prefix,
-                        const std::string& tenant,
-                        std::vector<std::unique_ptr<RGWRole>>& roles) override;
+  int list_roles(const DoutPrefixProvider *dpp,
+                 optional_yield y,
+                 const std::string& tenant,
+                 const std::string& path_prefix,
+                 const std::string& marker,
+                 uint32_t max_items,
+                 RoleList& listing) override;
   virtual std::unique_ptr<RGWOIDCProvider> get_oidc_provider() override;
   virtual int get_oidc_providers(
       const DoutPrefixProvider* dpp, const std::string& tenant,

diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index ee90ba42a38a67e93aac87ad8f42c95676423b43..07ccab041e8f72857ac924d9f5883348eacc2d5e 100644 (file)
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -3050,11 +3050,13 @@ std::unique_ptr<RGWRole> MotrStore::get_role(std::string id)
   return std::unique_ptr<RGWRole>(p);
 }
 
-int MotrStore::get_roles(const DoutPrefixProvider *dpp,
-    optional_yield y,
-    const std::string& path_prefix,
-    const std::string& tenant,
-    vector<std::unique_ptr<RGWRole>>& roles)
+int MotrStore::list_roles(const DoutPrefixProvider *dpp,
+                          optional_yield y,
+                          const std::string& tenant,
+                          const std::string& path_prefix,
+                          const std::string& marker,
+                          uint32_t max_items,
+                          RoleList& listing)
 {
   return 0;
 }

diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index c9ffcffc2d40ceaace38d16729ea06650597cf19..e856c34e0b2e9f741eb0a1d0fb9b39f731d238f0 100644 (file)
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -1064,11 +1064,13 @@ class MotrStore : public StoreDriver {
         std::multimap<std::string, std::string> tags={}) override;
     virtual std::unique_ptr<RGWRole> get_role(const RGWRoleInfo& info) override;
     virtual std::unique_ptr<RGWRole> get_role(std::string id) override;
-    virtual int get_roles(const DoutPrefixProvider *dpp,
-        optional_yield y,
-        const std::string& path_prefix,
-        const std::string& tenant,
-        std::vector<std::unique_ptr<RGWRole>>& roles) override;
+    int list_roles(const DoutPrefixProvider *dpp,
+                   optional_yield y,
+                   const std::string& tenant,
+                   const std::string& path_prefix,
+                   const std::string& marker,
+                   uint32_t max_items,
+                   RoleList& listing) override;
     virtual std::unique_ptr<RGWOIDCProvider> get_oidc_provider() override;
     virtual int get_oidc_providers(const DoutPrefixProvider *dpp,
         const std::string& tenant,

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 79431b3521e0d63802f4fec0f89773e841da230e..61b1159104ae32fe736f243fb8b2d4a7e223c62c 100644 (file)
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1804,13 +1804,17 @@ std::unique_ptr<RGWRole> RadosStore::get_role(const RGWRoleInfo& info)
   return std::make_unique<RadosRole>(this, info);
 }
 
-int RadosStore::get_roles(const DoutPrefixProvider *dpp,
-                         optional_yield y,
-                         const std::string& path_prefix,
-                         const std::string& tenant,
-                         vector<std::unique_ptr<RGWRole>>& roles)
+int RadosStore::list_roles(const DoutPrefixProvider *dpp,
+                          optional_yield y,
+                          const std::string& tenant,
+                          const std::string& path_prefix,
+                          const std::string& marker,
+                          uint32_t max_items,
+                          RoleList& listing)
 {
-  auto pool = svc()->zone->get_zone_params().roles_pool;
+  listing.roles.clear();
+
+  const auto& pool = svc()->zone->get_zone_params().roles_pool;
   std::string prefix;
 
   // List all roles if path prefix is empty
@@ -1821,46 +1825,53 @@ int RadosStore::get_roles(const DoutPrefixProvider *dpp,
   }
 
   //Get the filtered objects
-  list<std::string> result;
-  bool is_truncated;
   RGWListRawObjsCtx ctx;
-  do {
-    list<std::string> oids;
-    int r = rados->list_raw_objects(dpp, pool, prefix, 1000, ctx, oids, &is_truncated);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "ERROR: listing filtered objects failed: "
-                  << prefix << ": " << cpp_strerror(-r) << dendl;
-      return r;
-    }
-    for (const auto& iter : oids) {
-      result.push_back(iter.substr(RGWRole::role_path_oid_prefix.size()));
-    }
-  } while (is_truncated);
+  int r = rados->list_raw_objects_init(dpp, pool, marker, &ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  bool is_truncated = false;
+  list<std::string> oids;
+  r = rados->list_raw_objects(dpp, pool, prefix, max_items,
+                              ctx, oids, &is_truncated);
+  if (r == -ENOENT) {
+    r = 0;
+  } else if (r < 0) {
+    return r;
+  }
+
+  for (const auto& oid : oids) {
+    const std::string key = oid.substr(RGWRole::role_path_oid_prefix.size());
 
-  for (const auto& it : result) {
     //Find the role oid prefix from the end
-    size_t pos = it.rfind(RGWRole::role_oid_prefix);
+    size_t pos = key.rfind(RGWRole::role_oid_prefix);
     if (pos == std::string::npos) {
-        continue;
+      continue;
     }
     // Split the result into path and info_oid + id
-    std::string path = it.substr(0, pos);
+    std::string path = key.substr(0, pos);
 
     /*Make sure that prefix is part of path (False results could've been returned)
       because of the role info oid + id appended to the path)*/
     if(path_prefix.empty() || path.find(path_prefix) != std::string::npos) {
       //Get id from info oid prefix + id
-      std::string id = it.substr(pos + RGWRole::role_oid_prefix.length());
+      std::string id = key.substr(pos + RGWRole::role_oid_prefix.length());
 
       std::unique_ptr<rgw::sal::RGWRole> role = get_role(id);
-      int ret = role->read_info(dpp, y);
-      if (ret < 0) {
-        return ret;
+      r = role->read_info(dpp, y);
+      if (r < 0) {
+        return r;
       }
-      roles.push_back(std::move(role));
+      listing.roles.push_back(std::move(role->get_info()));
     }
   }
 
+  if (is_truncated) {
+    listing.next_marker = rados->list_raw_objs_get_cursor(ctx);
+  } else {
+    listing.next_marker.clear();
+  }
   return 0;
 }
 

diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 7702fd3a8f2ec818cb747c18057c9c4065487091..cfee6b872662c2a11c55eee68c0ac38f2ce39860 100644 (file)
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -337,11 +337,13 @@ class RadosStore : public StoreDriver {
                 std::multimap<std::string,std::string> tags={}) override;
     virtual std::unique_ptr<RGWRole> get_role(std::string id) override;
     virtual std::unique_ptr<RGWRole> get_role(const RGWRoleInfo& info) override;
-    virtual int get_roles(const DoutPrefixProvider *dpp,
-                         optional_yield y,
-                         const std::string& path_prefix,
-                         const std::string& tenant,
-                         std::vector<std::unique_ptr<RGWRole>>& roles) override;
+    int list_roles(const DoutPrefixProvider *dpp,
+                   optional_yield y,
+                   const std::string& tenant,
+                   const std::string& path_prefix,
+                   const std::string& marker,
+                   uint32_t max_items,
+                   RoleList& listing) override;
     virtual std::unique_ptr<RGWOIDCProvider> get_oidc_provider() override;
     virtual int get_oidc_providers(const DoutPrefixProvider *dpp,
                                   const std::string& tenant,

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index ddedb4f22de159001b08ec7fa3601bf5f79c43f4..badb43b3c1904d68f780db52c2e06490da7b766a 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1163,26 +1163,6 @@ static void show_policy_names(std::vector<string> policy_names, Formatter* forma
   formatter->flush(cout);
 }
 
-static void show_role_info(rgw::sal::RGWRole* role, Formatter* formatter)
-{
-  formatter->open_object_section("role");
-  role->dump(formatter);
-  formatter->close_section();
-  formatter->flush(cout);
-}
-
-static void show_roles_info(vector<std::unique_ptr<rgw::sal::RGWRole>>& roles, Formatter* formatter)
-{
-  formatter->open_array_section("Roles");
-  for (const auto& it : roles) {
-    formatter->open_object_section("role");
-    it->dump(formatter);
-    formatter->close_section();
-  }
-  formatter->close_section();
-  formatter->flush(cout);
-}
-
 static void show_reshard_status(
   const list<cls_rgw_bucket_instance_entry>& status, Formatter *formatter)
 {
@@ -6812,7 +6792,8 @@ int main(int argc, const char **argv)
       if (ret < 0) {
         return -ret;
       }
-      show_role_info(role.get(), formatter.get());
+      encode_json("role", *role, formatter.get());
+      formatter->flush(cout);
       return 0;
     }
   case OPT::ROLE_DELETE:
@@ -6840,7 +6821,8 @@ int main(int argc, const char **argv)
       if (ret < 0) {
         return -ret;
       }
-      show_role_info(role.get(), formatter.get());
+      encode_json("role", *role, formatter.get());
+      formatter->flush(cout);
       return 0;
     }
   case OPT::ROLE_TRUST_POLICY_MODIFY:
@@ -6880,12 +6862,41 @@ int main(int argc, const char **argv)
     }
   case OPT::ROLE_LIST:
     {
-      vector<std::unique_ptr<rgw::sal::RGWRole>> result;
-      ret = driver->get_roles(dpp(), null_yield, path_prefix, tenant, result);
-      if (ret < 0) {
-        return -ret;
+      rgw::sal::RoleList listing;
+      listing.next_marker = marker;
+
+      int32_t remaining = std::numeric_limits<int32_t>::max();
+      if (max_entries_specified) {
+        remaining = max_entries;
+        formatter->open_object_section("result");
       }
-      show_roles_info(result, formatter.get());
+      formatter->open_array_section("Roles");
+
+      do {
+        constexpr int32_t max_chunk = 100;
+        int32_t count = std::min(max_chunk, remaining);
+
+        ret = driver->list_roles(dpp(), null_yield, tenant, path_prefix,
+                                 listing.next_marker, count, listing);
+        if (ret < 0) {
+          return -ret;
+        }
+        for (const auto& info : listing.roles) {
+          encode_json("member", info, formatter.get());
+        }
+        formatter->flush(cout);
+        remaining -= listing.roles.size();
+      } while (!listing.next_marker.empty() && remaining > 0);
+
+      formatter->close_section(); // Roles
+
+      if (max_entries_specified) {
+        if (!listing.next_marker.empty()) {
+          encode_json("next-marker", listing.next_marker, formatter.get());
+        }
+        formatter->close_section(); // result
+      }
+      formatter->flush(cout);
       return 0;
     }
   case OPT::ROLE_POLICY_PUT:

diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc
index 4ee5707e0b05a5714c1f63b5d0ae9ffc3fb96a94..05450491f440e33a9c18e1d1c62b797b2d8e89c2 100644 (file)
--- a/src/rgw/rgw_rest_role.cc
+++ b/src/rgw/rgw_rest_role.cc
@@ -434,6 +434,13 @@ void RGWModifyRoleTrustPolicy::execute(optional_yield y)
 int RGWListRoles::init_processing(optional_yield y)
 {
   path_prefix = s->info.args.get("PathPrefix");
+  marker = s->info.args.get("Marker");
+
+  int r = s->info.args.get_int("MaxItems", &max_items, max_items);
+  if (r < 0 || max_items > 1000) {
+    s->err.message = "Invalid value for MaxItems";
+    return -EINVAL;
+  }
 
   return 0;
 }
@@ -441,24 +448,30 @@ int RGWListRoles::init_processing(optional_yield y)
 void RGWListRoles::execute(optional_yield y)
 {
   // TODO: list_account_roles() for account owner
-  vector<std::unique_ptr<rgw::sal::RGWRole>> result;
-  op_ret = driver->get_roles(s, y, path_prefix, s->user->get_tenant(), result);
+  rgw::sal::RoleList listing;
+  op_ret = driver->list_roles(s, y, s->user->get_tenant(), path_prefix,
+                              marker, max_items, listing);
 
   if (op_ret == 0) {
-    s->formatter->open_array_section("ListRolesResponse");
-    s->formatter->open_array_section("ListRolesResult");
-    s->formatter->open_object_section("Roles");
-    for (const auto& it : result) {
-      s->formatter->open_object_section("member");
-      it->dump(s->formatter);
-      s->formatter->close_section();
+    s->formatter->open_object_section("ListRolesResponse");
+    s->formatter->open_object_section("ListRolesResult");
+    s->formatter->open_array_section("Roles");
+    for (const auto& info : listing.roles) {
+      encode_json("member", info, s->formatter);
     }
-    s->formatter->close_section();
-    s->formatter->close_section();
+    s->formatter->close_section(); // Roles
+
+    const bool truncated = !listing.next_marker.empty();
+    encode_json("IsTruncated", truncated, s->formatter);
+    if (truncated) {
+      encode_json("Marker", listing.next_marker, s->formatter);
+    }
+
+    s->formatter->close_section(); // ListRolesResult
     s->formatter->open_object_section("ResponseMetadata");
     s->formatter->dump_string("RequestId", s->trans_id);
-    s->formatter->close_section();
-    s->formatter->close_section();
+    s->formatter->close_section(); // ResponseMetadata
+    s->formatter->close_section(); // ListRolesResponse
   }
 }
 

diff --git a/src/rgw/rgw_rest_role.h b/src/rgw/rgw_rest_role.h
index fff9406227b3226fdc2cab0426d233130682e562..be8f80b53ae43898efa1b2dabc474a6785fe1f85 100644 (file)
--- a/src/rgw/rgw_rest_role.h
+++ b/src/rgw/rgw_rest_role.h
@@ -81,6 +81,9 @@ public:
 
 class RGWListRoles : public RGWRestRole {
   std::string path_prefix;
+  std::string marker;
+  int max_items = 100;
+  std::string next_marker;
 public:
   RGWListRoles() : RGWRestRole(rgw::IAM::iamListRoles, RGW_CAP_READ) {}
   int init_processing(optional_yield y) override;

diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index d2e0e0f03d2ad78abbfd8de458dde370bad9ad09..910a4e5b142cb3364833f977cdc28b24a77ed8dd 100644 (file)
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -573,11 +573,13 @@ class Driver {
     virtual std::unique_ptr<RGWRole> get_role(std::string id) = 0;
     virtual std::unique_ptr<RGWRole> get_role(const RGWRoleInfo& info) = 0;
     /** Get all IAM Roles optionally filtered by path */
-    virtual int get_roles(const DoutPrefixProvider *dpp,
-                         optional_yield y,
-                         const std::string& path_prefix,
-                         const std::string& tenant,
-                         std::vector<std::unique_ptr<RGWRole>>& roles) = 0;
+    virtual int list_roles(const DoutPrefixProvider *dpp,
+                          optional_yield y,
+                          const std::string& tenant,
+                          const std::string& path_prefix,
+                          const std::string& marker,
+                          uint32_t max_items,
+                          RoleList& listing) = 0;
     /** Get an empty Open ID Connector provider */
     virtual std::unique_ptr<RGWOIDCProvider> get_oidc_provider() = 0;
     /** Get all Open ID Connector providers, optionally filtered by tenant  */

diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index bfd05e5d51112aa03eeb9c3919416aa073c2a582..4a0015e046d51b4744407a171e882c850f89c5b1 100644 (file)
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -1411,11 +1411,13 @@ namespace rgw::sal {
     return std::unique_ptr<RGWRole>(p);
   }
 
-  int DBStore::get_roles(const DoutPrefixProvider *dpp,
-      optional_yield y,
-      const std::string& path_prefix,
-      const std::string& tenant,
-      vector<std::unique_ptr<RGWRole>>& roles)
+  int DBStore::list_roles(const DoutPrefixProvider *dpp,
+                          optional_yield y,
+                          const std::string& tenant,
+                          const std::string& path_prefix,
+                          const std::string& marker,
+                          uint32_t max_items,
+                          RoleList& listing)
   {
     return 0;
   }

diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 7a28bcc1a4f64cdacb8c5174a3a8badcff853794..9a0f638e3da5b46651335fd7fcbaf5aca3da43e8 100644 (file)
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -910,11 +910,13 @@ public:
           std::multimap<std::string,std::string> tags={}) override;
       virtual std::unique_ptr<RGWRole> get_role(std::string id) override;
       virtual std::unique_ptr<RGWRole> get_role(const RGWRoleInfo& info) override;
-      virtual int get_roles(const DoutPrefixProvider *dpp,
-          optional_yield y,
-          const std::string& path_prefix,
-          const std::string& tenant,
-          std::vector<std::unique_ptr<RGWRole>>& roles) override;
+      int list_roles(const DoutPrefixProvider *dpp,
+                     optional_yield y,
+                     const std::string& tenant,
+                     const std::string& path_prefix,
+                     const std::string& marker,
+                     uint32_t max_items,
+                     RoleList& listing) override;
       virtual std::unique_ptr<RGWOIDCProvider> get_oidc_provider() override;
       virtual int get_oidc_providers(const DoutPrefixProvider *dpp,
           const std::string& tenant,

diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index 4fe26e1c7e27fc19b58cd31eb178eb22a823c479..6b17cd36d5349e05cf27781df6cebbd159b3c66c 100644 (file)
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -570,13 +570,16 @@ std::unique_ptr<RGWRole> FilterDriver::get_role(const RGWRoleInfo& info)
   return next->get_role(info);
 }
 
-int FilterDriver::get_roles(const DoutPrefixProvider *dpp,
-                          optional_yield y,
-                          const std::string& path_prefix,
-                          const std::string& tenant,
-                          std::vector<std::unique_ptr<RGWRole>>& roles)
-{
-  return next->get_roles(dpp, y, path_prefix, tenant, roles);
+int FilterDriver::list_roles(const DoutPrefixProvider *dpp,
+                            optional_yield y,
+                            const std::string& tenant,
+                            const std::string& path_prefix,
+                            const std::string& marker,
+                            uint32_t max_items,
+                            RoleList& listing)
+{
+  return next->list_roles(dpp, y, tenant, path_prefix,
+                          marker, max_items, listing);
 }
 
 std::unique_ptr<RGWOIDCProvider> FilterDriver::get_oidc_provider()

diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 5dfa6c209a93388e7c207ccfc7fbafe393de3529..7b31f9c751fdce2b7b90d22f138cc486b57caebc 100644 (file)
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -396,11 +396,13 @@ public:
                 std::multimap<std::string,std::string> tags={}) override;
   virtual std::unique_ptr<RGWRole> get_role(std::string id) override;
   virtual std::unique_ptr<RGWRole> get_role(const RGWRoleInfo& info) override;
-  virtual int get_roles(const DoutPrefixProvider *dpp,
-                       optional_yield y,
-                       const std::string& path_prefix,
-                       const std::string& tenant,
-                       std::vector<std::unique_ptr<RGWRole>>& roles) override;
+  virtual int list_roles(const DoutPrefixProvider *dpp,
+                        optional_yield y,
+                        const std::string& tenant,
+                        const std::string& path_prefix,
+                        const std::string& marker,
+                        uint32_t max_items,
+                        RoleList& listing) override;
   virtual std::unique_ptr<RGWOIDCProvider> get_oidc_provider() override;
   virtual int get_oidc_providers(const DoutPrefixProvider *dpp,
                                 const std::string& tenant,