From 411adabc9d6da39289416326db8c078a8f19e26c Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 4 Jun 2009 12:00:44 -0700 Subject: [PATCH] rados: fixes for dynamic oids --- src/include/librados.h | 6 +++-- src/librados.cc | 58 ++++++++++++++++++++++++++++------------- src/os/FileStore.cc | 11 +++++--- src/osd/PGLS.h | 2 +- src/osd/ReplicatedPG.cc | 10 ++++--- src/testrados.c | 9 +++++++ src/testradospp.cc | 8 +++--- 7 files changed, 73 insertions(+), 31 deletions(-) diff --git a/src/include/librados.h b/src/include/librados.h index 4812a0f99e386..b2bfb0e777175 100644 --- a/src/include/librados.h +++ b/src/include/librados.h @@ -26,7 +26,9 @@ typedef void *rados_list_ctx_t; typedef int rados_pool_t; int rados_open_pool(const char *name, rados_pool_t *pool); int rados_close_pool(rados_pool_t pool); -int rados_list(rados_pool_t pool, int max, char *entries, rados_list_ctx_t *ctx); +void rados_pool_init_ctx(rados_list_ctx_t *ctx); +void rados_pool_close_ctx(rados_list_ctx_t *ctx); +int rados_pool_list_next(rados_pool_t pool, const char **entry, rados_list_ctx_t *ctx); /* read/write objects */ int rados_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len); @@ -76,7 +78,7 @@ public: ListCtx() : ctx(NULL) {} }; - int list(rados_pool_t pool, int max, vector& entries, Rados::ListCtx& ctx); + int list(rados_pool_t pool, int max, std::list& entries, Rados::ListCtx& ctx); // -- aio -- struct AioCompletion { diff --git a/src/librados.cc b/src/librados.cc index e4395a5b7425d..03d533de72050 100644 --- a/src/librados.cc +++ b/src/librados.cc @@ -79,11 +79,15 @@ public: struct PGLSOp { int seed; __u64 cookie; + std::list list; + std::list::iterator iter; + __u64 pos; + __u64 total; - PGLSOp() : seed(0), cookie(0) {} + PGLSOp() : seed(0), cookie(0), pos(0), total(0) {} }; - int list(int pool, int max_entries, vector& entries, RadosClient::PGLSOp& op); + int list(int pool, int max_entries, std::list& entries, RadosClient::PGLSOp& op); // --- aio --- struct AioCompletion { @@ -314,7 +318,7 @@ bool RadosClient::_dispatch(Message *m) return true; } -int RadosClient::list(int pool, int max_entries, vector& entries, RadosClient::PGLSOp& op) +int RadosClient::list(int pool, int max_entries, std::list& entries, RadosClient::PGLSOp& op) { SnapContext snapc; utime_t ut = g_clock.now(); @@ -326,6 +330,7 @@ int RadosClient::list(int pool, int max_entries, vector& entries, Rado object_t oid; memset(&oid, 0, sizeof(oid)); + entries.clear(); ceph_object_layout layout; retry: @@ -362,7 +367,7 @@ retry: op.cookie = (__u64)response.handle; response_size = response.entries.size(); if (response_size) { - entries.swap(response.entries); + entries.merge(response.entries); max_entries -= response_size; if (!max_entries) return r; @@ -565,7 +570,7 @@ int Rados::initialize(int argc, const char *argv[]) return client->init() ? 0 : -1; } -int Rados::list(rados_pool_t pool, int max, vector& entries, Rados::ListCtx& ctx) +int Rados::list(rados_pool_t pool, int max, std::list& entries, Rados::ListCtx& ctx) { if (!client) return -EINVAL; @@ -804,8 +809,21 @@ extern "C" int rados_exec(rados_pool_t pool, const char *o, const char *cls, con } return ret; } +extern "C" void rados_pool_init_ctx(rados_list_ctx_t *ctx) +{ + *ctx = NULL; +} + +extern "C" void rados_pool_close_ctx(rados_list_ctx_t *ctx) +{ + if (*ctx) { + RadosClient::PGLSOp *op = (RadosClient::PGLSOp *)*ctx; + delete op; + *ctx = NULL; + } +} -extern "C" int rados_list(rados_pool_t pool, int max, char *entries, rados_list_ctx_t *ctx) +extern "C" int rados_pool_list_next(rados_pool_t pool, const char **entry, rados_list_ctx_t *ctx) { int ret; @@ -815,21 +833,25 @@ extern "C" int rados_list(rados_pool_t pool, int max, char *entries, rados_list_ return -ENOMEM; } RadosClient::PGLSOp *op = (RadosClient::PGLSOp *)*ctx; - - vector vec; - ret = radosp->list(pool, max, vec, *op); - if (!vec.size()) { - delete op; - *ctx = NULL; + if (op->pos == op->total) { + op->list.clear(); +#define MAX_ENTRIES 1024 + ret = radosp->list(pool, MAX_ENTRIES, op->list, *op); + if (!op->list.size()) { + delete op; + *ctx = NULL; + return -ENOENT; + } + op->pos = 0; + op->total = op->list.size(); + op->iter = op->list.begin(); } -#warning fixme - /*for (int i=0; iiter->name.c_str(); + op->pos++; + op->iter++; - return ret; + return 0; } diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index e884be269f03b..4ab23d2172121 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -1890,6 +1890,7 @@ int FileStore::collection_list_partial(coll_t c, vector& ls, int max_ DIR *dir = NULL; struct dirent *de; + bool end; // first, build (ino, object) list vector< pair > inolist; @@ -1902,19 +1903,23 @@ int FileStore::collection_list_partial(coll_t c, vector& ls, int max_ return -errno; } - if (handle) { + if (handle && *handle) { seekdir(dir, *(off_t *)handle); + *handle = 0; } for (int i=0; id_name[0] == '.') { @@ -1929,7 +1934,7 @@ int FileStore::collection_list_partial(coll_t c, vector& ls, int max_ } } - if (handle) + if (handle && !end) *handle = (collection_list_handle_t)telldir(dir); ::closedir(dir); diff --git a/src/osd/PGLS.h b/src/osd/PGLS.h index 13d1118e4dcb3..a36c50f43a736 100644 --- a/src/osd/PGLS.h +++ b/src/osd/PGLS.h @@ -7,7 +7,7 @@ struct PGLSResponse { collection_list_handle_t handle; - vector entries; + list entries; void encode(bufferlist& bl) const { ::encode((__u64)handle, bl); diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 7f33a12eba2df..873edc94dac42 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -383,10 +383,14 @@ void ReplicatedPG::do_pg_op(MOSDOp *op) // read into a buffer PGLSResponse response; response.handle = (collection_list_handle_t)(__u64)(p->pgls_cookie); - //result = osd->store->collection_list_partial(op->get_pg().to_coll(), response.entries, p->length, - //&response.handle); -#warning fixme + vector sentries; + result = osd->store->collection_list_partial(op->get_pg().to_coll(), sentries, p->length, + &response.handle); if (!result) { + vector::iterator iter; + for (iter = sentries.begin(); iter != sentries.end(); ++iter) { + response.entries.push_back(iter->oid); + } ::encode(response, outdata); } dout(10) << " pgls result=" << result << " outdata.length()=" << outdata.length() << dendl; diff --git a/src/testrados.c b/src/testrados.c index c4a25fddf5805..c9c7db27c804e 100644 --- a/src/testrados.c +++ b/src/testrados.c @@ -56,8 +56,17 @@ int main(int argc, const char **argv) rados_aio_release(a); rados_aio_release(b); + const char *entry; + rados_list_ctx_t ctx; + rados_pool_init_ctx(&ctx); + while (rados_pool_list_next(pool, &entry, &ctx) >= 0) { + printf("list entry: %s\n", entry); + } + rados_pool_close_ctx(&ctx); + rados_close_pool(pool); + rados_deinitialize(); return 0; diff --git a/src/testradospp.cc b/src/testradospp.cc index 8893caeb93055..0124e1027afc2 100644 --- a/src/testradospp.cc +++ b/src/testradospp.cc @@ -42,7 +42,7 @@ int main(int argc, const char **argv) snprintf(buf, 128, "%s", ctime(&tm)); bl.append(buf, strlen(buf)); - object_t oid("foo"); + object_t oid("bar"); rados_pool_t pool; int r = rados.open_pool("data", &pool); @@ -71,11 +71,11 @@ int main(int argc, const char **argv) Rados::ListCtx ctx; int entries; do { - vector vec; - r = rados.list(pool, 1, vec, ctx); + list vec; + r = rados.list(pool, 2, vec, ctx); entries = vec.size(); cout << "list result=" << r << " entries=" << entries << std::endl; - vector::iterator iter; + list::iterator iter; for (iter = vec.begin(); iter != vec.end(); ++iter) { cout << *iter << std::endl; } -- 2.39.5