From: Sebastien Ponce Date: Sat, 25 Jun 2016 16:04:43 +0000 (+0200) Subject: radosstriper : implementation of async remove/stat/stat2 X-Git-Tag: v11.1.0~355^2~6 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=cf5b105779b8254b30d77f5ea19439f69107494c;p=ceph.git radosstriper : implementation of async remove/stat/stat2 This completes the asynchronous API of libradosstriper for stats and remove Signed-off-by: Sebastien Ponce --- diff --git a/src/include/radosstriper/libradosstriper.h b/src/include/radosstriper/libradosstriper.h index 70595c933d51..5aa4a6bcf6b2 100644 --- a/src/include/radosstriper/libradosstriper.h +++ b/src/include/radosstriper/libradosstriper.h @@ -553,6 +553,26 @@ int rados_striper_aio_read(rados_striper_t striper, const size_t len, uint64_t off); +/** + * Asynchronously removes a striped object + * + * @note There is no atomicity of the deletion and the striped + * object may be left incomplete if an error is returned (metadata + * all present, but some stripes missing) + * However, there is a atomicity of the metadata deletion and + * the deletion can not happen if any I/O is ongoing (it + * will return EBUSY). Identically, no I/O will be able to start + * during deletion (same EBUSY return code) + * @param striper the striper in which the remove will occur + * @param soid the name of the striped object + * @param completion what to do when the remove is safe and complete + * @returns 0 on success, negative error code on failure + */ + +int rados_striper_aio_remove(rados_striper_t striper, + const char* soid, + rados_completion_t completion); + /** * Block until all pending writes in a striper are safe * @@ -565,6 +585,22 @@ int rados_striper_aio_read(rados_striper_t striper, */ void rados_striper_aio_flush(rados_striper_t striper); +/** + * Asynchronously get object stats (size/mtime) + * + * @param striper the striper in which the stat will occur + * @param soid the id of the striped object + * @param psize where to store object size + * @param pmtime where to store modification time + * @param completion what to do when the stats is complete + * @returns 0 on success, negative error code on failure + */ +int rados_striper_aio_stat(rados_striper_t striper, + const char* soid, + rados_completion_t completion, + uint64_t *psize, + time_t *pmtime); + /** @} Asynchronous I/O */ #ifdef __cplusplus diff --git a/src/include/radosstriper/libradosstriper.hpp b/src/include/radosstriper/libradosstriper.hpp index 6352d581e3b5..674a56b71e53 100644 --- a/src/include/radosstriper/libradosstriper.hpp +++ b/src/include/radosstriper/libradosstriper.hpp @@ -178,6 +178,15 @@ namespace libradosstriper * synchronously get striped object stats (size/mtime) */ int stat(const std::string& soid, uint64_t *psize, time_t *pmtime); + int stat2(const std::string& soid, uint64_t *psize, struct timespec *pts); + + /** + * asynchronously get striped object stats (size/mtime) + */ + int aio_stat(const std::string& soid, librados::AioCompletion *c, + uint64_t *psize, time_t *pmtime); + int aio_stat2(const std::string& soid, librados::AioCompletion *c, + uint64_t *psize, struct timespec *pts); /** * deletes a striped object. @@ -191,6 +200,14 @@ namespace libradosstriper */ int remove(const std::string& soid); int remove(const std::string& soid, int flags); + + /** + * asynchronous remove of striped objects + * See synchronous version for comments on (lack of) atomicity + */ + int aio_remove(const std::string& soid, librados::AioCompletion *c); + int aio_remove(const std::string& soid, librados::AioCompletion *c, int flags); + /** * Resizes a striped object * the truncation can not happen if any I/O is ongoing (it diff --git a/src/libradosstriper/RadosStriperImpl.cc b/src/libradosstriper/RadosStriperImpl.cc index 72d465ae156f..ca0a1bb8e262 100644 --- a/src/libradosstriper/RadosStriperImpl.cc +++ b/src/libradosstriper/RadosStriperImpl.cc @@ -76,7 +76,6 @@ * * There are a number of missing features/improvements that could be implemented. * Here are some ideas : - * - asynchronous stat * - implementation of missing entry points (compared to rados) * In particular : clone_range, sparse_read, exec, aio_flush_async, tmaps, omaps, ... * @@ -125,7 +124,10 @@ libradosstriper::RadosStriperImpl::CompletionData::CompletionData RefCountedObject(striper->cct(), n), m_striper(striper), m_soid(soid), m_lockCookie(lockCookie), m_ack(0) { m_striper->get(); - if (userCompletion) m_ack = new librados::IoCtxImpl::C_aio_Ack(userCompletion); + if (userCompletion) { + m_ack = new librados::IoCtxImpl::C_aio_Ack(userCompletion); + userCompletion->io = striper->m_ioCtxImpl; + } } libradosstriper::RadosStriperImpl::CompletionData::~CompletionData() { @@ -574,26 +576,154 @@ int libradosstriper::RadosStriperImpl::aio_flush() int libradosstriper::RadosStriperImpl::stat(const std::string& soid, uint64_t *psize, time_t *pmtime) { - // get pmtime as the pmtime of the first object + // create a completion object + librados::AioCompletionImpl c; + // call asynchronous version of stat + int rc = aio_stat(soid, &c, psize, pmtime); + if (rc == 0) { + // wait for completion of the remove + c.wait_for_complete(); + // get result + rc = c.get_return_value(); + } + return rc; +} + +static void striper_stat_aio_stat_complete(rados_completion_t c, void *arg) { + libradosstriper::RadosStriperImpl::BasicStatCompletionData *data = + reinterpret_cast(arg); + int rc = rados_aio_get_return_value(c); + if (rc == -ENOENT) { + // remember this has failed + data->m_statRC = rc; + } + data->m_multiCompletion->complete_request(rc); +} + +static void striper_stat_aio_getxattr_complete(rados_completion_t c, void *arg) { + libradosstriper::RadosStriperImpl::BasicStatCompletionData *data = + reinterpret_cast(arg); + int rc = rados_aio_get_return_value(c); + // We need to handle the case of sparse files here + if (rc < 0) { + // remember this has failed + data->m_getxattrRC = rc; + } else { + // this intermediate string allows to add a null terminator before calling strtol + std::string err; + std::string strsize(data->m_bl.c_str(), data->m_bl.length()); + *data->m_psize = strict_strtoll(strsize.c_str(), 10, &err); + if (!err.empty()) { + lderr(data->m_striper->cct()) << XATTR_SIZE << " : " << err << dendl; + data->m_getxattrRC = -EINVAL; + } + rc = 0; + } + data->m_multiCompletion->complete_request(rc); +} + +static void striper_stat_aio_req_complete(rados_striper_multi_completion_t c, + void *arg) { + libradosstriper::RadosStriperImpl::BasicStatCompletionData *data = + reinterpret_cast(arg); + if (data->m_statRC) { + data->complete(data->m_statRC); + } else { + if (data->m_getxattrRC < 0) { + data->complete(data->m_getxattrRC); + } else { + data->complete(0); + } + } + data->put(); +} + +template +int libradosstriper::RadosStriperImpl::aio_generic_stat +(const std::string& soid, + librados::AioCompletionImpl *c, + uint64_t *psize, + TimeType *pmtime, + typename libradosstriper::RadosStriperImpl::StatFunction::Type statFunction) +{ + // use a MultiAioCompletion object for dealing with the fact + // that we'll do 2 asynchronous calls in parallel + libradosstriper::MultiAioCompletionImpl *multi_completion = + new libradosstriper::MultiAioCompletionImpl; + // Data object used for passing context to asynchronous calls std::string firstObjOid = getObjectId(soid, 0); - uint64_t obj_size; - int rc = m_ioCtx.stat(firstObjOid, &obj_size, pmtime); - if (rc < 0) return rc; - // get the pmsize from the first object attributes - bufferlist bl; - rc = getxattr(soid, XATTR_SIZE, bl); - if (rc < 0) return rc; - std::string err; - // this intermediate string allows to add a null terminator before calling strtol - std::string strsize(bl.c_str(), bl.length()); - *psize = strict_strtoll(strsize.c_str(), 10, &err); - if (!err.empty()) { - lderr(cct()) << XATTR_SIZE << " : " << err << dendl; - return -EINVAL; + StatCompletionData *cdata = + new StatCompletionData(this, firstObjOid, c, + multi_completion, psize, pmtime); + multi_completion->set_complete_callback(cdata, striper_stat_aio_req_complete); + // use a regular AioCompletion for the stat async call + librados::AioCompletion *stat_completion = + librados::Rados::aio_create_completion(cdata, + striper_stat_aio_stat_complete, 0); + multi_completion->add_request(); + object_t obj(firstObjOid); + int rc = (m_ioCtxImpl->*statFunction)(obj, stat_completion->pc, + &cdata->m_objectSize, cdata->m_pmtime); + stat_completion->release(); + if (rc < 0) { + // nothing is really started so cancel everything + delete multi_completion; + delete cdata; + delete stat_completion; + return rc; + } + // use a regular AioCompletion for the getxattr async call + librados::AioCompletion *getxattr_completion = + librados::Rados::aio_create_completion(cdata, + striper_stat_aio_getxattr_complete, 0); + multi_completion->add_request(); + // in parallel, get the pmsize from the first object asynchronously + rc = m_ioCtxImpl->aio_getxattr(obj, getxattr_completion->pc, + XATTR_SIZE, cdata->m_bl); + getxattr_completion->release(); + multi_completion->finish_adding_requests(); + if (rc < 0) { + // the async stat is ongoing, so we need to go on + // we mark the getxattr as failed in the data object + cdata->m_getxattrRC = rc; + delete getxattr_completion; + multi_completion->complete_request(rc); + return rc; } return 0; } +int libradosstriper::RadosStriperImpl::aio_stat(const std::string& soid, + librados::AioCompletionImpl *c, + uint64_t *psize, + time_t *pmtime) +{ + return aio_generic_stat(soid, c, psize, pmtime, &librados::IoCtxImpl::aio_stat); +} + +int libradosstriper::RadosStriperImpl::stat2(const std::string& soid, uint64_t *psize, struct timespec *pts) +{ + // create a completion object + librados::AioCompletionImpl c; + // call asynchronous version of stat + int rc = aio_stat2(soid, &c, psize, pts); + if (rc == 0) { + // wait for completion of the remove + c.wait_for_complete_and_cb(); + // get result + rc = c.get_return_value(); + } + return rc; +} + +int libradosstriper::RadosStriperImpl::aio_stat2(const std::string& soid, + librados::AioCompletionImpl *c, + uint64_t *psize, + struct timespec *pts) +{ + return aio_generic_stat(soid, c, psize, pts, &librados::IoCtxImpl::aio_stat2); +} + static void rados_req_remove_complete(rados_completion_t c, void *arg) { libradosstriper::RadosStriperImpl::RadosRemoveCompletionData *cdata = @@ -663,6 +793,9 @@ int libradosstriper::RadosStriperImpl::aio_remove(const std::string& soid, new libradosstriper::MultiAioCompletionImpl; multi_completion->set_complete_callback(cdata, striper_remove_aio_req_complete); // call asynchronous internal version of remove + ldout(cct(), 10) + << "RadosStriperImpl : Aio_remove starting for " + << soid << dendl; return internal_aio_remove(soid, multi_completion); } catch (ErrorCode &e) { return e.m_code; @@ -1157,6 +1290,7 @@ int libradosstriper::RadosStriperImpl::truncate(const std::string& soid, // call asynchrous version of truncate int rc = aio_truncate(soid, multi_completion, original_size, size, layout); // wait for completion of the truncation + multi_completion->finish_adding_requests(); multi_completion->wait_for_complete_and_cb(); // return result if (rc == 0) { @@ -1200,6 +1334,7 @@ int libradosstriper::RadosStriperImpl::aio_truncate } if (exists) { // remove asynchronously + multi_completion->add_request(); librados::AioCompletion *rados_completion = librados::Rados::aio_create_completion(multi_completion, rados_req_remove_complete, @@ -1232,6 +1367,7 @@ int libradosstriper::RadosStriperImpl::aio_truncate rc = m_ioCtx.trunc(getObjectId(soid, objectno), new_object_size); } else { // removes are asynchronous in order to speed up truncations of big files + multi_completion->add_request(); librados::AioCompletion *rados_completion = librados::Rados::aio_create_completion(multi_completion, rados_req_remove_complete, diff --git a/src/libradosstriper/RadosStriperImpl.h b/src/libradosstriper/RadosStriperImpl.h index c71952abebfc..74a3062ce862 100644 --- a/src/libradosstriper/RadosStriperImpl.h +++ b/src/libradosstriper/RadosStriperImpl.h @@ -25,6 +25,7 @@ #include "include/radosstriper/libradosstriper.hpp" #include "librados/IoCtxImpl.h" +#include "librados/AioCompletionImpl.h" #include "common/RefCountedObj.h" struct libradosstriper::RadosStriperImpl { @@ -120,6 +121,61 @@ struct libradosstriper::RadosStriperImpl { bufferlist *m_bl; }; + /** + * struct handling (most of) the data needed to pass to the call back + * function in asynchronous stat operations. + * Inherited by the actual type for adding time information in different + * versions (time_t or struct timespec) + */ + struct BasicStatCompletionData : CompletionData { + /// constructor + BasicStatCompletionData(libradosstriper::RadosStriperImpl* striper, + const std::string& soid, + librados::AioCompletionImpl *userCompletion, + libradosstriper::MultiAioCompletionImpl *multiCompletion, + uint64_t *psize) : + CompletionData(striper, soid, "", userCompletion), + m_multiCompletion(multiCompletion), m_psize(psize), + m_statRC(0), m_getxattrRC(0) {}; + // MultiAioCompletionImpl used to handle the double aysnc + // call in the back (stat + getxattr) + libradosstriper::MultiAioCompletionImpl *m_multiCompletion; + // where to store the size of first objct + // this will be ignored but we need a place to store it when + // async stat is called + uint64_t m_objectSize; + // where to store the file size + uint64_t *m_psize; + /// the bufferlist object used for the getxattr call + bufferlist m_bl; + /// return code of the stat + int m_statRC; + /// return code of the getxattr + int m_getxattrRC; + }; + + /** + * struct handling the data needed to pass to the call back + * function in asynchronous stat operations. + * Simple templated extension of BasicStatCompletionData. + * The template parameter is the type of the time information + * (used with time_t for stat and struct timespec for stat2) + */ + template + struct StatCompletionData : BasicStatCompletionData { + /// constructor + StatCompletionData(libradosstriper::RadosStriperImpl* striper, + const std::string& soid, + librados::AioCompletionImpl *userCompletion, + libradosstriper::MultiAioCompletionImpl *multiCompletion, + uint64_t *psize, + TimeType *pmtime) : + BasicStatCompletionData(striper, soid, userCompletion, multiCompletion, psize), + m_pmtime(pmtime) {}; + // where to store the file time + TimeType *m_pmtime; + }; + /** * struct handling the data needed to pass to the call back * function in asynchronous remove operations of a Rados File @@ -239,6 +295,21 @@ struct libradosstriper::RadosStriperImpl { // stat, deletion and truncation int stat(const std::string& soid, uint64_t *psize, time_t *pmtime); + int stat2(const std::string& soid, uint64_t *psize, struct timespec *pts); + template + struct StatFunction { + typedef int (librados::IoCtxImpl::*Type) (const object_t& oid, + librados::AioCompletionImpl *c, + uint64_t *psize, TimeType *pmtime); + }; + template + int aio_generic_stat(const std::string& soid, librados::AioCompletionImpl *c, + uint64_t *psize, TimeType *pmtime, + typename StatFunction::Type statFunction); + int aio_stat(const std::string& soid, librados::AioCompletionImpl *c, + uint64_t *psize, time_t *pmtime); + int aio_stat2(const std::string& soid, librados::AioCompletionImpl *c, + uint64_t *psize, struct timespec *pts); int remove(const std::string& soid, int flags=0); int trunc(const std::string& soid, uint64_t size); diff --git a/src/libradosstriper/libradosstriper.cc b/src/libradosstriper/libradosstriper.cc index 352fe390bacc..c72899a5fa0a 100644 --- a/src/libradosstriper/libradosstriper.cc +++ b/src/libradosstriper/libradosstriper.cc @@ -269,15 +269,50 @@ int libradosstriper::RadosStriper::stat(const std::string& soid, uint64_t *psize return rados_striper_impl->stat(soid, psize, pmtime); } +int libradosstriper::RadosStriper::aio_stat(const std::string& soid, + librados::AioCompletion *c, + uint64_t *psize, + time_t *pmtime) +{ + return rados_striper_impl->aio_stat(soid, c->pc, psize, pmtime); +} + +int libradosstriper::RadosStriper::stat2(const std::string& soid, uint64_t *psize, struct timespec *pts) +{ + return rados_striper_impl->stat2(soid, psize, pts); +} + +int libradosstriper::RadosStriper::aio_stat2(const std::string& soid, + librados::AioCompletion *c, + uint64_t *psize, + struct timespec *pts) +{ + return rados_striper_impl->aio_stat2(soid, c->pc, psize, pts); +} + int libradosstriper::RadosStriper::remove(const std::string& soid) { return rados_striper_impl->remove(soid); } + +int libradosstriper::RadosStriper::aio_remove(const std::string& soid, + librados::AioCompletion *c) +{ + return rados_striper_impl->aio_remove(soid, c->pc); +} + int libradosstriper::RadosStriper::remove(const std::string& soid, int flags) { return rados_striper_impl->remove(soid, flags); } +int libradosstriper::RadosStriper::aio_remove(const std::string& soid, + librados::AioCompletion *c, + int flags) +{ + return rados_striper_impl->aio_remove(soid, c->pc, flags); +} + int libradosstriper::RadosStriper::trunc(const std::string& soid, uint64_t size) { return rados_striper_impl->trunc(soid, size); @@ -610,8 +645,26 @@ extern "C" int rados_striper_aio_read(rados_striper_t striper, return impl->aio_read(soid, (librados::AioCompletionImpl*)completion, buf, len, off); } +extern "C" int rados_striper_aio_remove(rados_striper_t striper, + const char* soid, + rados_completion_t completion) +{ + libradosstriper::RadosStriperImpl *impl = (libradosstriper::RadosStriperImpl *)striper; + return impl->aio_remove(soid, (librados::AioCompletionImpl*)completion); +} + extern "C" void rados_striper_aio_flush(rados_striper_t striper) { libradosstriper::RadosStriperImpl *impl = (libradosstriper::RadosStriperImpl *)striper; impl->aio_flush(); } + +extern "C" int rados_striper_aio_stat(rados_striper_t striper, + const char* soid, + rados_completion_t completion, + uint64_t *psize, + time_t *pmtime) +{ + libradosstriper::RadosStriperImpl *impl = (libradosstriper::RadosStriperImpl *)striper; + return impl->aio_stat(soid, (librados::AioCompletionImpl*)completion, psize, pmtime); +} diff --git a/src/test/libradosstriper/aio.cc b/src/test/libradosstriper/aio.cc index 847e01141df3..8d92cbbdf10e 100644 --- a/src/test/libradosstriper/aio.cc +++ b/src/test/libradosstriper/aio.cc @@ -5,6 +5,7 @@ #include "test/librados/test.h" #include "test/libradosstriper/TestCase.h" +#include #include #include #include @@ -573,3 +574,42 @@ TEST_F(StriperTestPP, RoundTripWriteFullPP) { my_completion2->release(); my_completion3->release(); } + +TEST_F(StriperTest, RemoveTest) { + char buf[128]; + char buf2[sizeof(buf)]; + // create oabject + memset(buf, 0xaa, sizeof(buf)); + ASSERT_EQ(0, rados_striper_write(striper, "RemoveTest", buf, sizeof(buf), 0)); + // async remove it + AioTestData test_data; + rados_completion_t my_completion; + ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data, + set_completion_complete, set_completion_safe, &my_completion)); + ASSERT_EQ(0, rados_striper_aio_remove(striper, "RemoveTest", my_completion)); + { + TestAlarm alarm; + ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion)); + } + ASSERT_EQ(0, rados_aio_get_return_value(my_completion)); + rados_aio_release(my_completion); + // check we get ENOENT on reading + ASSERT_EQ(-ENOENT, rados_striper_read(striper, "RemoveTest", buf2, sizeof(buf2), 0)); +} + +TEST_F(StriperTestPP, RemoveTestPP) { + char buf[128]; + memset(buf, 0xaa, sizeof(buf)); + bufferlist bl; + bl.append(buf, sizeof(buf)); + ASSERT_EQ(0, striper.write("RemoveTestPP", bl, sizeof(buf), 0)); + boost::scoped_ptr my_completion(cluster.aio_create_completion(0, 0, 0)); + ASSERT_EQ(0, striper.aio_remove("RemoveTestPP", my_completion.get())); + { + TestAlarm alarm; + ASSERT_EQ(0, my_completion->wait_for_complete()); + } + ASSERT_EQ(0, my_completion->get_return_value()); + bufferlist bl2; + ASSERT_EQ(-ENOENT, striper.read("RemoveTestPP", &bl2, sizeof(buf), 0)); +}