From: Kefu Chai Date: Mon, 5 Jun 2017 05:46:54 +0000 (+0800) Subject: radosstriper: use intrusive_ptr to manage lifecycle of MultiAioCompletionImpl X-Git-Tag: ses5-milestone8~1^2~22^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b7e937335856901eb481c4775193343192a70836;p=ceph.git radosstriper: use intrusive_ptr to manage lifecycle of MultiAioCompletionImpl this change also silence some cppcheck warnings: we fail to rlease references to MultiAioCompletionImpl in some of the error handling paths. Signed-off-by: Kefu Chai --- diff --git a/src/libradosstriper/MultiAioCompletionImpl.cc b/src/libradosstriper/MultiAioCompletionImpl.cc index 270182918cc7..7bb21bc0e0e3 100644 --- a/src/libradosstriper/MultiAioCompletionImpl.cc +++ b/src/libradosstriper/MultiAioCompletionImpl.cc @@ -59,3 +59,13 @@ void libradosstriper::MultiAioCompletionImpl::finish_adding_requests() safe(); lock.Unlock(); } + +void intrusive_ptr_add_ref(libradosstriper::MultiAioCompletionImpl* ptr) +{ + ptr->get(); +} + +void intrusive_ptr_release(libradosstriper::MultiAioCompletionImpl* ptr) +{ + ptr->put(); +} diff --git a/src/libradosstriper/MultiAioCompletionImpl.h b/src/libradosstriper/MultiAioCompletionImpl.h index a7d8ed4eb6b0..696f3def90e8 100644 --- a/src/libradosstriper/MultiAioCompletionImpl.h +++ b/src/libradosstriper/MultiAioCompletionImpl.h @@ -177,4 +177,7 @@ struct libradosstriper::MultiAioCompletionImpl { }; +void intrusive_ptr_add_ref(libradosstriper::MultiAioCompletionImpl*); +void intrusive_ptr_release(libradosstriper::MultiAioCompletionImpl*); + #endif // CEPH_LIBRADOSSTRIPERSTRIPER_MULTIAIOCOMPLETIONIMPL_H diff --git a/src/libradosstriper/RadosStriperImpl.cc b/src/libradosstriper/RadosStriperImpl.cc index fd5f13f0065f..52783b53cc79 100644 --- a/src/libradosstriper/RadosStriperImpl.cc +++ b/src/libradosstriper/RadosStriperImpl.cc @@ -26,7 +26,6 @@ #include "common/dout.h" #include "common/strtol.h" #include "osdc/Striper.h" -#include "libradosstriper/MultiAioCompletionImpl.h" #include "librados/AioCompletionImpl.h" #include @@ -112,10 +111,38 @@ struct ceph_file_layout default_file_layout = { init_le32(-1), // fl_pg_pool }; +using libradosstriper::MultiAioCompletionImplPtr; + +namespace { ///////////////////////// CompletionData ///////////////////////////// -libradosstriper::RadosStriperImpl::CompletionData::CompletionData +/** + * struct handling the data needed to pass to the call back + * function in asynchronous operations + */ +struct CompletionData : RefCountedObject { + /// constructor + CompletionData(libradosstriper::RadosStriperImpl * striper, + const std::string& soid, + const std::string& lockCookie, + librados::AioCompletionImpl *userCompletion = 0, + int n = 1); + /// destructor + ~CompletionData() override; + /// complete method + void complete(int r); + /// striper to be used to handle the write completion + libradosstriper::RadosStriperImpl *m_striper; + /// striped object concerned by the write operation + std::string m_soid; + /// shared lock to be released at completion + std::string m_lockCookie; + /// completion handler + librados::IoCtxImpl::C_aio_Complete *m_ack; +}; + +CompletionData::CompletionData (libradosstriper::RadosStriperImpl* striper, const std::string& soid, const std::string& lockCookie, @@ -130,16 +157,48 @@ libradosstriper::RadosStriperImpl::CompletionData::CompletionData } } -libradosstriper::RadosStriperImpl::CompletionData::~CompletionData() { +CompletionData::~CompletionData() { if (m_ack) delete m_ack; m_striper->put(); } -void libradosstriper::RadosStriperImpl::CompletionData::complete(int r) { +void CompletionData::complete(int r) { if (m_ack) m_ack->finish(r); } -libradosstriper::RadosStriperImpl::ReadCompletionData::ReadCompletionData +/** + * struct handling the data needed to pass to the call back + * function in asynchronous read operations + */ +struct ReadCompletionData : CompletionData { + /// bufferlist containing final result + bufferlist* m_bl; + /// extents that will be read + std::vector* m_extents; + /// intermediate results + std::vector* m_resultbl; + /// return code of read completion, to be remembered until unlocking happened + int m_readRc; + /// completion object for the unlocking of the striped object at the end of the read + librados::AioCompletion *m_unlockCompletion; + /// constructor + ReadCompletionData(libradosstriper::RadosStriperImpl * striper, + const std::string& soid, + const std::string& lockCookie, + librados::AioCompletionImpl *userCompletion, + bufferlist* bl, + std::vector* extents, + std::vector* resultbl, + int n); + /// destructor + ~ReadCompletionData() override; + /// complete method for when reading is over + void complete_read(int r); + /// complete method for when object is unlocked + void complete_unlock(int r); +}; + +ReadCompletionData::ReadCompletionData (libradosstriper::RadosStriperImpl* striper, const std::string& soid, const std::string& lockCookie, @@ -152,13 +211,13 @@ libradosstriper::RadosStriperImpl::ReadCompletionData::ReadCompletionData m_bl(bl), m_extents(extents), m_resultbl(resultbl), m_readRc(0), m_unlockCompletion(0) {} -libradosstriper::RadosStriperImpl::ReadCompletionData::~ReadCompletionData() { +ReadCompletionData::~ReadCompletionData() { m_unlockCompletion->release(); delete m_extents; delete m_resultbl; } -void libradosstriper::RadosStriperImpl::ReadCompletionData::complete_read(int r) { +void ReadCompletionData::complete_read(int r) { // gather data into final buffer Striper::StripedReadResult readResult; vector::iterator bit = m_resultbl->begin(); @@ -173,13 +232,40 @@ void libradosstriper::RadosStriperImpl::ReadCompletionData::complete_read(int r) m_readRc = r; } -void libradosstriper::RadosStriperImpl::ReadCompletionData::complete_unlock(int r) { +void ReadCompletionData::complete_unlock(int r) { // call parent's completion method // Note that we ignore the return code of the unlock as we cannot do much about it CompletionData::complete(m_readRc?m_readRc:m_bl->length()); } -libradosstriper::RadosStriperImpl::WriteCompletionData::WriteCompletionData +/** + * struct handling the data needed to pass to the call back + * function in asynchronous write operations + */ +struct WriteCompletionData : CompletionData { + /// safe completion handler + librados::IoCtxImpl::C_aio_Complete *m_safe; + /// return code of write completion, to be remembered until unlocking happened + int m_writeRc; + /// completion object for the unlocking of the striped object at the end of the write + librados::AioCompletion *m_unlockCompletion; + /// constructor + WriteCompletionData(libradosstriper::RadosStriperImpl * striper, + const std::string& soid, + const std::string& lockCookie, + librados::AioCompletionImpl *userCompletion, + int n); + /// destructor + ~WriteCompletionData() override; + /// complete method for when writing is over + void complete_write(int r); + /// complete method for when object is unlocked + void complete_unlock(int r); + /// safe method + void safe(int r); +}; + +WriteCompletionData::WriteCompletionData (libradosstriper::RadosStriperImpl* striper, const std::string& soid, const std::string& lockCookie, @@ -192,46 +278,160 @@ libradosstriper::RadosStriperImpl::WriteCompletionData::WriteCompletionData } } -libradosstriper::RadosStriperImpl::WriteCompletionData::~WriteCompletionData() { +WriteCompletionData::~WriteCompletionData() { m_unlockCompletion->release(); if (m_safe) delete m_safe; } -void libradosstriper::RadosStriperImpl::WriteCompletionData::complete_unlock(int r) { +void WriteCompletionData::complete_unlock(int r) { // call parent's completion method // Note that we ignore the return code of the unlock as we cannot do much about it CompletionData::complete(m_writeRc); } -void libradosstriper::RadosStriperImpl::WriteCompletionData::complete_write(int r) { +void WriteCompletionData::complete_write(int r) { // Remember return code m_writeRc = r; } -void libradosstriper::RadosStriperImpl::WriteCompletionData::safe(int r) { +void WriteCompletionData::safe(int r) { if (m_safe) m_safe->finish(r); } -libradosstriper::RadosStriperImpl::RemoveCompletionData::RemoveCompletionData -(libradosstriper::RadosStriperImpl* striper, - const std::string& soid, - const std::string& lockCookie, - librados::AioCompletionImpl *userCompletion, - int flags) : +struct RemoveCompletionData : CompletionData { + /// removal flags + int flags; + /** + * constructor + * note that the constructed object will take ownership of the lock + */ + RemoveCompletionData(libradosstriper::RadosStriperImpl * striper, + const std::string& soid, + const std::string& lockCookie, + librados::AioCompletionImpl *userCompletion, + int flags = 0) : CompletionData(striper, soid, lockCookie, userCompletion), flags(flags) {} +}; -libradosstriper::RadosStriperImpl::TruncateCompletionData::TruncateCompletionData -(libradosstriper::RadosStriperImpl* striper, - const std::string& soid, - uint64_t size) : - RefCountedObject(striper->cct()), - m_striper(striper), m_soid(soid), m_size(size) { - m_striper->get(); -} +/** + * struct handling the data needed to pass to the call back + * function in asynchronous truncate operations + */ +struct TruncateCompletionData : RefCountedObject { + /// constructor + TruncateCompletionData(libradosstriper::RadosStriperImpl* striper, + const std::string& soid, + uint64_t size) : + RefCountedObject(striper->cct()), + m_striper(striper), m_soid(soid), m_size(size) { + m_striper->get(); + } + /// destructor + ~TruncateCompletionData() override { + m_striper->put(); + } + /// striper to be used + libradosstriper::RadosStriperImpl *m_striper; + /// striped object concerned by the truncate operation + std::string m_soid; + /// the final size of the truncated object + uint64_t m_size; +}; -libradosstriper::RadosStriperImpl::TruncateCompletionData::~TruncateCompletionData() { - m_striper->put(); -} +/** + * struct handling the data needed to pass to the call back + * function in asynchronous read operations of a Rados File + */ +struct RadosReadCompletionData : RefCountedObject { + /// constructor + RadosReadCompletionData(MultiAioCompletionImplPtr multiAioCompl, + uint64_t expectedBytes, + bufferlist *bl, + CephContext *context, + int n = 1) : + RefCountedObject(context, n), + m_multiAioCompl(multiAioCompl), m_expectedBytes(expectedBytes), m_bl(bl) {} + /// the multi asynch io completion object to be used + MultiAioCompletionImplPtr m_multiAioCompl; + /// the expected number of bytes + uint64_t m_expectedBytes; + /// the bufferlist object where data have been written + bufferlist *m_bl; +}; + +/** + * struct handling (most of) the data needed to pass to the call back + * function in asynchronous stat operations. + * Inherited by the actual type for adding time information in different + * versions (time_t or struct timespec) + */ +struct BasicStatCompletionData : CompletionData { + /// constructor + BasicStatCompletionData(libradosstriper::RadosStriperImpl* striper, + const std::string& soid, + librados::AioCompletionImpl *userCompletion, + libradosstriper::MultiAioCompletionImpl *multiCompletion, + uint64_t *psize, + int n = 1) : + CompletionData(striper, soid, "", userCompletion, n), + m_multiCompletion(multiCompletion), m_psize(psize), + m_statRC(0), m_getxattrRC(0) {}; + // MultiAioCompletionImpl used to handle the double aysnc + // call in the back (stat + getxattr) + libradosstriper::MultiAioCompletionImpl *m_multiCompletion; + // where to store the size of first objct + // this will be ignored but we need a place to store it when + // async stat is called + uint64_t m_objectSize; + // where to store the file size + uint64_t *m_psize; + /// the bufferlist object used for the getxattr call + bufferlist m_bl; + /// return code of the stat + int m_statRC; + /// return code of the getxattr + int m_getxattrRC; +}; + +/** + * struct handling the data needed to pass to the call back + * function in asynchronous stat operations. + * Simple templated extension of BasicStatCompletionData. + * The template parameter is the type of the time information + * (used with time_t for stat and struct timespec for stat2) + */ +template +struct StatCompletionData : BasicStatCompletionData { + /// constructor + StatCompletionData(libradosstriper::RadosStriperImpl* striper, + const std::string& soid, + librados::AioCompletionImpl *userCompletion, + libradosstriper::MultiAioCompletionImpl *multiCompletion, + uint64_t *psize, + TimeType *pmtime, + int n = 1) : + BasicStatCompletionData(striper, soid, userCompletion, multiCompletion, psize, n), + m_pmtime(pmtime) {}; + // where to store the file time + TimeType *m_pmtime; +}; + +/** + * struct handling the data needed to pass to the call back + * function in asynchronous remove operations of a Rados File + */ +struct RadosRemoveCompletionData : RefCountedObject { + /// constructor + RadosRemoveCompletionData(MultiAioCompletionImplPtr multiAioCompl, + CephContext *context) : + RefCountedObject(context, 2), + m_multiAioCompl(multiAioCompl) {}; + /// the multi asynch io completion object to be used + MultiAioCompletionImplPtr m_multiAioCompl; +}; + + +} // namespace { ///////////////////////// constructor ///////////////////////////// @@ -412,8 +612,7 @@ int libradosstriper::RadosStriperImpl::aio_write_full(const std::string& soid, static void rados_read_aio_unlock_complete(rados_striper_multi_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::ReadCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); libradosstriper::MultiAioCompletionImpl *comp = reinterpret_cast(c); cdata->complete_unlock(comp->rval); @@ -422,8 +621,7 @@ static void rados_read_aio_unlock_complete(rados_striper_multi_completion_t c, v static void striper_read_aio_req_complete(rados_striper_multi_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::ReadCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); // launch the async unlocking of the object cdata->m_striper->aio_unlockObject(cdata->m_soid, cdata->m_lockCookie, cdata->m_unlockCompletion); // complete the read part in parallel @@ -434,21 +632,19 @@ static void striper_read_aio_req_complete(rados_striper_multi_completion_t c, vo static void rados_req_read_safe(rados_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::RadosReadCompletionData *data = - reinterpret_cast(arg); + auto data = reinterpret_cast(arg); int rc = rados_aio_get_return_value(c); // ENOENT means that we are dealing with a sparse file. This is fine, // data (0s) will be created on the fly by the rados_req_read_complete method if (rc == -ENOENT) rc = 0; - libradosstriper::MultiAioCompletionImpl *multiAioComp = data->m_multiAioCompl; + auto multiAioComp = data->m_multiAioCompl; multiAioComp->safe_request(rc); data->put(); } static void rados_req_read_complete(rados_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::RadosReadCompletionData *data = - reinterpret_cast(arg); + auto data = reinterpret_cast(arg); int rc = rados_aio_get_return_value(c); // We need to handle the case of sparse files here if (rc == -ENOENT) { @@ -472,7 +668,7 @@ static void rados_req_read_complete(rados_completion_t c, void *arg) } rc = data->m_expectedBytes; } - libradosstriper::MultiAioCompletionImpl * multiAioComp = data->m_multiAioCompl; + auto multiAioComp = data->m_multiAioCompl; multiAioComp->complete_request(rc); data->put(); } @@ -519,7 +715,8 @@ int libradosstriper::RadosStriperImpl::aio_read(const std::string& soid, librados::Rados::aio_create_completion(cdata, rados_read_aio_unlock_complete, 0); cdata->m_unlockCompletion = unlock_completion; // create the multiCompletion object handling the reads - libradosstriper::MultiAioCompletionImpl *nc = new libradosstriper::MultiAioCompletionImpl; + MultiAioCompletionImplPtr nc{new libradosstriper::MultiAioCompletionImpl, + false}; nc->set_complete_callback(cdata, striper_read_aio_req_complete); // go through the extents int r = 0, i = 0; @@ -546,7 +743,6 @@ int libradosstriper::RadosStriperImpl::aio_read(const std::string& soid, break; } nc->finish_adding_requests(); - nc->put(); return r; } @@ -596,8 +792,7 @@ int libradosstriper::RadosStriperImpl::stat(const std::string& soid, uint64_t *p } static void striper_stat_aio_stat_complete(rados_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::BasicStatCompletionData *data = - reinterpret_cast(arg); + auto data = reinterpret_cast(arg); int rc = rados_aio_get_return_value(c); if (rc == -ENOENT) { // remember this has failed @@ -608,8 +803,7 @@ static void striper_stat_aio_stat_complete(rados_completion_t c, void *arg) { } static void striper_stat_aio_getxattr_complete(rados_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::BasicStatCompletionData *data = - reinterpret_cast(arg); + auto data = reinterpret_cast(arg); int rc = rados_aio_get_return_value(c); // We need to handle the case of sparse files here if (rc < 0) { @@ -632,8 +826,7 @@ static void striper_stat_aio_getxattr_complete(rados_completion_t c, void *arg) static void striper_stat_aio_req_complete(rados_striper_multi_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::BasicStatCompletionData *data = - reinterpret_cast(arg); + auto data = reinterpret_cast(arg); if (data->m_statRC) { data->complete(data->m_statRC); } else { @@ -656,13 +849,13 @@ int libradosstriper::RadosStriperImpl::aio_generic_stat { // use a MultiAioCompletion object for dealing with the fact // that we'll do 2 asynchronous calls in parallel - libradosstriper::MultiAioCompletionImpl *multi_completion = - new libradosstriper::MultiAioCompletionImpl; + MultiAioCompletionImplPtr multi_completion{ + new libradosstriper::MultiAioCompletionImpl, false}; // Data object used for passing context to asynchronous calls std::string firstObjOid = getObjectId(soid, 0); StatCompletionData *cdata = new StatCompletionData(this, firstObjOid, c, - multi_completion, psize, pmtime, 4); + multi_completion.get(), psize, pmtime, 4); multi_completion->set_complete_callback(cdata, striper_stat_aio_req_complete); // use a regular AioCompletion for the stat async call librados::AioCompletion *stat_completion = @@ -674,7 +867,6 @@ int libradosstriper::RadosStriperImpl::aio_generic_stat stat_completion->release(); if (rc < 0) { // nothing is really started so cancel everything - delete multi_completion; delete cdata; return rc; } @@ -692,11 +884,9 @@ int libradosstriper::RadosStriperImpl::aio_generic_stat // we mark the getxattr as failed in the data object cdata->m_getxattrRC = rc; multi_completion->complete_request(rc); - multi_completion->put(); return rc; } cdata->put(); - multi_completion->put(); return 0; } @@ -733,8 +923,7 @@ int libradosstriper::RadosStriperImpl::aio_stat2(const std::string& soid, static void rados_req_remove_complete(rados_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::RadosRemoveCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); int rc = rados_aio_get_return_value(c); // in case the object did not exist, it means we had a sparse file, all is fine if (rc == -ENOENT) { @@ -746,8 +935,7 @@ static void rados_req_remove_complete(rados_completion_t c, void *arg) static void rados_req_remove_safe(rados_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::RadosRemoveCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); int rc = rados_aio_get_return_value(c); // in case the object did not exist, it means we had a sparse file, all is fine if (rc == -ENOENT) { @@ -759,8 +947,7 @@ static void rados_req_remove_safe(rados_completion_t c, void *arg) static void striper_remove_aio_req_complete(rados_striper_multi_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::RemoveCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); libradosstriper::MultiAioCompletionImpl *comp = reinterpret_cast(c); ldout(cdata->m_striper->cct(), 10) @@ -808,21 +995,20 @@ int libradosstriper::RadosStriperImpl::aio_remove(const std::string& soid, if (rc) return rc; // create CompletionData for the async remove call RemoveCompletionData *cdata = new RemoveCompletionData(this, soid, lockCookie, c, flags); - libradosstriper::MultiAioCompletionImpl *multi_completion = - new libradosstriper::MultiAioCompletionImpl; + MultiAioCompletionImplPtr multi_completion{ + new libradosstriper::MultiAioCompletionImpl, false}; multi_completion->set_complete_callback(cdata, striper_remove_aio_req_complete); // call asynchronous internal version of remove ldout(cct(), 10) << "RadosStriperImpl : Aio_remove starting for " << soid << dendl; rc = internal_aio_remove(soid, multi_completion); - multi_completion->put(); return rc; } -int libradosstriper::RadosStriperImpl::internal_aio_remove -(const std::string& soid, - libradosstriper::MultiAioCompletionImpl *multi_completion, +int libradosstriper::RadosStriperImpl::internal_aio_remove( + const std::string& soid, + MultiAioCompletionImplPtr multi_completion, int flags) { std::string firstObjOid = getObjectId(soid, 0); @@ -954,8 +1140,7 @@ void libradosstriper::RadosStriperImpl::aio_unlockObject(const std::string& soid static void rados_write_aio_unlock_complete(rados_striper_multi_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::WriteCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); libradosstriper::MultiAioCompletionImpl *comp = reinterpret_cast(c); cdata->complete_unlock(comp->rval); @@ -964,8 +1149,7 @@ static void rados_write_aio_unlock_complete(rados_striper_multi_completion_t c, static void striper_write_aio_req_complete(rados_striper_multi_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::WriteCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); // launch the async unlocking of the object cdata->m_striper->aio_unlockObject(cdata->m_soid, cdata->m_lockCookie, cdata->m_unlockCompletion); // complete the write part in parallel @@ -977,8 +1161,7 @@ static void striper_write_aio_req_complete(rados_striper_multi_completion_t c, v static void striper_write_aio_req_safe(rados_striper_multi_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::WriteCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); libradosstriper::MultiAioCompletionImpl *comp = reinterpret_cast(c); cdata->safe(comp->rval); @@ -1001,7 +1184,8 @@ int libradosstriper::RadosStriperImpl::write_in_open_object(const std::string& s librados::Rados::aio_create_completion(cdata, rados_write_aio_unlock_complete, 0); cdata->m_unlockCompletion = unlock_completion; // create the multicompletion that will handle the write completion - libradosstriper::MultiAioCompletionImpl *c = new libradosstriper::MultiAioCompletionImpl; + MultiAioCompletionImplPtr c{new libradosstriper::MultiAioCompletionImpl, + false}; c->set_complete_callback(cdata, striper_write_aio_req_complete); c->set_safe_callback(cdata, striper_write_aio_req_safe); // call the asynchronous API @@ -1015,7 +1199,6 @@ int libradosstriper::RadosStriperImpl::write_in_open_object(const std::string& s // return result rc = c->get_return_value(); } - c->put(); cdata->put(); return rc; } @@ -1039,12 +1222,12 @@ int libradosstriper::RadosStriperImpl::aio_write_in_open_object(const std::strin librados::Rados::aio_create_completion(cdata, rados_write_aio_unlock_complete, 0); cdata->m_unlockCompletion = unlock_completion; // create the multicompletion that will handle the write completion - libradosstriper::MultiAioCompletionImpl *nc = new libradosstriper::MultiAioCompletionImpl; + libradosstriper::MultiAioCompletionImplPtr nc{ + new libradosstriper::MultiAioCompletionImpl, false}; nc->set_complete_callback(cdata, striper_write_aio_req_complete); nc->set_safe_callback(cdata, striper_write_aio_req_safe); // internal asynchronous API int rc = internal_aio_write(soid, nc, bl, len, off, layout); - nc->put(); cdata->put(); return rc; } @@ -1065,7 +1248,7 @@ static void rados_req_write_complete(rados_completion_t c, void *arg) int libradosstriper::RadosStriperImpl::internal_aio_write(const std::string& soid, - libradosstriper::MultiAioCompletionImpl *c, + libradosstriper::MultiAioCompletionImplPtr c, const bufferlist& bl, size_t len, uint64_t off, @@ -1095,8 +1278,11 @@ libradosstriper::RadosStriperImpl::internal_aio_write(const std::string& soid, // and write the object c->add_request(); librados::AioCompletion *rados_completion = - librados::Rados::aio_create_completion(c, rados_req_write_complete, rados_req_write_safe); - r = m_ioCtx.aio_write(p->oid.name, rados_completion, oid_bl, p->length, p->offset); + librados::Rados::aio_create_completion(c.get(), + rados_req_write_complete, + rados_req_write_safe); + r = m_ioCtx.aio_write(p->oid.name, rados_completion, oid_bl, + p->length, p->offset); rados_completion->release(); if (r < 0) break; @@ -1313,8 +1499,7 @@ int libradosstriper::RadosStriperImpl::createAndOpenStripedObject(const std::str static void striper_truncate_aio_req_complete(rados_striper_multi_completion_t c, void *arg) { - libradosstriper::RadosStriperImpl::TruncateCompletionData *cdata = - reinterpret_cast(arg); + auto cdata = reinterpret_cast(arg); libradosstriper::MultiAioCompletionImpl *comp = reinterpret_cast(c); if (0 == comp->rval) { @@ -1334,8 +1519,8 @@ int libradosstriper::RadosStriperImpl::truncate(const std::string& soid, ceph_file_layout &layout) { TruncateCompletionData *cdata = new TruncateCompletionData(this, soid, size); - libradosstriper::MultiAioCompletionImpl *multi_completion = - new libradosstriper::MultiAioCompletionImpl; + libradosstriper::MultiAioCompletionImplPtr multi_completion{ + new libradosstriper::MultiAioCompletionImpl, false}; multi_completion->set_complete_callback(cdata, striper_truncate_aio_req_complete); // call asynchrous version of truncate int rc = aio_truncate(soid, multi_completion, original_size, size, layout); @@ -1346,13 +1531,12 @@ int libradosstriper::RadosStriperImpl::truncate(const std::string& soid, if (rc == 0) { rc = multi_completion->get_return_value(); } - multi_completion->put(); return rc; } int libradosstriper::RadosStriperImpl::aio_truncate (const std::string& soid, - libradosstriper::MultiAioCompletionImpl *multi_completion, + libradosstriper::MultiAioCompletionImplPtr multi_completion, uint64_t original_size, uint64_t size, ceph_file_layout &layout) diff --git a/src/libradosstriper/RadosStriperImpl.h b/src/libradosstriper/RadosStriperImpl.h index 332c37536d89..6622471ed48d 100644 --- a/src/libradosstriper/RadosStriperImpl.h +++ b/src/libradosstriper/RadosStriperImpl.h @@ -21,221 +21,18 @@ #include "include/rados/librados.hpp" #include "include/radosstriper/libradosstriper.h" #include "include/radosstriper/libradosstriper.hpp" +#include "MultiAioCompletionImpl.h" #include "librados/IoCtxImpl.h" #include "librados/AioCompletionImpl.h" #include "common/RefCountedObj.h" -struct libradosstriper::RadosStriperImpl { +namespace libradosstriper { - /** - * struct handling the data needed to pass to the call back - * function in asynchronous operations - */ - struct CompletionData : RefCountedObject { - /// constructor - CompletionData(libradosstriper::RadosStriperImpl * striper, - const std::string& soid, - const std::string& lockCookie, - librados::AioCompletionImpl *userCompletion = 0, - int n = 1); - /// destructor - ~CompletionData() override; - /// complete method - void complete(int r); - /// striper to be used to handle the write completion - libradosstriper::RadosStriperImpl *m_striper; - /// striped object concerned by the write operation - std::string m_soid; - /// shared lock to be released at completion - std::string m_lockCookie; - /// completion handler - librados::IoCtxImpl::C_aio_Complete *m_ack; - }; - - /** - * struct handling the data needed to pass to the call back - * function in asynchronous read operations - */ - struct ReadCompletionData : CompletionData { - /// bufferlist containing final result - bufferlist* m_bl; - /// extents that will be read - std::vector* m_extents; - /// intermediate results - std::vector* m_resultbl; - /// return code of read completion, to be remembered until unlocking happened - int m_readRc; - /// completion object for the unlocking of the striped object at the end of the read - librados::AioCompletion *m_unlockCompletion; - /// constructor - ReadCompletionData(libradosstriper::RadosStriperImpl * striper, - const std::string& soid, - const std::string& lockCookie, - librados::AioCompletionImpl *userCompletion, - bufferlist* bl, - std::vector* extents, - std::vector* resultbl, - int n); - /// destructor - ~ReadCompletionData() override; - /// complete method for when reading is over - void complete_read(int r); - /// complete method for when object is unlocked - void complete_unlock(int r); - }; - - /** - * struct handling the data needed to pass to the call back - * function in asynchronous write operations - */ - struct WriteCompletionData : CompletionData { - /// safe completion handler - librados::IoCtxImpl::C_aio_Complete *m_safe; - /// return code of write completion, to be remembered until unlocking happened - int m_writeRc; - /// completion object for the unlocking of the striped object at the end of the write - librados::AioCompletion *m_unlockCompletion; - /// constructor - WriteCompletionData(libradosstriper::RadosStriperImpl * striper, - const std::string& soid, - const std::string& lockCookie, - librados::AioCompletionImpl *userCompletion, - int n); - /// destructor - ~WriteCompletionData() override; - /// complete method for when writing is over - void complete_write(int r); - /// complete method for when object is unlocked - void complete_unlock(int r); - /// safe method - void safe(int r); - }; - - /** - * struct handling the data needed to pass to the call back - * function in asynchronous read operations of a Rados File - */ - struct RadosReadCompletionData : RefCountedObject { - /// constructor - RadosReadCompletionData(MultiAioCompletionImpl *multiAioCompl, - uint64_t expectedBytes, - bufferlist *bl, - CephContext *context, - int n = 1) : - RefCountedObject(context, n), - m_multiAioCompl(multiAioCompl), m_expectedBytes(expectedBytes), m_bl(bl) {}; - /// the multi asynch io completion object to be used - MultiAioCompletionImpl *m_multiAioCompl; - /// the expected number of bytes - uint64_t m_expectedBytes; - /// the bufferlist object where data have been written - bufferlist *m_bl; - }; - - /** - * struct handling (most of) the data needed to pass to the call back - * function in asynchronous stat operations. - * Inherited by the actual type for adding time information in different - * versions (time_t or struct timespec) - */ - struct BasicStatCompletionData : CompletionData { - /// constructor - BasicStatCompletionData(libradosstriper::RadosStriperImpl* striper, - const std::string& soid, - librados::AioCompletionImpl *userCompletion, - libradosstriper::MultiAioCompletionImpl *multiCompletion, - uint64_t *psize, - int n = 1) : - CompletionData(striper, soid, "", userCompletion, n), - m_multiCompletion(multiCompletion), m_psize(psize), - m_statRC(0), m_getxattrRC(0) {}; - // MultiAioCompletionImpl used to handle the double aysnc - // call in the back (stat + getxattr) - libradosstriper::MultiAioCompletionImpl *m_multiCompletion; - // where to store the size of first objct - // this will be ignored but we need a place to store it when - // async stat is called - uint64_t m_objectSize; - // where to store the file size - uint64_t *m_psize; - /// the bufferlist object used for the getxattr call - bufferlist m_bl; - /// return code of the stat - int m_statRC; - /// return code of the getxattr - int m_getxattrRC; - }; +using MultiAioCompletionImplPtr = + boost::intrusive_ptr; - /** - * struct handling the data needed to pass to the call back - * function in asynchronous stat operations. - * Simple templated extension of BasicStatCompletionData. - * The template parameter is the type of the time information - * (used with time_t for stat and struct timespec for stat2) - */ - template - struct StatCompletionData : BasicStatCompletionData { - /// constructor - StatCompletionData(libradosstriper::RadosStriperImpl* striper, - const std::string& soid, - librados::AioCompletionImpl *userCompletion, - libradosstriper::MultiAioCompletionImpl *multiCompletion, - uint64_t *psize, - TimeType *pmtime, - int n = 1) : - BasicStatCompletionData(striper, soid, userCompletion, multiCompletion, psize, n), - m_pmtime(pmtime) {}; - // where to store the file time - TimeType *m_pmtime; - }; - - /** - * struct handling the data needed to pass to the call back - * function in asynchronous remove operations of a Rados File - */ - struct RadosRemoveCompletionData : RefCountedObject { - /// constructor - RadosRemoveCompletionData(MultiAioCompletionImpl *multiAioCompl, - CephContext *context) : - RefCountedObject(context, 2), - m_multiAioCompl(multiAioCompl) {}; - /// the multi asynch io completion object to be used - MultiAioCompletionImpl *m_multiAioCompl; - }; - - struct RemoveCompletionData : CompletionData { - /// removal flags - int flags; - /** - * constructor - * note that the constructed object will take ownership of the lock - */ - RemoveCompletionData(libradosstriper::RadosStriperImpl * striper, - const std::string& soid, - const std::string& lockCookie, - librados::AioCompletionImpl *userCompletion, - int flags = 0); - }; - - /** - * struct handling the data needed to pass to the call back - * function in asynchronous truncate operations - */ - struct TruncateCompletionData : RefCountedObject { - /// constructor - TruncateCompletionData(libradosstriper::RadosStriperImpl* striper, - const std::string& soid, - uint64_t size); - /// destructor - ~TruncateCompletionData() override; - /// striper to be used - libradosstriper::RadosStriperImpl *m_striper; - /// striped object concerned by the truncate operation - std::string m_soid; - /// the final size of the truncated object - uint64_t m_size; - }; +struct RadosStriperImpl { /** * exception wrapper around an error code @@ -355,7 +152,7 @@ struct libradosstriper::RadosStriperImpl { size_t len, uint64_t off); int internal_aio_write(const std::string& soid, - libradosstriper::MultiAioCompletionImpl *c, + MultiAioCompletionImplPtr c, const bufferlist& bl, size_t len, uint64_t off, @@ -374,7 +171,7 @@ struct libradosstriper::RadosStriperImpl { uint64_t *size); int internal_aio_remove(const std::string& soid, - libradosstriper::MultiAioCompletionImpl *multi_completion, + MultiAioCompletionImplPtr multi_completion, int flags=0); /** @@ -437,7 +234,7 @@ struct libradosstriper::RadosStriperImpl { * point is synchronous for lack of asynchronous truncation in the rados layer */ int aio_truncate(const std::string& soid, - libradosstriper::MultiAioCompletionImpl *c, + MultiAioCompletionImplPtr c, uint64_t original_size, uint64_t size, ceph_file_layout &layout); @@ -473,5 +270,5 @@ struct libradosstriper::RadosStriperImpl { // Default layout ceph_file_layout m_layout; }; - +} #endif