const size_t len,
uint64_t off);
+/**
+ * Asynchronously removes a striped object
+ *
+ * @note There is no atomicity of the deletion and the striped
+ * object may be left incomplete if an error is returned (metadata
+ * all present, but some stripes missing)
+ * However, there is a atomicity of the metadata deletion and
+ * the deletion can not happen if any I/O is ongoing (it
+ * will return EBUSY). Identically, no I/O will be able to start
+ * during deletion (same EBUSY return code)
+ * @param striper the striper in which the remove will occur
+ * @param soid the name of the striped object
+ * @param completion what to do when the remove is safe and complete
+ * @returns 0 on success, negative error code on failure
+ */
+
+int rados_striper_aio_remove(rados_striper_t striper,
+ const char* soid,
+ rados_completion_t completion);
+
/**
* Block until all pending writes in a striper are safe
*
*/
void rados_striper_aio_flush(rados_striper_t striper);
+/**
+ * Asynchronously get object stats (size/mtime)
+ *
+ * @param striper the striper in which the stat will occur
+ * @param soid the id of the striped object
+ * @param psize where to store object size
+ * @param pmtime where to store modification time
+ * @param completion what to do when the stats is complete
+ * @returns 0 on success, negative error code on failure
+ */
+int rados_striper_aio_stat(rados_striper_t striper,
+ const char* soid,
+ rados_completion_t completion,
+ uint64_t *psize,
+ time_t *pmtime);
+
/** @} Asynchronous I/O */
#ifdef __cplusplus
* synchronously get striped object stats (size/mtime)
*/
int stat(const std::string& soid, uint64_t *psize, time_t *pmtime);
+ int stat2(const std::string& soid, uint64_t *psize, struct timespec *pts);
+
+ /**
+ * asynchronously get striped object stats (size/mtime)
+ */
+ int aio_stat(const std::string& soid, librados::AioCompletion *c,
+ uint64_t *psize, time_t *pmtime);
+ int aio_stat2(const std::string& soid, librados::AioCompletion *c,
+ uint64_t *psize, struct timespec *pts);
/**
* deletes a striped object.
*/
int remove(const std::string& soid);
int remove(const std::string& soid, int flags);
+
+ /**
+ * asynchronous remove of striped objects
+ * See synchronous version for comments on (lack of) atomicity
+ */
+ int aio_remove(const std::string& soid, librados::AioCompletion *c);
+ int aio_remove(const std::string& soid, librados::AioCompletion *c, int flags);
+
/**
* Resizes a striped object
* the truncation can not happen if any I/O is ongoing (it
*
* There are a number of missing features/improvements that could be implemented.
* Here are some ideas :
- * - asynchronous stat
* - implementation of missing entry points (compared to rados)
* In particular : clone_range, sparse_read, exec, aio_flush_async, tmaps, omaps, ...
*
RefCountedObject(striper->cct(), n),
m_striper(striper), m_soid(soid), m_lockCookie(lockCookie), m_ack(0) {
m_striper->get();
- if (userCompletion) m_ack = new librados::IoCtxImpl::C_aio_Ack(userCompletion);
+ if (userCompletion) {
+ m_ack = new librados::IoCtxImpl::C_aio_Ack(userCompletion);
+ userCompletion->io = striper->m_ioCtxImpl;
+ }
}
libradosstriper::RadosStriperImpl::CompletionData::~CompletionData() {
int libradosstriper::RadosStriperImpl::stat(const std::string& soid, uint64_t *psize, time_t *pmtime)
{
- // get pmtime as the pmtime of the first object
+ // create a completion object
+ librados::AioCompletionImpl c;
+ // call asynchronous version of stat
+ int rc = aio_stat(soid, &c, psize, pmtime);
+ if (rc == 0) {
+ // wait for completion of the remove
+ c.wait_for_complete();
+ // get result
+ rc = c.get_return_value();
+ }
+ return rc;
+}
+
+static void striper_stat_aio_stat_complete(rados_completion_t c, void *arg) {
+ libradosstriper::RadosStriperImpl::BasicStatCompletionData *data =
+ reinterpret_cast<libradosstriper::RadosStriperImpl::BasicStatCompletionData*>(arg);
+ int rc = rados_aio_get_return_value(c);
+ if (rc == -ENOENT) {
+ // remember this has failed
+ data->m_statRC = rc;
+ }
+ data->m_multiCompletion->complete_request(rc);
+}
+
+static void striper_stat_aio_getxattr_complete(rados_completion_t c, void *arg) {
+ libradosstriper::RadosStriperImpl::BasicStatCompletionData *data =
+ reinterpret_cast<libradosstriper::RadosStriperImpl::BasicStatCompletionData*>(arg);
+ int rc = rados_aio_get_return_value(c);
+ // We need to handle the case of sparse files here
+ if (rc < 0) {
+ // remember this has failed
+ data->m_getxattrRC = rc;
+ } else {
+ // this intermediate string allows to add a null terminator before calling strtol
+ std::string err;
+ std::string strsize(data->m_bl.c_str(), data->m_bl.length());
+ *data->m_psize = strict_strtoll(strsize.c_str(), 10, &err);
+ if (!err.empty()) {
+ lderr(data->m_striper->cct()) << XATTR_SIZE << " : " << err << dendl;
+ data->m_getxattrRC = -EINVAL;
+ }
+ rc = 0;
+ }
+ data->m_multiCompletion->complete_request(rc);
+}
+
+static void striper_stat_aio_req_complete(rados_striper_multi_completion_t c,
+ void *arg) {
+ libradosstriper::RadosStriperImpl::BasicStatCompletionData *data =
+ reinterpret_cast<libradosstriper::RadosStriperImpl::BasicStatCompletionData*>(arg);
+ if (data->m_statRC) {
+ data->complete(data->m_statRC);
+ } else {
+ if (data->m_getxattrRC < 0) {
+ data->complete(data->m_getxattrRC);
+ } else {
+ data->complete(0);
+ }
+ }
+ data->put();
+}
+
+template<class TimeType>
+int libradosstriper::RadosStriperImpl::aio_generic_stat
+(const std::string& soid,
+ librados::AioCompletionImpl *c,
+ uint64_t *psize,
+ TimeType *pmtime,
+ typename libradosstriper::RadosStriperImpl::StatFunction<TimeType>::Type statFunction)
+{
+ // use a MultiAioCompletion object for dealing with the fact
+ // that we'll do 2 asynchronous calls in parallel
+ libradosstriper::MultiAioCompletionImpl *multi_completion =
+ new libradosstriper::MultiAioCompletionImpl;
+ // Data object used for passing context to asynchronous calls
std::string firstObjOid = getObjectId(soid, 0);
- uint64_t obj_size;
- int rc = m_ioCtx.stat(firstObjOid, &obj_size, pmtime);
- if (rc < 0) return rc;
- // get the pmsize from the first object attributes
- bufferlist bl;
- rc = getxattr(soid, XATTR_SIZE, bl);
- if (rc < 0) return rc;
- std::string err;
- // this intermediate string allows to add a null terminator before calling strtol
- std::string strsize(bl.c_str(), bl.length());
- *psize = strict_strtoll(strsize.c_str(), 10, &err);
- if (!err.empty()) {
- lderr(cct()) << XATTR_SIZE << " : " << err << dendl;
- return -EINVAL;
+ StatCompletionData<TimeType> *cdata =
+ new StatCompletionData<TimeType>(this, firstObjOid, c,
+ multi_completion, psize, pmtime);
+ multi_completion->set_complete_callback(cdata, striper_stat_aio_req_complete);
+ // use a regular AioCompletion for the stat async call
+ librados::AioCompletion *stat_completion =
+ librados::Rados::aio_create_completion(cdata,
+ striper_stat_aio_stat_complete, 0);
+ multi_completion->add_request();
+ object_t obj(firstObjOid);
+ int rc = (m_ioCtxImpl->*statFunction)(obj, stat_completion->pc,
+ &cdata->m_objectSize, cdata->m_pmtime);
+ stat_completion->release();
+ if (rc < 0) {
+ // nothing is really started so cancel everything
+ delete multi_completion;
+ delete cdata;
+ delete stat_completion;
+ return rc;
+ }
+ // use a regular AioCompletion for the getxattr async call
+ librados::AioCompletion *getxattr_completion =
+ librados::Rados::aio_create_completion(cdata,
+ striper_stat_aio_getxattr_complete, 0);
+ multi_completion->add_request();
+ // in parallel, get the pmsize from the first object asynchronously
+ rc = m_ioCtxImpl->aio_getxattr(obj, getxattr_completion->pc,
+ XATTR_SIZE, cdata->m_bl);
+ getxattr_completion->release();
+ multi_completion->finish_adding_requests();
+ if (rc < 0) {
+ // the async stat is ongoing, so we need to go on
+ // we mark the getxattr as failed in the data object
+ cdata->m_getxattrRC = rc;
+ delete getxattr_completion;
+ multi_completion->complete_request(rc);
+ return rc;
}
return 0;
}
+int libradosstriper::RadosStriperImpl::aio_stat(const std::string& soid,
+ librados::AioCompletionImpl *c,
+ uint64_t *psize,
+ time_t *pmtime)
+{
+ return aio_generic_stat<time_t>(soid, c, psize, pmtime, &librados::IoCtxImpl::aio_stat);
+}
+
+int libradosstriper::RadosStriperImpl::stat2(const std::string& soid, uint64_t *psize, struct timespec *pts)
+{
+ // create a completion object
+ librados::AioCompletionImpl c;
+ // call asynchronous version of stat
+ int rc = aio_stat2(soid, &c, psize, pts);
+ if (rc == 0) {
+ // wait for completion of the remove
+ c.wait_for_complete_and_cb();
+ // get result
+ rc = c.get_return_value();
+ }
+ return rc;
+}
+
+int libradosstriper::RadosStriperImpl::aio_stat2(const std::string& soid,
+ librados::AioCompletionImpl *c,
+ uint64_t *psize,
+ struct timespec *pts)
+{
+ return aio_generic_stat<struct timespec>(soid, c, psize, pts, &librados::IoCtxImpl::aio_stat2);
+}
+
static void rados_req_remove_complete(rados_completion_t c, void *arg)
{
libradosstriper::RadosStriperImpl::RadosRemoveCompletionData *cdata =
new libradosstriper::MultiAioCompletionImpl;
multi_completion->set_complete_callback(cdata, striper_remove_aio_req_complete);
// call asynchronous internal version of remove
+ ldout(cct(), 10)
+ << "RadosStriperImpl : Aio_remove starting for "
+ << soid << dendl;
return internal_aio_remove(soid, multi_completion);
} catch (ErrorCode &e) {
return e.m_code;
// call asynchrous version of truncate
int rc = aio_truncate(soid, multi_completion, original_size, size, layout);
// wait for completion of the truncation
+ multi_completion->finish_adding_requests();
multi_completion->wait_for_complete_and_cb();
// return result
if (rc == 0) {
}
if (exists) {
// remove asynchronously
+ multi_completion->add_request();
librados::AioCompletion *rados_completion =
librados::Rados::aio_create_completion(multi_completion,
rados_req_remove_complete,
rc = m_ioCtx.trunc(getObjectId(soid, objectno), new_object_size);
} else {
// removes are asynchronous in order to speed up truncations of big files
+ multi_completion->add_request();
librados::AioCompletion *rados_completion =
librados::Rados::aio_create_completion(multi_completion,
rados_req_remove_complete,
#include "include/radosstriper/libradosstriper.hpp"
#include "librados/IoCtxImpl.h"
+#include "librados/AioCompletionImpl.h"
#include "common/RefCountedObj.h"
struct libradosstriper::RadosStriperImpl {
bufferlist *m_bl;
};
+ /**
+ * struct handling (most of) the data needed to pass to the call back
+ * function in asynchronous stat operations.
+ * Inherited by the actual type for adding time information in different
+ * versions (time_t or struct timespec)
+ */
+ struct BasicStatCompletionData : CompletionData {
+ /// constructor
+ BasicStatCompletionData(libradosstriper::RadosStriperImpl* striper,
+ const std::string& soid,
+ librados::AioCompletionImpl *userCompletion,
+ libradosstriper::MultiAioCompletionImpl *multiCompletion,
+ uint64_t *psize) :
+ CompletionData(striper, soid, "", userCompletion),
+ m_multiCompletion(multiCompletion), m_psize(psize),
+ m_statRC(0), m_getxattrRC(0) {};
+ // MultiAioCompletionImpl used to handle the double aysnc
+ // call in the back (stat + getxattr)
+ libradosstriper::MultiAioCompletionImpl *m_multiCompletion;
+ // where to store the size of first objct
+ // this will be ignored but we need a place to store it when
+ // async stat is called
+ uint64_t m_objectSize;
+ // where to store the file size
+ uint64_t *m_psize;
+ /// the bufferlist object used for the getxattr call
+ bufferlist m_bl;
+ /// return code of the stat
+ int m_statRC;
+ /// return code of the getxattr
+ int m_getxattrRC;
+ };
+
+ /**
+ * struct handling the data needed to pass to the call back
+ * function in asynchronous stat operations.
+ * Simple templated extension of BasicStatCompletionData.
+ * The template parameter is the type of the time information
+ * (used with time_t for stat and struct timespec for stat2)
+ */
+ template<class TimeType>
+ struct StatCompletionData : BasicStatCompletionData {
+ /// constructor
+ StatCompletionData(libradosstriper::RadosStriperImpl* striper,
+ const std::string& soid,
+ librados::AioCompletionImpl *userCompletion,
+ libradosstriper::MultiAioCompletionImpl *multiCompletion,
+ uint64_t *psize,
+ TimeType *pmtime) :
+ BasicStatCompletionData(striper, soid, userCompletion, multiCompletion, psize),
+ m_pmtime(pmtime) {};
+ // where to store the file time
+ TimeType *m_pmtime;
+ };
+
/**
* struct handling the data needed to pass to the call back
* function in asynchronous remove operations of a Rados File
// stat, deletion and truncation
int stat(const std::string& soid, uint64_t *psize, time_t *pmtime);
+ int stat2(const std::string& soid, uint64_t *psize, struct timespec *pts);
+ template<class TimeType>
+ struct StatFunction {
+ typedef int (librados::IoCtxImpl::*Type) (const object_t& oid,
+ librados::AioCompletionImpl *c,
+ uint64_t *psize, TimeType *pmtime);
+ };
+ template<class TimeType>
+ int aio_generic_stat(const std::string& soid, librados::AioCompletionImpl *c,
+ uint64_t *psize, TimeType *pmtime,
+ typename StatFunction<TimeType>::Type statFunction);
+ int aio_stat(const std::string& soid, librados::AioCompletionImpl *c,
+ uint64_t *psize, time_t *pmtime);
+ int aio_stat2(const std::string& soid, librados::AioCompletionImpl *c,
+ uint64_t *psize, struct timespec *pts);
int remove(const std::string& soid, int flags=0);
int trunc(const std::string& soid, uint64_t size);
return rados_striper_impl->stat(soid, psize, pmtime);
}
+int libradosstriper::RadosStriper::aio_stat(const std::string& soid,
+ librados::AioCompletion *c,
+ uint64_t *psize,
+ time_t *pmtime)
+{
+ return rados_striper_impl->aio_stat(soid, c->pc, psize, pmtime);
+}
+
+int libradosstriper::RadosStriper::stat2(const std::string& soid, uint64_t *psize, struct timespec *pts)
+{
+ return rados_striper_impl->stat2(soid, psize, pts);
+}
+
+int libradosstriper::RadosStriper::aio_stat2(const std::string& soid,
+ librados::AioCompletion *c,
+ uint64_t *psize,
+ struct timespec *pts)
+{
+ return rados_striper_impl->aio_stat2(soid, c->pc, psize, pts);
+}
+
int libradosstriper::RadosStriper::remove(const std::string& soid)
{
return rados_striper_impl->remove(soid);
}
+
+int libradosstriper::RadosStriper::aio_remove(const std::string& soid,
+ librados::AioCompletion *c)
+{
+ return rados_striper_impl->aio_remove(soid, c->pc);
+}
+
int libradosstriper::RadosStriper::remove(const std::string& soid, int flags)
{
return rados_striper_impl->remove(soid, flags);
}
+int libradosstriper::RadosStriper::aio_remove(const std::string& soid,
+ librados::AioCompletion *c,
+ int flags)
+{
+ return rados_striper_impl->aio_remove(soid, c->pc, flags);
+}
+
int libradosstriper::RadosStriper::trunc(const std::string& soid, uint64_t size)
{
return rados_striper_impl->trunc(soid, size);
return impl->aio_read(soid, (librados::AioCompletionImpl*)completion, buf, len, off);
}
+extern "C" int rados_striper_aio_remove(rados_striper_t striper,
+ const char* soid,
+ rados_completion_t completion)
+{
+ libradosstriper::RadosStriperImpl *impl = (libradosstriper::RadosStriperImpl *)striper;
+ return impl->aio_remove(soid, (librados::AioCompletionImpl*)completion);
+}
+
extern "C" void rados_striper_aio_flush(rados_striper_t striper)
{
libradosstriper::RadosStriperImpl *impl = (libradosstriper::RadosStriperImpl *)striper;
impl->aio_flush();
}
+
+extern "C" int rados_striper_aio_stat(rados_striper_t striper,
+ const char* soid,
+ rados_completion_t completion,
+ uint64_t *psize,
+ time_t *pmtime)
+{
+ libradosstriper::RadosStriperImpl *impl = (libradosstriper::RadosStriperImpl *)striper;
+ return impl->aio_stat(soid, (librados::AioCompletionImpl*)completion, psize, pmtime);
+}
#include "test/librados/test.h"
#include "test/libradosstriper/TestCase.h"
+#include <boost/scoped_ptr.hpp>
#include <fcntl.h>
#include <semaphore.h>
#include <errno.h>
my_completion2->release();
my_completion3->release();
}
+
+TEST_F(StriperTest, RemoveTest) {
+ char buf[128];
+ char buf2[sizeof(buf)];
+ // create oabject
+ memset(buf, 0xaa, sizeof(buf));
+ ASSERT_EQ(0, rados_striper_write(striper, "RemoveTest", buf, sizeof(buf), 0));
+ // async remove it
+ AioTestData test_data;
+ rados_completion_t my_completion;
+ ASSERT_EQ(0, rados_aio_create_completion((void*)&test_data,
+ set_completion_complete, set_completion_safe, &my_completion));
+ ASSERT_EQ(0, rados_striper_aio_remove(striper, "RemoveTest", my_completion));
+ {
+ TestAlarm alarm;
+ ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+ }
+ ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+ rados_aio_release(my_completion);
+ // check we get ENOENT on reading
+ ASSERT_EQ(-ENOENT, rados_striper_read(striper, "RemoveTest", buf2, sizeof(buf2), 0));
+}
+
+TEST_F(StriperTestPP, RemoveTestPP) {
+ char buf[128];
+ memset(buf, 0xaa, sizeof(buf));
+ bufferlist bl;
+ bl.append(buf, sizeof(buf));
+ ASSERT_EQ(0, striper.write("RemoveTestPP", bl, sizeof(buf), 0));
+ boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0, 0));
+ ASSERT_EQ(0, striper.aio_remove("RemoveTestPP", my_completion.get()));
+ {
+ TestAlarm alarm;
+ ASSERT_EQ(0, my_completion->wait_for_complete());
+ }
+ ASSERT_EQ(0, my_completion->get_return_value());
+ bufferlist bl2;
+ ASSERT_EQ(-ENOENT, striper.read("RemoveTestPP", &bl2, sizeof(buf), 0));
+}
projects / ceph.git / commitdiff