OP_COLL_HINT = 40, // cid, type, bl
OP_TRY_RENAME = 41, // oldcid, oldoid, newoid
- OP_MERGE_DELETE = 42, //move tempobj to base object. cid, oid, newoid, vector of tuple <src offset, dest offset, len>
};
// Transaction hint type
case OP_CLONERANGE2:
case OP_CLONE:
- case OP_MERGE_DELETE:
assert(op->cid < cm.size());
assert(op->oid < om.size());
assert(op->dest_oid < om.size());
void decode_attrset(map<string,bufferlist>& aset) {
::decode(aset, data_bl_p);
}
- void decode_move_info(vector<std::pair<uint64_t, uint64_t >>& move_info) {
- ::decode(move_info, data_bl_p);
- }
void decode_attrset_bl(bufferlist *pbl) {
decode_str_str_map_to_bl(data_bl_p, pbl);
}
data.ops++;
}
- /*
- * Move source object to base object.
- * Data portion is only copied from source object to base object.
- * The copy is done according to the move_info vector of tuple, which
- * has information of offset and length.
- * Finally, the source object is deleted.
- */
- void move_ranges_destroy_src(
- const coll_t& cid,
- const ghobject_t& src_oid,
- ghobject_t oid,
- const vector<std::pair<uint64_t, uint64_t>>& move_info) {
- Op* _op = _get_next_op();
- _op->op = OP_MERGE_DELETE;
- _op->cid = _get_coll_id(cid);
- _op->oid = _get_object_id(src_oid);
- _op->dest_oid = _get_object_id(oid);
- ::encode(move_info, data_bl);
- data.ops++;
- }
-
/// Create the collection
void create_collection(const coll_t& cid, int bits) {
Op* _op = _get_next_op();
}
break;
- case Transaction::OP_MERGE_DELETE:
- {
- const ghobject_t& noid = i.get_oid(op->dest_oid);
- OnodeRef& no = ovec[op->dest_oid];
- if (!no) {
- no = c->get_onode(noid, true);
- }
- vector<std::pair<uint64_t, uint64_t>> move_info;
- i.decode_move_info(move_info);
- r = _move_ranges_destroy_src(txc, c, o, no, move_info);
- }
- break;
-
case Transaction::OP_COLL_ADD:
assert(0 == "not implemented");
break;
return r;
}
-/* Move contents of src object according to move_info to base object.
- * Once the move_info is traversed completely, delete the src object.
- */
-int BlueStore::_move_ranges_destroy_src(
- TransContext *txc,
- CollectionRef& c,
- OnodeRef& srco,
- OnodeRef& baseo,
- const vector<std::pair<uint64_t, uint64_t>> move_info)
-{
- dout(15) << __func__ << " " << c->cid << " "
- << srco->oid << " -> " << baseo->oid
- << dendl;
-
- int r = 0;
-
- // Traverse move_info completely, move contents from src object
- // to base object.
- for (unsigned i = 0; i < move_info.size(); ++i) {
- uint64_t off = move_info[i].first;
- uint64_t len = move_info[i].second;
-
- dout(15) << __func__ << " " << c->cid << " " << srco->oid << " -> "
- << baseo->oid << " 0x" << std::hex << off << "~" << len
- << dendl;
-
- r = _clone_range(txc, c, srco, baseo, off, len, off);
- if (r < 0)
- goto out;
- }
-
- // delete the src object
- r = _do_remove(txc, c, srco);
-
- out:
- return r;
-}
-
int BlueStore::_rename(TransContext *txc,
CollectionRef& c,
OnodeRef& oldo,
OnodeRef& oldo,
OnodeRef& newo,
uint64_t srcoff, uint64_t length, uint64_t dstoff);
- int _move_ranges_destroy_src(
- TransContext *txc,
- CollectionRef& c,
- OnodeRef& oldo,
- OnodeRef& newo,
- const vector<std::pair<uint64_t,uint64_t>> move_info);
int _rename(TransContext *txc,
CollectionRef& c,
OnodeRef& oldo,
}
break;
- case Transaction::OP_MERGE_DELETE:
- {
- ghobject_t src_oid = i.get_oid(op->oid);
- coll_t cid = i.get_cid(op->cid);
- ghobject_t oid = i.get_oid(op->dest_oid);
- coll_t src_cid = i.get_cid(op->cid);
- _kludge_temp_object_collection(cid, oid);
- _kludge_temp_object_collection(src_cid, src_oid);
- vector<std::pair<uint64_t, uint64_t>> move_info;
- i.decode_move_info(move_info);
- tracepoint(objectstore, move_ranges_destroy_src_enter, osr_name);
- r = _move_ranges_destroy_src(src_cid, src_oid, cid, oid, move_info, spos);
- tracepoint(objectstore, move_ranges_destroy_src_exit, r);
- }
- break;
-
case Transaction::OP_MKCOLL:
{
coll_t cid = i.get_cid(op->cid);
return r;
}
-/*
- * Move contents of src object according to move_info to base object. Once the move_info is traversed completely, delete the src object.
- */
-int FileStore::_move_ranges_destroy_src(
- const coll_t& src_cid, const ghobject_t& src_oid,
- const coll_t& cid, const ghobject_t& oid,
- const vector<std::pair<uint64_t, uint64_t>> move_info,
- const SequencerPosition& spos)
-{
- int r = 0;
-
- dout(10) << __func__ << src_cid << "/" << src_oid << " -> "
- << cid << "/" << oid << dendl;
-
- // check replay guard for base object. If not possible to replay, return.
- int dstcmp = _check_replay_guard(cid, oid, spos);
- if (dstcmp < 0)
- return 0;
-
- // check the src name too; it might have a newer guard, and we don't
- // want to clobber it
- int srccmp = _check_replay_guard(src_cid, src_oid, spos);
- if (srccmp < 0)
- return 0;
-
- FDRef b;
- r = lfn_open(cid, oid, true, &b);
- if (r < 0) {
- return 0;
- }
-
- FDRef t;
- r = lfn_open(src_cid, src_oid, false, &t);
- //If we are replaying, it is possible that we do not find src obj as
- //it is deleted before crashing.
- if (r < 0) {
- lfn_close(b);
- dout(10) << __func__ << " replaying -->" << replaying << dendl;
- if (replaying) {
- _set_replay_guard(**b, spos, &oid);
- return 0;
- } else {
- return -ENOENT;
- }
- }
-
- for (unsigned i = 0; i < move_info.size(); ++i) {
- uint64_t off = move_info[i].first;
- uint64_t len = move_info[i].second;
- r = _do_clone_range(**t, **b, off, len, off);
- if (r < 0)
- break;
- }
-
- dout(10) << __func__ << cid << "/" << oid << " " << " = " << r << dendl;
-
- lfn_close(t);
-
- //In case crash occurs here, replay will have to do cloning again.
- //Only if do_clone_range is successful, go ahead with deleting the source object.
- if (r < 0)
- goto out;
-
- r = lfn_unlink(src_cid, src_oid, spos, true);
- // If crash occurs between unlink and set guard, correct the error.
- // as during next time, it might not find the already deleted object.
- if (r < 0 && replaying) {
- r = 0;
- }
-
- if (r < 0)
- goto out;
-
- //set replay guard for base obj coll_t, as this api is not idempotent.
- _set_replay_guard(**b, spos, &oid);
-
-out:
- lfn_close(b);
- dout(10) << __func__ << cid << "/" << oid << " " << " = " << r << dendl;
- return r;
-}
-
class SyncEntryTimeout : public Context {
public:
explicit SyncEntryTimeout(int commit_timeo)
int _clone_range(const coll_t& oldcid, const ghobject_t& oldoid, const coll_t& newcid, const ghobject_t& newoid,
uint64_t srcoff, uint64_t len, uint64_t dstoff,
const SequencerPosition& spos);
- int _move_ranges_destroy_src(
- const coll_t& temp_cid, const ghobject_t& temp_oid,
- const coll_t& cid, const ghobject_t& oid,
- const vector<std::pair<uint64_t, uint64_t> > move_info,
- const SequencerPosition& spos);
int _do_clone_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff);
int _do_sparse_copy_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff);
int _do_copy_range(int from, int to, uint64_t srcoff, uint64_t len, uint64_t dstoff, bool skip_sloppycrc=false);
}
break;
- case Transaction::OP_MERGE_DELETE:
- {
- const ghobject_t& boid = i.get_oid(op->dest_oid);
- OnodeRef bo = c->get_onode(boid, true);
- vector<std::pair<uint64_t, uint64_t>> move_info;
- i.decode_move_info(move_info);
- r = _move_ranges_destroy_src(txc, c, o, cvec[op->dest_cid], bo, move_info);
- }
- break;
-
case Transaction::OP_COLL_ADD:
assert(0 == "not implemented");
break;
return r;
}
-/* Move contents of src object according to move_info to base object.
- * Once the move_info is traversed completely, delete the src object.
- */
-int KStore::_move_ranges_destroy_src(
- TransContext *txc,
- CollectionRef& c,
- OnodeRef& srco,
- CollectionRef& basec,
- OnodeRef& baseo,
- vector<std::pair<uint64_t, uint64_t>> move_info)
-{
- int r = 0;
- bufferlist bl;
- baseo->exists = true;
- _assign_nid(txc, baseo);
-
- // Traverse move_info completely, move contents from src to base object.
- for (unsigned i = 0; i < move_info.size(); ++i) {
- uint64_t off = move_info[i].first;
- uint64_t len = move_info[i].second;
-
- r = _do_read(srco, off, len, bl, 0);
- if (r < 0)
- goto out;
-
- r = _do_write(txc, baseo, off, bl.length(), bl, 0);
- txc->write_onode(baseo);
- r = 0;
- }
-
- // After for loop ends, remove src obj
- r = _do_remove(txc, srco);
-
- out:
- return r;
-}
-
-
int KStore::_rename(TransContext *txc,
CollectionRef& c,
OnodeRef& oldo,
OnodeRef& oldo,
OnodeRef& newo,
uint64_t srcoff, uint64_t length, uint64_t dstoff);
- int _move_ranges_destroy_src(
- TransContext *txc,
- CollectionRef& c,
- OnodeRef& oldo,
- CollectionRef& bc,
- OnodeRef& newo,
- vector<std::pair<uint64_t, uint64_t>> move_info);
int _rename(TransContext *txc,
CollectionRef& c,
OnodeRef& oldo,
}
break;
- case Transaction::OP_MERGE_DELETE:
- {
- coll_t cid = i.get_cid(op->cid);
- ghobject_t oid = i.get_oid(op->oid);
- ghobject_t noid = i.get_oid(op->dest_oid);
- vector<std::pair<uint64_t, uint64_t>> move_info;
- i.decode_move_info(move_info);
- r = _move_ranges_destroy_src(cid, oid, noid, move_info);
- }
- break;
-
case Transaction::OP_MKCOLL:
{
coll_t cid = i.get_cid(op->cid);
return len;
}
-/* Move contents of src object according to move_info to base object.
- * Once the move_info is traversed completely, delete the src object.
- */
-int MemStore::_move_ranges_destroy_src(
- const coll_t& cid, const ghobject_t& srcoid,
- const ghobject_t& baseoid,
- const vector<std::pair<uint64_t, uint64_t> > move_info)
-{
- dout(10) << __func__ << " " << cid << " " << srcoid << " -> "
- << baseoid << dendl;
- CollectionRef c = get_collection(cid);
- if (!c)
- return -ENOENT;
-
- ObjectRef oo = c->get_object(srcoid);
- if (!oo)
- return -ENOENT;
- ObjectRef no = c->get_or_create_object(baseoid);
-
- for (unsigned i = 0; i < move_info.size(); ++i) {
- uint64_t off = move_info[i].first;
- uint64_t len = move_info[i].second;
- if (off >= oo->get_size())
- return 0;
- if (off + len >= oo->get_size())
- len = oo->get_size() - off;
- const ssize_t old_size = no->get_size();
- no->clone(oo.get(), off, len, off);
- used_bytes += (no->get_size() - old_size);
- }
-
- // delete the src object
- _remove(cid, srcoid);
- return 0;
-}
-
int MemStore::_omap_clear(const coll_t& cid, const ghobject_t &oid)
{
dout(10) << __func__ << " " << cid << " " << oid << dendl;
int _clone_range(const coll_t& cid, const ghobject_t& oldoid,
const ghobject_t& newoid,
uint64_t srcoff, uint64_t len, uint64_t dstoff);
- int _move_ranges_destroy_src(const coll_t& cid, const ghobject_t& oldoid,
- const ghobject_t& newoid,
- const vector<pair<uint64_t, uint64_t> > move_info);
int _omap_clear(const coll_t& cid, const ghobject_t &oid);
int _omap_setkeys(const coll_t& cid, const ghobject_t &oid, bufferlist& aset_bl);
int _omap_rmkeys(const coll_t& cid, const ghobject_t &oid, bufferlist& keys_bl);
case DSOP_COLL_CREATE:
ok = do_coll_create(gen);
break;
- case DSOP_MERGE_DELETE:
- ok = do_move_ranges_delete_srcobj(gen);
- break;
default:
assert(0 == "bad op");
return true;
}
-bool DeterministicOpSequence::do_move_ranges_delete_srcobj(rngen_t& gen)
-{
- coll_t coll;
- hobject_t orig_obj, new_obj;
- if (!_prepare_clone(gen, coll, orig_obj, new_obj)) {
- return false;
- }
-
- /* Whenever we have to make a merge_delete() operation, just write to the
- * object first, so we know we have something to move in the said range.
- */
-
- boost::uniform_int<> write_size_rng(100, (2 << 19));
- size_t size = (size_t) write_size_rng(gen);
- bufferlist bl;
- _gen_random(gen, size, bl);
-
- boost::uniform_int<> move_len(1, bl.length());
- size = (size_t) move_len(gen);
-
- vector<std::pair<uint64_t, uint64_t>> move_info = {
- make_pair(0, size)
- };
-
- dout(0) << "do_move_ranges_delete_srcobj " << coll.to_str() << "/" << orig_obj.oid.name
- << " (0~" << size << ")"
- << " => " << coll.to_str() << "/" << new_obj.oid.name
- << " (0)" << dendl;
- _do_write_and_merge_delete(coll, orig_obj, new_obj, move_info, bl);
- return true;
-}
-
-
bool DeterministicOpSequence::_prepare_colls(rngen_t& gen,
coll_entry_t* &orig_coll, coll_entry_t* &new_coll)
{
m_store->apply_transaction(&m_osr, std::move(t));
}
-void DeterministicOpSequence::_do_write_and_merge_delete(
- coll_t coll,
- hobject_t& orig_obj,
- hobject_t& new_obj,
- vector<std::pair<uint64_t, uint64_t>> move_info,
- bufferlist& bl)
-{
- ObjectStore::Transaction t;
-
- note_txn(&t);
- for (unsigned i = 0; i < move_info.size(); ++i) {
- uint64_t off = move_info[i].first;
- t.write(coll, ghobject_t(orig_obj), off, bl.length(), bl);
- }
- t.move_ranges_destroy_src(coll, ghobject_t(orig_obj), ghobject_t(new_obj), move_info);
- m_store->apply_transaction(&m_osr, std::move(t));
-}
-
void DeterministicOpSequence::_do_coll_move(coll_t orig_coll, coll_t new_coll,
hobject_t& obj)
{
DSOP_CLONE = 2,
DSOP_CLONE_RANGE = 3,
DSOP_OBJ_REMOVE = 4,
- DSOP_COLL_MOVE = 5,
- DSOP_SET_ATTRS = 6,
- DSOP_COLL_CREATE = 7,
- DSOP_MERGE_DELETE = 8,
+ DSOP_COLL_MOVE = 6,
+ DSOP_SET_ATTRS = 7,
+ DSOP_COLL_CREATE = 8,
DSOP_FIRST = DSOP_TOUCH,
- DSOP_LAST = DSOP_MERGE_DELETE,
+ DSOP_LAST = DSOP_COLL_CREATE,
};
int32_t txn;
bool do_coll_move(rngen_t& gen);
bool do_set_attrs(rngen_t& gen);
bool do_coll_create(rngen_t& gen);
- bool do_move_ranges_delete_srcobj(rngen_t& gen);
virtual void _do_touch(coll_t coll, hobject_t& obj);
virtual void _do_remove(coll_t coll, hobject_t& obj);
virtual void _do_write_and_clone_range(coll_t coll, hobject_t& orig_obj,
hobject_t& new_obj, uint64_t srcoff, uint64_t srclen,
uint64_t dstoff, bufferlist& bl);
- virtual void _do_write_and_merge_delete(
- coll_t coll, hobject_t& orig_obj,
- hobject_t& new_obj, vector<std::pair<uint64_t, uint64_t>> move_info,
- bufferlist& bl);
virtual void _do_coll_move(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
virtual void _do_coll_create(coll_t cid, uint32_t pg_num, uint64_t num_objs);
}
}
-TEST_P(StoreTest, SimpleMoveRangeDelSrcTest) {
- ObjectStore::Sequencer osr("test");
- int r;
- coll_t cid;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid, 0);
- cerr << "Creating collection " << cid << std::endl;
- r = apply_transaction(store, &osr, std::move(t));
- ASSERT_EQ(r, 0);
- }
-
- ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
- hoid.hobj.pool = -1;
- ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP)));
- hoid2.hobj.pool = -1;
-
- bufferlist small, newdata;
- small.append("small");
- {
- ObjectStore::Transaction t;
- t.write(cid, hoid2, 0, 5, small);
- cerr << "Creating object2 and write bl " << hoid << std::endl;
- r = apply_transaction(store, &osr, std::move(t));
- ASSERT_EQ(r, 0);
- }
- {
- ObjectStore::Transaction t;
- t.write(cid, hoid2, 10, 5, small);
- cerr << "Writing object2 again " << hoid << std::endl;
- r = apply_transaction(store, &osr, std::move(t));
- ASSERT_EQ(r, 0);
-
- }
- {
- vector<std::pair<uint64_t, uint64_t>> move_info = {
- make_pair(0, 5),
- make_pair(10, 5)
- };
-
- ObjectStore::Transaction t;
- t.move_ranges_destroy_src(cid, hoid2, hoid, move_info);
- cerr << "move temp object" << std::endl;
- r = apply_transaction(store, &osr, std::move(t));
- ASSERT_EQ(r, 0);
-
- r = store->read(cid, hoid, 0, 5, newdata);
- ASSERT_EQ(r, 5);
- ASSERT_TRUE(newdata.contents_equal(small));
-
- r = store->read(cid, hoid, 10, 5, newdata);
- ASSERT_EQ(r, 5);
- ASSERT_TRUE(newdata.contents_equal(small));
-
- }
- {
- ObjectStore::Transaction t;
- t.remove(cid, hoid);
- t.remove_collection(cid);
- cerr << "Cleaning" << std::endl;
- r = apply_transaction(store, &osr, std::move(t));
- ASSERT_EQ(r, 0);
- }
-}
-
TEST_P(StoreTest, OmapSimple) {
ObjectStore::Sequencer osr("test");
int r;
return status;
}
- int move_ranges_destroy_src() {
- Mutex::Locker locker(lock);
- EnterExit ee("move_ranges_destroy_src");
- if (!can_unlink())
- return -ENOENT;
- if (!can_create())
- return -ENOSPC;
- wait_for_ready();
-
- ghobject_t old_obj;
- int max = 20;
- do {
- old_obj = get_uniform_random_object();
- } while (--max && !contents[old_obj].data.length());
- bufferlist &srcdata = contents[old_obj].data;
- if (srcdata.length() == 0) {
- return 0;
- }
- available_objects.erase(old_obj);
- ghobject_t new_obj = get_uniform_random_object();
- available_objects.erase(new_obj);
-
- boost::uniform_int<> u1(0, max_object_len - max_write_len);
- boost::uniform_int<> u2(0, max_write_len);
- uint64_t off = u1(*rng);
- uint64_t len = u2(*rng);
- if (write_alignment) {
- off = ROUND_UP_TO(off, write_alignment);
- len = ROUND_UP_TO(len, write_alignment);
- }
-
- if (off > srcdata.length() - 1) {
- off = srcdata.length() - 1;
- }
- if (off + len > srcdata.length()) {
- len = srcdata.length() - off;
- }
- if (0)
- cout << __func__ << " " << off << "~" << len
- << " (size " << srcdata.length() << ")" << std::endl;
-
- ObjectStore::Transaction t;
- vector<std::pair<uint64_t,uint64_t>> extents;
- extents.emplace_back(
- std::pair<uint64_t,uint64_t>(off, len));
- t.move_ranges_destroy_src(cid, old_obj, new_obj, extents);
- ++in_flight;
- in_flight_objects.insert(old_obj);
-
- bufferlist bl;
- if (off < srcdata.length()) {
- if (off + len > srcdata.length()) {
- bl.substr_of(srcdata, off, srcdata.length() - off);
- } else {
- bl.substr_of(srcdata, off, len);
- }
- }
-
- // *copy* the data buffer, since we may modify it later.
- {
- bufferlist t;
- t.append(bl.c_str(), bl.length());
- t.swap(bl);
- }
-
- bufferlist& dstdata = contents[new_obj].data;
- if (dstdata.length() <= off) {
- if (bl.length() > 0) {
- dstdata.append_zero(off - dstdata.length());
- dstdata.append(bl);
- }
- } else {
- bufferlist value;
- assert(dstdata.length() > off);
- dstdata.copy(0, off, value);
- value.append(bl);
- if (value.length() < dstdata.length())
- dstdata.copy(value.length(),
- dstdata.length() - value.length(), value);
- value.swap(dstdata);
- }
-
- // remove source
- contents.erase(old_obj);
-
- int status = store->queue_transaction(osr, std::move(t), new C_SyntheticOnClone(this, old_obj, new_obj));
- return status;
- }
-
int setattrs() {
Mutex::Locker locker(lock);
EnterExit ee("setattrs");
test_obj.write();
} else if (val > 500) {
test_obj.clone();
- } else if (val > 475) {
- test_obj.move_ranges_destroy_src();
} else if (val > 450) {
test_obj.clone_range();
} else if (val > 300) {
t.clone(c, o1, o2);
bl.append("some other data");
t.write(c, o2, 1, bl.length(), bl);
-
- t.move_ranges_destroy_src(c, o1, o2, move_info);
}
TEST(Transaction, GetNumBytes)
projects / ceph.git / commitdiff