#include "common/common_init.h"
#include "common/hobject.h"
#include "global/global_init.h"
+#include "include/buffer_fwd.h"
#include "os/ObjectStore.h"
#include "test/objectstore/ObjectStoreImitator.h"
#include <gtest/gtest.h>
#define dout_context g_ceph_context
+constexpr uint64_t _1Kb = 1024;
+constexpr uint64_t _1Mb = 1024 * _1Kb;
+constexpr uint64_t _1Gb = 1024 * _1Mb;
+
+static bufferlist make_bl(size_t len, char c) {
+ bufferlist bl;
+ if (len > 0) {
+ bl.reserve(len);
+ bl.append(std::string(len, c));
+ }
+
+ return bl;
+}
+
+// --------- FragmentationSimulator ----------
+
class FragmentationSimulator {
public:
struct WorkloadGenerator {
};
using WorkloadGeneratorRef = std::shared_ptr<WorkloadGenerator>;
std::vector<WorkloadGeneratorRef> generators;
+
void add_generator(WorkloadGeneratorRef gen) {
std::cout << "Generator: " << gen->name() << " added\n";
generators.push_back(gen);
return 0;
}
- FragmentationSimulator() {
- std::cout << "Initializing simulator\n" << std::endl;
- os = new ObjectStoreImitator(g_ceph_context, "", 4096);
- os->init_alloc("btree", 1024 * 1024 * 1024, 4096);
+ FragmentationSimulator(const std::string &alloc_type, uint64_t size,
+ uint64_t min_alloc_size = 4096) {
+ std::cout << "Initializing simulator" << std::endl;
+
+ os = new ObjectStoreImitator(g_ceph_context, "", min_alloc_size);
+ std::cout << "Initializing allocator: " << alloc_type << " size: 0x"
+ << std::hex << size << std::dec << "\n"
+ << std::endl;
+ os->init_alloc(alloc_type, size);
}
~FragmentationSimulator() { delete os; }
ObjectStoreImitator *os;
};
+// --------- SimpleCWGenerator ----------
+
struct SimpleCWGenerator : public FragmentationSimulator::WorkloadGenerator {
std::string name() override { return "SimpleCW"; }
int generate_txns(ObjectStore::CollectionHandle &ch,
h1.oid = "obj_1";
h1.set_hash(1);
h1.pool = 1;
+ auto oid = ghobject_t(h1);
ObjectStore::Transaction t1;
- t1.create(ch->cid, ghobject_t(h1));
+ t1.create(ch->get_cid(), oid);
os->queue_transaction(ch, std::move(t1));
+ ObjectStore::Transaction t2;
+ t2.write(ch->get_cid(), oid, 0, _1Mb, make_bl(_1Mb, 'c'));
+ os->queue_transaction(ch, std::move(t2));
+
return 0;
}
};
+// ----------- Tests -----------
+
TEST(FragmentationSimulator, simple) {
- auto sim = FragmentationSimulator();
+ auto sim = FragmentationSimulator("stupid", _1Gb);
sim.add_generator(std::make_shared<SimpleCWGenerator>());
sim.begin_simulation_with_generators();
}
+// ----------- main -----------
+
int main(int argc, char **argv) {
auto args = argv_to_vec(argc, argv);
auto cct =
#define OBJECT_MAX_SIZE 0xffffffff // 32 bits
void ObjectStoreImitator::init_alloc(const std::string &alloc_type,
- int64_t size, uint64_t min_alloc_size) {
+ int64_t size) {
alloc.reset(Allocator::create(cct, alloc_type, size, min_alloc_size));
}
+void ObjectStoreImitator::print_status() {
+ std::cout << std::hex
+ << "Fragmentation score: " << alloc->get_fragmentation_score()
+ << " , fragmentation: " << alloc->get_fragmentation()
+ << ", allocator name " << alloc->get_name() << ", allocator type "
+ << alloc->get_type() << ", capacity 0x" << alloc->get_capacity()
+ << ", block size 0x" << alloc->get_block_size() << ", free 0x"
+ << alloc->get_free() << std::dec << std::endl;
+}
+
// ------- Transactions -------
int ObjectStoreImitator::queue_transactions(CollectionHandle &ch,
std::vector<ObjectRef> ovec(i.objects.size());
- while (i.have_op()) {
+ for (int pos = 0; i.have_op(); ++pos) {
Transaction::Op *op = i.decode_op();
int r = 0;
switch (op->op) {
case Transaction::OP_RMCOLL: {
- if (coll_map.find(c->cid) != coll_map.end())
- coll_map.erase(c->cid);
+ const coll_t &cid = i.get_cid(op->cid);
+ r = _remove_collection(cid, &c);
+ if (!r)
+ continue;
} break;
case Transaction::OP_MKCOLL: {
switch (op->op) {
case Transaction::OP_CREATE:
- case Transaction::OP_TOUCH:
+ case Transaction::OP_TOUCH: {
_assign_nid(o);
r = 0;
- break;
+ } break;
case Transaction::OP_WRITE: {
uint64_t off = op->off;
uint64_t dstoff = op->dest_off;
r = _clone_range(c, o, no, srcoff, len, dstoff);
} break;
- {
- case Transaction::OP_COLL_ADD:
- ceph_abort_msg("not implemented");
- break;
-
- case Transaction::OP_COLL_REMOVE:
- ceph_abort_msg("not implemented");
- break;
-
- case Transaction::OP_COLL_MOVE:
- ceph_abort_msg("deprecated");
- break;
-
- case Transaction::OP_COLL_MOVE_RENAME:
- case Transaction::OP_TRY_RENAME: {
- ceph_assert(op->cid == op->dest_cid);
- const ghobject_t &noid = i.get_oid(op->dest_oid);
- ObjectRef &no = ovec[op->dest_oid];
- if (!no) {
- no = c->get_obj(noid, false);
- }
- r = _rename(c, o, no, noid);
- } break;
-
- case Transaction::OP_OMAP_CLEAR:
- case Transaction::OP_OMAP_SETKEYS:
- case Transaction::OP_OMAP_RMKEYS:
- case Transaction::OP_OMAP_RMKEYRANGE:
- case Transaction::OP_OMAP_SETHEADER:
- break;
-
- case Transaction::OP_SETALLOCHINT: {
- r = _set_alloc_hint(c, o, op->expected_object_size,
- op->expected_write_size, op->hint);
- } break;
-
- default:
- derr << __func__ << " bad op " << op->op << dendl;
- ceph_abort();
+
+ case Transaction::OP_COLL_ADD:
+ case Transaction::OP_COLL_REMOVE:
+ ceph_abort_msg("not implemented");
+ break;
+
+ case Transaction::OP_COLL_MOVE:
+ ceph_abort_msg("deprecated");
+ break;
+
+ case Transaction::OP_COLL_MOVE_RENAME:
+ case Transaction::OP_TRY_RENAME: {
+ ceph_assert(op->cid == op->dest_cid);
+ const ghobject_t &noid = i.get_oid(op->dest_oid);
+ ObjectRef &no = ovec[op->dest_oid];
+ if (!no) {
+ no = c->get_obj(noid, false);
}
- endop:
- return;
+ r = _rename(c, o, no, noid);
+ } break;
+
+ case Transaction::OP_OMAP_CLEAR:
+ case Transaction::OP_OMAP_SETKEYS:
+ case Transaction::OP_OMAP_RMKEYS:
+ case Transaction::OP_OMAP_RMKEYRANGE:
+ case Transaction::OP_OMAP_SETHEADER:
+ break;
+
+ case Transaction::OP_SETALLOCHINT: {
+ r = _set_alloc_hint(c, o, op->expected_object_size,
+ op->expected_write_size, op->hint);
+ } break;
+
+ default:
+ derr << __func__ << " bad op " << op->op << dendl;
+ ceph_abort();
+ }
+
+ endop:
+ if (r < 0) {
+ derr << __func__ << " error " << cpp_strerror(r)
+ << " not handled on operation " << op->op << " (op " << pos
+ << ", counting from 0)" << dendl;
+ ceph_abort_msg("unexpected error");
}
}
}
int64_t prealloc_left =
alloc->allocate(need, min_alloc_size, need, 0, &prealloc);
if (prealloc_left < 0 || prealloc_left < (int64_t)need) {
- if (prealloc.size()) {
+ derr << __func__ << " failed to allocate 0x" << std::hex << need
+ << " allocated 0x " << (prealloc_left < 0 ? 0 : prealloc_left)
+ << " min_alloc_size 0x" << min_alloc_size << " available 0x "
+ << alloc->get_free() << std::dec << dendl;
+ if (prealloc.size())
alloc->release(prealloc);
- }
+
return -ENOSPC;
}
int r =
collection_list(ch, ghobject_t(), ghobject_t::get_max(), 1, &ls, &next);
if (r < 0) {
+ derr << __func__ << " collection_list returned: " << cpp_strerror(r)
+ << dendl;
return r;
}
typedef boost::intrusive_ptr<Object> ObjectRef;
struct Collection : public CollectionImpl {
- ObjectStoreImitator *sim;
bluestore_cnode_t cnode;
std::map<ghobject_t, ObjectRef> objects;
}
Collection(ObjectStoreImitator *sim_, coll_t cid_)
- : CollectionImpl(sim_->cct, cid_), sim(sim_), exists(true),
- commit_queue(nullptr) {}
+ : CollectionImpl(sim_->cct, cid_), exists(true), commit_queue(nullptr) {
+ }
};
CollectionRef _get_collection(const coll_t &cid);
~ObjectStoreImitator() = default;
- void init_alloc(const std::string &alloc_type, int64_t size,
- uint64_t min_alloc_size);
+ void init_alloc(const std::string &alloc_type, int64_t size);
+ void print_status();
// Overrides
+ // This is often not called directly but through queue_transaction
int queue_transactions(CollectionHandle &ch, std::vector<Transaction> &tls,
TrackedOpRef op = TrackedOpRef(),
ThreadPool::TPHandle *handle = NULL) override;
projects / ceph.git / commitdiff