: ${EXT3:=$(which mkfs.ext3)}
: ${XFS:=$(which mkfs.xfs)}
: ${BTRFS:=$(which mkfs.btrfs)}
-: ${CEPH_TEST_FILESTORE:=ceph_test_filestore}
+: ${CEPH_TEST_FILESTORE:=ceph_test_objectstore}
: ${FILE_SYSTEMS:=EXT4} # EXT3 XFS BTRFS
: ${DEBUG:=}
main
fi
# Local Variables:
-# compile-command: "CEPH_TEST_FILESTORE=../../../src/ceph_test_filestore filestore.sh TEST"
+# compile-command: "CEPH_TEST_FILESTORE=../../../src/ceph_test_objectstore filestore.sh TEST"
# End:
unittest_escape_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_escape
-unittest_chain_xattr_SOURCES = test/filestore/chain_xattr.cc
+unittest_chain_xattr_SOURCES = test/objectstore/chain_xattr.cc
unittest_chain_xattr_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
unittest_chain_xattr_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_chain_xattr
bin_DEBUGPROGRAMS += ceph_test_libcephfs
if LINUX
-ceph_test_filestore_SOURCES = test/filestore/store_test.cc
-ceph_test_filestore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-ceph_test_filestore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-bin_DEBUGPROGRAMS += ceph_test_filestore
+ceph_test_objectstore_SOURCES = test/objectstore/store_test.cc
+ceph_test_objectstore_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+ceph_test_objectstore_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph_test_objectstore
endif
-ceph_test_filestore_workloadgen_SOURCES = \
- test/filestore/workload_generator.cc \
- test/filestore/TestFileStoreState.cc
-ceph_test_filestore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL)
-bin_DEBUGPROGRAMS += ceph_test_filestore_workloadgen
+ceph_test_objectstore_workloadgen_SOURCES = \
+ test/objectstore/workload_generator.cc \
+ test/objectstore/TestObjectStoreState.cc
+ceph_test_objectstore_workloadgen_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+bin_DEBUGPROGRAMS += ceph_test_objectstore_workloadgen
ceph_test_filestore_idempotent_SOURCES = \
- test/filestore/test_idempotent.cc \
- test/filestore/FileStoreTracker.cc \
+ test/objectstore/test_idempotent.cc \
+ test/objectstore/FileStoreTracker.cc \
test/common/ObjectContents.cc
ceph_test_filestore_idempotent_LDADD = $(LIBOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent
ceph_test_filestore_idempotent_sequence_SOURCES = \
- test/filestore/test_idempotent_sequence.cc \
- test/filestore/DeterministicOpSequence.cc \
- test/filestore/TestFileStoreState.cc \
- test/filestore/FileStoreDiff.cc
+ test/objectstore/test_idempotent_sequence.cc \
+ test/objectstore/DeterministicOpSequence.cc \
+ test/objectstore/TestObjectStoreState.cc \
+ test/objectstore/FileStoreDiff.cc
ceph_test_filestore_idempotent_sequence_LDADD = $(LIBOS) $(CEPH_GLOBAL)
bin_DEBUGPROGRAMS += ceph_test_filestore_idempotent_sequence
test/bench/testfilestore_backend.h \
test/common/ObjectContents.h \
test/encoding/types.h \
- test/filestore/DeterministicOpSequence.h \
- test/filestore/FileStoreDiff.h \
- test/filestore/FileStoreTracker.h \
- test/filestore/TestFileStoreState.h \
- test/filestore/workload_generator.h \
+ test/objectstore/DeterministicOpSequence.h \
+ test/objectstore/FileStoreDiff.h \
+ test/objectstore/FileStoreTracker.h \
+ test/objectstore/TestObjectStoreState.h \
+ test/objectstore/workload_generator.h \
test/kv_store_bench.h \
test/librados/test.h \
test/ObjectMap/KeyValueDBMemory.h \
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include <fstream>
-#include <time.h>
-#include <stdlib.h>
-#include <signal.h>
-#include <sstream>
-#include "os/FileStore.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "common/debug.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/lexical_cast.hpp>
-
-#include "DeterministicOpSequence.h"
-
-#include "common/config.h"
-#include "include/assert.h"
-
-#define dout_subsys ceph_subsys_filestore
-#undef dout_prefix
-#define dout_prefix *_dout << "deterministic_seq "
-
-DeterministicOpSequence::DeterministicOpSequence(FileStore *store,
- std::string status)
- : TestFileStoreState(store),
- txn(0),
- m_osr("OSR")
-{
- txn_coll = coll_t("meta");
- txn_object = hobject_t(sobject_t("txn", CEPH_NOSNAP));
-
- if (!status.empty())
- m_status.open(status.c_str());
-}
-
-DeterministicOpSequence::~DeterministicOpSequence()
-{
- // TODO Auto-generated destructor stub
-}
-
-bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen)
-{
- bool ok = false;
- switch (op) {
- case DSOP_TOUCH:
- ok = do_touch(gen);
- break;
- case DSOP_WRITE:
- ok = do_write(gen);
- break;
- case DSOP_CLONE:
- ok = do_clone(gen);
- break;
- case DSOP_CLONE_RANGE:
- ok = do_clone_range(gen);
- break;
- case DSOP_OBJ_REMOVE:
- ok = do_remove(gen);
- break;
- case DSOP_COLL_ADD:
- ok = do_coll_add(gen);
- break;
- case DSOP_COLL_RENAME:
- //do_coll_rename(gen);
- break;
- case DSOP_SET_ATTRS:
- ok = do_set_attrs(gen);
- break;
- default:
- assert(0 == "bad op");
- }
- return ok;
-}
-
-void DeterministicOpSequence::generate(int seed, int num_txs)
-{
- std::ostringstream ss;
- ss << "generate run " << num_txs << " --seed " << seed;
-
- if (m_status.is_open()) {
- m_status << ss.str() << std::endl;
- m_status.flush();
- }
-
- dout(0) << ss.str() << dendl;
-
- rngen_t gen(seed);
- boost::uniform_int<> op_rng(DSOP_FIRST, DSOP_LAST);
-
- for (txn = 1; txn <= num_txs; ) {
- int op = op_rng(gen);
- _print_status(txn, op);
- dout(0) << "generate seq " << txn << " op " << op << dendl;
- if (run_one_op(op, gen))
- txn++;
- }
-}
-
-void DeterministicOpSequence::_print_status(int seq, int op)
-{
- if (!m_status.is_open())
- return;
- m_status << seq << " " << op << std::endl;
- m_status.flush();
-}
-
-int DeterministicOpSequence::_gen_coll_id(rngen_t& gen)
-{
- boost::uniform_int<> coll_rng(0, m_collections_ids.size()-1);
- return coll_rng(gen);
-}
-
-int DeterministicOpSequence::_gen_obj_id(rngen_t& gen)
-{
- boost::uniform_int<> obj_rng(0, m_num_objects - 1);
- return obj_rng(gen);
-}
-
-void DeterministicOpSequence::note_txn(ObjectStore::Transaction *t)
-{
- bufferlist bl;
- ::encode(txn, bl);
- t->truncate(txn_coll, txn_object, 0);
- t->write(txn_coll, txn_object, 0, bl.length(), bl);
- dout(10) << __func__ << " " << txn << dendl;
-}
-
-bool DeterministicOpSequence::do_touch(rngen_t& gen)
-{
- int coll_id = _gen_coll_id(gen);
- int obj_id = _gen_obj_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- // Don't care about other collections if already exists
- if (!entry->check_for_obj(obj_id)) {
- bool other_found = false;
- map<int, coll_entry_t*>::iterator it = m_collections.begin();
- for (; it != m_collections.end(); ++it) {
- if (it->second->check_for_obj(obj_id)) {
- ceph_assert(it->first != coll_id);
- other_found = true;
- }
- }
- if (other_found) {
- dout(0) << "do_touch new object in collection and exists in another" << dendl;
- return false;
- }
- }
- hobject_t *obj = entry->touch_obj(obj_id);
-
- dout(0) << "do_touch " << entry->m_coll.to_str() << "/" << obj->oid.name << dendl;
-
- _do_touch(entry->m_coll, *obj);
- return true;
-}
-
-bool DeterministicOpSequence::do_remove(rngen_t& gen)
-{
- int coll_id = _gen_coll_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- if (entry->m_objects.size() == 0) {
- dout(0) << "do_remove no objects in collection" << dendl;
- return false;
- }
- int obj_id = entry->get_random_obj_id(gen);
- hobject_t *obj = entry->touch_obj(obj_id);
- ceph_assert(obj);
-
- dout(0) << "do_remove " << entry->m_coll.to_str() << "/" << obj->oid.name << dendl;
-
- _do_remove(entry->m_coll, *obj);
- hobject_t *rmobj = entry->remove_obj(obj_id);
- ceph_assert(rmobj);
- delete rmobj;
- return true;
-}
-
-static void _gen_random(rngen_t& gen,
- size_t size, bufferlist& bl) {
-
- static const char alphanum[] = "0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
- "abcdefghijklmnopqrstuvwxyz";
-
- boost::uniform_int<> char_rng(0, sizeof(alphanum));
- bufferptr bp(size);
- for (unsigned int i = 0; i < size - 1; i++) {
- bp[i] = alphanum[char_rng(gen)];
- }
- bp[size - 1] = '\0';
- bl.append(bp);
-}
-
-static void gen_attrs(rngen_t &gen,
- map<string, bufferlist> *out) {
- boost::uniform_int<> num_rng(10, 30);
- boost::uniform_int<> key_size_rng(5, 10);
- boost::uniform_int<> val_size_rng(100, 1000);
- size_t num_attrs = static_cast<size_t>(num_rng(gen));
- for (size_t i = 0; i < num_attrs; ++i) {
- size_t key_size = static_cast<size_t>(num_rng(gen));
- size_t val_size = static_cast<size_t>(num_rng(gen));
- bufferlist keybl;
- _gen_random(gen, key_size, keybl);
- string key(keybl.c_str(), keybl.length());
- _gen_random(gen, val_size, (*out)[key]);
- }
-}
-
-bool DeterministicOpSequence::do_set_attrs(rngen_t& gen)
-{
- int coll_id = _gen_coll_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- if (entry->m_objects.size() == 0) {
- dout(0) << "do_set_attrs no objects in collection" << dendl;
- return false;
- }
- int obj_id = entry->get_random_obj_id(gen);
- hobject_t *obj = entry->touch_obj(obj_id);
- ceph_assert(obj);
-
- map<string, bufferlist> out;
- gen_attrs(gen, &out);
-
- dout(0) << "do_set_attrs " << out.size() << " entries" << dendl;
- _do_set_attrs(entry->m_coll, *obj, out);
- return true;
-}
-
-bool DeterministicOpSequence::do_write(rngen_t& gen)
-{
- int coll_id = _gen_coll_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- if (entry->m_objects.size() == 0) {
- dout(0) << "do_write no objects in collection" << dendl;
- return false;
- }
- int obj_id = entry->get_random_obj_id(gen);
- hobject_t *obj = entry->touch_obj(obj_id);
- ceph_assert(obj);
-
- boost::uniform_int<> size_rng(100, (2 << 19));
- size_t size = (size_t) size_rng(gen);
- bufferlist bl;
- _gen_random(gen, size, bl);
-
- dout(0) << "do_write " << entry->m_coll.to_str() << "/" << obj->oid.name
- << " 0~" << size << dendl;
-
- _do_write(entry->m_coll, *obj, 0, bl.length(), bl);
- return true;
-}
-
-bool DeterministicOpSequence::_prepare_clone(rngen_t& gen,
- coll_t& coll_ret, hobject_t& orig_obj_ret, hobject_t& new_obj_ret)
-{
- int coll_id = _gen_coll_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- if (entry->m_objects.size() >= 2) {
- dout(0) << "_prepare_clone coll " << entry->m_coll.to_str()
- << " doesn't have 2 or more objects" << dendl;
- return false;
- }
-
- int orig_obj_id = entry->get_random_obj_id(gen);
- hobject_t *orig_obj = entry->touch_obj(orig_obj_id);
- ceph_assert(orig_obj);
-
- int id;
- do {
- id = entry->get_random_obj_id(gen);
- } while (id == orig_obj_id);
- hobject_t *new_obj = entry->touch_obj(id);
- ceph_assert(new_obj);
-
- coll_ret = entry->m_coll;
- orig_obj_ret = *orig_obj;
- new_obj_ret = *new_obj;
-
- return true;
-}
-
-bool DeterministicOpSequence::do_clone(rngen_t& gen)
-{
- coll_t coll;
- hobject_t orig_obj, new_obj;
- if (!_prepare_clone(gen, coll, orig_obj, new_obj)) {
- return false;
- }
-
- dout(0) << "do_clone " << coll.to_str() << "/" << orig_obj.oid.name
- << " => " << coll.to_str() << "/" << new_obj.oid.name << dendl;
-
- _do_clone(coll, orig_obj, new_obj);
- return true;
-}
-
-bool DeterministicOpSequence::do_clone_range(rngen_t& gen)
-{
- coll_t coll;
- hobject_t orig_obj, new_obj;
- if (!_prepare_clone(gen, coll, orig_obj, new_obj)) {
- return false;
- }
-
- /* Whenever we have to make a clone_range() operation, just write to the
- * object first, so we know we have something to clone in the said range.
- * This may not be the best solution ever, but currently we're not keeping
- * track of the written-to objects, and until we know for sure we really
- * need to, let's just focus on the task at hand.
- */
-
- boost::uniform_int<> write_size_rng(100, (2 << 19));
- size_t size = (size_t) write_size_rng(gen);
- bufferlist bl;
- _gen_random(gen, size, bl);
-
- boost::uniform_int<> clone_len(1, bl.length());
- size = (size_t) clone_len(gen);
-
- dout(0) << "do_clone_range " << coll.to_str() << "/" << orig_obj.oid.name
- << " (0~" << size << ")"
- << " => " << coll.to_str() << "/" << new_obj.oid.name
- << " (0)" << dendl;
- _do_write_and_clone_range(coll, orig_obj, new_obj, 0, size, 0, bl);
- return true;
-}
-
-bool DeterministicOpSequence::_prepare_colls(rngen_t& gen,
- coll_entry_t* &orig_coll, coll_entry_t* &new_coll)
-{
- ceph_assert(m_collections_ids.size() > 1);
- int orig_coll_id = _gen_coll_id(gen);
- int new_coll_id;
- do {
- new_coll_id = _gen_coll_id(gen);
- } while (new_coll_id == orig_coll_id);
-
- dout(0) << "_prepare_colls from coll id " << orig_coll_id
- << " to coll id " << new_coll_id << dendl;
-
- orig_coll = get_coll_at(orig_coll_id);
- ceph_assert(orig_coll != NULL);
- new_coll = get_coll_at(new_coll_id);
- ceph_assert(new_coll != NULL);
-
- if (!orig_coll->m_objects.size()) {
- dout(0) << "_prepare_colls coll " << orig_coll->m_coll.to_str()
- << " has no objects to use" << dendl;
- return false;
- }
-
- return true;
-}
-
-
-bool DeterministicOpSequence::do_coll_rename(rngen_t& gen)
-{
- int coll_pos = _gen_coll_id(gen);
- dout(0) << "do_coll_rename coll pos #" << coll_pos << dendl;
-
- coll_entry_t *coll_entry = get_coll_at(coll_pos);
- if (!coll_entry) {
- dout(0) << "do_coll_rename no collection at pos #" << coll_pos << dendl;
- return false;
- }
-
- coll_t orig_coll = coll_entry->m_coll;
- char buf[100];
- memset(buf, 0, 100);
- snprintf(buf, 100, "0.%d_head", m_next_coll_nr++);
- coll_t new_coll(buf);
- coll_entry->m_coll = new_coll;
-
- dout(0) << "do_coll_rename " << orig_coll.to_str()
- << " => " << new_coll.to_str() << dendl;
- _do_coll_rename(orig_coll, new_coll);
- return true;
-}
-
-bool DeterministicOpSequence::do_coll_add(rngen_t& gen)
-{
- coll_entry_t *orig_coll = NULL, *new_coll = NULL;
- if (!_prepare_colls(gen, orig_coll, new_coll))
- return false;
-
- ceph_assert(orig_coll && new_coll);
-
- boost::uniform_int<> obj_rng(0, orig_coll->m_objects.size()-1);
- int obj_pos = obj_rng(gen);
- int obj_key = -1;
- hobject_t *obj = orig_coll->get_obj_at(obj_pos, &obj_key);
- if (!obj) {
- dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
- << " has no object as pos #" << obj_pos << " (key " << obj_key << ")"
- << dendl;
- return false;
- }
- if (new_coll->check_for_obj(obj_key)) {
- dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
- << " already has object as pos #" << obj_pos << " (key " << obj_key << ")"
- << dendl;
- return false;
- }
- dout(0) << "do_coll_add " << orig_coll->m_coll.to_str() << "/" << obj->oid.name
- << " => " << new_coll->m_coll.to_str() << "/" << obj->oid.name << dendl;
- new_coll->touch_obj(obj_key);
-
- _do_coll_add(orig_coll->m_coll, new_coll->m_coll, *obj);
- return true;
-}
-
-void DeterministicOpSequence::_do_touch(coll_t coll, hobject_t& obj)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.touch(coll, obj);
- m_store->apply_transaction(t);
-}
-
-void DeterministicOpSequence::_do_remove(coll_t coll, hobject_t& obj)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.remove(coll, obj);
- m_store->apply_transaction(t);
-}
-
-void DeterministicOpSequence::_do_set_attrs(coll_t coll,
- hobject_t &obj,
- const map<string, bufferlist> &attrs)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.omap_setkeys(coll, obj, attrs);
- m_store->apply_transaction(t);
-}
-
-void DeterministicOpSequence::_do_write(coll_t coll, hobject_t& obj,
- uint64_t off, uint64_t len, const bufferlist& data)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.write(coll, obj, off, len, data);
- m_store->apply_transaction(t);
-}
-
-void DeterministicOpSequence::_do_clone(coll_t coll, hobject_t& orig_obj,
- hobject_t& new_obj)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.clone(coll, orig_obj, new_obj);
- m_store->apply_transaction(t);
-}
-
-void DeterministicOpSequence::_do_clone_range(coll_t coll,
- hobject_t& orig_obj, hobject_t& new_obj, uint64_t srcoff,
- uint64_t srclen, uint64_t dstoff)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.clone_range(coll, orig_obj, new_obj, srcoff, srclen, dstoff);
- m_store->apply_transaction(t);
-}
-
-void DeterministicOpSequence::_do_write_and_clone_range(coll_t coll,
- hobject_t& orig_obj,
- hobject_t& new_obj,
- uint64_t srcoff,
- uint64_t srclen,
- uint64_t dstoff,
- bufferlist& bl)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.write(coll, orig_obj, srcoff, bl.length(), bl);
- t.clone_range(coll, orig_obj, new_obj, srcoff, srclen, dstoff);
- m_store->apply_transaction(t);
-}
-
-void DeterministicOpSequence::_do_coll_add(coll_t orig_coll, coll_t new_coll,
- hobject_t& obj)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.remove(new_coll, obj);
- t.collection_add(new_coll, orig_coll, obj);
- m_store->apply_transaction(t);
-}
-
-void DeterministicOpSequence::_do_coll_rename(coll_t orig_coll, coll_t new_coll)
-{
- ObjectStore::Transaction t;
- note_txn(&t);
- t.collection_rename(orig_coll, new_coll);
- m_store->apply_transaction(t);
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#ifndef FILESTORE_DTRMNSTC_SEQ_OPS_H_
-#define FILESTORE_DTRMNSTC_SEQ_OPS_H_
-
-#include <iostream>
-#include <fstream>
-#include <set>
-#include "os/FileStore.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/random/mersenne_twister.hpp>
-#include <boost/random/uniform_int.hpp>
-
-#include "TestFileStoreState.h"
-
-typedef boost::mt11213b rngen_t;
-
-class DeterministicOpSequence : public TestFileStoreState {
- public:
- DeterministicOpSequence(FileStore *store, std::string status = std::string());
- virtual ~DeterministicOpSequence();
-
- virtual void generate(int seed, int num_txs);
-
- protected:
- enum {
- DSOP_TOUCH = 0,
- DSOP_WRITE = 1,
- DSOP_CLONE = 2,
- DSOP_CLONE_RANGE = 3,
- DSOP_OBJ_REMOVE = 4,
- DSOP_COLL_RENAME = 5,
- DSOP_COLL_ADD = 6,
- DSOP_SET_ATTRS = 7,
-
- DSOP_FIRST = DSOP_TOUCH,
- DSOP_LAST = DSOP_SET_ATTRS,
- };
-
- int32_t txn;
-
- coll_t txn_coll;
- hobject_t txn_object;
-
- ObjectStore::Sequencer m_osr;
- std::ofstream m_status;
-
- bool run_one_op(int op, rngen_t& gen);
-
- void note_txn(ObjectStore::Transaction *t);
- bool do_touch(rngen_t& gen);
- bool do_remove(rngen_t& gen);
- bool do_write(rngen_t& gen);
- bool do_clone(rngen_t& gen);
- bool do_clone_range(rngen_t& gen);
- bool do_coll_rename(rngen_t& gen);
- bool do_coll_add(rngen_t& gen);
- bool do_set_attrs(rngen_t& gen);
-
- virtual void _do_touch(coll_t coll, hobject_t& obj);
- virtual void _do_remove(coll_t coll, hobject_t& obj);
- virtual void _do_write(coll_t coll, hobject_t& obj, uint64_t off,
- uint64_t len, const bufferlist& data);
- virtual void _do_set_attrs(coll_t coll,
- hobject_t &obj,
- const map<string, bufferlist> &attrs);
- virtual void _do_clone(coll_t coll, hobject_t& orig_obj, hobject_t& new_obj);
- virtual void _do_clone_range(coll_t coll, hobject_t& orig_obj,
- hobject_t& new_obj, uint64_t srcoff, uint64_t srclen, uint64_t dstoff);
- virtual void _do_write_and_clone_range(coll_t coll, hobject_t& orig_obj,
- hobject_t& new_obj, uint64_t srcoff, uint64_t srclen,
- uint64_t dstoff, bufferlist& bl);
- virtual void _do_coll_add(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
- virtual void _do_coll_rename(coll_t orig_coll, coll_t new_coll);
-
- int _gen_coll_id(rngen_t& gen);
- int _gen_obj_id(rngen_t& gen);
- void _print_status(int seq, int op);
-
- private:
- bool _prepare_clone(rngen_t& gen, coll_t& coll_ret,
- hobject_t& orig_obj_ret, hobject_t& new_obj_ret);
- bool _prepare_colls(rngen_t& gen,
- coll_entry_t* &orig_coll, coll_entry_t* &new_coll);
-};
-
-
-#endif /* FILESTORE_DTRMNSTC_SEQ_OPS_H_ */
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <stdio.h>
-#include <stdlib.h>
-#include <map>
-#include <boost/scoped_ptr.hpp>
-#include "common/debug.h"
-#include "os/FileStore.h"
-#include "common/config.h"
-
-#include "FileStoreDiff.h"
-
-#define dout_subsys ceph_subsys_filestore
-#undef dout_prefix
-#define dout_prefix *_dout << "filestore_diff "
-
-FileStoreDiff::FileStoreDiff(FileStore *a, FileStore *b)
- : a_store(a), b_store(b)
-{
- int err;
- err = a_store->mount();
- ceph_assert(err == 0);
-
- err = b_store->mount();
- ceph_assert(err == 0);
-}
-
-FileStoreDiff::~FileStoreDiff()
-{
- a_store->umount();
- b_store->umount();
-}
-
-
-bool FileStoreDiff::diff_attrs(std::map<std::string,bufferptr>& b,
- std::map<std::string,bufferptr>& a)
-{
- bool ret = false;
- std::map<std::string, bufferptr>::iterator b_it = b.begin();
- std::map<std::string, bufferptr>::iterator a_it = a.begin();
- for (; b_it != b.end(); ++b_it, ++a_it) {
- if (b_it->first != a_it->first) {
- dout(0) << "diff_attrs name mismatch (verify: " << b_it->first
- << ", store: " << a_it->first << ")" << dendl;
- ret = true;
- continue;
- }
-
- if (!b_it->second.cmp(a_it->second)) {
- dout(0) << "diff_attrs contents mismatch on attr " << b_it->first << dendl;
- ret = true;
- continue;
- }
- }
- return ret;
-}
-
-static bool diff_omap(std::map<std::string,bufferlist>& b,
- std::map<std::string,bufferlist>& a)
-{
- bool ret = false;
- std::map<std::string, bufferlist>::iterator b_it = b.begin();
- std::map<std::string, bufferlist>::iterator a_it = a.begin();
- for (; b_it != b.end(); ++b_it, ++a_it) {
- if (b_it->first != a_it->first) {
- dout(0) << "diff_attrs name mismatch (verify: " << b_it->first
- << ", store: " << a_it->first << ")" << dendl;
- ret = true;
- continue;
- }
-
- if (!(b_it->second == a_it->second)) {
- dout(0) << "diff_attrs contents mismatch on attr " << b_it->first << dendl;
- ret = true;
- continue;
- }
- }
- return ret;
-}
-
-bool FileStoreDiff::diff_objects_stat(struct stat& a, struct stat& b)
-{
- bool ret = false;
-
- if (a.st_uid != b.st_uid) {
- dout(0) << "diff_objects_stat uid mismatch (A: "
- << a.st_uid << " != B: " << b.st_uid << ")" << dendl;
- ret = true;
- }
-
- if (a.st_gid != b.st_gid) {
- dout(0) << "diff_objects_stat gid mismatch (A: "
- << a.st_gid << " != B: " << b.st_gid << ")" << dendl;
- ret = true;
- }
-
- if (a.st_mode != b.st_mode) {
- dout(0) << "diff_objects_stat mode mismatch (A: "
- << a.st_mode << " != B: " << b.st_mode << ")" << dendl;
- ret = true;
- }
-
- if (a.st_nlink != b.st_nlink) {
- dout(0) << "diff_objects_stat nlink mismatch (A: "
- << a.st_nlink << " != B: " << b.st_nlink << ")" << dendl;
- ret = true;
- }
-
- if (a.st_size != b.st_size) {
- dout(0) << "diff_objects_stat size mismatch (A: "
- << a.st_size << " != B: " << b.st_size << ")" << dendl;
- ret = true;
- }
- return ret;
-}
-
-bool FileStoreDiff::diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll)
-{
- dout(2) << __func__ << " coll " << coll << dendl;
-
- bool ret = false;
-
- int err;
- std::vector<ghobject_t> b_objects, a_objects;
- err = b_store->collection_list(coll, b_objects);
- if (err < 0) {
- dout(0) << "diff_objects list on verify coll " << coll.to_str()
- << " returns " << err << dendl;
- return true;
- }
- err = a_store->collection_list(coll, a_objects);
- if (err < 0) {
- dout(0) << "diff_objects list on store coll " << coll.to_str()
- << " returns " << err << dendl;
- return true;
- }
-
- if (b_objects.size() != a_objects.size()) {
- dout(0) << "diff_objects num objs mismatch (A: " << a_objects.size()
- << ", B: " << b_objects.size() << ")" << dendl;
- ret = true;
- }
-
- std::vector<ghobject_t>::iterator b_it = b_objects.begin();
- std::vector<ghobject_t>::iterator a_it = b_objects.begin();
- for (; b_it != b_objects.end(); ++b_it, ++a_it) {
- ghobject_t b_obj = *b_it, a_obj = *a_it;
- if (b_obj.hobj.oid.name != a_obj.hobj.oid.name) {
- dout(0) << "diff_objects name mismatch on A object "
- << coll << "/" << a_obj << " and B object "
- << coll << "/" << b_obj << dendl;
- ret = true;
- continue;
- }
-
- struct stat b_stat, a_stat;
- err = b_store->stat(coll, b_obj, &b_stat);
- if (err < 0) {
- dout(0) << "diff_objects error stating B object "
- << coll.to_str() << "/" << b_obj.hobj.oid.name << dendl;
- ret = true;
- }
- err = a_store->stat(coll, a_obj, &a_stat);
- if (err < 0) {
- dout(0) << "diff_objects error stating A object "
- << coll << "/" << a_obj << dendl;
- ret = true;
- }
-
- if (diff_objects_stat(a_stat, b_stat)) {
- dout(0) << "diff_objects stat mismatch on "
- << coll << "/" << b_obj << dendl;
- ret = true;
- }
-
- bufferlist a_obj_bl, b_obj_bl;
- b_store->read(coll, b_obj, 0, b_stat.st_size, b_obj_bl);
- a_store->read(coll, a_obj, 0, a_stat.st_size, a_obj_bl);
-
- if (!a_obj_bl.contents_equal(b_obj_bl)) {
- dout(0) << "diff_objects content mismatch on "
- << coll << "/" << b_obj << dendl;
- ret = true;
- }
-
- std::map<std::string, bufferptr> a_obj_attrs_map, b_obj_attrs_map;
- err = a_store->getattrs(coll, a_obj, a_obj_attrs_map);
- if (err < 0) {
- dout(0) << "diff_objects getattrs on A object " << coll << "/" << a_obj
- << " returns " << err << dendl;
- ret = true;
- }
- err = b_store->getattrs(coll, b_obj, b_obj_attrs_map);
- if (err < 0) {
- dout(0) << "diff_objects getattrs on B object " << coll << "/" << b_obj
- << "returns " << err << dendl;
- ret = true;
- }
-
- if (diff_attrs(b_obj_attrs_map, a_obj_attrs_map)) {
- dout(0) << "diff_objects attrs mismatch on A object "
- << coll << "/" << a_obj << " and B object "
- << coll << "/" << b_obj << dendl;
- ret = true;
- }
-
- std::map<std::string, bufferlist> a_obj_omap, b_obj_omap;
- std::set<std::string> a_omap_keys, b_omap_keys;
- err = a_store->omap_get_keys(coll, a_obj, &a_omap_keys);
- if (err < 0) {
- dout(0) << "diff_objects getomap on A object " << coll << "/" << a_obj
- << " returns " << err << dendl;
- ret = true;
- }
- err = a_store->omap_get_values(coll, a_obj, a_omap_keys, &a_obj_omap);
- if (err < 0) {
- dout(0) << "diff_objects getomap on A object " << coll << "/" << a_obj
- << " returns " << err << dendl;
- ret = true;
- }
- err = b_store->omap_get_keys(coll, b_obj, &b_omap_keys);
- if (err < 0) {
- dout(0) << "diff_objects getomap on A object " << coll << "/" << b_obj
- << " returns " << err << dendl;
- ret = true;
- }
- err = b_store->omap_get_values(coll, b_obj, b_omap_keys, &b_obj_omap);
- if (err < 0) {
- dout(0) << "diff_objects getomap on A object " << coll << "/" << b_obj
- << " returns " << err << dendl;
- ret = true;
- }
- if (diff_omap(a_obj_omap, b_obj_omap)) {
- dout(0) << "diff_objects omap mismatch on A object "
- << coll << "/" << a_obj << " and B object "
- << coll << "/" << b_obj << dendl;
- ret = true;
- }
- }
-
- return ret;
-}
-
-bool FileStoreDiff::diff_coll_attrs(FileStore *a_store, FileStore *b_store, coll_t coll)
-{
- bool ret = false;
-
- int err;
- std::map<std::string, bufferptr> b_coll_attrs, a_coll_attrs;
- err = b_store->collection_getattrs(coll, b_coll_attrs);
- if (err < 0) {
- dout(0) << "diff_attrs getattrs on verify coll " << coll.to_str()
- << "returns " << err << dendl;
- ret = true;
- }
- err = a_store->collection_getattrs(coll, a_coll_attrs);
- if (err < 0) {
- dout(0) << "diff_attrs getattrs on A coll " << coll.to_str()
- << "returns " << err << dendl;
- ret = true;
- }
-
- if (b_coll_attrs.size() != a_coll_attrs.size()) {
- dout(0) << "diff_attrs size mismatch (A: " << a_coll_attrs.size()
- << ", B: " << a_coll_attrs.size() << ")" << dendl;
- ret = true;
- }
-
- return diff_attrs(b_coll_attrs, a_coll_attrs) || ret;
-}
-
-bool FileStoreDiff::diff()
-{
- bool ret = false;
-
- std::vector<coll_t> a_coll_list, b_coll_list;
- a_store->list_collections(a_coll_list);
- b_store->list_collections(b_coll_list);
-
- std::vector<coll_t>::iterator it = b_coll_list.begin();
- for (; it != b_coll_list.end(); ++it) {
- coll_t b_coll = *it;
- if (!a_store->collection_exists(b_coll)) {
- dout(0) << "diff B coll " << b_coll.to_str() << " DNE on A" << dendl;
- ret = true;
- continue;
- }
- for (std::vector<coll_t>::iterator j = a_coll_list.begin();
- j != a_coll_list.end(); ++j) {
- if (*j == *it) {
- a_coll_list.erase(j);
- break;
- }
- }
-
- if (diff_coll_attrs(a_store, b_store, b_coll))
- ret = true;
-
- if (diff_objects(a_store, b_store, b_coll))
- ret = true;
- }
- for (std::vector<coll_t>::iterator it = a_coll_list.begin();
- it != a_coll_list.end(); ++it) {
- dout(0) << "diff A coll " << *it << " DNE on B" << dendl;
- ret = true;
- }
-
- return ret;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#ifndef FILESTORE_DIFF_H_
-#define FILESTORE_DIFF_H_
-
-#include <iostream>
-#include <stdlib.h>
-#include <map>
-#include <boost/scoped_ptr.hpp>
-#include "common/debug.h"
-#include "os/FileStore.h"
-#include "common/config.h"
-
-class FileStoreDiff {
-
- private:
- FileStore *a_store;
- FileStore *b_store;
-
- bool diff_coll_attrs(FileStore *a_store, FileStore *b_store, coll_t coll);
- bool diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll);
- bool diff_objects_stat(struct stat& a, struct stat& b);
- bool diff_attrs(std::map<std::string,bufferptr>& b,
- std::map<std::string,bufferptr>& a);
-
-public:
- FileStoreDiff(FileStore *a, FileStore *b);
- virtual ~FileStoreDiff();
-
- bool diff();
-};
-
-#endif /* FILESTOREDIFF_H_ */
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-#include "FileStoreTracker.h"
-#include <stdlib.h>
-#include <iostream>
-#include <boost/scoped_ptr.hpp>
-#include "include/Context.h"
-#include "common/Mutex.h"
-
-class OnApplied : public Context {
- FileStoreTracker *tracker;
- list<pair<pair<string, string>, uint64_t> > in_flight;
- ObjectStore::Transaction *t;
-public:
- OnApplied(FileStoreTracker *tracker,
- list<pair<pair<string, string>, uint64_t> > in_flight,
- ObjectStore::Transaction *t)
- : tracker(tracker), in_flight(in_flight), t(t) {}
-
- void finish(int r) {
- for (list<pair<pair<string, string>, uint64_t> >::iterator i =
- in_flight.begin();
- i != in_flight.end();
- ++i) {
- tracker->applied(i->first, i->second);
- }
- delete t;
- }
-};
-
-class OnCommitted : public Context {
- FileStoreTracker *tracker;
- list<pair<pair<string, string>, uint64_t> > in_flight;
-public:
- OnCommitted(FileStoreTracker *tracker,
- list<pair<pair<string, string>, uint64_t> > in_flight)
- : tracker(tracker), in_flight(in_flight) {}
-
- void finish(int r) {
- for (list<pair<pair<string, string>, uint64_t> >::iterator i =
- in_flight.begin();
- i != in_flight.end();
- ++i) {
- tracker->committed(i->first, i->second);
- }
- }
-};
-
-int FileStoreTracker::init()
-{
- set<string> to_get;
- to_get.insert("STATUS");
- map<string, bufferlist> got;
- db->get("STATUS", to_get, &got);
- restart_seq = 0;
- if (!got.empty()) {
- bufferlist::iterator bp = got.begin()->second.begin();
- ::decode(restart_seq, bp);
- }
- ++restart_seq;
- KeyValueDB::Transaction t = db->get_transaction();
- got.clear();
- ::encode(restart_seq, got["STATUS"]);
- t->set("STATUS", got);
- db->submit_transaction(t);
- return 0;
-}
-
-void FileStoreTracker::submit_transaction(Transaction &t)
-{
- list<pair<pair<string, string>, uint64_t> > in_flight;
- OutTransaction out;
- out.t = new ObjectStore::Transaction;
- out.in_flight = &in_flight;
- for (list<Transaction::Op*>::iterator i = t.ops.begin();
- i != t.ops.end();
- ++i) {
- (**i)(this, &out);
- }
- store->queue_transaction(
- 0, out.t,
- new OnApplied(this, in_flight, out.t),
- new OnCommitted(this, in_flight));
-}
-
-void FileStoreTracker::write(const pair<string, string> &obj,
- OutTransaction *out)
-{
- Mutex::Locker l(lock);
- std::cerr << "Writing " << obj << std::endl;
- ObjectContents contents = get_current_content(obj);
-
- uint64_t offset = rand() % (SIZE/2);
- uint64_t len = rand() % (SIZE/2);
- if (!len) len = 10;
- contents.write(rand(), offset, len);
-
- bufferlist to_write;
- ObjectContents::Iterator iter = contents.get_iterator();
- iter.seek_to(offset);
- for (uint64_t i = offset;
- i < offset + len;
- ++i, ++iter) {
- assert(iter.valid());
- to_write.append(*iter);
- }
- out->t->write(coll_t(obj.first),
- hobject_t(sobject_t(obj.second, CEPH_NOSNAP)),
- offset,
- len,
- to_write);
- out->in_flight->push_back(make_pair(obj, set_content(obj, contents)));
-}
-
-void FileStoreTracker::remove(const pair<string, string> &obj,
- OutTransaction *out)
-{
- std::cerr << "Deleting " << obj << std::endl;
- Mutex::Locker l(lock);
- ObjectContents old_contents = get_current_content(obj);
- if (!old_contents.exists())
- return;
- out->t->remove(coll_t(obj.first),
- hobject_t(sobject_t(obj.second, CEPH_NOSNAP)));
- ObjectContents contents;
- out->in_flight->push_back(make_pair(obj, set_content(obj, contents)));
-}
-
-void FileStoreTracker::clone_range(const pair<string, string> &from,
- const pair<string, string> &to,
- OutTransaction *out) {
- Mutex::Locker l(lock);
- std::cerr << "CloningRange " << from << " to " << to << std::endl;
- assert(from.first == to.first);
- ObjectContents from_contents = get_current_content(from);
- ObjectContents to_contents = get_current_content(to);
- if (!from_contents.exists()) {
- return;
- }
- if (from.second == to.second) {
- return;
- }
-
- uint64_t new_size = from_contents.size();
- interval_set<uint64_t> interval_to_clone;
- uint64_t offset = rand() % (new_size/2);
- uint64_t len = rand() % (new_size/2);
- if (!len) len = 10;
- interval_to_clone.insert(offset, len);
- to_contents.clone_range(from_contents, interval_to_clone);
- out->t->clone_range(coll_t(from.first),
- hobject_t(sobject_t(from.second, CEPH_NOSNAP)),
- hobject_t(sobject_t(to.second, CEPH_NOSNAP)),
- offset,
- len,
- offset);
- out->in_flight->push_back(make_pair(to, set_content(to, to_contents)));
-}
-
-void FileStoreTracker::clone(const pair<string, string> &from,
- const pair<string, string> &to,
- OutTransaction *out) {
- Mutex::Locker l(lock);
- std::cerr << "Cloning " << from << " to " << to << std::endl;
- assert(from.first == to.first);
- if (from.second == to.second) {
- return;
- }
- ObjectContents from_contents = get_current_content(from);
- ObjectContents to_contents = get_current_content(to);
- if (!from_contents.exists()) {
- return;
- }
-
- if (to_contents.exists())
- out->t->remove(coll_t(to.first),
- hobject_t(sobject_t(to.second, CEPH_NOSNAP)));
- out->t->clone(coll_t(from.first),
- hobject_t(sobject_t(from.second, CEPH_NOSNAP)),
- hobject_t(sobject_t(to.second, CEPH_NOSNAP)));
- out->in_flight->push_back(make_pair(to, set_content(to, from_contents)));
-}
-
-
-string obj_to_prefix(const pair<string, string> &obj) {
- string sep;
- sep.push_back('^');
- return obj.first + sep + obj.second + "_CONTENTS_";
-}
-
-string obj_to_meta_prefix(const pair<string, string> &obj) {
- string sep;
- sep.push_back('^');
- return obj.first + sep + obj.second;
-}
-
-string seq_to_key(uint64_t seq) {
- char buf[50];
- snprintf(buf, sizeof(buf), "%*llu", 20, (unsigned long long int)seq);
- return string(buf);
-}
-
-struct ObjStatus {
- uint64_t last_applied;
- uint64_t last_committed;
- uint64_t restart_seq;
- ObjStatus() : last_applied(0), last_committed(0), restart_seq(0) {}
-
- uint64_t get_last_applied(uint64_t seq) const {
- if (seq > restart_seq)
- return last_committed;
- else
- return last_applied;
- }
- void set_last_applied(uint64_t _last_applied, uint64_t seq) {
- last_applied = _last_applied;
- restart_seq = seq;
- }
- uint64_t trim_to() const {
- return last_applied < last_committed ?
- last_applied : last_committed;
- }
-};
-void encode(const ObjStatus &obj, bufferlist &bl) {
- ::encode(obj.last_applied, bl);
- ::encode(obj.last_committed, bl);
- ::encode(obj.restart_seq, bl);
-}
-void decode(ObjStatus &obj, bufferlist::iterator &bl) {
- ::decode(obj.last_applied, bl);
- ::decode(obj.last_committed, bl);
- ::decode(obj.restart_seq, bl);
-}
-
-
-ObjStatus get_obj_status(const pair<string, string> &obj,
- KeyValueDB *db)
-{
- set<string> to_get;
- to_get.insert("META");
- map<string, bufferlist> got;
- db->get(obj_to_meta_prefix(obj), to_get, &got);
- ObjStatus retval;
- if (!got.empty()) {
- bufferlist::iterator bp = got.begin()->second.begin();
- ::decode(retval, bp);
- }
- return retval;
-}
-
-void set_obj_status(const pair<string, string> &obj,
- const ObjStatus &status,
- KeyValueDB::Transaction t)
-{
- map<string, bufferlist> to_set;
- ::encode(status, to_set["META"]);
- t->set(obj_to_meta_prefix(obj), to_set);
-}
-
-void _clean_forward(const pair<string, string> &obj,
- uint64_t last_valid,
- KeyValueDB *db)
-{
- KeyValueDB::Transaction t = db->get_transaction();
- KeyValueDB::Iterator i = db->get_iterator(obj_to_prefix(obj));
- set<string> to_remove;
- i->upper_bound(seq_to_key(last_valid));
- for (; i->valid(); i->next()) {
- to_remove.insert(i->key());
- }
- t->rmkeys(obj_to_prefix(obj), to_remove);
- db->submit_transaction(t);
-}
-
-
-void FileStoreTracker::verify(const string &coll, const string &obj,
- bool on_start) {
- Mutex::Locker l(lock);
- std::cerr << "Verifying " << make_pair(coll, obj) << std::endl;
-
- pair<uint64_t, uint64_t> valid_reads = get_valid_reads(make_pair(coll, obj));
- std::cerr << "valid_reads is " << valid_reads << std::endl;
- bufferlist contents;
- int r = store->read(coll_t(coll),
- hobject_t(sobject_t(obj, CEPH_NOSNAP)),
- 0,
- 2*SIZE,
- contents);
- std::cerr << "exists: " << r << std::endl;
-
-
- for (uint64_t i = valid_reads.first;
- i < valid_reads.second;
- ++i) {
- ObjectContents old_contents = get_content(make_pair(coll, obj), i);
-
- std::cerr << "old_contents exists " << old_contents.exists() << std::endl;
- if (!old_contents.exists() && (r == -ENOENT))
- return;
-
- if (old_contents.exists() && (r == -ENOENT))
- continue;
-
- if (!old_contents.exists() && (r != -ENOENT))
- continue;
-
- if (contents.length() != old_contents.size()) {
- std::cerr << "old_contents.size() is "
- << old_contents.size() << std::endl;
- continue;
- }
-
- bufferlist::iterator bp = contents.begin();
- ObjectContents::Iterator iter = old_contents.get_iterator();
- iter.seek_to_first();
- bool matches = true;
- uint64_t pos = 0;
- for (; !bp.end() && iter.valid();
- ++iter, ++bp, ++pos) {
- if (*iter != *bp) {
- std::cerr << "does not match at pos " << pos << std::endl;
- matches = false;
- break;
- }
- }
- if (matches) {
- if (on_start)
- _clean_forward(make_pair(coll, obj), i, db);
- return;
- }
- }
- std::cerr << "Verifying " << make_pair(coll, obj) << " failed " << std::endl;
- assert(0);
-}
-
-ObjectContents FileStoreTracker::get_current_content(
- const pair<string, string> &obj)
-{
- KeyValueDB::Iterator iter = db->get_iterator(
- obj_to_prefix(obj));
- iter->seek_to_last();
- if (iter->valid()) {
- bufferlist bl = iter->value();
- bufferlist::iterator bp = bl.begin();
- pair<uint64_t, bufferlist> val;
- ::decode(val, bp);
- assert(seq_to_key(val.first) == iter->key());
- bp = val.second.begin();
- return ObjectContents(bp);
- }
- return ObjectContents();
-}
-
-ObjectContents FileStoreTracker::get_content(
- const pair<string, string> &obj, uint64_t version)
-{
- set<string> to_get;
- map<string, bufferlist> got;
- to_get.insert(seq_to_key(version));
- db->get(obj_to_prefix(obj), to_get, &got);
- if (got.empty())
- return ObjectContents();
- pair<uint64_t, bufferlist> val;
- bufferlist::iterator bp = got.begin()->second.begin();
- ::decode(val, bp);
- bp = val.second.begin();
- assert(val.first == version);
- return ObjectContents(bp);
-}
-
-pair<uint64_t, uint64_t> FileStoreTracker::get_valid_reads(
- const pair<string, string> &obj)
-{
- pair<uint64_t, uint64_t> bounds = make_pair(0,1);
- KeyValueDB::Iterator iter = db->get_iterator(
- obj_to_prefix(obj));
- iter->seek_to_last();
- if (iter->valid()) {
- pair<uint64_t, bufferlist> val;
- bufferlist bl = iter->value();
- bufferlist::iterator bp = bl.begin();
- ::decode(val, bp);
- bounds.second = val.first + 1;
- }
-
- ObjStatus obj_status = get_obj_status(obj, db);
- bounds.first = obj_status.get_last_applied(restart_seq);
- return bounds;
-}
-
-void clear_obsolete(const pair<string, string> &obj,
- const ObjStatus &status,
- KeyValueDB *db,
- KeyValueDB::Transaction t)
-{
- KeyValueDB::Iterator iter = db->get_iterator(obj_to_prefix(obj));
- set<string> to_remove;
- iter->seek_to_first();
- for (; iter->valid() && iter->key() < seq_to_key(status.trim_to());
- iter->next())
- to_remove.insert(iter->key());
- t->rmkeys(obj_to_prefix(obj), to_remove);
-}
-
-void FileStoreTracker::committed(const pair<string, string> &obj,
- uint64_t seq) {
- Mutex::Locker l(lock);
- ObjStatus status = get_obj_status(obj, db);
- assert(status.last_committed < seq);
- status.last_committed = seq;
- KeyValueDB::Transaction t = db->get_transaction();
- clear_obsolete(obj, status, db, t);
- set_obj_status(obj, status, t);
- db->submit_transaction(t);
-}
-
-void FileStoreTracker::applied(const pair<string, string> &obj,
- uint64_t seq) {
- Mutex::Locker l(lock);
- std::cerr << "Applied " << obj << " version " << seq << std::endl;
- ObjStatus status = get_obj_status(obj, db);
- assert(status.last_applied < seq);
- status.set_last_applied(seq, restart_seq);
- KeyValueDB::Transaction t = db->get_transaction();
- clear_obsolete(obj, status, db, t);
- set_obj_status(obj, status, t);
- db->submit_transaction(t);
-}
-
-
-uint64_t FileStoreTracker::set_content(const pair<string, string> &obj,
- ObjectContents &content) {
- KeyValueDB::Transaction t = db->get_transaction();
- KeyValueDB::Iterator iter = db->get_iterator(
- obj_to_prefix(obj));
- iter->seek_to_last();
- uint64_t most_recent = 0;
- if (iter->valid()) {
- pair<uint64_t, bufferlist> val;
- bufferlist bl = iter->value();
- bufferlist::iterator bp = bl.begin();
- ::decode(val, bp);
- most_recent = val.first;
- }
- bufferlist buf_content;
- content.encode(buf_content);
- map<string, bufferlist> to_set;
- ::encode(make_pair(most_recent + 1, buf_content),
- to_set[seq_to_key(most_recent + 1)]);
- t->set(obj_to_prefix(obj), to_set);
- db->submit_transaction(t);
- return most_recent + 1;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-
-#ifndef FILESTORE_TRACKER_H
-#define FILESTORE_TRACKER_H
-#include "test/common/ObjectContents.h"
-#include "os/FileStore.h"
-#include "os/KeyValueDB.h"
-#include <boost/scoped_ptr.hpp>
-#include <list>
-#include <map>
-#include "common/Mutex.h"
-
-class FileStoreTracker {
- const static uint64_t SIZE = 4 * 1024;
- ObjectStore *store;
- KeyValueDB *db;
- Mutex lock;
- uint64_t restart_seq;
-
- struct OutTransaction {
- list<pair<pair<string, string>, uint64_t> > *in_flight;
- ObjectStore::Transaction *t;
- };
-public:
- FileStoreTracker(ObjectStore *store, KeyValueDB *db)
- : store(store), db(db),
- lock("Tracker Lock"), restart_seq(0) {}
-
- class Transaction {
- class Op {
- public:
- virtual void operator()(FileStoreTracker *harness,
- OutTransaction *out) = 0;
- virtual ~Op() {};
- };
- list<Op*> ops;
- class Write : public Op {
- public:
- string coll;
- string oid;
- Write(const string &coll,
- const string &oid)
- : coll(coll), oid(oid) {}
- void operator()(FileStoreTracker *harness,
- OutTransaction *out) {
- harness->write(make_pair(coll, oid), out);
- }
- };
- class CloneRange : public Op {
- public:
- string coll;
- string from;
- string to;
- CloneRange(const string &coll,
- const string &from,
- const string &to)
- : coll(coll), from(from), to(to) {}
- void operator()(FileStoreTracker *harness,
- OutTransaction *out) {
- harness->clone_range(make_pair(coll, from), make_pair(coll, to),
- out);
- }
- };
- class Clone : public Op {
- public:
- string coll;
- string from;
- string to;
- Clone(const string &coll,
- const string &from,
- const string &to)
- : coll(coll), from(from), to(to) {}
- void operator()(FileStoreTracker *harness,
- OutTransaction *out) {
- harness->clone(make_pair(coll, from), make_pair(coll, to),
- out);
- }
- };
- class Remove: public Op {
- public:
- string coll;
- string obj;
- Remove(const string &coll,
- const string &obj)
- : coll(coll), obj(obj) {}
- void operator()(FileStoreTracker *harness,
- OutTransaction *out) {
- harness->remove(make_pair(coll, obj),
- out);
- }
- };
- public:
- void write(const string &coll, const string &oid) {
- ops.push_back(new Write(coll, oid));
- }
- void clone_range(const string &coll, const string &from,
- const string &to) {
- ops.push_back(new CloneRange(coll, from, to));
- }
- void clone(const string &coll, const string &from,
- const string &to) {
- ops.push_back(new Clone(coll, from, to));
- }
- void remove(const string &coll, const string &oid) {
- ops.push_back(new Remove(coll, oid));
- }
- friend class FileStoreTracker;
- };
-
- int init();
- void submit_transaction(Transaction &t);
- void verify(const string &coll,
- const string &from,
- bool on_start = false);
-
-private:
- ObjectContents get_current_content(const pair<string, string> &obj);
- pair<uint64_t, uint64_t> get_valid_reads(const pair<string, string> &obj);
- ObjectContents get_content(const pair<string, string> &obj, uint64_t version);
-
- void committed(const pair<string, string> &obj, uint64_t seq);
- void applied(const pair<string, string> &obj, uint64_t seq);
- uint64_t set_content(const pair<string, string> &obj, ObjectContents &content);
-
- // ObjectContents Operations
- void write(const pair<string, string> &obj, OutTransaction *out);
- void remove(const pair<string, string> &obj, OutTransaction *out);
- void clone_range(const pair<string, string> &from,
- const pair<string, string> &to,
- OutTransaction *out);
- void clone(const pair<string, string> &from,
- const pair<string, string> &to,
- OutTransaction *out);
- friend class OnApplied;
- friend class OnCommitted;
-};
-
-#endif
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include <time.h>
-#include <stdlib.h>
-#include <signal.h>
-#include "os/FileStore.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "common/debug.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/lexical_cast.hpp>
-#include "TestFileStoreState.h"
-#include "include/assert.h"
-
-#define dout_subsys ceph_subsys_filestore
-#undef dout_prefix
-#define dout_prefix *_dout << "ceph_test_filestore_state "
-
-const coll_t TestFileStoreState::META_COLL("meta");
-const coll_t TestFileStoreState::TEMP_COLL("temp");
-
-void TestFileStoreState::init(int colls, int objs)
-{
- dout(5) << "init " << colls << " colls " << objs << " objs" << dendl;
-
- ObjectStore::Transaction *t;
- t = new ObjectStore::Transaction;
-
- t->create_collection(META_COLL);
- t->create_collection(TEMP_COLL);
- m_store->apply_transaction(*t);
-
- wait_for_ready();
-
- int baseid = 0;
- for (int i = 0; i < colls; i++) {
- int coll_id = i;
- coll_entry_t *entry = coll_create(coll_id);
- dout(5) << "init create collection " << entry->m_coll.to_str()
- << " meta " << entry->m_meta_obj.oid.name << dendl;
-
- t = new ObjectStore::Transaction;
- t->create_collection(entry->m_coll);
- t->touch(META_COLL, entry->m_meta_obj);
-
- for (int i = 0; i < objs; i++) {
- hobject_t *obj = entry->touch_obj(i + baseid);
- t->touch(entry->m_coll, *obj);
- ceph_assert(i + baseid == m_num_objects);
- m_num_objects++;
- }
- baseid += objs;
-
- m_store->queue_transaction(&(entry->m_osr), t,
- new C_OnFinished(this, t));
- inc_in_flight();
-
- m_collections.insert(make_pair(coll_id, entry));
- m_collections_ids.push_back(coll_id);
- m_next_coll_nr++;
- }
- dout(5) << "init has " << m_in_flight.read() << "in-flight transactions" << dendl;
- wait_for_done();
- dout(5) << "init finished" << dendl;
-}
-
-TestFileStoreState::coll_entry_t *TestFileStoreState::coll_create(int id)
-{
- char buf[100];
- char meta_buf[100];
- memset(buf, 0, 100);
- memset(meta_buf, 0, 100);
- snprintf(buf, 100, "0.%d_head", id);
- snprintf(meta_buf, 100, "pglog_0.%d_head", id);
- return (new coll_entry_t(id, buf, meta_buf));
-}
-
-TestFileStoreState::coll_entry_t*
-TestFileStoreState::get_coll(int key, bool erase)
-{
- dout(5) << "get_coll id " << key << dendl;
-
- coll_entry_t *entry = NULL;
- map<int, coll_entry_t*>::iterator it = m_collections.find(key);
- if (it != m_collections.end()) {
- entry = it->second;
- if (erase) {
- m_collections.erase(it);
- vector<int>::iterator cid_it = m_collections_ids.begin()+(entry->m_id);
- dout(20) << __func__ << " removing key " << key << " coll_id " << entry->m_id
- << " iterator's entry id " << (*cid_it) << dendl;
- m_collections_ids.erase(cid_it);
- }
- }
-
- dout(5) << "get_coll id " << key;
- if (!entry)
- *_dout << " non-existent";
- else
- *_dout << " name " << entry->m_coll.to_str();
- *_dout << dendl;
- return entry;
-}
-
-TestFileStoreState::coll_entry_t*
-TestFileStoreState::get_coll_at(int pos, bool erase)
-{
- dout(5) << "get_coll_at pos " << pos << dendl;
-
- if (m_collections.empty())
- return NULL;
-
- assert((size_t) pos < m_collections_ids.size());
-
- int coll_id = m_collections_ids[pos];
- coll_entry_t *entry = m_collections[coll_id];
-
- if (entry == NULL) {
- dout(5) << "get_coll_at pos " << pos << " non-existent" << dendl;
- return NULL;
- }
-
- if (erase) {
- m_collections.erase(coll_id);
- vector<int>::iterator it = m_collections_ids.begin()+(pos);
- dout(20) << __func__ << " removing pos " << pos << " coll_id " << coll_id
- << " iterator's entry id " << (*it) << dendl;
- m_collections_ids.erase(it);
- }
-
- dout(5) << "get_coll_at pos " << pos << ": "
- << entry->m_coll << "(removed: " << erase << ")" << dendl;
-
- return entry;
-}
-
-TestFileStoreState::coll_entry_t::~coll_entry_t()
-{
- if (m_objects.size() > 0) {
- map<int, hobject_t*>::iterator it = m_objects.begin();
- for (; it != m_objects.end(); ++it) {
- hobject_t *obj = it->second;
- if (obj) {
- delete obj;
- }
- }
- m_objects.clear();
- }
-}
-
-bool TestFileStoreState::coll_entry_t::check_for_obj(int id)
-{
- if (m_objects.count(id))
- return true;
- return false;
-}
-
-hobject_t *TestFileStoreState::coll_entry_t::touch_obj(int id)
-{
- map<int, hobject_t*>::iterator it = m_objects.find(id);
- if (it != m_objects.end()) {
- dout(5) << "touch_obj coll id " << m_id
- << " name " << it->second->oid.name << dendl;
- return it->second;
- }
-
- char buf[100];
- memset(buf, 0, 100);
- snprintf(buf, 100, "obj%d", id);
-
- hobject_t *obj = new hobject_t(sobject_t(object_t(buf), CEPH_NOSNAP));
- m_objects.insert(make_pair(id, obj));
-
- dout(5) << "touch_obj coll id " << m_id << " name " << buf << dendl;
- return obj;
-}
-
-hobject_t *TestFileStoreState::coll_entry_t::get_obj(int id)
-{
- return get_obj(id, false);
-}
-
-/**
- * remove_obj - Removes object without freeing it.
- * @param id Object's id in the map.
- * @return The object or NULL in case of error.
- */
-hobject_t *TestFileStoreState::coll_entry_t::remove_obj(int id)
-{
- return get_obj(id, true);
-}
-
-hobject_t *TestFileStoreState::coll_entry_t::get_obj(int id, bool remove)
-{
- map<int, hobject_t*>::iterator it = m_objects.find(id);
- if (it == m_objects.end()) {
- dout(5) << "get_obj coll " << m_coll.to_str()
- << " obj #" << id << " non-existent" << dendl;
- return NULL;
- }
-
- hobject_t *obj = it->second;
- if (remove)
- m_objects.erase(it);
-
- dout(5) << "get_obj coll " << m_coll.to_str() << " id " << id
- << ": " << obj->oid.name << "(removed: " << remove << ")" << dendl;
-
- return obj;
-}
-
-hobject_t *TestFileStoreState::coll_entry_t::get_obj_at(int pos, int *key)
-{
- return get_obj_at(pos, false, key);
-}
-
-/**
- * remove_obj_at - Removes object without freeing it.
- * @param pos The map's position in which the object lies.
- * @return The object or NULL in case of error.
- */
-hobject_t *TestFileStoreState::coll_entry_t::remove_obj_at(int pos, int *key)
-{
- return get_obj_at(pos, true, key);
-}
-
-hobject_t *TestFileStoreState::coll_entry_t::get_obj_at(int pos,
- bool remove, int *key)
-{
- if (m_objects.empty()) {
- dout(5) << "get_obj_at coll " << m_coll.to_str() << " pos " << pos
- << " in an empty collection" << dendl;
- return NULL;
- }
-
- hobject_t *ret = NULL;
- map<int, hobject_t*>::iterator it = m_objects.begin();
- for (int i = 0; it != m_objects.end(); ++it, i++) {
- if (i == pos) {
- ret = it->second;
- break;
- }
- }
-
- if (ret == NULL) {
- dout(5) << "get_obj_at coll " << m_coll.to_str() << " pos " << pos
- << " non-existent" << dendl;
- return NULL;
- }
-
- if (key != NULL)
- *key = it->first;
-
- if (remove)
- m_objects.erase(it);
-
- dout(5) << "get_obj_at coll id " << m_id << " pos " << pos
- << ": " << ret->oid.name << "(removed: " << remove << ")" << dendl;
-
- return ret;
-}
-
-hobject_t*
-TestFileStoreState::coll_entry_t::replace_obj(int id, hobject_t *obj) {
- hobject_t *old_obj = remove_obj(id);
- m_objects.insert(make_pair(id, obj));
- return old_obj;
-}
-
-int TestFileStoreState::coll_entry_t::get_random_obj_id(rngen_t& gen)
-{
- ceph_assert(!m_objects.empty());
-
- boost::uniform_int<> orig_obj_rng(0, m_objects.size()-1);
- int pos = orig_obj_rng(gen);
- map<int, hobject_t*>::iterator it = m_objects.begin();
- for (int i = 0; it != m_objects.end(); ++it, i++) {
- if (i == pos) {
- return it->first;
- }
- }
- ceph_assert(0 == "INTERNAL ERROR");
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#ifndef TEST_FILESTORE_STATE_H_
-#define TEST_FILESTORE_STATE_H_
-
-#include "os/FileStore.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/random/mersenne_twister.hpp>
-#include <boost/random/uniform_int.hpp>
-#include <map>
-#include <vector>
-
-typedef boost::mt11213b rngen_t;
-
-class TestFileStoreState {
-public:
- struct coll_entry_t {
- int m_id;
- coll_t m_coll;
- hobject_t m_meta_obj;
- ObjectStore::Sequencer m_osr;
- map<int, hobject_t*> m_objects;
- int m_next_object_id;
-
- coll_entry_t(int i, char *coll_buf, char *meta_obj_buf)
- : m_id(i), m_coll(coll_buf),
- m_meta_obj(sobject_t(object_t(meta_obj_buf), CEPH_NOSNAP)),
- m_osr(coll_buf), m_next_object_id(0) {
- }
- ~coll_entry_t();
-
- hobject_t *touch_obj(int id);
- bool check_for_obj(int id);
- hobject_t *get_obj(int id);
- hobject_t *remove_obj(int id);
- hobject_t *get_obj_at(int pos, int *key = NULL);
- hobject_t *remove_obj_at(int pos, int *key = NULL);
- hobject_t *replace_obj(int id, hobject_t *obj);
- int get_random_obj_id(rngen_t& gen);
-
- private:
- hobject_t *get_obj(int id, bool remove);
- hobject_t *get_obj_at(int pos, bool remove, int *key = NULL);
- };
-
- /* kept in upper case for consistency with coll_t's */
- static const coll_t META_COLL;
- static const coll_t TEMP_COLL;
-
- protected:
- boost::shared_ptr<ObjectStore> m_store;
- map<int, coll_entry_t*> m_collections;
- vector<int> m_collections_ids;
- int m_next_coll_nr;
- int m_num_objs_per_coll;
- int m_num_objects;
-
- int m_max_in_flight;
- atomic_t m_in_flight;
- Mutex m_finished_lock;
- Cond m_finished_cond;
-
- void wait_for_ready() {
- Mutex::Locker locker(m_finished_lock);
- while ((m_max_in_flight > 0) && ((int)m_in_flight.read() >= m_max_in_flight))
- m_finished_cond.Wait(m_finished_lock);
- }
-
- void wait_for_done() {
- Mutex::Locker locker(m_finished_lock);
- while (m_in_flight.read())
- m_finished_cond.Wait(m_finished_lock);
- }
-
- void set_max_in_flight(int max) {
- m_max_in_flight = max;
- }
- void set_num_objs_per_coll(int val) {
- m_num_objs_per_coll = val;
- }
-
- coll_entry_t *get_coll(int key, bool erase = false);
- coll_entry_t *get_coll_at(int pos, bool erase = false);
-
- private:
- static const int m_default_num_colls = 30;
-
- public:
- TestFileStoreState(FileStore *store) :
- m_next_coll_nr(0), m_num_objs_per_coll(10), m_num_objects(0),
- m_max_in_flight(0), m_finished_lock("Finished Lock") {
- m_in_flight.set(0);
- m_store.reset(store);
- }
- ~TestFileStoreState() {
- map<int, coll_entry_t*>::iterator it = m_collections.begin();
- while (it != m_collections.end()) {
- if (it->second)
- delete it->second;
- m_collections.erase(it++);
- }
- }
-
- void init(int colls, int objs);
- void init() {
- init(m_default_num_colls, 0);
- }
-
- int inc_in_flight() {
- return ((int) m_in_flight.inc());
- }
-
- int dec_in_flight() {
- return ((int) m_in_flight.dec());
- }
-
- coll_entry_t *coll_create(int id);
-
- class C_OnFinished: public Context {
- protected:
- TestFileStoreState *m_state;
- ObjectStore::Transaction *m_tx;
-
- public:
- C_OnFinished(TestFileStoreState *state,
- ObjectStore::Transaction *t) : m_state(state), m_tx(t) { }
-
- void finish(int r) {
- Mutex::Locker locker(m_state->m_finished_lock);
- m_state->dec_in_flight();
- m_state->m_finished_cond.Signal();
-
- delete m_tx;
- }
- };
-};
-
-#endif /* TEST_FILESTORE_STATE_H_ */
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Library Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Library Public License for more details.
- *
- */
-
-#include <stdio.h>
-#include <signal.h>
-#include "os/chain_xattr.h"
-#include "include/Context.h"
-#include "common/errno.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include <gtest/gtest.h>
-
-#define LARGE_BLOCK_LEN CHAIN_XATTR_MAX_BLOCK_LEN + 1024
-
-TEST(chain_xattr, get_and_set) {
- const char* file = "testfile";
- ::unlink(file);
- int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
- const string user("user.");
-
- {
- const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
- const string x(LARGE_BLOCK_LEN, 'X');
-
- {
- char y[LARGE_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0));
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN));
- ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
- ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN));
- }
-
- {
- char y[LARGE_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0));
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN));
- ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
- ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN));
- }
- }
-
- //
- // when chain_setxattr is used to store value that is
- // CHAIN_XATTR_MAX_BLOCK_LEN * 2 + 10 bytes long it
- //
- // add user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes
- // add user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes
- // add user.foo@2 => 10 bytes
- //
- // then ( no chain_removexattr in between ) when it is used to
- // override with a value that is exactly CHAIN_XATTR_MAX_BLOCK_LEN
- // bytes long it will
- //
- // replace user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes
- // remove user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes
- // leak user.foo@2 => 10 bytes
- //
- // see http://marc.info/?l=ceph-devel&m=136027076615853&w=4 for the
- // discussion
- //
- {
- const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
- const string x(LARGE_BLOCK_LEN, 'X');
-
- {
- char y[CHAIN_XATTR_MAX_NAME_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
- ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- }
-
- {
- char y[CHAIN_XATTR_MAX_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
- ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- }
- }
-
- {
- int x = 0;
- ASSERT_EQ(-ENOENT, chain_setxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x)));
- ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", 0, 0));
- ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x)));
- ASSERT_EQ(-ENOENT, chain_removexattr("UNLIKELY_TO_EXIST", "NAME"));
- int unlikely_to_be_a_valid_fd = 400;
- ASSERT_EQ(-EBADF, chain_fsetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x)));
- ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", 0, 0));
- ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x)));
- ASSERT_EQ(-EBADF, chain_fremovexattr(unlikely_to_be_a_valid_fd, "NAME"));
- }
-
- {
- int x;
- const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN * 2, '@');
- ASSERT_THROW(chain_setxattr(file, name.c_str(), &x, sizeof(x)), FailedAssertion);
- ASSERT_THROW(chain_fsetxattr(fd, name.c_str(), &x, sizeof(x)), FailedAssertion);
- }
-
- {
- const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
- const string x(LARGE_BLOCK_LEN, 'X');
- {
- char y[LARGE_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN - 1));
- ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
- }
-
- {
- char y[LARGE_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN - 1));
- ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
- }
- }
-
- ::close(fd);
- ::unlink(file);
-}
-
-TEST(chain_xattr, listxattr) {
- const char* file = "testfile";
- ::unlink(file);
- int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
- const string user("user.");
- const string name1 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '1');
- const string name2 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
- const string x(LARGE_BLOCK_LEN, 'X');
- const int y = 1234;
-
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name1.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ((int)sizeof(y), chain_setxattr(file, name2.c_str(), &y, sizeof(y)));
-
- int buffer_size = name1.size() + sizeof('\0') + name2.size() + sizeof('\0');
- char* expected = (char*)malloc(buffer_size);
- ::strcpy(expected, name1.c_str());
- ::strcpy(expected + name1.size() + 1, name2.c_str());
- char* actual = (char*)calloc(1, buffer_size);
- ASSERT_LT(buffer_size, chain_listxattr(file, NULL, 0)); // size evaluation is conservative
- chain_listxattr(file, actual, buffer_size);
- ::memset(actual, '\0', buffer_size);
- chain_flistxattr(fd, actual, buffer_size);
- ASSERT_EQ(0, ::memcmp(expected, actual, buffer_size));
-
- int unlikely_to_be_a_valid_fd = 400;
- ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, 0));
- ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, buffer_size));
- ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, 0));
- ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, buffer_size));
- ASSERT_EQ(-ERANGE, chain_listxattr(file, actual, 1));
- ASSERT_EQ(-ERANGE, chain_flistxattr(fd, actual, 1));
-
- ASSERT_EQ(0, chain_removexattr(file, name1.c_str()));
- ASSERT_EQ(0, chain_removexattr(file, name2.c_str()));
-
- ::unlink(file);
-}
-
-int main(int argc, char **argv) {
- vector<const char*> args;
- argv_to_vec(argc, (const char **)argv, args);
-
- global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->set_val("err_to_stderr", "false");
- g_ceph_context->_conf->set_val("log_to_stderr", "false");
- g_ceph_context->_conf->apply_changes(NULL);
-
- const char* file = "testfile";
- int x = 1234;
- int y = 0;
- int tmpfd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
- int ret = ::ceph_os_fsetxattr(tmpfd, "user.test", &x, sizeof(x));
- if (ret >= 0)
- ret = ::ceph_os_fgetxattr(tmpfd, "user.test", &y, sizeof(y));
- ::close(tmpfd);
- ::unlink(file);
- if ((ret < 0) || (x != y)) {
- cerr << "SKIP all tests because extended attributes don't appear to work in the file system in which the tests are run: " << cpp_strerror(ret) << std::endl;
- } else {
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
- }
-}
-
-// Local Variables:
-// compile-command: "cd ../.. ; make unittest_chain_xattr ; valgrind --tool=memcheck ./unittest_chain_xattr # --gtest_filter=chain_xattr.get_and_set"
-// End:
+++ /dev/null
-#!/bin/bash
-# vim: ts=8 sw=2 smarttab
-#
-# run_seed_to.sh - Run ceph_test_filestore_idempotent_sequence up until an
-# injection point, generating a sequence of operations based on a
-# provided seed.
-#
-# We also perform three additional tests, focused on assessing if
-# replaying a larger chunck of the journal affects the expected store
-# behavior. These tests will be performed by increasing the store's
-# journal sync interval to a very large value, allowing the store to
-# finish execution before the first sync (unless the store runs for
-# over 10 hours, case on which the interval variables must be changed
-# to an appropriate value). Unless the '--no-journal-test' option is
-# specified, we will run the 3 following scenarios:
-#
-# 1) journal sync'ing for both stores is good as disabled
-# (we call it '00', for store naming purposes)
-# 2) journal sync'ing for store A is as good as disabled
-# (we call it '01', for store naming purposes)
-# 3) journal sync'ing for store B is as good as disabled
-# (we call it '10', for store naming purposes)
-#
-# All log files are also appropriately named accordingly (i.e., a.00.fail,
-# a.10.recover, or b.01.clean).
-#
-# By default, the test will not exit on error, although it will show the
-# fail message. This behavior is so defined so we run the whole battery of
-# tests, and obtain as many mismatches as possible in one go. We may force
-# the test to exit on error by specifying the '--exit-on-error' option.
-#
-#
-set -e
-
-test_opts=""
-
-usage() {
- echo "usage: $1 [options..] <seed> <kill-at>"
- echo
- echo "options:"
- echo " -c, --colls <VAL> # of collections"
- echo " -o, --objs <VAL> # of objects"
- echo " -b, --btrfs <VAL> seq number for btrfs stores"
- echo " --no-journal-test don't perform journal replay tests"
- echo " -e, --exit-on-error exit with 1 on error"
- echo " -v, --valgrind run commands through valgrind"
- echo
- echo "env vars:"
- echo " OPTS_STORE additional opts for both stores"
- echo " OPTS_STORE_A additional opts for store A"
- echo " OPTS_STORE_B additional opts for store B"
- echo
-}
-
-die_on_missing_arg() {
- if [[ "$2" == "" ]]; then
- echo "$1: missing required parameter"
- exit 1
- fi
-}
-
-
-required_args=2
-obtained_args=0
-
-seed=""
-killat=""
-on_btrfs=0
-on_btrfs_seq=0
-journal_test=1
-min_sync_interval="36000" # ten hours, yes.
-max_sync_interval="36001"
-exit_on_error=0
-v=""
-
-do_rm() {
- if [[ $on_btrfs -eq 0 ]]; then
- rm -fr $*
- fi
-}
-
-set_arg() {
- if [[ $1 -eq 1 ]]; then
- seed=$2
- elif [[ $1 -eq 2 ]]; then
- killat=$2
- else
- echo "error: unknown purpose for '$2'"
- usage $0
- exit 1
- fi
-}
-
-while [[ $# -gt 0 ]];
-do
- case "$1" in
- -c | --colls)
- die_on_missing_arg "$1" "$2"
- test_opts="$test_opts --test-num-colls $2"
- shift 2
- ;;
- -o | --objs)
- die_on_missing_arg "$1" "$2"
- test_opts="$test_opts --test-num-objs $2"
- shift 2
- ;;
- -h | --help)
- usage $0 ;
- exit 0
- ;;
- -b | --btrfs)
- die_on_missing_arg "$1" "$2"
- on_btrfs=1
- on_btrfs_seq=$2
- shift 2
- ;;
- --no-journal-test)
- journal_test=0
- shift
- ;;
- -e | --exit-on-error)
- exit_on_error=1
- shift
- ;;
- -v | --valgrind)
- v="valgrind --leak-check=full"
- shift
- ;;
- --)
- shift
- break
- ;;
- -*)
- echo "$1: unknown option" >&2
- usage $0
- exit 1
- ;;
- *)
- obtained_args=$(($obtained_args+1))
- set_arg $obtained_args $1
- shift
- ;;
- esac
-done
-
-if [[ $obtained_args -ne $required_args ]]; then
- echo "error: missing argument"
- usage $0 ;
- exit 1
-fi
-
-if [[ "$OPTS_STORE" != "" ]]; then
- test_opts="$test_opts $OPTS_STORE"
-fi
-
-test_opts_a="$test_opts"
-test_opts_b="$test_opts"
-
-if [[ "$OPTS_STORE_A" != "" ]]; then
- test_opts_a="$test_opts_a $OPTS_STORE_A"
-fi
-if [[ "$OPTS_STORE_B" != "" ]]; then
- test_opts_b="$test_opts_b $OPTS_STORE_B"
-fi
-
-echo seed $seed
-echo kill at $killat
-
-# run forever, until $killat...
-to=1000000000
-
-#
-# store names
-#
-# We need these for two reasons:
-# 1) if we are running the tests on a btrfs volume, then we need to use
-# a seq number for each run. Being on btrfs means we will fail when
-# removing the store's directories and it's far more simple to just
-# specify differente store names such as 'a.$seq' or 'b.$seq'.
-#
-# 2) unless the '--no-journal-test' option is specified, we will run
-# three additional tests for each store, and we will reuse the same
-# command for each one of the runs, but varying the store's name and
-# arguments.
-#
-store_a="a"
-store_b="b"
-
-if [[ $on_btrfs -eq 1 ]]; then
- store_a="$store_a.$on_btrfs_seq"
- store_b="$store_b.$on_btrfs_seq"
-fi
-
-total_runs=1
-
-if [[ $journal_test -eq 1 ]]; then
- total_runs=$(($total_runs + 3))
-fi
-
-num_runs=0
-
-opt_min_sync="--filestore-min-sync-interval $min_sync_interval"
-opt_max_sync="--filestore-max-sync-interval $max_sync_interval"
-
-ret=0
-
-while [[ $num_runs -lt $total_runs ]];
-do
- tmp_name_a=$store_a
- tmp_name_b=$store_b
- tmp_opts_a=$test_opts_a
- tmp_opts_b=$test_opts_b
-
- #
- # We have already tested whether there are diffs when both journals
- # are properly working. Now let's try on three other scenarios:
- # 1) journal sync'ing for both stores is good as disabled
- # (we call it '00')
- # 2) journal sync'ing for store A is as good as disabled
- # (we call it '01')
- # 3) journal sync'ing for store B is as good as disabled
- # (we call it '10')
- #
- if [[ $num_runs -gt 0 && $journal_test -eq 1 ]]; then
- echo "run #$num_runs"
- case $num_runs in
- 1)
- tmp_name_a="$tmp_name_a.00"
- tmp_name_b="$tmp_name_b.00"
- tmp_opts_a="$tmp_opts_a $opt_min_sync $opt_max_sync"
- tmp_opts_b="$tmp_opts_b $opt_min_sync $opt_max_sync"
- ;;
- 2)
- tmp_name_a="$tmp_name_a.01"
- tmp_name_b="$tmp_name_b.01"
- tmp_opts_a="$tmp_opts_a $opt_min_sync $opt_max_sync"
- ;;
- 3)
- tmp_name_a="$tmp_name_a.10"
- tmp_name_b="$tmp_name_b.10"
- tmp_opts_b="$tmp_opts_b $opt_min_sync $opt_max_sync"
- ;;
- esac
- fi
-
- do_rm $tmp_name_a $tmp_name_a.fail $tmp_name_a.recover
- $v ceph_test_filestore_idempotent_sequence run-sequence-to $to \
- $tmp_name_a $tmp_name_a/journal \
- --test-seed $seed --osd-journal-size 100 \
- --filestore-kill-at $killat $tmp_opts_a \
- --log-file $tmp_name_a.fail --debug-filestore 20 || true
-
- stop_at=`ceph_test_filestore_idempotent_sequence get-last-op \
- $tmp_name_a $tmp_name_a/journal \
- --log-file $tmp_name_a.recover \
- --debug-filestore 20 --debug-journal 20`
-
- if [[ "`expr $stop_at - $stop_at 2>/dev/null`" != "0" ]]; then
- echo "error: get-last-op returned '$stop_at'"
- exit 1
- fi
-
- echo stopped at $stop_at
-
- do_rm $tmp_name_b $tmp_name_b.clean
- $v ceph_test_filestore_idempotent_sequence run-sequence-to \
- $stop_at $tmp_name_b $tmp_name_b/journal \
- --test-seed $seed --osd-journal-size 100 \
- --log-file $tmp_name_b.clean --debug-filestore 20 $tmp_opts_b
-
- if $v ceph_test_filestore_idempotent_sequence diff \
- $tmp_name_a $tmp_name_a/journal $tmp_name_b $tmp_name_b/journal ; then
- echo OK
- else
- echo "FAIL"
- echo " see:"
- echo " $tmp_name_a.fail -- leading up to failure"
- echo " $tmp_name_a.recover -- journal replay"
- echo " $tmp_name_b.clean -- the clean reference"
-
- ret=1
- if [[ $exit_on_error -eq 1 ]]; then
- exit 1
- fi
- fi
-
- num_runs=$(($num_runs+1))
-done
-
-exit $ret
+++ /dev/null
-#!/bin/sh
-
-set -e
-
-seed=$1
-from=$2
-to=$3
-dir=$4
-
-mydir=`dirname $0`
-
-for f in `seq $from $to`
-do
- if ! $mydir/run_seed_to.sh $seed $f; then
- if [ -d $dir ]; then
- echo copying evidence to $dir
- cp -a . $dir
- else
- echo no dir provided for evidence disposal
- fi
- exit 1
- fi
-done
\ No newline at end of file
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include <time.h>
-#include <sys/mount.h>
-#include "os/FileStore.h"
-#include "os/KeyValueStore.h"
-#include "include/Context.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "common/Mutex.h"
-#include "common/Cond.h"
-#include "common/errno.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/random/mersenne_twister.hpp>
-#include <boost/random/uniform_int.hpp>
-#include <boost/random/binomial_distribution.hpp>
-#include <gtest/gtest.h>
-
-#include "include/unordered_map.h"
-typedef boost::mt11213b gen_type;
-
-#if GTEST_HAS_PARAM_TEST
-
-class StoreTest : public ::testing::TestWithParam<const char*> {
-public:
- boost::scoped_ptr<ObjectStore> store;
-
- StoreTest() : store(0) {}
- virtual void SetUp() {
- int r = ::mkdir("store_test_temp_dir", 0777);
- if (r < 0 && errno != EEXIST) {
- r = -errno;
- cerr << __func__ << ": unable to create store_test_temp_dir" << ": " << cpp_strerror(r) << std::endl;
- return;
- }
-
- ObjectStore *store_ = ObjectStore::create(g_ceph_context,
- string(GetParam()),
- string("store_test_temp_dir"),
- string("store_test_temp_journal"));
- store.reset(store_);
- EXPECT_EQ(store->mkfs(), 0);
- EXPECT_EQ(store->mount(), 0);
- }
-
- virtual void TearDown() {
- store->umount();
- }
-};
-
-bool sorted(const vector<ghobject_t> &in) {
- ghobject_t start;
- for (vector<ghobject_t>::const_iterator i = in.begin();
- i != in.end();
- ++i) {
- if (start > *i) return false;
- start = *i;
- }
- return true;
-}
-
-TEST_P(StoreTest, SimpleColTest) {
- coll_t cid = coll_t("initial");
- int r = 0;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- cerr << "create collection" << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- {
- ObjectStore::Transaction t;
- t.remove_collection(cid);
- cerr << "remove collection" << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- cerr << "add collection" << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- {
- ObjectStore::Transaction t;
- t.remove_collection(cid);
- cerr << "remove collection" << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-}
-
-TEST_P(StoreTest, SimpleObjectTest) {
- int r;
- coll_t cid = coll_t("coll");
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- cerr << "Creating collection " << cid << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
- {
- ObjectStore::Transaction t;
- t.touch(cid, hoid);
- cerr << "Creating object " << hoid << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- {
- ObjectStore::Transaction t;
- t.remove(cid, hoid);
- t.remove_collection(cid);
- cerr << "Cleaning" << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-}
-
-TEST_P(StoreTest, SimpleObjectLongnameTest) {
- int r;
- coll_t cid = coll_t("coll");
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- cerr << "Creating collection " << cid << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- ghobject_t hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP)));
- {
- ObjectStore::Transaction t;
- t.touch(cid, hoid);
- cerr << "Creating object " << hoid << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- {
- ObjectStore::Transaction t;
- t.remove(cid, hoid);
- t.remove_collection(cid);
- cerr << "Cleaning" << std::endl;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-}
-
-TEST_P(StoreTest, ManyObjectTest) {
- int NUM_OBJS = 2000;
- int r = 0;
- coll_t cid("blah");
- string base = "";
- for (int i = 0; i < 100; ++i) base.append("aaaaa");
- set<ghobject_t> created;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- for (int i = 0; i < NUM_OBJS; ++i) {
- if (!(i % 5)) {
- cerr << "Object " << i << std::endl;
- }
- ObjectStore::Transaction t;
- char buf[100];
- snprintf(buf, sizeof(buf), "%d", i);
- ghobject_t hoid(hobject_t(sobject_t(string(buf) + base, CEPH_NOSNAP)));
- t.touch(cid, hoid);
- created.insert(hoid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-
- for (set<ghobject_t>::iterator i = created.begin();
- i != created.end();
- ++i) {
- struct stat buf;
- ASSERT_TRUE(!store->stat(cid, *i, &buf));
- }
-
- set<ghobject_t> listed;
- vector<ghobject_t> objects;
- r = store->collection_list(cid, objects);
- ASSERT_EQ(r, 0);
-
- cerr << "objects.size() is " << objects.size() << std::endl;
- for (vector<ghobject_t> ::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- listed.insert(*i);
- ASSERT_TRUE(created.count(*i));
- }
- ASSERT_TRUE(listed.size() == created.size());
-
- ghobject_t start, next;
- objects.clear();
- r = store->collection_list_partial(
- cid,
- ghobject_t::get_max(),
- 50,
- 60,
- 0,
- &objects,
- &next
- );
- ASSERT_EQ(r, 0);
- ASSERT_TRUE(objects.empty());
-
- objects.clear();
- listed.clear();
- while (1) {
- r = store->collection_list_partial(cid, start,
- 50,
- 60,
- 0,
- &objects,
- &next);
- ASSERT_TRUE(sorted(objects));
- ASSERT_EQ(r, 0);
- listed.insert(objects.begin(), objects.end());
- if (objects.size() < 50) {
- ASSERT_TRUE(next.is_max());
- break;
- }
- objects.clear();
- start = next;
- }
- cerr << "listed.size() is " << listed.size() << std::endl;
- ASSERT_TRUE(listed.size() == created.size());
- for (set<ghobject_t>::iterator i = listed.begin();
- i != listed.end();
- ++i) {
- ASSERT_TRUE(created.count(*i));
- }
-
- for (set<ghobject_t>::iterator i = created.begin();
- i != created.end();
- ++i) {
- ObjectStore::Transaction t;
- t.remove(cid, *i);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- cerr << "cleaning up" << std::endl;
- {
- ObjectStore::Transaction t;
- t.remove_collection(cid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-}
-
-class ObjectGenerator {
-public:
- virtual ghobject_t create_object(gen_type *gen) = 0;
- virtual ~ObjectGenerator() {}
-};
-
-class MixedGenerator : public ObjectGenerator {
-public:
- unsigned seq;
- MixedGenerator() : seq(0) {}
- ghobject_t create_object(gen_type *gen) {
- char buf[100];
- snprintf(buf, sizeof(buf), "%u", seq);
-
- boost::uniform_int<> true_false(0, 1);
- string name(buf);
- if (true_false(*gen)) {
- // long
- for (int i = 0; i < 100; ++i) name.append("aaaaa");
- } else if (true_false(*gen)) {
- name = "DIR_" + name;
- }
-
- // hash
- //boost::binomial_distribution<uint32_t> bin(0xFFFFFF, 0.5);
- ++seq;
- return ghobject_t(hobject_t(name, string(), rand() & 2 ? CEPH_NOSNAP : rand(), rand() & 0xFF, 0, ""));
- }
-};
-
-class SyntheticWorkloadState {
-public:
- static const unsigned max_in_flight = 16;
- static const unsigned max_objects = 3000;
- coll_t cid;
- unsigned in_flight;
- set<ghobject_t> available_objects;
- set<ghobject_t> in_use_objects;
- ObjectGenerator *object_gen;
- gen_type *rng;
- ObjectStore *store;
- ObjectStore::Sequencer *osr;
-
- Mutex lock;
- Cond cond;
-
- class C_SyntheticOnReadable : public Context {
- public:
- SyntheticWorkloadState *state;
- ObjectStore::Transaction *t;
- ghobject_t hoid;
- C_SyntheticOnReadable(SyntheticWorkloadState *state,
- ObjectStore::Transaction *t, ghobject_t hoid)
- : state(state), t(t), hoid(hoid) {}
-
- void finish(int r) {
- ASSERT_TRUE(r >= 0);
- Mutex::Locker locker(state->lock);
- if (state->in_use_objects.count(hoid)) {
- state->available_objects.insert(hoid);
- state->in_use_objects.erase(hoid);
- }
- --(state->in_flight);
- state->cond.Signal();
- }
- };
-
-
- SyntheticWorkloadState(ObjectStore *store,
- ObjectGenerator *gen,
- gen_type *rng,
- ObjectStore::Sequencer *osr,
- coll_t cid)
- : cid(cid), in_flight(0), object_gen(gen), rng(rng), store(store), osr(osr),
- lock("State lock") {}
-
- int init() {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- return store->apply_transaction(t);
- }
-
- ghobject_t get_uniform_random_object() {
- while (in_flight >= max_in_flight || available_objects.empty())
- cond.Wait(lock);
- boost::uniform_int<> choose(0, available_objects.size() - 1);
- int index = choose(*rng);
- set<ghobject_t>::iterator i = available_objects.begin();
- for ( ; index > 0; --index, ++i) ;
- ghobject_t ret = *i;
- available_objects.erase(i);
- return ret;
- }
-
- void wait_for_ready() {
- while (in_flight >= max_in_flight)
- cond.Wait(lock);
- }
-
- void wait_for_done() {
- Mutex::Locker locker(lock);
- while (in_flight)
- cond.Wait(lock);
- }
-
- bool can_create() {
- return (available_objects.size() + in_use_objects.size()) < max_objects;
- }
-
- bool can_unlink() {
- return (available_objects.size() + in_use_objects.size()) > 0;
- }
-
- int touch() {
- Mutex::Locker locker(lock);
- if (!can_create())
- return -ENOSPC;
- wait_for_ready();
- ghobject_t new_obj = object_gen->create_object(rng);
- in_use_objects.insert(new_obj);
- available_objects.erase(new_obj);
- ObjectStore::Transaction *t = new ObjectStore::Transaction;
- t->touch(cid, new_obj);
- ++in_flight;
- return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, new_obj));
- }
-
- void scan() {
- Mutex::Locker locker(lock);
- while (in_flight)
- cond.Wait(lock);
- vector<ghobject_t> objects;
- set<ghobject_t> objects_set, objects_set2;
- ghobject_t next, current;
- while (1) {
- cerr << "scanning..." << std::endl;
- int r = store->collection_list_partial(cid, current, 50, 100,
- 0, &objects, &next);
- ASSERT_EQ(r, 0);
- ASSERT_TRUE(sorted(objects));
- objects_set.insert(objects.begin(), objects.end());
- objects.clear();
- if (next.is_max()) break;
- current = next;
- }
- ASSERT_EQ(objects_set.size(), available_objects.size());
- for (set<ghobject_t>::iterator i = objects_set.begin();
- i != objects_set.end();
- ++i) {
- ASSERT_GT(available_objects.count(*i), (unsigned)0);
- }
-
- int r = store->collection_list(cid, objects);
- ASSERT_EQ(r, 0);
- objects_set2.insert(objects.begin(), objects.end());
- ASSERT_EQ(objects_set2.size(), available_objects.size());
- for (set<ghobject_t>::iterator i = objects_set2.begin();
- i != objects_set2.end();
- ++i) {
- ASSERT_GT(available_objects.count(*i), (unsigned)0);
- }
- }
-
- int stat() {
- ghobject_t hoid;
- {
- Mutex::Locker locker(lock);
- if (!can_unlink())
- return -ENOENT;
- hoid = get_uniform_random_object();
- in_use_objects.insert(hoid);
- ++in_flight;
- }
- struct stat buf;
- int r = store->stat(cid, hoid, &buf);
- {
- Mutex::Locker locker(lock);
- --in_flight;
- cond.Signal();
- in_use_objects.erase(hoid);
- available_objects.insert(hoid);
- }
- return r;
- }
-
- int unlink() {
- Mutex::Locker locker(lock);
- if (!can_unlink())
- return -ENOENT;
- ghobject_t to_remove = get_uniform_random_object();
- ObjectStore::Transaction *t = new ObjectStore::Transaction;
- t->remove(cid, to_remove);
- ++in_flight;
- return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, to_remove));
- }
-
- void print_internal_state() {
- Mutex::Locker locker(lock);
- cerr << "available_objects: " << available_objects.size()
- << " in_use_objects: " << in_use_objects.size()
- << " total objects: " << in_use_objects.size() + available_objects.size()
- << " in_flight " << in_flight << std::endl;
- }
-};
-
-TEST_P(StoreTest, Synthetic) {
- ObjectStore::Sequencer osr("test");
- MixedGenerator gen;
- gen_type rng(time(NULL));
- coll_t cid("synthetic_1");
-
- SyntheticWorkloadState test_obj(store.get(), &gen, &rng, &osr, cid);
- test_obj.init();
- for (int i = 0; i < 1000; ++i) {
- if (!(i % 10)) cerr << "seeding object " << i << std::endl;
- test_obj.touch();
- }
- for (int i = 0; i < 10000; ++i) {
- if (!(i % 10)) {
- cerr << "Op " << i << std::endl;
- test_obj.print_internal_state();
- }
- boost::uniform_int<> true_false(0, 99);
- int val = true_false(rng);
- if (val > 97) {
- test_obj.scan();
- } else if (val > 50) {
- test_obj.stat();
- } else if (val > 30) {
- test_obj.unlink();
- } else {
- test_obj.touch();
- }
- }
- test_obj.wait_for_done();
-}
-
-TEST_P(StoreTest, HashCollisionTest) {
- coll_t cid("blah");
- int r;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- string base = "";
- for (int i = 0; i < 100; ++i) base.append("aaaaa");
- set<ghobject_t> created;
- for (int n = 0; n < 10; ++n) {
- char nbuf[100];
- sprintf(nbuf, "n%d", n);
- for (int i = 0; i < 1000; ++i) {
- char buf[100];
- sprintf(buf, "%d", i);
- if (!(i % 5)) {
- cerr << "Object n" << n << " "<< i << std::endl;
- }
- ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, 0, 0, string(nbuf)));
- {
- ObjectStore::Transaction t;
- t.touch(cid, hoid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- created.insert(hoid);
- }
- }
- vector<ghobject_t> objects;
- r = store->collection_list(cid, objects);
- ASSERT_EQ(r, 0);
- set<ghobject_t> listed(objects.begin(), objects.end());
- cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl;
- ASSERT_TRUE(listed.size() == created.size());
- objects.clear();
- listed.clear();
- ghobject_t current, next;
- while (1) {
- r = store->collection_list_partial(cid, current, 50, 60,
- 0, &objects, &next);
- ASSERT_EQ(r, 0);
- ASSERT_TRUE(sorted(objects));
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- if (listed.count(*i))
- cerr << *i << " repeated" << std::endl;
- listed.insert(*i);
- }
- if (objects.size() < 50) {
- ASSERT_TRUE(next.is_max());
- break;
- }
- objects.clear();
- current = next;
- }
- cerr << "listed.size() is " << listed.size() << std::endl;
- ASSERT_TRUE(listed.size() == created.size());
- for (set<ghobject_t>::iterator i = listed.begin();
- i != listed.end();
- ++i) {
- ASSERT_TRUE(created.count(*i));
- }
-
- for (set<ghobject_t>::iterator i = created.begin();
- i != created.end();
- ++i) {
- ObjectStore::Transaction t;
- t.collection_remove(cid, *i);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- ObjectStore::Transaction t;
- t.remove_collection(cid);
- store->apply_transaction(t);
-}
-
-TEST_P(StoreTest, OMapTest) {
- coll_t cid("blah");
- ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
- int r;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-
- map<string, bufferlist> attrs;
- {
- ObjectStore::Transaction t;
- t.touch(cid, hoid);
- t.omap_clear(cid, hoid);
- map<string, bufferlist> start_set;
- t.omap_setkeys(cid, hoid, start_set);
- store->apply_transaction(t);
- }
-
- for (int i = 0; i < 100; i++) {
- if (!(i%5)) {
- std::cout << "On iteration " << i << std::endl;
- }
- ObjectStore::Transaction t;
- bufferlist bl;
- map<string, bufferlist> cur_attrs;
- r = store->omap_get(cid, hoid, &bl, &cur_attrs);
- ASSERT_EQ(r, 0);
- for (map<string, bufferlist>::iterator j = attrs.begin();
- j != attrs.end();
- ++j) {
- bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str());
- if (!correct) {
- std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl;
- if (cur_attrs.count(j->first) > 0) {
- std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl;
- }
- }
- ASSERT_EQ(correct, true);
- }
- ASSERT_EQ(attrs.size(), cur_attrs.size());
-
- char buf[100];
- snprintf(buf, sizeof(buf), "%d", i);
- bl.clear();
- bufferptr bp(buf, strlen(buf) + 1);
- bl.append(bp);
- map<string, bufferlist> to_add;
- to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl));
- attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl));
- t.omap_setkeys(cid, hoid, to_add);
- store->apply_transaction(t);
- }
-
- int i = 0;
- while (attrs.size()) {
- if (!(i%5)) {
- std::cout << "removal: On iteration " << i << std::endl;
- }
- ObjectStore::Transaction t;
- bufferlist bl;
- map<string, bufferlist> cur_attrs;
- r = store->omap_get(cid, hoid, &bl, &cur_attrs);
- ASSERT_EQ(r, 0);
- for (map<string, bufferlist>::iterator j = attrs.begin();
- j != attrs.end();
- ++j) {
- bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str());
- if (!correct) {
- std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl;
- if (cur_attrs.count(j->first) > 0) {
- std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl;
- }
- }
- ASSERT_EQ(correct, true);
- }
-
- string to_remove = attrs.begin()->first;
- set<string> keys_to_remove;
- keys_to_remove.insert(to_remove);
- t.omap_rmkeys(cid, hoid, keys_to_remove);
- store->apply_transaction(t);
-
- attrs.erase(to_remove);
-
- ++i;
- }
-
- ObjectStore::Transaction t;
- t.remove(cid, hoid);
- t.remove_collection(cid);
- store->apply_transaction(t);
-}
-
-TEST_P(StoreTest, XattrTest) {
- coll_t cid("blah");
- ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
- bufferlist big;
- for (unsigned i = 0; i < 10000; ++i) {
- big.append('\0');
- }
- bufferlist small;
- for (unsigned i = 0; i < 10; ++i) {
- small.append('\0');
- }
- int r;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- t.touch(cid, hoid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-
- map<string, bufferlist> attrs;
- {
- ObjectStore::Transaction t;
- t.setattr(cid, hoid, "attr1", small);
- attrs["attr1"] = small;
- t.setattr(cid, hoid, "attr2", big);
- attrs["attr2"] = big;
- t.setattr(cid, hoid, "attr3", small);
- attrs["attr3"] = small;
- t.setattr(cid, hoid, "attr1", small);
- attrs["attr1"] = small;
- t.setattr(cid, hoid, "attr4", big);
- attrs["attr4"] = big;
- t.setattr(cid, hoid, "attr3", big);
- attrs["attr3"] = big;
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-
- map<string, bufferptr> aset;
- store->getattrs(cid, hoid, aset);
- ASSERT_EQ(aset.size(), attrs.size());
- for (map<string, bufferptr>::iterator i = aset.begin();
- i != aset.end();
- ++i) {
- bufferlist bl;
- bl.push_back(i->second);
- ASSERT_TRUE(attrs[i->first] == bl);
- }
-
- {
- ObjectStore::Transaction t;
- t.rmattr(cid, hoid, "attr2");
- attrs.erase("attr2");
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-
- aset.clear();
- store->getattrs(cid, hoid, aset);
- ASSERT_EQ(aset.size(), attrs.size());
- for (map<string, bufferptr>::iterator i = aset.begin();
- i != aset.end();
- ++i) {
- bufferlist bl;
- bl.push_back(i->second);
- ASSERT_TRUE(attrs[i->first] == bl);
- }
-
- bufferptr bp;
- r = store->getattr(cid, hoid, "attr2", bp);
- ASSERT_EQ(r, -ENODATA);
-
- r = store->getattr(cid, hoid, "attr3", bp);
- ASSERT_GE(r, 0);
- bufferlist bl2;
- bl2.push_back(bp);
- ASSERT_TRUE(bl2 == attrs["attr3"]);
-}
-
-void colsplittest(
- ObjectStore *store,
- unsigned num_objects,
- unsigned common_suffix_size
- ) {
- coll_t cid("from");
- coll_t tid("to");
- int r = 0;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- {
- ObjectStore::Transaction t;
- for (uint32_t i = 0; i < 2*num_objects; ++i) {
- stringstream objname;
- objname << "obj" << i;
- t.touch(cid, ghobject_t(hobject_t(
- objname.str(),
- "",
- CEPH_NOSNAP,
- i<<common_suffix_size,
- 0, "")));
- }
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- {
- ObjectStore::Transaction t;
- t.create_collection(tid);
- t.split_collection(cid, common_suffix_size+1, 0, tid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
-
- ObjectStore::Transaction t;
- vector<ghobject_t> objects;
- r = store->collection_list(cid, objects);
- ASSERT_EQ(r, 0);
- ASSERT_EQ(objects.size(), num_objects);
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- ASSERT_EQ(!(i->hobj.hash & (1<<common_suffix_size)), 0u);
- t.remove(cid, *i);
- }
-
- objects.clear();
- r = store->collection_list(tid, objects);
- ASSERT_EQ(r, 0);
- ASSERT_EQ(objects.size(), num_objects);
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- ASSERT_EQ(i->hobj.hash & (1<<common_suffix_size), 0u);
- t.remove(tid, *i);
- }
-
- t.remove_collection(cid);
- t.remove_collection(tid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
-}
-
-TEST_P(StoreTest, ColSplitTest1) {
- colsplittest(store.get(), 10000, 11);
-}
-TEST_P(StoreTest, ColSplitTest2) {
- colsplittest(store.get(), 100, 7);
-}
-
-#if 0
-TEST_P(StoreTest, ColSplitTest3) {
- colsplittest(store.get(), 100000, 25);
-}
-#endif
-
-/**
- * This test tests adding two different groups
- * of objects, each with 1 common prefix and 1
- * different prefix. We then remove half
- * in order to verify that the merging correctly
- * stops at the common prefix subdir. See bug
- * #5273 */
-TEST_P(StoreTest, TwoHash) {
- coll_t cid("asdf");
- int r;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- std::cout << "Making objects" << std::endl;
- for (int i = 0; i < 360; ++i) {
- ObjectStore::Transaction t;
- ghobject_t o;
- if (i < 8) {
- o.hobj.hash = (i << 16) | 0xA1;
- t.touch(cid, o);
- }
- o.hobj.hash = (i << 16) | 0xB1;
- t.touch(cid, o);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- std::cout << "Removing half" << std::endl;
- for (int i = 1; i < 8; ++i) {
- ObjectStore::Transaction t;
- ghobject_t o;
- o.hobj.hash = (i << 16) | 0xA1;
- t.remove(cid, o);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- std::cout << "Checking" << std::endl;
- for (int i = 1; i < 8; ++i) {
- ObjectStore::Transaction t;
- ghobject_t o;
- o.hobj.hash = (i << 16) | 0xA1;
- bool exists = store->exists(cid, o);
- ASSERT_EQ(exists, false);
- }
- {
- ghobject_t o;
- o.hobj.hash = 0xA1;
- bool exists = store->exists(cid, o);
- ASSERT_EQ(exists, true);
- }
- std::cout << "Cleanup" << std::endl;
- for (int i = 0; i < 360; ++i) {
- ObjectStore::Transaction t;
- ghobject_t o;
- o.hobj.hash = (i << 16) | 0xA1;
- t.remove(cid, o);
- o.hobj.hash = (i << 16) | 0xB1;
- t.remove(cid, o);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- ObjectStore::Transaction t;
- t.remove_collection(cid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
-}
-
-TEST_P(StoreTest, MoveRename) {
- coll_t temp_cid("mytemp");
- hobject_t temp_oid("tmp_oid", "", CEPH_NOSNAP, 0, 0, "");
- coll_t cid("dest");
- hobject_t oid("dest_oid", "", CEPH_NOSNAP, 0, 0, "");
- int r;
- {
- ObjectStore::Transaction t;
- t.create_collection(cid);
- t.touch(cid, oid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- ASSERT_TRUE(store->exists(cid, oid));
- bufferlist data, attr;
- map<string, bufferlist> omap;
- data.append("data payload");
- attr.append("attr value");
- omap["omap_key"].append("omap value");
- {
- ObjectStore::Transaction t;
- t.create_collection(temp_cid);
- t.touch(temp_cid, temp_oid);
- t.write(temp_cid, temp_oid, 0, data.length(), data);
- t.setattr(temp_cid, temp_oid, "attr", attr);
- t.omap_setkeys(temp_cid, temp_oid, omap);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- ASSERT_TRUE(store->exists(temp_cid, temp_oid));
- {
- ObjectStore::Transaction t;
- t.remove(cid, oid);
- t.collection_move_rename(temp_cid, temp_oid, cid, oid);
- r = store->apply_transaction(t);
- ASSERT_EQ(r, 0);
- }
- ASSERT_TRUE(store->exists(cid, oid));
- ASSERT_FALSE(store->exists(temp_cid, temp_oid));
- {
- bufferlist newdata;
- r = store->read(cid, oid, 0, 1000, newdata);
- ASSERT_GE(r, 0);
- ASSERT_TRUE(newdata.contents_equal(data));
- bufferlist newattr;
- r = store->getattr(cid, oid, "attr", newattr);
- ASSERT_GE(r, 0);
- ASSERT_TRUE(newattr.contents_equal(attr));
- set<string> keys;
- keys.insert("omap_key");
- map<string, bufferlist> newomap;
- r = store->omap_get_values(cid, oid, keys, &newomap);
- ASSERT_GE(r, 0);
- ASSERT_EQ(1u, newomap.size());
- ASSERT_TRUE(newomap.count("omap_key"));
- ASSERT_TRUE(newomap["omap_key"].contents_equal(omap["omap_key"]));
- }
-}
-
-INSTANTIATE_TEST_CASE_P(
- ObjectStore,
- StoreTest,
- ::testing::Values("filestore", "keyvaluestore-dev"));
-
-#else
-
-// Google Test may not support value-parameterized tests with some
-// compilers. If we use conditional compilation to compile out all
-// code referring to the gtest_main library, MSVC linker will not link
-// that library at all and consequently complain about missing entry
-// point defined in that library (fatal error LNK1561: entry point
-// must be defined). This dummy test keeps gtest_main linked in.
-TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {}
-
-#endif
-
-
-//
-// support tests for qa/workunits/filestore/filestore.sh
-//
-TEST(EXT4StoreTest, _detect_fs) {
- if (::getenv("DISK") == NULL || ::getenv("MOUNTPOINT") == NULL) {
- cerr << "SKIP because DISK and MOUNTPOINT environment variables are not set. It is meant to run from qa/workunits/filestore/filestore.sh " << std::endl;
- return;
- }
- const string disk(::getenv("DISK"));
- EXPECT_LT((unsigned)0, disk.size());
- const string mnt(::getenv("MOUNTPOINT"));
- EXPECT_LT((unsigned)0, mnt.size());
- ::umount(mnt.c_str());
-
- const string dir("store_test_temp_dir");
- const string journal("store_test_temp_journal");
-
- //
- // without user_xattr, ext4 fails
- //
- {
- g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "true");
- EXPECT_EQ(::system((string("mount -o loop,nouser_xattr ") + disk + " " + mnt).c_str()), 0);
- EXPECT_EQ(::chdir(mnt.c_str()), 0);
- EXPECT_EQ(::mkdir(dir.c_str(), 0755), 0);
- FileStore store(dir, journal);
- EXPECT_EQ(store._detect_fs(), -ENOTSUP);
- EXPECT_EQ(::chdir(".."), 0);
- EXPECT_EQ(::umount(mnt.c_str()), 0);
- }
- //
- // mounted with user_xattr, ext4 fails if filestore_xattr_use_omap is false
- //
- {
- g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "false");
- EXPECT_EQ(::system((string("mount -o loop,user_xattr ") + disk + " " + mnt).c_str()), 0);
- EXPECT_EQ(::chdir(mnt.c_str()), 0);
- FileStore store(dir, journal);
- EXPECT_EQ(store._detect_fs(), -ENOTSUP);
- EXPECT_EQ(::chdir(".."), 0);
- EXPECT_EQ(::umount(mnt.c_str()), 0);
- }
- //
- // mounted with user_xattr, ext4 succeeds if filestore_xattr_use_omap is true
- //
- {
- g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "true");
- EXPECT_EQ(::system((string("mount -o loop,user_xattr ") + disk + " " + mnt).c_str()), 0);
- EXPECT_EQ(::chdir(mnt.c_str()), 0);
- FileStore store(dir, journal);
- EXPECT_EQ(store._detect_fs(), 0);
- EXPECT_EQ(::chdir(".."), 0);
- EXPECT_EQ(::umount(mnt.c_str()), 0);
- }
-}
-
-
-int main(int argc, char **argv) {
- vector<const char*> args;
- argv_to_vec(argc, (const char **)argv, args);
-
- global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->set_val("osd_journal_size", "400");
- g_ceph_context->_conf->set_val("filestore_index_retry_probability", "0.5");
- g_ceph_context->_conf->set_val("filestore_op_thread_timeout", "1000");
- g_ceph_context->_conf->set_val("filestore_op_thread_suicide_timeout", "10000");
- g_ceph_context->_conf->apply_changes(NULL);
-
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
-
-// Local Variables:
-// compile-command: "cd ../.. ; make ceph_test_filestore ; ./ceph_test_filestore --gtest_filter=StoreTest.* --log-to-stderr=true --debug-filestore=20"
-// End:
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#include <iostream>
-#include <sstream>
-#include <boost/scoped_ptr.hpp>
-#include "os/FileStore.h"
-#include "global/global_init.h"
-#include "common/ceph_argparse.h"
-#include "common/debug.h"
-#include "test/common/ObjectContents.h"
-#include "FileStoreTracker.h"
-#include "os/LevelDBStore.h"
-#include "os/KeyValueDB.h"
-#include "os/ObjectStore.h"
-
-void usage(const string &name) {
- std::cerr << "Usage: " << name << " [new|continue] store_path store_journal db_path"
- << std::endl;
-}
-
-template <typename T>
-typename T::iterator rand_choose(T &cont) {
- if (cont.size() == 0) {
- return cont.end();
- }
- int index = rand() % cont.size();
- typename T::iterator retval = cont.begin();
-
- for (; index > 0; --index) ++retval;
- return retval;
-}
-
-int main(int argc, char **argv) {
- vector<const char*> args;
- argv_to_vec(argc, (const char **)argv, args);
-
- global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->apply_changes(NULL);
-
- std::cerr << "args: " << args << std::endl;
- if (args.size() < 4) {
- usage(argv[0]);
- return 1;
- }
-
- string store_path(args[1]);
- string store_dev(args[2]);
- string db_path(args[3]);
-
- bool start_new = false;
- if (string(args[0]) == string("new")) start_new = true;
-
- LevelDBStore *_db = new LevelDBStore(g_ceph_context, db_path);
- assert(!_db->create_and_open(std::cerr));
- boost::scoped_ptr<KeyValueDB> db(_db);
- boost::scoped_ptr<ObjectStore> store(new FileStore(store_path, store_dev));
-
-
- if (start_new) {
- std::cerr << "mkfs" << std::endl;
- assert(!store->mkfs());
- ObjectStore::Transaction t;
- assert(!store->mount());
- t.create_collection(coll_t("coll"));
- store->apply_transaction(t);
- } else {
- assert(!store->mount());
- }
-
- FileStoreTracker tracker(store.get(), db.get());
-
- set<string> objects;
- for (unsigned i = 0; i < 10; ++i) {
- stringstream stream;
- stream << "Object_" << i;
- tracker.verify("coll", stream.str(), true);
- objects.insert(stream.str());
- }
-
- while (1) {
- FileStoreTracker::Transaction t;
- for (unsigned j = 0; j < 100; ++j) {
- int val = rand() % 100;
- if (val < 30) {
- t.write("coll", *rand_choose(objects));
- } else if (val < 60) {
- t.clone("coll", *rand_choose(objects),
- *rand_choose(objects));
- } else if (val < 70) {
- t.remove("coll", *rand_choose(objects));
- } else {
- t.clone_range("coll", *rand_choose(objects),
- *rand_choose(objects));
- }
- }
- tracker.submit_transaction(t);
- tracker.verify("coll", *rand_choose(objects));
- }
- return 0;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include <sstream>
-#include <time.h>
-#include <stdlib.h>
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "common/debug.h"
-#include "os/FileStore.h"
-
-#include "DeterministicOpSequence.h"
-#include "FileStoreDiff.h"
-
-#include "common/config.h"
-#include "include/assert.h"
-
-#define dout_subsys ceph_subsys_
-#undef dout_prefix
-#define dout_prefix *_dout << "test_idempotent_sequence "
-
-void usage(const char *name, std::string command = "") {
- assert(name != NULL);
-
- std::string more = "cmd <args...>";
- std::string diff = "diff <filestoreA> <journalA> <filestoreB> <journalB>";
- std::string get_last_op = "get-last-op <filestore> <journal>";
- std::string run_seq_to = "run-sequence-to <num-ops> <filestore> <journal>";
-
- if (!command.empty()) {
- if (command == "diff")
- more = diff;
- else if (command == "get-last-op")
- more = get_last_op;
- else if (command == "run-sequence-to")
- more = run_seq_to;
- }
- std::cout << "usage: " << name << " " << more << " [options]" << std::endl;
-
- std::cout << "\n\
-Commands:\n\
- " << diff << "\n\
- " << get_last_op << "\n\
- " << run_seq_to << "\n\
-\n\
-Global Options:\n\
- -c FILE Read configuration from FILE\n\
- --osd-data PATH Set OSD Data path\n\
- --osd-journal PATH Set OSD Journal path\n\
- --osd-journal-size VAL Set Journal size\n\
- --help This message\n\
-\n\
-Test-specific Options:\n\
- --test-seed VAL Seed to run the test\n\
- --test-status-file PATH Path to keep the status file\n\
- --test-num-colls VAL Number of collections to create on init\n\
- --test-num-objs VAL Number of objects to create on init\n\
-" << std::endl;
-}
-
-const char *our_name = NULL;
-int seed = 0, num_txs = 100, num_colls = 30, num_objs = 0;
-bool is_seed_set = false;
-int verify_at = 0;
-std::string status_file;
-
-int run_diff(std::string& a_path, std::string& a_journal,
- std::string& b_path, std::string& b_journal)
-{
- FileStore *a = new FileStore(a_path, a_journal, "a");
- FileStore *b = new FileStore(b_path, b_journal, "b");
-
- int ret = 0;
- {
- FileStoreDiff fsd(a, b);
- if (fsd.diff()) {
- dout(0) << "diff found an difference" << dendl;
- ret = -1;
- } else {
- dout(0) << "no diff" << dendl;
- }
- }
-
- delete a;
- delete b;
- return ret;
-}
-
-int run_get_last_op(std::string& filestore_path, std::string& journal_path)
-{
- FileStore *store = new FileStore(filestore_path, journal_path);
-
- int err = store->mount();
- if (err) {
- store->umount();
- delete store;
- return err;
- }
-
- coll_t txn_coll("meta");
- hobject_t txn_object(sobject_t("txn", CEPH_NOSNAP));
- bufferlist bl;
- store->read(txn_coll, txn_object, 0, 100, bl);
- int32_t txn = 0;
- if (bl.length()) {
- bufferlist::iterator p = bl.begin();
- ::decode(txn, p);
- }
-
- store->umount();
- delete store;
-
- cout << txn << std::endl;
- return 0;
-}
-
-int run_sequence_to(int val, std::string& filestore_path,
- std::string& journal_path)
-{
- num_txs = val;
-
- if (!is_seed_set)
- seed = (int) time(NULL);
-
- FileStore *store = new FileStore(filestore_path, journal_path);
-
- int err;
-
- // mkfs iff directory dne
- err = ::mkdir(filestore_path.c_str(), 0755);
- if (err) {
- cerr << filestore_path << " already exists" << std::endl;
- store->umount();
- delete store;
- return err;
- }
-
- err = store->mkfs();
- ceph_assert(err == 0);
-
- err = store->mount();
- ceph_assert(err == 0);
-
- DeterministicOpSequence op_sequence(store, status_file);
- op_sequence.init(num_colls, num_objs);
- op_sequence.generate(seed, num_txs);
- store->umount();
- return 0;
-}
-
-int run_command(std::string& command, std::vector<std::string>& args)
-{
- if (command.empty()) {
- usage(our_name);
- exit(0);
- }
-
- /* We'll have a class that will handle the options, the command
- * and its arguments. For the time being, and so we can move on, let's
- * tolerate this big, ugly code.
- */
- if (command == "diff") {
- /* expect 4 arguments: (filestore path + journal path)*2 */
- if (args.size() == 4) {
- return run_diff(args[0], args[1], args[2], args[3]);
- }
- } else if (command == "get-last-op") {
- /* expect 2 arguments: a filestore path + journal */
- if (args.size() == 2) {
- return run_get_last_op(args[0], args[1]);
- }
- } else if (command == "run-sequence-to") {
- /* expect 3 arguments: # of operations and a filestore path + journal. */
- if (args.size() == 3) {
- return run_sequence_to(strtoll(args[0].c_str(), NULL, 10), args[1], args[2]);
- }
- } else {
- std::cout << "unknown command " << command << std::endl;
- usage(our_name);
- exit(1);
- }
-
- usage(our_name, command);
- exit(1);
-}
-
-int main(int argc, const char *argv[])
-{
- vector<const char*> def_args;
- vector<const char*> args;
- our_name = argv[0];
- argv_to_vec(argc, argv, args);
-
- global_init(&def_args, args,
- CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->apply_changes(NULL);
-
- std::string command;
- std::vector<std::string> command_args;
-
- for (std::vector<const char*>::iterator i = args.begin(); i != args.end();) {
- string val;
-
- if (ceph_argparse_double_dash(args, i)) {
- break;
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-seed", (char*) NULL)) {
- seed = strtoll(val.c_str(), NULL, 10);
- is_seed_set = true;
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-num-colls", (char*) NULL)) {
- num_colls = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-num-objs", (char*) NULL)) {
- num_objs = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-status-file", (char*) NULL)) {
- status_file = val;
- } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) {
- usage(our_name);
- exit(0);
- } else {
- if (command.empty())
- command = *i++;
- else
- command_args.push_back(string(*i++));
- }
- }
-
- int ret = run_command(command, command_args);
-
- return ret;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2012 New Dream Network
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- */
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include <assert.h>
-#include <time.h>
-#include <stdlib.h>
-#include <signal.h>
-#include <cctype>
-#include <errno.h>
-#include <sys/time.h>
-#include "os/FileStore.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "common/debug.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/lexical_cast.hpp>
-#include "workload_generator.h"
-#include "include/assert.h"
-
-#include "TestFileStoreState.h"
-
-static const char *our_name = NULL;
-void usage();
-
-boost::scoped_ptr<WorkloadGenerator> wrkldgen;
-
-#define dout_subsys ceph_subsys_
-
-
-WorkloadGenerator::WorkloadGenerator(vector<const char*> args)
- : TestFileStoreState(NULL),
- m_max_in_flight(def_max_in_flight),
- m_num_ops(-1),
- m_destroy_coll_every_nr_runs(def_destroy_coll_every_nr_runs),
- m_num_colls(def_num_colls),
- m_write_data_bytes(0), m_write_xattr_obj_bytes(0),
- m_write_xattr_coll_bytes(0), m_write_pglog_bytes(0),
- m_suppress_write_data(false), m_suppress_write_xattr_obj(false),
- m_suppress_write_xattr_coll(false), m_suppress_write_log(false),
- m_do_stats(false),
- m_stats_finished_txs(0),
- m_stats_lock("WorldloadGenerator::m_stats_lock"),
- m_stats_show_secs(5),
- m_stats_total_written(0),
- m_stats_begin()
-{
- int err = 0;
-
- m_nr_runs.set(0);
-
- init_args(args);
- dout(0) << "data = " << g_conf->osd_data << dendl;
- dout(0) << "journal = " << g_conf->osd_journal << dendl;
- dout(0) << "journal size = " << g_conf->osd_journal_size << dendl;
-
- err = ::mkdir(g_conf->osd_data.c_str(), 0755);
- ceph_assert(err == 0 || (err < 0 && errno == EEXIST));
- ObjectStore *store_ptr = new FileStore(g_conf->osd_data, g_conf->osd_journal);
- m_store.reset(store_ptr);
- err = m_store->mkfs();
- ceph_assert(err == 0);
- err = m_store->mount();
- ceph_assert(err == 0);
-
- set_max_in_flight(m_max_in_flight);
- set_num_objs_per_coll(def_num_obj_per_coll);
-
- init(m_num_colls, 0);
-
- dout(0) << "#colls = " << m_num_colls << dendl;
- dout(0) << "#objs per coll = " << m_num_objs_per_coll << dendl;
- dout(0) << "#txs per destr = " << m_destroy_coll_every_nr_runs << dendl;
-
-}
-
-size_t WorkloadGenerator::_parse_size_or_die(std::string& val)
-{
- size_t s = 0;
- int multiplier = 0;
- size_t i = 0;
-
- if (val.empty()) // this should never happen, but catch it anyway.
- goto die;
-
-
- for (i = 0; i < val.length(); i++) {
- if (!isdigit(val[i])) {
- if (isalpha(val[i])) {
- val[i] = tolower(val[i]);
- switch (val[i]) {
- case 'b': break;
- case 'k': multiplier = 10; break;
- case 'm': multiplier = 20; break;
- case 'g': multiplier = 30; break;
- default:
- goto die;
- }
- val[i] = '\0';
- break;
- } else {
- goto die;
- }
- }
- }
-
- s = strtoll(val.c_str(), NULL, 10) * (1 << multiplier);
- return s;
-
-die:
- usage();
- exit(1);
-}
-
-void WorkloadGenerator::_suppress_ops_or_die(std::string& val)
-{
- for (size_t i = 0; i < val.length(); i++) {
- switch (val[i]) {
- case 'c': m_suppress_write_xattr_coll = true; break;
- case 'o': m_suppress_write_xattr_obj = true; break;
- case 'l': m_suppress_write_log = true; break;
- case 'd': m_suppress_write_data = true; break;
- default:
- usage();
- exit(1);
- }
- }
-}
-
-void WorkloadGenerator::init_args(vector<const char*> args)
-{
- for (std::vector<const char*>::iterator i = args.begin(); i != args.end();) {
- string val;
-
- if (ceph_argparse_double_dash(args, i)) {
- break;
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-num-colls", (char*) NULL)) {
- m_num_colls = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-objs-per-coll", (char*) NULL)) {
- m_num_objs_per_coll = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-destroy-coll-per-N-trans", (char*) NULL)) {
- m_destroy_coll_every_nr_runs = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-num-ops", (char*) NULL)) {
- m_num_ops = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-max-in-flight", (char*) NULL)) {
- m_max_in_flight = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-write-data-size", (char*) NULL)) {
- m_write_data_bytes = _parse_size_or_die(val);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-write-xattr-obj-size", (char*) NULL)) {
- m_write_xattr_obj_bytes = _parse_size_or_die(val);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-write-xattr-coll-size", (char*) NULL)) {
- m_write_xattr_coll_bytes = _parse_size_or_die(val);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-write-pglog-size", (char*) NULL)) {
- m_write_pglog_bytes = _parse_size_or_die(val);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-suppress-ops", (char*) NULL)) {
- _suppress_ops_or_die(val);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-show-stats-period", (char*) NULL)) {
- m_stats_show_secs = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_flag(args, i, "--test-show-stats", (char*) NULL)) {
- m_do_stats = true;
- } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) {
- usage();
- exit(0);
- }
- }
-}
-
-int WorkloadGenerator::get_uniform_random_value(int min, int max)
-{
- boost::uniform_int<> value(min, max);
- return value(m_rng);
-}
-
-TestFileStoreState::coll_entry_t *WorkloadGenerator::get_rnd_coll_entry(bool erase = false)
-{
- int index = get_uniform_random_value(0, m_collections_ids.size()-1);
- coll_entry_t *entry = get_coll_at(index, erase);
- return entry;
-}
-
-hobject_t *WorkloadGenerator::get_rnd_obj(coll_entry_t *entry)
-{
- assert(entry != NULL);
-
- bool create =
- (get_uniform_random_value(0,100) < 50 || !entry->m_objects.size());
-
- if (create && ((int) entry->m_objects.size() < m_num_objs_per_coll)) {
- return (entry->touch_obj(entry->m_next_object_id++));
- }
-
- int idx = get_uniform_random_value(0, entry->m_objects.size()-1);
- return entry->get_obj_at(idx);
-}
-
-/**
- * We'll generate a random amount of bytes, ranging from a single byte up to
- * a couple of MB.
- */
-size_t WorkloadGenerator::get_random_byte_amount(size_t min, size_t max)
-{
- size_t diff = max - min;
- return (size_t) (min + (rand() % diff));
-}
-
-void WorkloadGenerator::get_filled_byte_array(bufferlist& bl, size_t size)
-{
- static const char alphanum[] = "0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
- "abcdefghijklmnopqrstuvwxyz";
-
- bufferptr bp(size);
- if (false) {
- for (unsigned int i = 0; i < size - 1; i++) {
- bp[i] = alphanum[rand() % sizeof(alphanum)];
- }
- bp[size - 1] = '\0';
- } else {
- bp.zero();
- }
- bl.append(bp);
-}
-
-void WorkloadGenerator::do_write_object(ObjectStore::Transaction *t,
- coll_t coll, hobject_t obj,
- C_StatState *stat)
-{
- if (m_suppress_write_data) {
- dout(5) << __func__ << " suppressed" << dendl;
- return;
- }
-
- size_t size = m_write_data_bytes;
- if (!size)
- size = get_random_byte_amount(min_write_bytes, max_write_bytes);
-
- bufferlist bl;
- get_filled_byte_array(bl, size);
-
- dout(2) << __func__ << " " << coll << "/" << obj
- << " size " << bl.length() << dendl;
-
- if (m_do_stats && (stat != NULL))
- stat->written_data += bl.length();
-
- t->write(coll, obj, 0, bl.length(), bl);
-}
-
-void WorkloadGenerator::do_setattr_object(ObjectStore::Transaction *t,
- coll_t coll, hobject_t obj,
- C_StatState *stat)
-{
- if (m_suppress_write_xattr_obj) {
- dout(5) << __func__ << " suppressed" << dendl;
- return;
- }
-
- size_t size = m_write_xattr_obj_bytes;
- if (!size)
- size = get_random_byte_amount(min_xattr_obj_bytes, max_xattr_obj_bytes);
-
- bufferlist bl;
- get_filled_byte_array(bl, size);
-
- dout(2) << __func__ << " " << coll << "/" << obj << " size " << size << dendl;
-
- if (m_do_stats && (stat != NULL))
- stat->written_data += bl.length();
-
- t->setattr(coll, obj, "objxattr", bl);
-}
-
-void WorkloadGenerator::do_setattr_collection(ObjectStore::Transaction *t,
- coll_t coll, C_StatState *stat)
-{
- if (m_suppress_write_xattr_coll) {
- dout(5) << __func__ << " suppressed" << dendl;
- return;
- }
-
- size_t size = m_write_xattr_coll_bytes;
- if (!size)
- size = get_random_byte_amount(min_xattr_coll_bytes, max_xattr_coll_bytes);
-
- bufferlist bl;
- get_filled_byte_array(bl, size);
- dout(2) << __func__ << " coll " << coll << " size " << size << dendl;
-
- if (m_do_stats && (stat != NULL))
- stat->written_data += bl.length();
-
- t->collection_setattr(coll, "collxattr", bl);
-}
-
-void WorkloadGenerator::do_append_log(ObjectStore::Transaction *t,
- coll_entry_t *entry, C_StatState *stat)
-{
- if (m_suppress_write_log) {
- dout(5) << __func__ << " suppressed" << dendl;
- return;
- }
-
- size_t size = (m_write_pglog_bytes ? m_write_pglog_bytes : log_append_bytes);
-
- bufferlist bl;
- get_filled_byte_array(bl, size);
- hobject_t log_obj = entry->m_meta_obj;
-
- dout(2) << __func__ << " coll " << entry->m_coll << " "
- << META_COLL << " /" << log_obj << " (" << bl.length() << ")" << dendl;
-
- if (m_do_stats && (stat != NULL))
- stat->written_data += bl.length();
-
- uint64_t s = pg_log_size[entry->m_coll];
- t->write(META_COLL, log_obj, s, bl.length(), bl);
- pg_log_size[entry->m_coll] += bl.length();
-}
-
-void WorkloadGenerator::do_destroy_collection(ObjectStore::Transaction *t,
- coll_entry_t *entry,
- C_StatState *stat)
-{
- m_nr_runs.set(0);
- entry->m_osr.flush();
- vector<ghobject_t> ls;
- m_store->collection_list(entry->m_coll, ls);
- dout(2) << __func__ << " coll " << entry->m_coll
- << " (" << ls.size() << " objects)" << dendl;
-
- for (vector<ghobject_t>::iterator it = ls.begin(); it < ls.end(); ++it) {
- t->remove(entry->m_coll, *it);
- }
-
- t->remove_collection(entry->m_coll);
- t->remove(META_COLL, entry->m_meta_obj);
-}
-
-TestFileStoreState::coll_entry_t
-*WorkloadGenerator::do_create_collection(ObjectStore::Transaction *t,
- C_StatState *stat)
-{
- coll_entry_t *entry = coll_create(m_next_coll_nr++);
- if (!entry) {
- dout(0) << __func__ << " failed to create coll id "
- << m_next_coll_nr << dendl;
- return NULL;
- }
- m_collections.insert(make_pair(entry->m_id, entry));
-
- dout(2) << __func__ << " id " << entry->m_id << " coll " << entry->m_coll << dendl;
- t->create_collection(entry->m_coll);
- dout(2) << __func__ << " meta " << META_COLL << "/" << entry->m_meta_obj << dendl;
- t->touch(META_COLL, entry->m_meta_obj);
- return entry;
-}
-
-void WorkloadGenerator::do_stats()
-{
- utime_t now = ceph_clock_now(NULL);
- m_stats_lock.Lock();
-
- utime_t duration = (now - m_stats_begin);
-
- // when cast to double, a utime_t behaves properly
- double throughput = (m_stats_total_written / ((double) duration));
- double tx_throughput (m_stats_finished_txs / ((double) duration));
-
- dout(0) << __func__
- << " written: " << m_stats_total_written
- << " duration: " << duration << " sec"
- << " bandwidth: " << prettybyte_t(throughput) << "/s"
- << " iops: " << tx_throughput << "/s"
- << dendl;
-
- m_stats_lock.Unlock();
-}
-
-void WorkloadGenerator::run()
-{
- bool create_coll = false;
- int ops_run = 0;
-
- utime_t stats_interval(m_stats_show_secs, 0);
- utime_t now = ceph_clock_now(NULL);
- utime_t stats_time = now;
- m_stats_begin = now;
-
- do {
- C_StatState *stat_state = NULL;
-
- if (m_num_ops && (ops_run == m_num_ops))
- break;
-
- if (!create_coll && !m_collections.size()) {
- dout(0) << "We ran out of collections!" << dendl;
- break;
- }
-
- dout(5) << __func__
- << " m_finished_lock is-locked: " << m_finished_lock.is_locked()
- << " in-flight: " << m_in_flight.read()
- << dendl;
-
- wait_for_ready();
-
- ObjectStore::Transaction *t = new ObjectStore::Transaction;
- Context *c;
- bool destroy_collection = false;
- TestFileStoreState::coll_entry_t *entry = NULL;
-
-
- if (m_do_stats) {
- utime_t now = ceph_clock_now(NULL);
- utime_t elapsed = now - stats_time;
- if (elapsed >= stats_interval) {
- do_stats();
- stats_time = now;
- }
- stat_state = new C_StatState(this, now);
- }
-
- if (create_coll) {
- create_coll = false;
-
- entry = do_create_collection(t, stat_state);
- if (!entry) {
- dout(0) << __func__ << " something went terribly wrong creating coll" << dendl;
- break;
- }
-
- c = new C_OnReadable(this, t);
- goto queue_tx;
- }
-
- destroy_collection = should_destroy_collection();
- entry = get_rnd_coll_entry(destroy_collection);
- assert(entry != NULL);
-
- if (destroy_collection) {
- do_destroy_collection(t, entry, stat_state);
- c = new C_OnDestroyed(this, t, entry);
- if (!m_num_ops)
- create_coll = true;
- } else {
- hobject_t *obj = get_rnd_obj(entry);
-
- do_write_object(t, entry->m_coll, *obj, stat_state);
- do_setattr_object(t, entry->m_coll, *obj, stat_state);
- do_setattr_collection(t, entry->m_coll, stat_state);
- do_append_log(t, entry, stat_state);
-
- c = new C_OnReadable(this, t);
- }
-
-queue_tx:
-
- if (m_do_stats) {
- Context *tmp = c;
- c = new C_StatWrapper(stat_state, tmp);
- }
-
- m_store->queue_transaction(&(entry->m_osr), t, c);
-
- inc_in_flight();
-
- ops_run ++;
-
- } while (true);
-
- dout(2) << __func__ << " waiting for "
- << m_in_flight.read() << " in-flight transactions" << dendl;
-
- wait_for_done();
-
- do_stats();
-
- dout(0) << __func__ << " finishing" << dendl;
-}
-
-void usage()
-{
- cout << "usage: " << our_name << "[options]" << std::endl;
-
- cout << "\
-\n\
-Global Options:\n\
- -c FILE Read configuration from FILE\n\
- --osd-data PATH Set OSD Data path\n\
- --osd-journal PATH Set OSD Journal path\n\
- --osd-journal-size VAL Set Journal size\n\
- --help This message\n\
-\n\
-Test-specific Options:\n\
- --test-num-colls VAL Set the number of collections\n\
- --test-num-objs-per-coll VAL Set the number of objects per collection\n\
- --test-destroy-coll-per-N-trans VAL Set how many transactions to run before\n\
- destroying a collection.\n\
- --test-num-ops VAL Run a certain number of operations\n\
- (a VAL of 0 runs the test forever)\n\
- --test-max-in-flight VAL Maximum number of in-flight transactions\n\
- (default: 50)\n\
- --test-suppress-ops OPS Suppress ops specified in OPS\n\
- --test-write-data-size SIZE Specify SIZE for all data writes\n\
- --test-write-xattr-obj-size SIZE Specify SIZE for all xattrs on objects\n\
- --test-write-xattr-coll-size SIZE Specify SIZE for all xattrs on colls\n\
- --test-write-pglog-size SIZE Specify SIZE for all pglog writes\n\
- --test-show-stats Show stats as we go\n\
- --test-show-stats-period SECS Show stats every SECS (default: 5)\n\
-\n\
- SIZE is a numeric value that can be assumed as being bytes, or may be any\n\
- other unit if specified: B or b, K or k, M or m, G or g.\n\
- e.g., 1G = 1024M = 1048576k = 1073741824\n\
-\n\
- OPS can be one or more of the following options:\n\
- c writes on collection's xattrs\n\
- o writes on object's xattr\n\
- l writes on pglog\n\
- d data writes on objects\n\
-\n\
-" << std::endl;
-}
-
-int main(int argc, const char *argv[])
-{
- vector<const char*> def_args;
- vector<const char*> args;
-
- our_name = argv[0];
-
- def_args.push_back("--osd-journal-size");
- def_args.push_back("400");
-// def_args.push_back("--osd-data");
-// def_args.push_back("workload_gen_dir");
-// def_args.push_back("--osd-journal");
-// def_args.push_back("workload_gen_dir/journal");
- argv_to_vec(argc, argv, args);
-
- global_init(&def_args, args,
- CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->apply_changes(NULL);
-
- WorkloadGenerator *wrkldgen_ptr = new WorkloadGenerator(args);
- wrkldgen.reset(wrkldgen_ptr);
- wrkldgen->run();
- return 0;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2012 New Dream Network
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- */
-#ifndef WORKLOAD_GENERATOR_H_
-#define WORKLOAD_GENERATOR_H_
-
-#include "os/FileStore.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/random/mersenne_twister.hpp>
-#include <boost/random/uniform_int.hpp>
-#include <map>
-#include <sys/time.h>
-
-#include "TestFileStoreState.h"
-
-typedef boost::mt11213b rngen_t;
-
-class WorkloadGenerator : public TestFileStoreState {
- public:
- static const int def_max_in_flight = 50;
-
- static const int def_destroy_coll_every_nr_runs = 100;
- static const int def_num_obj_per_coll = 6000;
- static const int def_num_colls = 30;
-
- static const size_t min_write_bytes = 1;
- static const size_t max_write_mb = 5;
- static const size_t max_write_bytes = (max_write_mb * 1024 * 1024);
-
- static const size_t min_xattr_obj_bytes = 2;
- static const size_t max_xattr_obj_bytes = 300;
- static const size_t min_xattr_coll_bytes = 4;
- static const size_t max_xattr_coll_bytes = 600;
-
- static const size_t log_append_bytes = 1024;
-
- struct C_StatState {
- utime_t start;
- unsigned int written_data;
- WorkloadGenerator *wrkldgen;
-
- C_StatState(WorkloadGenerator *state, utime_t s)
- : start(s), written_data(0), wrkldgen(state) { }
- };
-
-
- protected:
- int m_max_in_flight;
- int m_num_ops;
- int m_destroy_coll_every_nr_runs;
- atomic_t m_nr_runs;
-
- int m_num_colls;
-
- rngen_t m_rng;
-
- map<coll_t, uint64_t> pg_log_size;
-
- size_t m_write_data_bytes;
- size_t m_write_xattr_obj_bytes;
- size_t m_write_xattr_coll_bytes;
- size_t m_write_pglog_bytes;
-
- bool m_suppress_write_data;
- bool m_suppress_write_xattr_obj;
- bool m_suppress_write_xattr_coll;
- bool m_suppress_write_log;
-
- bool m_do_stats;
-
- int m_stats_finished_txs;
- Mutex m_stats_lock;
- int m_stats_show_secs;
-
- size_t m_stats_total_written;
- utime_t m_stats_begin;
-
- private:
-
- void _suppress_ops_or_die(std::string& val);
- size_t _parse_size_or_die(std::string& val);
- void init_args(vector<const char*> args);
-
- int get_uniform_random_value(int min, int max);
- coll_entry_t *get_rnd_coll_entry(bool erase);
- hobject_t *get_rnd_obj(coll_entry_t *entry);
- int get_random_collection_nr();
- int get_random_object_nr(int coll_nr);
-
- size_t get_random_byte_amount(size_t min, size_t max);
- void get_filled_byte_array(bufferlist& bl, size_t size);
-
- void do_write_object(ObjectStore::Transaction *t,
- coll_t coll, hobject_t obj, C_StatState *stat);
- void do_setattr_object(ObjectStore::Transaction *t,
- coll_t coll, hobject_t obj, C_StatState *stat);
- void do_setattr_collection(ObjectStore::Transaction *t, coll_t coll,
- C_StatState *stat);
- void do_append_log(ObjectStore::Transaction *t, coll_entry_t *entry,
- C_StatState *stat);
-
- bool should_destroy_collection() {
- return ((m_destroy_coll_every_nr_runs > 0) &&
- ((int)m_nr_runs.read() >= m_destroy_coll_every_nr_runs));
- }
- void do_destroy_collection(ObjectStore::Transaction *t, coll_entry_t *entry,
- C_StatState *stat);
- coll_entry_t *do_create_collection(ObjectStore::Transaction *t,
- C_StatState *stat);
-
- void do_stats();
-
-public:
- WorkloadGenerator(vector<const char*> args);
- ~WorkloadGenerator() {
- m_store->umount();
- }
-
- class C_OnReadable: public TestFileStoreState::C_OnFinished {
- WorkloadGenerator *wrkldgen_state;
-
- public:
- C_OnReadable(WorkloadGenerator *state,
- ObjectStore::Transaction *t)
- :TestFileStoreState::C_OnFinished(state, t), wrkldgen_state(state) { }
-
- void finish(int r)
- {
- TestFileStoreState::C_OnFinished::finish(r);
- wrkldgen_state->m_nr_runs.inc();
- }
- };
-
- class C_OnDestroyed: public C_OnReadable {
- coll_entry_t *m_entry;
-
- public:
- C_OnDestroyed(WorkloadGenerator *state,
- ObjectStore::Transaction *t, coll_entry_t *entry) :
- C_OnReadable(state, t), m_entry(entry) {}
-
- void finish(int r) {
- C_OnReadable::finish(r);
- delete m_entry;
- }
- };
-
- class C_StatWrapper : public Context {
- C_StatState *stat_state;
- Context *ctx;
-
- public:
- C_StatWrapper(C_StatState *state, Context *context)
- : stat_state(state), ctx(context) { }
-
- void finish(int r) {
- ctx->complete(r);
-
- stat_state->wrkldgen->m_stats_lock.Lock();
-
- stat_state->wrkldgen->m_stats_total_written += stat_state->written_data;
- stat_state->wrkldgen->m_stats_finished_txs ++;
- stat_state->wrkldgen->m_stats_lock.Unlock();
- }
- };
-
- void run(void);
-};
-
-bool operator<(const WorkloadGenerator::coll_entry_t& l,
- const WorkloadGenerator::coll_entry_t& r) {
- return (l.m_id < r.m_id);
-}
-
-#endif /* WORKLOAD_GENERATOR_H_ */
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <iostream>
+#include <fstream>
+#include <time.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <sstream>
+#include "os/ObjectStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "DeterministicOpSequence.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "deterministic_seq "
+
+DeterministicOpSequence::DeterministicOpSequence(ObjectStore *store,
+ std::string status)
+ : TestObjectStoreState(store),
+ txn(0),
+ m_osr("OSR")
+{
+ txn_coll = coll_t("meta");
+ txn_object = hobject_t(sobject_t("txn", CEPH_NOSNAP));
+
+ if (!status.empty())
+ m_status.open(status.c_str());
+}
+
+DeterministicOpSequence::~DeterministicOpSequence()
+{
+ // TODO Auto-generated destructor stub
+}
+
+bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen)
+{
+ bool ok = false;
+ switch (op) {
+ case DSOP_TOUCH:
+ ok = do_touch(gen);
+ break;
+ case DSOP_WRITE:
+ ok = do_write(gen);
+ break;
+ case DSOP_CLONE:
+ ok = do_clone(gen);
+ break;
+ case DSOP_CLONE_RANGE:
+ ok = do_clone_range(gen);
+ break;
+ case DSOP_OBJ_REMOVE:
+ ok = do_remove(gen);
+ break;
+ case DSOP_COLL_ADD:
+ ok = do_coll_add(gen);
+ break;
+ case DSOP_COLL_RENAME:
+ //do_coll_rename(gen);
+ break;
+ case DSOP_SET_ATTRS:
+ ok = do_set_attrs(gen);
+ break;
+ default:
+ assert(0 == "bad op");
+ }
+ return ok;
+}
+
+void DeterministicOpSequence::generate(int seed, int num_txs)
+{
+ std::ostringstream ss;
+ ss << "generate run " << num_txs << " --seed " << seed;
+
+ if (m_status.is_open()) {
+ m_status << ss.str() << std::endl;
+ m_status.flush();
+ }
+
+ dout(0) << ss.str() << dendl;
+
+ rngen_t gen(seed);
+ boost::uniform_int<> op_rng(DSOP_FIRST, DSOP_LAST);
+
+ for (txn = 1; txn <= num_txs; ) {
+ int op = op_rng(gen);
+ _print_status(txn, op);
+ dout(0) << "generate seq " << txn << " op " << op << dendl;
+ if (run_one_op(op, gen))
+ txn++;
+ }
+}
+
+void DeterministicOpSequence::_print_status(int seq, int op)
+{
+ if (!m_status.is_open())
+ return;
+ m_status << seq << " " << op << std::endl;
+ m_status.flush();
+}
+
+int DeterministicOpSequence::_gen_coll_id(rngen_t& gen)
+{
+ boost::uniform_int<> coll_rng(0, m_collections_ids.size()-1);
+ return coll_rng(gen);
+}
+
+int DeterministicOpSequence::_gen_obj_id(rngen_t& gen)
+{
+ boost::uniform_int<> obj_rng(0, m_num_objects - 1);
+ return obj_rng(gen);
+}
+
+void DeterministicOpSequence::note_txn(ObjectStore::Transaction *t)
+{
+ bufferlist bl;
+ ::encode(txn, bl);
+ t->truncate(txn_coll, txn_object, 0);
+ t->write(txn_coll, txn_object, 0, bl.length(), bl);
+ dout(10) << __func__ << " " << txn << dendl;
+}
+
+bool DeterministicOpSequence::do_touch(rngen_t& gen)
+{
+ int coll_id = _gen_coll_id(gen);
+ int obj_id = _gen_obj_id(gen);
+
+ coll_entry_t *entry = get_coll_at(coll_id);
+ ceph_assert(entry != NULL);
+
+ // Don't care about other collections if already exists
+ if (!entry->check_for_obj(obj_id)) {
+ bool other_found = false;
+ map<int, coll_entry_t*>::iterator it = m_collections.begin();
+ for (; it != m_collections.end(); ++it) {
+ if (it->second->check_for_obj(obj_id)) {
+ ceph_assert(it->first != coll_id);
+ other_found = true;
+ }
+ }
+ if (other_found) {
+ dout(0) << "do_touch new object in collection and exists in another" << dendl;
+ return false;
+ }
+ }
+ hobject_t *obj = entry->touch_obj(obj_id);
+
+ dout(0) << "do_touch " << entry->m_coll.to_str() << "/" << obj->oid.name << dendl;
+
+ _do_touch(entry->m_coll, *obj);
+ return true;
+}
+
+bool DeterministicOpSequence::do_remove(rngen_t& gen)
+{
+ int coll_id = _gen_coll_id(gen);
+
+ coll_entry_t *entry = get_coll_at(coll_id);
+ ceph_assert(entry != NULL);
+
+ if (entry->m_objects.size() == 0) {
+ dout(0) << "do_remove no objects in collection" << dendl;
+ return false;
+ }
+ int obj_id = entry->get_random_obj_id(gen);
+ hobject_t *obj = entry->touch_obj(obj_id);
+ ceph_assert(obj);
+
+ dout(0) << "do_remove " << entry->m_coll.to_str() << "/" << obj->oid.name << dendl;
+
+ _do_remove(entry->m_coll, *obj);
+ hobject_t *rmobj = entry->remove_obj(obj_id);
+ ceph_assert(rmobj);
+ delete rmobj;
+ return true;
+}
+
+static void _gen_random(rngen_t& gen,
+ size_t size, bufferlist& bl) {
+
+ static const char alphanum[] = "0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz";
+
+ boost::uniform_int<> char_rng(0, sizeof(alphanum));
+ bufferptr bp(size);
+ for (unsigned int i = 0; i < size - 1; i++) {
+ bp[i] = alphanum[char_rng(gen)];
+ }
+ bp[size - 1] = '\0';
+ bl.append(bp);
+}
+
+static void gen_attrs(rngen_t &gen,
+ map<string, bufferlist> *out) {
+ boost::uniform_int<> num_rng(10, 30);
+ boost::uniform_int<> key_size_rng(5, 10);
+ boost::uniform_int<> val_size_rng(100, 1000);
+ size_t num_attrs = static_cast<size_t>(num_rng(gen));
+ for (size_t i = 0; i < num_attrs; ++i) {
+ size_t key_size = static_cast<size_t>(num_rng(gen));
+ size_t val_size = static_cast<size_t>(num_rng(gen));
+ bufferlist keybl;
+ _gen_random(gen, key_size, keybl);
+ string key(keybl.c_str(), keybl.length());
+ _gen_random(gen, val_size, (*out)[key]);
+ }
+}
+
+bool DeterministicOpSequence::do_set_attrs(rngen_t& gen)
+{
+ int coll_id = _gen_coll_id(gen);
+
+ coll_entry_t *entry = get_coll_at(coll_id);
+ ceph_assert(entry != NULL);
+
+ if (entry->m_objects.size() == 0) {
+ dout(0) << "do_set_attrs no objects in collection" << dendl;
+ return false;
+ }
+ int obj_id = entry->get_random_obj_id(gen);
+ hobject_t *obj = entry->touch_obj(obj_id);
+ ceph_assert(obj);
+
+ map<string, bufferlist> out;
+ gen_attrs(gen, &out);
+
+ dout(0) << "do_set_attrs " << out.size() << " entries" << dendl;
+ _do_set_attrs(entry->m_coll, *obj, out);
+ return true;
+}
+
+bool DeterministicOpSequence::do_write(rngen_t& gen)
+{
+ int coll_id = _gen_coll_id(gen);
+
+ coll_entry_t *entry = get_coll_at(coll_id);
+ ceph_assert(entry != NULL);
+
+ if (entry->m_objects.size() == 0) {
+ dout(0) << "do_write no objects in collection" << dendl;
+ return false;
+ }
+ int obj_id = entry->get_random_obj_id(gen);
+ hobject_t *obj = entry->touch_obj(obj_id);
+ ceph_assert(obj);
+
+ boost::uniform_int<> size_rng(100, (2 << 19));
+ size_t size = (size_t) size_rng(gen);
+ bufferlist bl;
+ _gen_random(gen, size, bl);
+
+ dout(0) << "do_write " << entry->m_coll.to_str() << "/" << obj->oid.name
+ << " 0~" << size << dendl;
+
+ _do_write(entry->m_coll, *obj, 0, bl.length(), bl);
+ return true;
+}
+
+bool DeterministicOpSequence::_prepare_clone(rngen_t& gen,
+ coll_t& coll_ret, hobject_t& orig_obj_ret, hobject_t& new_obj_ret)
+{
+ int coll_id = _gen_coll_id(gen);
+
+ coll_entry_t *entry = get_coll_at(coll_id);
+ ceph_assert(entry != NULL);
+
+ if (entry->m_objects.size() >= 2) {
+ dout(0) << "_prepare_clone coll " << entry->m_coll.to_str()
+ << " doesn't have 2 or more objects" << dendl;
+ return false;
+ }
+
+ int orig_obj_id = entry->get_random_obj_id(gen);
+ hobject_t *orig_obj = entry->touch_obj(orig_obj_id);
+ ceph_assert(orig_obj);
+
+ int id;
+ do {
+ id = entry->get_random_obj_id(gen);
+ } while (id == orig_obj_id);
+ hobject_t *new_obj = entry->touch_obj(id);
+ ceph_assert(new_obj);
+
+ coll_ret = entry->m_coll;
+ orig_obj_ret = *orig_obj;
+ new_obj_ret = *new_obj;
+
+ return true;
+}
+
+bool DeterministicOpSequence::do_clone(rngen_t& gen)
+{
+ coll_t coll;
+ hobject_t orig_obj, new_obj;
+ if (!_prepare_clone(gen, coll, orig_obj, new_obj)) {
+ return false;
+ }
+
+ dout(0) << "do_clone " << coll.to_str() << "/" << orig_obj.oid.name
+ << " => " << coll.to_str() << "/" << new_obj.oid.name << dendl;
+
+ _do_clone(coll, orig_obj, new_obj);
+ return true;
+}
+
+bool DeterministicOpSequence::do_clone_range(rngen_t& gen)
+{
+ coll_t coll;
+ hobject_t orig_obj, new_obj;
+ if (!_prepare_clone(gen, coll, orig_obj, new_obj)) {
+ return false;
+ }
+
+ /* Whenever we have to make a clone_range() operation, just write to the
+ * object first, so we know we have something to clone in the said range.
+ * This may not be the best solution ever, but currently we're not keeping
+ * track of the written-to objects, and until we know for sure we really
+ * need to, let's just focus on the task at hand.
+ */
+
+ boost::uniform_int<> write_size_rng(100, (2 << 19));
+ size_t size = (size_t) write_size_rng(gen);
+ bufferlist bl;
+ _gen_random(gen, size, bl);
+
+ boost::uniform_int<> clone_len(1, bl.length());
+ size = (size_t) clone_len(gen);
+
+ dout(0) << "do_clone_range " << coll.to_str() << "/" << orig_obj.oid.name
+ << " (0~" << size << ")"
+ << " => " << coll.to_str() << "/" << new_obj.oid.name
+ << " (0)" << dendl;
+ _do_write_and_clone_range(coll, orig_obj, new_obj, 0, size, 0, bl);
+ return true;
+}
+
+bool DeterministicOpSequence::_prepare_colls(rngen_t& gen,
+ coll_entry_t* &orig_coll, coll_entry_t* &new_coll)
+{
+ ceph_assert(m_collections_ids.size() > 1);
+ int orig_coll_id = _gen_coll_id(gen);
+ int new_coll_id;
+ do {
+ new_coll_id = _gen_coll_id(gen);
+ } while (new_coll_id == orig_coll_id);
+
+ dout(0) << "_prepare_colls from coll id " << orig_coll_id
+ << " to coll id " << new_coll_id << dendl;
+
+ orig_coll = get_coll_at(orig_coll_id);
+ ceph_assert(orig_coll != NULL);
+ new_coll = get_coll_at(new_coll_id);
+ ceph_assert(new_coll != NULL);
+
+ if (!orig_coll->m_objects.size()) {
+ dout(0) << "_prepare_colls coll " << orig_coll->m_coll.to_str()
+ << " has no objects to use" << dendl;
+ return false;
+ }
+
+ return true;
+}
+
+
+bool DeterministicOpSequence::do_coll_rename(rngen_t& gen)
+{
+ int coll_pos = _gen_coll_id(gen);
+ dout(0) << "do_coll_rename coll pos #" << coll_pos << dendl;
+
+ coll_entry_t *coll_entry = get_coll_at(coll_pos);
+ if (!coll_entry) {
+ dout(0) << "do_coll_rename no collection at pos #" << coll_pos << dendl;
+ return false;
+ }
+
+ coll_t orig_coll = coll_entry->m_coll;
+ char buf[100];
+ memset(buf, 0, 100);
+ snprintf(buf, 100, "0.%d_head", m_next_coll_nr++);
+ coll_t new_coll(buf);
+ coll_entry->m_coll = new_coll;
+
+ dout(0) << "do_coll_rename " << orig_coll.to_str()
+ << " => " << new_coll.to_str() << dendl;
+ _do_coll_rename(orig_coll, new_coll);
+ return true;
+}
+
+bool DeterministicOpSequence::do_coll_add(rngen_t& gen)
+{
+ coll_entry_t *orig_coll = NULL, *new_coll = NULL;
+ if (!_prepare_colls(gen, orig_coll, new_coll))
+ return false;
+
+ ceph_assert(orig_coll && new_coll);
+
+ boost::uniform_int<> obj_rng(0, orig_coll->m_objects.size()-1);
+ int obj_pos = obj_rng(gen);
+ int obj_key = -1;
+ hobject_t *obj = orig_coll->get_obj_at(obj_pos, &obj_key);
+ if (!obj) {
+ dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
+ << " has no object as pos #" << obj_pos << " (key " << obj_key << ")"
+ << dendl;
+ return false;
+ }
+ if (new_coll->check_for_obj(obj_key)) {
+ dout(0) << "do_coll_add coll " << orig_coll->m_coll.to_str()
+ << " already has object as pos #" << obj_pos << " (key " << obj_key << ")"
+ << dendl;
+ return false;
+ }
+ dout(0) << "do_coll_add " << orig_coll->m_coll.to_str() << "/" << obj->oid.name
+ << " => " << new_coll->m_coll.to_str() << "/" << obj->oid.name << dendl;
+ new_coll->touch_obj(obj_key);
+
+ _do_coll_add(orig_coll->m_coll, new_coll->m_coll, *obj);
+ return true;
+}
+
+void DeterministicOpSequence::_do_touch(coll_t coll, hobject_t& obj)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.touch(coll, obj);
+ m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_remove(coll_t coll, hobject_t& obj)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.remove(coll, obj);
+ m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_set_attrs(coll_t coll,
+ hobject_t &obj,
+ const map<string, bufferlist> &attrs)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.omap_setkeys(coll, obj, attrs);
+ m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_write(coll_t coll, hobject_t& obj,
+ uint64_t off, uint64_t len, const bufferlist& data)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.write(coll, obj, off, len, data);
+ m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_clone(coll_t coll, hobject_t& orig_obj,
+ hobject_t& new_obj)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.clone(coll, orig_obj, new_obj);
+ m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_clone_range(coll_t coll,
+ hobject_t& orig_obj, hobject_t& new_obj, uint64_t srcoff,
+ uint64_t srclen, uint64_t dstoff)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.clone_range(coll, orig_obj, new_obj, srcoff, srclen, dstoff);
+ m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_write_and_clone_range(coll_t coll,
+ hobject_t& orig_obj,
+ hobject_t& new_obj,
+ uint64_t srcoff,
+ uint64_t srclen,
+ uint64_t dstoff,
+ bufferlist& bl)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.write(coll, orig_obj, srcoff, bl.length(), bl);
+ t.clone_range(coll, orig_obj, new_obj, srcoff, srclen, dstoff);
+ m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_coll_add(coll_t orig_coll, coll_t new_coll,
+ hobject_t& obj)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.remove(new_coll, obj);
+ t.collection_add(new_coll, orig_coll, obj);
+ m_store->apply_transaction(t);
+}
+
+void DeterministicOpSequence::_do_coll_rename(coll_t orig_coll, coll_t new_coll)
+{
+ ObjectStore::Transaction t;
+ note_txn(&t);
+ t.collection_rename(orig_coll, new_coll);
+ m_store->apply_transaction(t);
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef FILESTORE_DTRMNSTC_SEQ_OPS_H_
+#define FILESTORE_DTRMNSTC_SEQ_OPS_H_
+
+#include <iostream>
+#include <fstream>
+#include <set>
+#include "os/ObjectStore.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/random/mersenne_twister.hpp>
+#include <boost/random/uniform_int.hpp>
+
+#include "TestObjectStoreState.h"
+
+typedef boost::mt11213b rngen_t;
+
+class DeterministicOpSequence : public TestObjectStoreState {
+ public:
+ DeterministicOpSequence(ObjectStore *store, std::string status = std::string());
+ virtual ~DeterministicOpSequence();
+
+ virtual void generate(int seed, int num_txs);
+
+ protected:
+ enum {
+ DSOP_TOUCH = 0,
+ DSOP_WRITE = 1,
+ DSOP_CLONE = 2,
+ DSOP_CLONE_RANGE = 3,
+ DSOP_OBJ_REMOVE = 4,
+ DSOP_COLL_RENAME = 5,
+ DSOP_COLL_ADD = 6,
+ DSOP_SET_ATTRS = 7,
+
+ DSOP_FIRST = DSOP_TOUCH,
+ DSOP_LAST = DSOP_SET_ATTRS,
+ };
+
+ int32_t txn;
+
+ coll_t txn_coll;
+ hobject_t txn_object;
+
+ ObjectStore::Sequencer m_osr;
+ std::ofstream m_status;
+
+ bool run_one_op(int op, rngen_t& gen);
+
+ void note_txn(ObjectStore::Transaction *t);
+ bool do_touch(rngen_t& gen);
+ bool do_remove(rngen_t& gen);
+ bool do_write(rngen_t& gen);
+ bool do_clone(rngen_t& gen);
+ bool do_clone_range(rngen_t& gen);
+ bool do_coll_rename(rngen_t& gen);
+ bool do_coll_add(rngen_t& gen);
+ bool do_set_attrs(rngen_t& gen);
+
+ virtual void _do_touch(coll_t coll, hobject_t& obj);
+ virtual void _do_remove(coll_t coll, hobject_t& obj);
+ virtual void _do_write(coll_t coll, hobject_t& obj, uint64_t off,
+ uint64_t len, const bufferlist& data);
+ virtual void _do_set_attrs(coll_t coll,
+ hobject_t &obj,
+ const map<string, bufferlist> &attrs);
+ virtual void _do_clone(coll_t coll, hobject_t& orig_obj, hobject_t& new_obj);
+ virtual void _do_clone_range(coll_t coll, hobject_t& orig_obj,
+ hobject_t& new_obj, uint64_t srcoff, uint64_t srclen, uint64_t dstoff);
+ virtual void _do_write_and_clone_range(coll_t coll, hobject_t& orig_obj,
+ hobject_t& new_obj, uint64_t srcoff, uint64_t srclen,
+ uint64_t dstoff, bufferlist& bl);
+ virtual void _do_coll_add(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
+ virtual void _do_coll_rename(coll_t orig_coll, coll_t new_coll);
+
+ int _gen_coll_id(rngen_t& gen);
+ int _gen_obj_id(rngen_t& gen);
+ void _print_status(int seq, int op);
+
+ private:
+ bool _prepare_clone(rngen_t& gen, coll_t& coll_ret,
+ hobject_t& orig_obj_ret, hobject_t& new_obj_ret);
+ bool _prepare_colls(rngen_t& gen,
+ coll_entry_t* &orig_coll, coll_entry_t* &new_coll);
+};
+
+
+#endif /* FILESTORE_DTRMNSTC_SEQ_OPS_H_ */
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <map>
+#include <boost/scoped_ptr.hpp>
+#include "common/debug.h"
+#include "os/FileStore.h"
+#include "common/config.h"
+
+#include "FileStoreDiff.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "filestore_diff "
+
+FileStoreDiff::FileStoreDiff(FileStore *a, FileStore *b)
+ : a_store(a), b_store(b)
+{
+ int err;
+ err = a_store->mount();
+ ceph_assert(err == 0);
+
+ err = b_store->mount();
+ ceph_assert(err == 0);
+}
+
+FileStoreDiff::~FileStoreDiff()
+{
+ a_store->umount();
+ b_store->umount();
+}
+
+
+bool FileStoreDiff::diff_attrs(std::map<std::string,bufferptr>& b,
+ std::map<std::string,bufferptr>& a)
+{
+ bool ret = false;
+ std::map<std::string, bufferptr>::iterator b_it = b.begin();
+ std::map<std::string, bufferptr>::iterator a_it = a.begin();
+ for (; b_it != b.end(); ++b_it, ++a_it) {
+ if (b_it->first != a_it->first) {
+ dout(0) << "diff_attrs name mismatch (verify: " << b_it->first
+ << ", store: " << a_it->first << ")" << dendl;
+ ret = true;
+ continue;
+ }
+
+ if (!b_it->second.cmp(a_it->second)) {
+ dout(0) << "diff_attrs contents mismatch on attr " << b_it->first << dendl;
+ ret = true;
+ continue;
+ }
+ }
+ return ret;
+}
+
+static bool diff_omap(std::map<std::string,bufferlist>& b,
+ std::map<std::string,bufferlist>& a)
+{
+ bool ret = false;
+ std::map<std::string, bufferlist>::iterator b_it = b.begin();
+ std::map<std::string, bufferlist>::iterator a_it = a.begin();
+ for (; b_it != b.end(); ++b_it, ++a_it) {
+ if (b_it->first != a_it->first) {
+ dout(0) << "diff_attrs name mismatch (verify: " << b_it->first
+ << ", store: " << a_it->first << ")" << dendl;
+ ret = true;
+ continue;
+ }
+
+ if (!(b_it->second == a_it->second)) {
+ dout(0) << "diff_attrs contents mismatch on attr " << b_it->first << dendl;
+ ret = true;
+ continue;
+ }
+ }
+ return ret;
+}
+
+bool FileStoreDiff::diff_objects_stat(struct stat& a, struct stat& b)
+{
+ bool ret = false;
+
+ if (a.st_uid != b.st_uid) {
+ dout(0) << "diff_objects_stat uid mismatch (A: "
+ << a.st_uid << " != B: " << b.st_uid << ")" << dendl;
+ ret = true;
+ }
+
+ if (a.st_gid != b.st_gid) {
+ dout(0) << "diff_objects_stat gid mismatch (A: "
+ << a.st_gid << " != B: " << b.st_gid << ")" << dendl;
+ ret = true;
+ }
+
+ if (a.st_mode != b.st_mode) {
+ dout(0) << "diff_objects_stat mode mismatch (A: "
+ << a.st_mode << " != B: " << b.st_mode << ")" << dendl;
+ ret = true;
+ }
+
+ if (a.st_nlink != b.st_nlink) {
+ dout(0) << "diff_objects_stat nlink mismatch (A: "
+ << a.st_nlink << " != B: " << b.st_nlink << ")" << dendl;
+ ret = true;
+ }
+
+ if (a.st_size != b.st_size) {
+ dout(0) << "diff_objects_stat size mismatch (A: "
+ << a.st_size << " != B: " << b.st_size << ")" << dendl;
+ ret = true;
+ }
+ return ret;
+}
+
+bool FileStoreDiff::diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll)
+{
+ dout(2) << __func__ << " coll " << coll << dendl;
+
+ bool ret = false;
+
+ int err;
+ std::vector<ghobject_t> b_objects, a_objects;
+ err = b_store->collection_list(coll, b_objects);
+ if (err < 0) {
+ dout(0) << "diff_objects list on verify coll " << coll.to_str()
+ << " returns " << err << dendl;
+ return true;
+ }
+ err = a_store->collection_list(coll, a_objects);
+ if (err < 0) {
+ dout(0) << "diff_objects list on store coll " << coll.to_str()
+ << " returns " << err << dendl;
+ return true;
+ }
+
+ if (b_objects.size() != a_objects.size()) {
+ dout(0) << "diff_objects num objs mismatch (A: " << a_objects.size()
+ << ", B: " << b_objects.size() << ")" << dendl;
+ ret = true;
+ }
+
+ std::vector<ghobject_t>::iterator b_it = b_objects.begin();
+ std::vector<ghobject_t>::iterator a_it = b_objects.begin();
+ for (; b_it != b_objects.end(); ++b_it, ++a_it) {
+ ghobject_t b_obj = *b_it, a_obj = *a_it;
+ if (b_obj.hobj.oid.name != a_obj.hobj.oid.name) {
+ dout(0) << "diff_objects name mismatch on A object "
+ << coll << "/" << a_obj << " and B object "
+ << coll << "/" << b_obj << dendl;
+ ret = true;
+ continue;
+ }
+
+ struct stat b_stat, a_stat;
+ err = b_store->stat(coll, b_obj, &b_stat);
+ if (err < 0) {
+ dout(0) << "diff_objects error stating B object "
+ << coll.to_str() << "/" << b_obj.hobj.oid.name << dendl;
+ ret = true;
+ }
+ err = a_store->stat(coll, a_obj, &a_stat);
+ if (err < 0) {
+ dout(0) << "diff_objects error stating A object "
+ << coll << "/" << a_obj << dendl;
+ ret = true;
+ }
+
+ if (diff_objects_stat(a_stat, b_stat)) {
+ dout(0) << "diff_objects stat mismatch on "
+ << coll << "/" << b_obj << dendl;
+ ret = true;
+ }
+
+ bufferlist a_obj_bl, b_obj_bl;
+ b_store->read(coll, b_obj, 0, b_stat.st_size, b_obj_bl);
+ a_store->read(coll, a_obj, 0, a_stat.st_size, a_obj_bl);
+
+ if (!a_obj_bl.contents_equal(b_obj_bl)) {
+ dout(0) << "diff_objects content mismatch on "
+ << coll << "/" << b_obj << dendl;
+ ret = true;
+ }
+
+ std::map<std::string, bufferptr> a_obj_attrs_map, b_obj_attrs_map;
+ err = a_store->getattrs(coll, a_obj, a_obj_attrs_map);
+ if (err < 0) {
+ dout(0) << "diff_objects getattrs on A object " << coll << "/" << a_obj
+ << " returns " << err << dendl;
+ ret = true;
+ }
+ err = b_store->getattrs(coll, b_obj, b_obj_attrs_map);
+ if (err < 0) {
+ dout(0) << "diff_objects getattrs on B object " << coll << "/" << b_obj
+ << "returns " << err << dendl;
+ ret = true;
+ }
+
+ if (diff_attrs(b_obj_attrs_map, a_obj_attrs_map)) {
+ dout(0) << "diff_objects attrs mismatch on A object "
+ << coll << "/" << a_obj << " and B object "
+ << coll << "/" << b_obj << dendl;
+ ret = true;
+ }
+
+ std::map<std::string, bufferlist> a_obj_omap, b_obj_omap;
+ std::set<std::string> a_omap_keys, b_omap_keys;
+ err = a_store->omap_get_keys(coll, a_obj, &a_omap_keys);
+ if (err < 0) {
+ dout(0) << "diff_objects getomap on A object " << coll << "/" << a_obj
+ << " returns " << err << dendl;
+ ret = true;
+ }
+ err = a_store->omap_get_values(coll, a_obj, a_omap_keys, &a_obj_omap);
+ if (err < 0) {
+ dout(0) << "diff_objects getomap on A object " << coll << "/" << a_obj
+ << " returns " << err << dendl;
+ ret = true;
+ }
+ err = b_store->omap_get_keys(coll, b_obj, &b_omap_keys);
+ if (err < 0) {
+ dout(0) << "diff_objects getomap on A object " << coll << "/" << b_obj
+ << " returns " << err << dendl;
+ ret = true;
+ }
+ err = b_store->omap_get_values(coll, b_obj, b_omap_keys, &b_obj_omap);
+ if (err < 0) {
+ dout(0) << "diff_objects getomap on A object " << coll << "/" << b_obj
+ << " returns " << err << dendl;
+ ret = true;
+ }
+ if (diff_omap(a_obj_omap, b_obj_omap)) {
+ dout(0) << "diff_objects omap mismatch on A object "
+ << coll << "/" << a_obj << " and B object "
+ << coll << "/" << b_obj << dendl;
+ ret = true;
+ }
+ }
+
+ return ret;
+}
+
+bool FileStoreDiff::diff_coll_attrs(FileStore *a_store, FileStore *b_store, coll_t coll)
+{
+ bool ret = false;
+
+ int err;
+ std::map<std::string, bufferptr> b_coll_attrs, a_coll_attrs;
+ err = b_store->collection_getattrs(coll, b_coll_attrs);
+ if (err < 0) {
+ dout(0) << "diff_attrs getattrs on verify coll " << coll.to_str()
+ << "returns " << err << dendl;
+ ret = true;
+ }
+ err = a_store->collection_getattrs(coll, a_coll_attrs);
+ if (err < 0) {
+ dout(0) << "diff_attrs getattrs on A coll " << coll.to_str()
+ << "returns " << err << dendl;
+ ret = true;
+ }
+
+ if (b_coll_attrs.size() != a_coll_attrs.size()) {
+ dout(0) << "diff_attrs size mismatch (A: " << a_coll_attrs.size()
+ << ", B: " << a_coll_attrs.size() << ")" << dendl;
+ ret = true;
+ }
+
+ return diff_attrs(b_coll_attrs, a_coll_attrs) || ret;
+}
+
+bool FileStoreDiff::diff()
+{
+ bool ret = false;
+
+ std::vector<coll_t> a_coll_list, b_coll_list;
+ a_store->list_collections(a_coll_list);
+ b_store->list_collections(b_coll_list);
+
+ std::vector<coll_t>::iterator it = b_coll_list.begin();
+ for (; it != b_coll_list.end(); ++it) {
+ coll_t b_coll = *it;
+ if (!a_store->collection_exists(b_coll)) {
+ dout(0) << "diff B coll " << b_coll.to_str() << " DNE on A" << dendl;
+ ret = true;
+ continue;
+ }
+ for (std::vector<coll_t>::iterator j = a_coll_list.begin();
+ j != a_coll_list.end(); ++j) {
+ if (*j == *it) {
+ a_coll_list.erase(j);
+ break;
+ }
+ }
+
+ if (diff_coll_attrs(a_store, b_store, b_coll))
+ ret = true;
+
+ if (diff_objects(a_store, b_store, b_coll))
+ ret = true;
+ }
+ for (std::vector<coll_t>::iterator it = a_coll_list.begin();
+ it != a_coll_list.end(); ++it) {
+ dout(0) << "diff A coll " << *it << " DNE on B" << dendl;
+ ret = true;
+ }
+
+ return ret;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef FILESTORE_DIFF_H_
+#define FILESTORE_DIFF_H_
+
+#include <iostream>
+#include <stdlib.h>
+#include <map>
+#include <boost/scoped_ptr.hpp>
+#include "common/debug.h"
+#include "os/FileStore.h"
+#include "common/config.h"
+
+class FileStoreDiff {
+
+ private:
+ FileStore *a_store;
+ FileStore *b_store;
+
+ bool diff_coll_attrs(FileStore *a_store, FileStore *b_store, coll_t coll);
+ bool diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll);
+ bool diff_objects_stat(struct stat& a, struct stat& b);
+ bool diff_attrs(std::map<std::string,bufferptr>& b,
+ std::map<std::string,bufferptr>& a);
+
+public:
+ FileStoreDiff(FileStore *a, FileStore *b);
+ virtual ~FileStoreDiff();
+
+ bool diff();
+};
+
+#endif /* FILESTOREDIFF_H_ */
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+#include "FileStoreTracker.h"
+#include <stdlib.h>
+#include <iostream>
+#include <boost/scoped_ptr.hpp>
+#include "include/Context.h"
+#include "common/Mutex.h"
+
+class OnApplied : public Context {
+ FileStoreTracker *tracker;
+ list<pair<pair<string, string>, uint64_t> > in_flight;
+ ObjectStore::Transaction *t;
+public:
+ OnApplied(FileStoreTracker *tracker,
+ list<pair<pair<string, string>, uint64_t> > in_flight,
+ ObjectStore::Transaction *t)
+ : tracker(tracker), in_flight(in_flight), t(t) {}
+
+ void finish(int r) {
+ for (list<pair<pair<string, string>, uint64_t> >::iterator i =
+ in_flight.begin();
+ i != in_flight.end();
+ ++i) {
+ tracker->applied(i->first, i->second);
+ }
+ delete t;
+ }
+};
+
+class OnCommitted : public Context {
+ FileStoreTracker *tracker;
+ list<pair<pair<string, string>, uint64_t> > in_flight;
+public:
+ OnCommitted(FileStoreTracker *tracker,
+ list<pair<pair<string, string>, uint64_t> > in_flight)
+ : tracker(tracker), in_flight(in_flight) {}
+
+ void finish(int r) {
+ for (list<pair<pair<string, string>, uint64_t> >::iterator i =
+ in_flight.begin();
+ i != in_flight.end();
+ ++i) {
+ tracker->committed(i->first, i->second);
+ }
+ }
+};
+
+int FileStoreTracker::init()
+{
+ set<string> to_get;
+ to_get.insert("STATUS");
+ map<string, bufferlist> got;
+ db->get("STATUS", to_get, &got);
+ restart_seq = 0;
+ if (!got.empty()) {
+ bufferlist::iterator bp = got.begin()->second.begin();
+ ::decode(restart_seq, bp);
+ }
+ ++restart_seq;
+ KeyValueDB::Transaction t = db->get_transaction();
+ got.clear();
+ ::encode(restart_seq, got["STATUS"]);
+ t->set("STATUS", got);
+ db->submit_transaction(t);
+ return 0;
+}
+
+void FileStoreTracker::submit_transaction(Transaction &t)
+{
+ list<pair<pair<string, string>, uint64_t> > in_flight;
+ OutTransaction out;
+ out.t = new ObjectStore::Transaction;
+ out.in_flight = &in_flight;
+ for (list<Transaction::Op*>::iterator i = t.ops.begin();
+ i != t.ops.end();
+ ++i) {
+ (**i)(this, &out);
+ }
+ store->queue_transaction(
+ 0, out.t,
+ new OnApplied(this, in_flight, out.t),
+ new OnCommitted(this, in_flight));
+}
+
+void FileStoreTracker::write(const pair<string, string> &obj,
+ OutTransaction *out)
+{
+ Mutex::Locker l(lock);
+ std::cerr << "Writing " << obj << std::endl;
+ ObjectContents contents = get_current_content(obj);
+
+ uint64_t offset = rand() % (SIZE/2);
+ uint64_t len = rand() % (SIZE/2);
+ if (!len) len = 10;
+ contents.write(rand(), offset, len);
+
+ bufferlist to_write;
+ ObjectContents::Iterator iter = contents.get_iterator();
+ iter.seek_to(offset);
+ for (uint64_t i = offset;
+ i < offset + len;
+ ++i, ++iter) {
+ assert(iter.valid());
+ to_write.append(*iter);
+ }
+ out->t->write(coll_t(obj.first),
+ hobject_t(sobject_t(obj.second, CEPH_NOSNAP)),
+ offset,
+ len,
+ to_write);
+ out->in_flight->push_back(make_pair(obj, set_content(obj, contents)));
+}
+
+void FileStoreTracker::remove(const pair<string, string> &obj,
+ OutTransaction *out)
+{
+ std::cerr << "Deleting " << obj << std::endl;
+ Mutex::Locker l(lock);
+ ObjectContents old_contents = get_current_content(obj);
+ if (!old_contents.exists())
+ return;
+ out->t->remove(coll_t(obj.first),
+ hobject_t(sobject_t(obj.second, CEPH_NOSNAP)));
+ ObjectContents contents;
+ out->in_flight->push_back(make_pair(obj, set_content(obj, contents)));
+}
+
+void FileStoreTracker::clone_range(const pair<string, string> &from,
+ const pair<string, string> &to,
+ OutTransaction *out) {
+ Mutex::Locker l(lock);
+ std::cerr << "CloningRange " << from << " to " << to << std::endl;
+ assert(from.first == to.first);
+ ObjectContents from_contents = get_current_content(from);
+ ObjectContents to_contents = get_current_content(to);
+ if (!from_contents.exists()) {
+ return;
+ }
+ if (from.second == to.second) {
+ return;
+ }
+
+ uint64_t new_size = from_contents.size();
+ interval_set<uint64_t> interval_to_clone;
+ uint64_t offset = rand() % (new_size/2);
+ uint64_t len = rand() % (new_size/2);
+ if (!len) len = 10;
+ interval_to_clone.insert(offset, len);
+ to_contents.clone_range(from_contents, interval_to_clone);
+ out->t->clone_range(coll_t(from.first),
+ hobject_t(sobject_t(from.second, CEPH_NOSNAP)),
+ hobject_t(sobject_t(to.second, CEPH_NOSNAP)),
+ offset,
+ len,
+ offset);
+ out->in_flight->push_back(make_pair(to, set_content(to, to_contents)));
+}
+
+void FileStoreTracker::clone(const pair<string, string> &from,
+ const pair<string, string> &to,
+ OutTransaction *out) {
+ Mutex::Locker l(lock);
+ std::cerr << "Cloning " << from << " to " << to << std::endl;
+ assert(from.first == to.first);
+ if (from.second == to.second) {
+ return;
+ }
+ ObjectContents from_contents = get_current_content(from);
+ ObjectContents to_contents = get_current_content(to);
+ if (!from_contents.exists()) {
+ return;
+ }
+
+ if (to_contents.exists())
+ out->t->remove(coll_t(to.first),
+ hobject_t(sobject_t(to.second, CEPH_NOSNAP)));
+ out->t->clone(coll_t(from.first),
+ hobject_t(sobject_t(from.second, CEPH_NOSNAP)),
+ hobject_t(sobject_t(to.second, CEPH_NOSNAP)));
+ out->in_flight->push_back(make_pair(to, set_content(to, from_contents)));
+}
+
+
+string obj_to_prefix(const pair<string, string> &obj) {
+ string sep;
+ sep.push_back('^');
+ return obj.first + sep + obj.second + "_CONTENTS_";
+}
+
+string obj_to_meta_prefix(const pair<string, string> &obj) {
+ string sep;
+ sep.push_back('^');
+ return obj.first + sep + obj.second;
+}
+
+string seq_to_key(uint64_t seq) {
+ char buf[50];
+ snprintf(buf, sizeof(buf), "%*llu", 20, (unsigned long long int)seq);
+ return string(buf);
+}
+
+struct ObjStatus {
+ uint64_t last_applied;
+ uint64_t last_committed;
+ uint64_t restart_seq;
+ ObjStatus() : last_applied(0), last_committed(0), restart_seq(0) {}
+
+ uint64_t get_last_applied(uint64_t seq) const {
+ if (seq > restart_seq)
+ return last_committed;
+ else
+ return last_applied;
+ }
+ void set_last_applied(uint64_t _last_applied, uint64_t seq) {
+ last_applied = _last_applied;
+ restart_seq = seq;
+ }
+ uint64_t trim_to() const {
+ return last_applied < last_committed ?
+ last_applied : last_committed;
+ }
+};
+void encode(const ObjStatus &obj, bufferlist &bl) {
+ ::encode(obj.last_applied, bl);
+ ::encode(obj.last_committed, bl);
+ ::encode(obj.restart_seq, bl);
+}
+void decode(ObjStatus &obj, bufferlist::iterator &bl) {
+ ::decode(obj.last_applied, bl);
+ ::decode(obj.last_committed, bl);
+ ::decode(obj.restart_seq, bl);
+}
+
+
+ObjStatus get_obj_status(const pair<string, string> &obj,
+ KeyValueDB *db)
+{
+ set<string> to_get;
+ to_get.insert("META");
+ map<string, bufferlist> got;
+ db->get(obj_to_meta_prefix(obj), to_get, &got);
+ ObjStatus retval;
+ if (!got.empty()) {
+ bufferlist::iterator bp = got.begin()->second.begin();
+ ::decode(retval, bp);
+ }
+ return retval;
+}
+
+void set_obj_status(const pair<string, string> &obj,
+ const ObjStatus &status,
+ KeyValueDB::Transaction t)
+{
+ map<string, bufferlist> to_set;
+ ::encode(status, to_set["META"]);
+ t->set(obj_to_meta_prefix(obj), to_set);
+}
+
+void _clean_forward(const pair<string, string> &obj,
+ uint64_t last_valid,
+ KeyValueDB *db)
+{
+ KeyValueDB::Transaction t = db->get_transaction();
+ KeyValueDB::Iterator i = db->get_iterator(obj_to_prefix(obj));
+ set<string> to_remove;
+ i->upper_bound(seq_to_key(last_valid));
+ for (; i->valid(); i->next()) {
+ to_remove.insert(i->key());
+ }
+ t->rmkeys(obj_to_prefix(obj), to_remove);
+ db->submit_transaction(t);
+}
+
+
+void FileStoreTracker::verify(const string &coll, const string &obj,
+ bool on_start) {
+ Mutex::Locker l(lock);
+ std::cerr << "Verifying " << make_pair(coll, obj) << std::endl;
+
+ pair<uint64_t, uint64_t> valid_reads = get_valid_reads(make_pair(coll, obj));
+ std::cerr << "valid_reads is " << valid_reads << std::endl;
+ bufferlist contents;
+ int r = store->read(coll_t(coll),
+ hobject_t(sobject_t(obj, CEPH_NOSNAP)),
+ 0,
+ 2*SIZE,
+ contents);
+ std::cerr << "exists: " << r << std::endl;
+
+
+ for (uint64_t i = valid_reads.first;
+ i < valid_reads.second;
+ ++i) {
+ ObjectContents old_contents = get_content(make_pair(coll, obj), i);
+
+ std::cerr << "old_contents exists " << old_contents.exists() << std::endl;
+ if (!old_contents.exists() && (r == -ENOENT))
+ return;
+
+ if (old_contents.exists() && (r == -ENOENT))
+ continue;
+
+ if (!old_contents.exists() && (r != -ENOENT))
+ continue;
+
+ if (contents.length() != old_contents.size()) {
+ std::cerr << "old_contents.size() is "
+ << old_contents.size() << std::endl;
+ continue;
+ }
+
+ bufferlist::iterator bp = contents.begin();
+ ObjectContents::Iterator iter = old_contents.get_iterator();
+ iter.seek_to_first();
+ bool matches = true;
+ uint64_t pos = 0;
+ for (; !bp.end() && iter.valid();
+ ++iter, ++bp, ++pos) {
+ if (*iter != *bp) {
+ std::cerr << "does not match at pos " << pos << std::endl;
+ matches = false;
+ break;
+ }
+ }
+ if (matches) {
+ if (on_start)
+ _clean_forward(make_pair(coll, obj), i, db);
+ return;
+ }
+ }
+ std::cerr << "Verifying " << make_pair(coll, obj) << " failed " << std::endl;
+ assert(0);
+}
+
+ObjectContents FileStoreTracker::get_current_content(
+ const pair<string, string> &obj)
+{
+ KeyValueDB::Iterator iter = db->get_iterator(
+ obj_to_prefix(obj));
+ iter->seek_to_last();
+ if (iter->valid()) {
+ bufferlist bl = iter->value();
+ bufferlist::iterator bp = bl.begin();
+ pair<uint64_t, bufferlist> val;
+ ::decode(val, bp);
+ assert(seq_to_key(val.first) == iter->key());
+ bp = val.second.begin();
+ return ObjectContents(bp);
+ }
+ return ObjectContents();
+}
+
+ObjectContents FileStoreTracker::get_content(
+ const pair<string, string> &obj, uint64_t version)
+{
+ set<string> to_get;
+ map<string, bufferlist> got;
+ to_get.insert(seq_to_key(version));
+ db->get(obj_to_prefix(obj), to_get, &got);
+ if (got.empty())
+ return ObjectContents();
+ pair<uint64_t, bufferlist> val;
+ bufferlist::iterator bp = got.begin()->second.begin();
+ ::decode(val, bp);
+ bp = val.second.begin();
+ assert(val.first == version);
+ return ObjectContents(bp);
+}
+
+pair<uint64_t, uint64_t> FileStoreTracker::get_valid_reads(
+ const pair<string, string> &obj)
+{
+ pair<uint64_t, uint64_t> bounds = make_pair(0,1);
+ KeyValueDB::Iterator iter = db->get_iterator(
+ obj_to_prefix(obj));
+ iter->seek_to_last();
+ if (iter->valid()) {
+ pair<uint64_t, bufferlist> val;
+ bufferlist bl = iter->value();
+ bufferlist::iterator bp = bl.begin();
+ ::decode(val, bp);
+ bounds.second = val.first + 1;
+ }
+
+ ObjStatus obj_status = get_obj_status(obj, db);
+ bounds.first = obj_status.get_last_applied(restart_seq);
+ return bounds;
+}
+
+void clear_obsolete(const pair<string, string> &obj,
+ const ObjStatus &status,
+ KeyValueDB *db,
+ KeyValueDB::Transaction t)
+{
+ KeyValueDB::Iterator iter = db->get_iterator(obj_to_prefix(obj));
+ set<string> to_remove;
+ iter->seek_to_first();
+ for (; iter->valid() && iter->key() < seq_to_key(status.trim_to());
+ iter->next())
+ to_remove.insert(iter->key());
+ t->rmkeys(obj_to_prefix(obj), to_remove);
+}
+
+void FileStoreTracker::committed(const pair<string, string> &obj,
+ uint64_t seq) {
+ Mutex::Locker l(lock);
+ ObjStatus status = get_obj_status(obj, db);
+ assert(status.last_committed < seq);
+ status.last_committed = seq;
+ KeyValueDB::Transaction t = db->get_transaction();
+ clear_obsolete(obj, status, db, t);
+ set_obj_status(obj, status, t);
+ db->submit_transaction(t);
+}
+
+void FileStoreTracker::applied(const pair<string, string> &obj,
+ uint64_t seq) {
+ Mutex::Locker l(lock);
+ std::cerr << "Applied " << obj << " version " << seq << std::endl;
+ ObjStatus status = get_obj_status(obj, db);
+ assert(status.last_applied < seq);
+ status.set_last_applied(seq, restart_seq);
+ KeyValueDB::Transaction t = db->get_transaction();
+ clear_obsolete(obj, status, db, t);
+ set_obj_status(obj, status, t);
+ db->submit_transaction(t);
+}
+
+
+uint64_t FileStoreTracker::set_content(const pair<string, string> &obj,
+ ObjectContents &content) {
+ KeyValueDB::Transaction t = db->get_transaction();
+ KeyValueDB::Iterator iter = db->get_iterator(
+ obj_to_prefix(obj));
+ iter->seek_to_last();
+ uint64_t most_recent = 0;
+ if (iter->valid()) {
+ pair<uint64_t, bufferlist> val;
+ bufferlist bl = iter->value();
+ bufferlist::iterator bp = bl.begin();
+ ::decode(val, bp);
+ most_recent = val.first;
+ }
+ bufferlist buf_content;
+ content.encode(buf_content);
+ map<string, bufferlist> to_set;
+ ::encode(make_pair(most_recent + 1, buf_content),
+ to_set[seq_to_key(most_recent + 1)]);
+ t->set(obj_to_prefix(obj), to_set);
+ db->submit_transaction(t);
+ return most_recent + 1;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+
+#ifndef FILESTORE_TRACKER_H
+#define FILESTORE_TRACKER_H
+#include "test/common/ObjectContents.h"
+#include "os/FileStore.h"
+#include "os/KeyValueDB.h"
+#include <boost/scoped_ptr.hpp>
+#include <list>
+#include <map>
+#include "common/Mutex.h"
+
+class FileStoreTracker {
+ const static uint64_t SIZE = 4 * 1024;
+ ObjectStore *store;
+ KeyValueDB *db;
+ Mutex lock;
+ uint64_t restart_seq;
+
+ struct OutTransaction {
+ list<pair<pair<string, string>, uint64_t> > *in_flight;
+ ObjectStore::Transaction *t;
+ };
+public:
+ FileStoreTracker(ObjectStore *store, KeyValueDB *db)
+ : store(store), db(db),
+ lock("Tracker Lock"), restart_seq(0) {}
+
+ class Transaction {
+ class Op {
+ public:
+ virtual void operator()(FileStoreTracker *harness,
+ OutTransaction *out) = 0;
+ virtual ~Op() {};
+ };
+ list<Op*> ops;
+ class Write : public Op {
+ public:
+ string coll;
+ string oid;
+ Write(const string &coll,
+ const string &oid)
+ : coll(coll), oid(oid) {}
+ void operator()(FileStoreTracker *harness,
+ OutTransaction *out) {
+ harness->write(make_pair(coll, oid), out);
+ }
+ };
+ class CloneRange : public Op {
+ public:
+ string coll;
+ string from;
+ string to;
+ CloneRange(const string &coll,
+ const string &from,
+ const string &to)
+ : coll(coll), from(from), to(to) {}
+ void operator()(FileStoreTracker *harness,
+ OutTransaction *out) {
+ harness->clone_range(make_pair(coll, from), make_pair(coll, to),
+ out);
+ }
+ };
+ class Clone : public Op {
+ public:
+ string coll;
+ string from;
+ string to;
+ Clone(const string &coll,
+ const string &from,
+ const string &to)
+ : coll(coll), from(from), to(to) {}
+ void operator()(FileStoreTracker *harness,
+ OutTransaction *out) {
+ harness->clone(make_pair(coll, from), make_pair(coll, to),
+ out);
+ }
+ };
+ class Remove: public Op {
+ public:
+ string coll;
+ string obj;
+ Remove(const string &coll,
+ const string &obj)
+ : coll(coll), obj(obj) {}
+ void operator()(FileStoreTracker *harness,
+ OutTransaction *out) {
+ harness->remove(make_pair(coll, obj),
+ out);
+ }
+ };
+ public:
+ void write(const string &coll, const string &oid) {
+ ops.push_back(new Write(coll, oid));
+ }
+ void clone_range(const string &coll, const string &from,
+ const string &to) {
+ ops.push_back(new CloneRange(coll, from, to));
+ }
+ void clone(const string &coll, const string &from,
+ const string &to) {
+ ops.push_back(new Clone(coll, from, to));
+ }
+ void remove(const string &coll, const string &oid) {
+ ops.push_back(new Remove(coll, oid));
+ }
+ friend class FileStoreTracker;
+ };
+
+ int init();
+ void submit_transaction(Transaction &t);
+ void verify(const string &coll,
+ const string &from,
+ bool on_start = false);
+
+private:
+ ObjectContents get_current_content(const pair<string, string> &obj);
+ pair<uint64_t, uint64_t> get_valid_reads(const pair<string, string> &obj);
+ ObjectContents get_content(const pair<string, string> &obj, uint64_t version);
+
+ void committed(const pair<string, string> &obj, uint64_t seq);
+ void applied(const pair<string, string> &obj, uint64_t seq);
+ uint64_t set_content(const pair<string, string> &obj, ObjectContents &content);
+
+ // ObjectContents Operations
+ void write(const pair<string, string> &obj, OutTransaction *out);
+ void remove(const pair<string, string> &obj, OutTransaction *out);
+ void clone_range(const pair<string, string> &from,
+ const pair<string, string> &to,
+ OutTransaction *out);
+ void clone(const pair<string, string> &from,
+ const pair<string, string> &to,
+ OutTransaction *out);
+ friend class OnApplied;
+ friend class OnCommitted;
+};
+
+#endif
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <iostream>
+#include <time.h>
+#include <stdlib.h>
+#include <signal.h>
+#include "os/ObjectStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include "TestObjectStoreState.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "ceph_test_filestore_state "
+
+const coll_t TestObjectStoreState::META_COLL("meta");
+const coll_t TestObjectStoreState::TEMP_COLL("temp");
+
+void TestObjectStoreState::init(int colls, int objs)
+{
+ dout(5) << "init " << colls << " colls " << objs << " objs" << dendl;
+
+ ObjectStore::Transaction *t;
+ t = new ObjectStore::Transaction;
+
+ t->create_collection(META_COLL);
+ t->create_collection(TEMP_COLL);
+ m_store->apply_transaction(*t);
+
+ wait_for_ready();
+
+ int baseid = 0;
+ for (int i = 0; i < colls; i++) {
+ int coll_id = i;
+ coll_entry_t *entry = coll_create(coll_id);
+ dout(5) << "init create collection " << entry->m_coll.to_str()
+ << " meta " << entry->m_meta_obj.oid.name << dendl;
+
+ t = new ObjectStore::Transaction;
+ t->create_collection(entry->m_coll);
+ t->touch(META_COLL, entry->m_meta_obj);
+
+ for (int i = 0; i < objs; i++) {
+ hobject_t *obj = entry->touch_obj(i + baseid);
+ t->touch(entry->m_coll, *obj);
+ ceph_assert(i + baseid == m_num_objects);
+ m_num_objects++;
+ }
+ baseid += objs;
+
+ m_store->queue_transaction(&(entry->m_osr), t,
+ new C_OnFinished(this, t));
+ inc_in_flight();
+
+ m_collections.insert(make_pair(coll_id, entry));
+ m_collections_ids.push_back(coll_id);
+ m_next_coll_nr++;
+ }
+ dout(5) << "init has " << m_in_flight.read() << "in-flight transactions" << dendl;
+ wait_for_done();
+ dout(5) << "init finished" << dendl;
+}
+
+TestObjectStoreState::coll_entry_t *TestObjectStoreState::coll_create(int id)
+{
+ char buf[100];
+ char meta_buf[100];
+ memset(buf, 0, 100);
+ memset(meta_buf, 0, 100);
+ snprintf(buf, 100, "0.%d_head", id);
+ snprintf(meta_buf, 100, "pglog_0.%d_head", id);
+ return (new coll_entry_t(id, buf, meta_buf));
+}
+
+TestObjectStoreState::coll_entry_t*
+TestObjectStoreState::get_coll(int key, bool erase)
+{
+ dout(5) << "get_coll id " << key << dendl;
+
+ coll_entry_t *entry = NULL;
+ map<int, coll_entry_t*>::iterator it = m_collections.find(key);
+ if (it != m_collections.end()) {
+ entry = it->second;
+ if (erase) {
+ m_collections.erase(it);
+ vector<int>::iterator cid_it = m_collections_ids.begin()+(entry->m_id);
+ dout(20) << __func__ << " removing key " << key << " coll_id " << entry->m_id
+ << " iterator's entry id " << (*cid_it) << dendl;
+ m_collections_ids.erase(cid_it);
+ }
+ }
+
+ dout(5) << "get_coll id " << key;
+ if (!entry)
+ *_dout << " non-existent";
+ else
+ *_dout << " name " << entry->m_coll.to_str();
+ *_dout << dendl;
+ return entry;
+}
+
+TestObjectStoreState::coll_entry_t*
+TestObjectStoreState::get_coll_at(int pos, bool erase)
+{
+ dout(5) << "get_coll_at pos " << pos << dendl;
+
+ if (m_collections.empty())
+ return NULL;
+
+ assert((size_t) pos < m_collections_ids.size());
+
+ int coll_id = m_collections_ids[pos];
+ coll_entry_t *entry = m_collections[coll_id];
+
+ if (entry == NULL) {
+ dout(5) << "get_coll_at pos " << pos << " non-existent" << dendl;
+ return NULL;
+ }
+
+ if (erase) {
+ m_collections.erase(coll_id);
+ vector<int>::iterator it = m_collections_ids.begin()+(pos);
+ dout(20) << __func__ << " removing pos " << pos << " coll_id " << coll_id
+ << " iterator's entry id " << (*it) << dendl;
+ m_collections_ids.erase(it);
+ }
+
+ dout(5) << "get_coll_at pos " << pos << ": "
+ << entry->m_coll << "(removed: " << erase << ")" << dendl;
+
+ return entry;
+}
+
+TestObjectStoreState::coll_entry_t::~coll_entry_t()
+{
+ if (m_objects.size() > 0) {
+ map<int, hobject_t*>::iterator it = m_objects.begin();
+ for (; it != m_objects.end(); ++it) {
+ hobject_t *obj = it->second;
+ if (obj) {
+ delete obj;
+ }
+ }
+ m_objects.clear();
+ }
+}
+
+bool TestObjectStoreState::coll_entry_t::check_for_obj(int id)
+{
+ if (m_objects.count(id))
+ return true;
+ return false;
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::touch_obj(int id)
+{
+ map<int, hobject_t*>::iterator it = m_objects.find(id);
+ if (it != m_objects.end()) {
+ dout(5) << "touch_obj coll id " << m_id
+ << " name " << it->second->oid.name << dendl;
+ return it->second;
+ }
+
+ char buf[100];
+ memset(buf, 0, 100);
+ snprintf(buf, 100, "obj%d", id);
+
+ hobject_t *obj = new hobject_t(sobject_t(object_t(buf), CEPH_NOSNAP));
+ m_objects.insert(make_pair(id, obj));
+
+ dout(5) << "touch_obj coll id " << m_id << " name " << buf << dendl;
+ return obj;
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj(int id)
+{
+ return get_obj(id, false);
+}
+
+/**
+ * remove_obj - Removes object without freeing it.
+ * @param id Object's id in the map.
+ * @return The object or NULL in case of error.
+ */
+hobject_t *TestObjectStoreState::coll_entry_t::remove_obj(int id)
+{
+ return get_obj(id, true);
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj(int id, bool remove)
+{
+ map<int, hobject_t*>::iterator it = m_objects.find(id);
+ if (it == m_objects.end()) {
+ dout(5) << "get_obj coll " << m_coll.to_str()
+ << " obj #" << id << " non-existent" << dendl;
+ return NULL;
+ }
+
+ hobject_t *obj = it->second;
+ if (remove)
+ m_objects.erase(it);
+
+ dout(5) << "get_obj coll " << m_coll.to_str() << " id " << id
+ << ": " << obj->oid.name << "(removed: " << remove << ")" << dendl;
+
+ return obj;
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj_at(int pos, int *key)
+{
+ return get_obj_at(pos, false, key);
+}
+
+/**
+ * remove_obj_at - Removes object without freeing it.
+ * @param pos The map's position in which the object lies.
+ * @return The object or NULL in case of error.
+ */
+hobject_t *TestObjectStoreState::coll_entry_t::remove_obj_at(int pos, int *key)
+{
+ return get_obj_at(pos, true, key);
+}
+
+hobject_t *TestObjectStoreState::coll_entry_t::get_obj_at(int pos,
+ bool remove, int *key)
+{
+ if (m_objects.empty()) {
+ dout(5) << "get_obj_at coll " << m_coll.to_str() << " pos " << pos
+ << " in an empty collection" << dendl;
+ return NULL;
+ }
+
+ hobject_t *ret = NULL;
+ map<int, hobject_t*>::iterator it = m_objects.begin();
+ for (int i = 0; it != m_objects.end(); ++it, i++) {
+ if (i == pos) {
+ ret = it->second;
+ break;
+ }
+ }
+
+ if (ret == NULL) {
+ dout(5) << "get_obj_at coll " << m_coll.to_str() << " pos " << pos
+ << " non-existent" << dendl;
+ return NULL;
+ }
+
+ if (key != NULL)
+ *key = it->first;
+
+ if (remove)
+ m_objects.erase(it);
+
+ dout(5) << "get_obj_at coll id " << m_id << " pos " << pos
+ << ": " << ret->oid.name << "(removed: " << remove << ")" << dendl;
+
+ return ret;
+}
+
+hobject_t*
+TestObjectStoreState::coll_entry_t::replace_obj(int id, hobject_t *obj) {
+ hobject_t *old_obj = remove_obj(id);
+ m_objects.insert(make_pair(id, obj));
+ return old_obj;
+}
+
+int TestObjectStoreState::coll_entry_t::get_random_obj_id(rngen_t& gen)
+{
+ ceph_assert(!m_objects.empty());
+
+ boost::uniform_int<> orig_obj_rng(0, m_objects.size()-1);
+ int pos = orig_obj_rng(gen);
+ map<int, hobject_t*>::iterator it = m_objects.begin();
+ for (int i = 0; it != m_objects.end(); ++it, i++) {
+ if (i == pos) {
+ return it->first;
+ }
+ }
+ ceph_assert(0 == "INTERNAL ERROR");
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef TEST_OBJECTSTORE_STATE_H_
+#define TEST_OBJECTSTORE_STATE_H_
+
+#include "os/ObjectStore.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/random/mersenne_twister.hpp>
+#include <boost/random/uniform_int.hpp>
+#include <map>
+#include <vector>
+
+typedef boost::mt11213b rngen_t;
+
+class TestObjectStoreState {
+public:
+ struct coll_entry_t {
+ int m_id;
+ coll_t m_coll;
+ hobject_t m_meta_obj;
+ ObjectStore::Sequencer m_osr;
+ map<int, hobject_t*> m_objects;
+ int m_next_object_id;
+
+ coll_entry_t(int i, char *coll_buf, char *meta_obj_buf)
+ : m_id(i), m_coll(coll_buf),
+ m_meta_obj(sobject_t(object_t(meta_obj_buf), CEPH_NOSNAP)),
+ m_osr(coll_buf), m_next_object_id(0) {
+ }
+ ~coll_entry_t();
+
+ hobject_t *touch_obj(int id);
+ bool check_for_obj(int id);
+ hobject_t *get_obj(int id);
+ hobject_t *remove_obj(int id);
+ hobject_t *get_obj_at(int pos, int *key = NULL);
+ hobject_t *remove_obj_at(int pos, int *key = NULL);
+ hobject_t *replace_obj(int id, hobject_t *obj);
+ int get_random_obj_id(rngen_t& gen);
+
+ private:
+ hobject_t *get_obj(int id, bool remove);
+ hobject_t *get_obj_at(int pos, bool remove, int *key = NULL);
+ };
+
+ /* kept in upper case for consistency with coll_t's */
+ static const coll_t META_COLL;
+ static const coll_t TEMP_COLL;
+
+ protected:
+ boost::shared_ptr<ObjectStore> m_store;
+ map<int, coll_entry_t*> m_collections;
+ vector<int> m_collections_ids;
+ int m_next_coll_nr;
+ int m_num_objs_per_coll;
+ int m_num_objects;
+
+ int m_max_in_flight;
+ atomic_t m_in_flight;
+ Mutex m_finished_lock;
+ Cond m_finished_cond;
+
+ void wait_for_ready() {
+ Mutex::Locker locker(m_finished_lock);
+ while ((m_max_in_flight > 0) && ((int)m_in_flight.read() >= m_max_in_flight))
+ m_finished_cond.Wait(m_finished_lock);
+ }
+
+ void wait_for_done() {
+ Mutex::Locker locker(m_finished_lock);
+ while (m_in_flight.read())
+ m_finished_cond.Wait(m_finished_lock);
+ }
+
+ void set_max_in_flight(int max) {
+ m_max_in_flight = max;
+ }
+ void set_num_objs_per_coll(int val) {
+ m_num_objs_per_coll = val;
+ }
+
+ coll_entry_t *get_coll(int key, bool erase = false);
+ coll_entry_t *get_coll_at(int pos, bool erase = false);
+
+ private:
+ static const int m_default_num_colls = 30;
+
+ public:
+ TestObjectStoreState(ObjectStore *store) :
+ m_next_coll_nr(0), m_num_objs_per_coll(10), m_num_objects(0),
+ m_max_in_flight(0), m_finished_lock("Finished Lock") {
+ m_in_flight.set(0);
+ m_store.reset(store);
+ }
+ ~TestObjectStoreState() {
+ map<int, coll_entry_t*>::iterator it = m_collections.begin();
+ while (it != m_collections.end()) {
+ if (it->second)
+ delete it->second;
+ m_collections.erase(it++);
+ }
+ }
+
+ void init(int colls, int objs);
+ void init() {
+ init(m_default_num_colls, 0);
+ }
+
+ int inc_in_flight() {
+ return ((int) m_in_flight.inc());
+ }
+
+ int dec_in_flight() {
+ return ((int) m_in_flight.dec());
+ }
+
+ coll_entry_t *coll_create(int id);
+
+ class C_OnFinished: public Context {
+ protected:
+ TestObjectStoreState *m_state;
+ ObjectStore::Transaction *m_tx;
+
+ public:
+ C_OnFinished(TestObjectStoreState *state,
+ ObjectStore::Transaction *t) : m_state(state), m_tx(t) { }
+
+ void finish(int r) {
+ Mutex::Locker locker(m_state->m_finished_lock);
+ m_state->dec_in_flight();
+ m_state->m_finished_cond.Signal();
+
+ delete m_tx;
+ }
+ };
+};
+
+#endif /* TEST_OBJECTSTORE_STATE_H_ */
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include <stdio.h>
+#include <signal.h>
+#include "os/chain_xattr.h"
+#include "include/Context.h"
+#include "common/errno.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include <gtest/gtest.h>
+
+#define LARGE_BLOCK_LEN CHAIN_XATTR_MAX_BLOCK_LEN + 1024
+
+TEST(chain_xattr, get_and_set) {
+ const char* file = "testfile";
+ ::unlink(file);
+ int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
+ const string user("user.");
+
+ {
+ const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
+ const string x(LARGE_BLOCK_LEN, 'X');
+
+ {
+ char y[LARGE_BLOCK_LEN];
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0));
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN));
+ ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
+ ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN));
+ }
+
+ {
+ char y[LARGE_BLOCK_LEN];
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0));
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN));
+ ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
+ ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN));
+ }
+ }
+
+ //
+ // when chain_setxattr is used to store value that is
+ // CHAIN_XATTR_MAX_BLOCK_LEN * 2 + 10 bytes long it
+ //
+ // add user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes
+ // add user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes
+ // add user.foo@2 => 10 bytes
+ //
+ // then ( no chain_removexattr in between ) when it is used to
+ // override with a value that is exactly CHAIN_XATTR_MAX_BLOCK_LEN
+ // bytes long it will
+ //
+ // replace user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes
+ // remove user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes
+ // leak user.foo@2 => 10 bytes
+ //
+ // see http://marc.info/?l=ceph-devel&m=136027076615853&w=4 for the
+ // discussion
+ //
+ {
+ const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
+ const string x(LARGE_BLOCK_LEN, 'X');
+
+ {
+ char y[CHAIN_XATTR_MAX_NAME_LEN];
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+ ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN));
+ ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0));
+ ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+ ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
+ ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+ }
+
+ {
+ char y[CHAIN_XATTR_MAX_BLOCK_LEN];
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+ ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN));
+ ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0));
+ ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+ ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
+ ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+ }
+ }
+
+ {
+ int x = 0;
+ ASSERT_EQ(-ENOENT, chain_setxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x)));
+ ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", 0, 0));
+ ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x)));
+ ASSERT_EQ(-ENOENT, chain_removexattr("UNLIKELY_TO_EXIST", "NAME"));
+ int unlikely_to_be_a_valid_fd = 400;
+ ASSERT_EQ(-EBADF, chain_fsetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x)));
+ ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", 0, 0));
+ ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x)));
+ ASSERT_EQ(-EBADF, chain_fremovexattr(unlikely_to_be_a_valid_fd, "NAME"));
+ }
+
+ {
+ int x;
+ const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN * 2, '@');
+ ASSERT_THROW(chain_setxattr(file, name.c_str(), &x, sizeof(x)), FailedAssertion);
+ ASSERT_THROW(chain_fsetxattr(fd, name.c_str(), &x, sizeof(x)), FailedAssertion);
+ }
+
+ {
+ const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
+ const string x(LARGE_BLOCK_LEN, 'X');
+ {
+ char y[LARGE_BLOCK_LEN];
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+ ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN - 1));
+ ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+ ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
+ }
+
+ {
+ char y[LARGE_BLOCK_LEN];
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+ ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN - 1));
+ ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
+ ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
+ }
+ }
+
+ ::close(fd);
+ ::unlink(file);
+}
+
+TEST(chain_xattr, listxattr) {
+ const char* file = "testfile";
+ ::unlink(file);
+ int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
+ const string user("user.");
+ const string name1 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '1');
+ const string name2 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
+ const string x(LARGE_BLOCK_LEN, 'X');
+ const int y = 1234;
+
+ ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name1.c_str(), x.c_str(), LARGE_BLOCK_LEN));
+ ASSERT_EQ((int)sizeof(y), chain_setxattr(file, name2.c_str(), &y, sizeof(y)));
+
+ int buffer_size = name1.size() + sizeof('\0') + name2.size() + sizeof('\0');
+ char* expected = (char*)malloc(buffer_size);
+ ::strcpy(expected, name1.c_str());
+ ::strcpy(expected + name1.size() + 1, name2.c_str());
+ char* actual = (char*)calloc(1, buffer_size);
+ ASSERT_LT(buffer_size, chain_listxattr(file, NULL, 0)); // size evaluation is conservative
+ chain_listxattr(file, actual, buffer_size);
+ ::memset(actual, '\0', buffer_size);
+ chain_flistxattr(fd, actual, buffer_size);
+ ASSERT_EQ(0, ::memcmp(expected, actual, buffer_size));
+
+ int unlikely_to_be_a_valid_fd = 400;
+ ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, 0));
+ ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, buffer_size));
+ ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, 0));
+ ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, buffer_size));
+ ASSERT_EQ(-ERANGE, chain_listxattr(file, actual, 1));
+ ASSERT_EQ(-ERANGE, chain_flistxattr(fd, actual, 1));
+
+ ASSERT_EQ(0, chain_removexattr(file, name1.c_str()));
+ ASSERT_EQ(0, chain_removexattr(file, name2.c_str()));
+
+ ::unlink(file);
+}
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->set_val("err_to_stderr", "false");
+ g_ceph_context->_conf->set_val("log_to_stderr", "false");
+ g_ceph_context->_conf->apply_changes(NULL);
+
+ const char* file = "testfile";
+ int x = 1234;
+ int y = 0;
+ int tmpfd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
+ int ret = ::ceph_os_fsetxattr(tmpfd, "user.test", &x, sizeof(x));
+ if (ret >= 0)
+ ret = ::ceph_os_fgetxattr(tmpfd, "user.test", &y, sizeof(y));
+ ::close(tmpfd);
+ ::unlink(file);
+ if ((ret < 0) || (x != y)) {
+ cerr << "SKIP all tests because extended attributes don't appear to work in the file system in which the tests are run: " << cpp_strerror(ret) << std::endl;
+ } else {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+ }
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make unittest_chain_xattr ; valgrind --tool=memcheck ./unittest_chain_xattr # --gtest_filter=chain_xattr.get_and_set"
+// End:
--- /dev/null
+#!/bin/bash
+# vim: ts=8 sw=2 smarttab
+#
+# run_seed_to.sh - Run ceph_test_filestore_idempotent_sequence up until an
+# injection point, generating a sequence of operations based on a
+# provided seed.
+#
+# We also perform three additional tests, focused on assessing if
+# replaying a larger chunck of the journal affects the expected store
+# behavior. These tests will be performed by increasing the store's
+# journal sync interval to a very large value, allowing the store to
+# finish execution before the first sync (unless the store runs for
+# over 10 hours, case on which the interval variables must be changed
+# to an appropriate value). Unless the '--no-journal-test' option is
+# specified, we will run the 3 following scenarios:
+#
+# 1) journal sync'ing for both stores is good as disabled
+# (we call it '00', for store naming purposes)
+# 2) journal sync'ing for store A is as good as disabled
+# (we call it '01', for store naming purposes)
+# 3) journal sync'ing for store B is as good as disabled
+# (we call it '10', for store naming purposes)
+#
+# All log files are also appropriately named accordingly (i.e., a.00.fail,
+# a.10.recover, or b.01.clean).
+#
+# By default, the test will not exit on error, although it will show the
+# fail message. This behavior is so defined so we run the whole battery of
+# tests, and obtain as many mismatches as possible in one go. We may force
+# the test to exit on error by specifying the '--exit-on-error' option.
+#
+#
+set -e
+
+test_opts=""
+
+usage() {
+ echo "usage: $1 [options..] <seed> <kill-at>"
+ echo
+ echo "options:"
+ echo " -c, --colls <VAL> # of collections"
+ echo " -o, --objs <VAL> # of objects"
+ echo " -b, --btrfs <VAL> seq number for btrfs stores"
+ echo " --no-journal-test don't perform journal replay tests"
+ echo " -e, --exit-on-error exit with 1 on error"
+ echo " -v, --valgrind run commands through valgrind"
+ echo
+ echo "env vars:"
+ echo " OPTS_STORE additional opts for both stores"
+ echo " OPTS_STORE_A additional opts for store A"
+ echo " OPTS_STORE_B additional opts for store B"
+ echo
+}
+
+die_on_missing_arg() {
+ if [[ "$2" == "" ]]; then
+ echo "$1: missing required parameter"
+ exit 1
+ fi
+}
+
+
+required_args=2
+obtained_args=0
+
+seed=""
+killat=""
+on_btrfs=0
+on_btrfs_seq=0
+journal_test=1
+min_sync_interval="36000" # ten hours, yes.
+max_sync_interval="36001"
+exit_on_error=0
+v=""
+
+do_rm() {
+ if [[ $on_btrfs -eq 0 ]]; then
+ rm -fr $*
+ fi
+}
+
+set_arg() {
+ if [[ $1 -eq 1 ]]; then
+ seed=$2
+ elif [[ $1 -eq 2 ]]; then
+ killat=$2
+ else
+ echo "error: unknown purpose for '$2'"
+ usage $0
+ exit 1
+ fi
+}
+
+while [[ $# -gt 0 ]];
+do
+ case "$1" in
+ -c | --colls)
+ die_on_missing_arg "$1" "$2"
+ test_opts="$test_opts --test-num-colls $2"
+ shift 2
+ ;;
+ -o | --objs)
+ die_on_missing_arg "$1" "$2"
+ test_opts="$test_opts --test-num-objs $2"
+ shift 2
+ ;;
+ -h | --help)
+ usage $0 ;
+ exit 0
+ ;;
+ -b | --btrfs)
+ die_on_missing_arg "$1" "$2"
+ on_btrfs=1
+ on_btrfs_seq=$2
+ shift 2
+ ;;
+ --no-journal-test)
+ journal_test=0
+ shift
+ ;;
+ -e | --exit-on-error)
+ exit_on_error=1
+ shift
+ ;;
+ -v | --valgrind)
+ v="valgrind --leak-check=full"
+ shift
+ ;;
+ --)
+ shift
+ break
+ ;;
+ -*)
+ echo "$1: unknown option" >&2
+ usage $0
+ exit 1
+ ;;
+ *)
+ obtained_args=$(($obtained_args+1))
+ set_arg $obtained_args $1
+ shift
+ ;;
+ esac
+done
+
+if [[ $obtained_args -ne $required_args ]]; then
+ echo "error: missing argument"
+ usage $0 ;
+ exit 1
+fi
+
+if [[ "$OPTS_STORE" != "" ]]; then
+ test_opts="$test_opts $OPTS_STORE"
+fi
+
+test_opts_a="$test_opts"
+test_opts_b="$test_opts"
+
+if [[ "$OPTS_STORE_A" != "" ]]; then
+ test_opts_a="$test_opts_a $OPTS_STORE_A"
+fi
+if [[ "$OPTS_STORE_B" != "" ]]; then
+ test_opts_b="$test_opts_b $OPTS_STORE_B"
+fi
+
+echo seed $seed
+echo kill at $killat
+
+# run forever, until $killat...
+to=1000000000
+
+#
+# store names
+#
+# We need these for two reasons:
+# 1) if we are running the tests on a btrfs volume, then we need to use
+# a seq number for each run. Being on btrfs means we will fail when
+# removing the store's directories and it's far more simple to just
+# specify differente store names such as 'a.$seq' or 'b.$seq'.
+#
+# 2) unless the '--no-journal-test' option is specified, we will run
+# three additional tests for each store, and we will reuse the same
+# command for each one of the runs, but varying the store's name and
+# arguments.
+#
+store_a="a"
+store_b="b"
+
+if [[ $on_btrfs -eq 1 ]]; then
+ store_a="$store_a.$on_btrfs_seq"
+ store_b="$store_b.$on_btrfs_seq"
+fi
+
+total_runs=1
+
+if [[ $journal_test -eq 1 ]]; then
+ total_runs=$(($total_runs + 3))
+fi
+
+num_runs=0
+
+opt_min_sync="--filestore-min-sync-interval $min_sync_interval"
+opt_max_sync="--filestore-max-sync-interval $max_sync_interval"
+
+ret=0
+
+while [[ $num_runs -lt $total_runs ]];
+do
+ tmp_name_a=$store_a
+ tmp_name_b=$store_b
+ tmp_opts_a=$test_opts_a
+ tmp_opts_b=$test_opts_b
+
+ #
+ # We have already tested whether there are diffs when both journals
+ # are properly working. Now let's try on three other scenarios:
+ # 1) journal sync'ing for both stores is good as disabled
+ # (we call it '00')
+ # 2) journal sync'ing for store A is as good as disabled
+ # (we call it '01')
+ # 3) journal sync'ing for store B is as good as disabled
+ # (we call it '10')
+ #
+ if [[ $num_runs -gt 0 && $journal_test -eq 1 ]]; then
+ echo "run #$num_runs"
+ case $num_runs in
+ 1)
+ tmp_name_a="$tmp_name_a.00"
+ tmp_name_b="$tmp_name_b.00"
+ tmp_opts_a="$tmp_opts_a $opt_min_sync $opt_max_sync"
+ tmp_opts_b="$tmp_opts_b $opt_min_sync $opt_max_sync"
+ ;;
+ 2)
+ tmp_name_a="$tmp_name_a.01"
+ tmp_name_b="$tmp_name_b.01"
+ tmp_opts_a="$tmp_opts_a $opt_min_sync $opt_max_sync"
+ ;;
+ 3)
+ tmp_name_a="$tmp_name_a.10"
+ tmp_name_b="$tmp_name_b.10"
+ tmp_opts_b="$tmp_opts_b $opt_min_sync $opt_max_sync"
+ ;;
+ esac
+ fi
+
+ do_rm $tmp_name_a $tmp_name_a.fail $tmp_name_a.recover
+ $v ceph_test_filestore_idempotent_sequence run-sequence-to $to \
+ $tmp_name_a $tmp_name_a/journal \
+ --test-seed $seed --osd-journal-size 100 \
+ --filestore-kill-at $killat $tmp_opts_a \
+ --log-file $tmp_name_a.fail --debug-filestore 20 || true
+
+ stop_at=`ceph_test_filestore_idempotent_sequence get-last-op \
+ $tmp_name_a $tmp_name_a/journal \
+ --log-file $tmp_name_a.recover \
+ --debug-filestore 20 --debug-journal 20`
+
+ if [[ "`expr $stop_at - $stop_at 2>/dev/null`" != "0" ]]; then
+ echo "error: get-last-op returned '$stop_at'"
+ exit 1
+ fi
+
+ echo stopped at $stop_at
+
+ do_rm $tmp_name_b $tmp_name_b.clean
+ $v ceph_test_filestore_idempotent_sequence run-sequence-to \
+ $stop_at $tmp_name_b $tmp_name_b/journal \
+ --test-seed $seed --osd-journal-size 100 \
+ --log-file $tmp_name_b.clean --debug-filestore 20 $tmp_opts_b
+
+ if $v ceph_test_filestore_idempotent_sequence diff \
+ $tmp_name_a $tmp_name_a/journal $tmp_name_b $tmp_name_b/journal ; then
+ echo OK
+ else
+ echo "FAIL"
+ echo " see:"
+ echo " $tmp_name_a.fail -- leading up to failure"
+ echo " $tmp_name_a.recover -- journal replay"
+ echo " $tmp_name_b.clean -- the clean reference"
+
+ ret=1
+ if [[ $exit_on_error -eq 1 ]]; then
+ exit 1
+ fi
+ fi
+
+ num_runs=$(($num_runs+1))
+done
+
+exit $ret
--- /dev/null
+#!/bin/sh
+
+set -e
+
+seed=$1
+from=$2
+to=$3
+dir=$4
+
+mydir=`dirname $0`
+
+for f in `seq $from $to`
+do
+ if ! $mydir/run_seed_to.sh $seed $f; then
+ if [ -d $dir ]; then
+ echo copying evidence to $dir
+ cp -a . $dir
+ else
+ echo no dir provided for evidence disposal
+ fi
+ exit 1
+ fi
+done
\ No newline at end of file
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <iostream>
+#include <time.h>
+#include <sys/mount.h>
+#include "os/ObjectStore.h"
+#include "os/KeyValueStore.h"
+#include "include/Context.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/errno.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/random/mersenne_twister.hpp>
+#include <boost/random/uniform_int.hpp>
+#include <boost/random/binomial_distribution.hpp>
+#include <gtest/gtest.h>
+
+#include "include/unordered_map.h"
+typedef boost::mt11213b gen_type;
+
+#if GTEST_HAS_PARAM_TEST
+
+class StoreTest : public ::testing::TestWithParam<const char*> {
+public:
+ boost::scoped_ptr<ObjectStore> store;
+
+ StoreTest() : store(0) {}
+ virtual void SetUp() {
+ int r = ::mkdir("store_test_temp_dir", 0777);
+ if (r < 0 && errno != EEXIST) {
+ r = -errno;
+ cerr << __func__ << ": unable to create store_test_temp_dir" << ": " << cpp_strerror(r) << std::endl;
+ return;
+ }
+
+ ObjectStore *store_ = ObjectStore::create(g_ceph_context,
+ string(GetParam()),
+ string("store_test_temp_dir"),
+ string("store_test_temp_journal"));
+ store.reset(store_);
+ EXPECT_EQ(store->mkfs(), 0);
+ EXPECT_EQ(store->mount(), 0);
+ }
+
+ virtual void TearDown() {
+ store->umount();
+ }
+};
+
+bool sorted(const vector<ghobject_t> &in) {
+ ghobject_t start;
+ for (vector<ghobject_t>::const_iterator i = in.begin();
+ i != in.end();
+ ++i) {
+ if (start > *i) return false;
+ start = *i;
+ }
+ return true;
+}
+
+TEST_P(StoreTest, SimpleColTest) {
+ coll_t cid = coll_t("initial");
+ int r = 0;
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ cerr << "create collection" << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ {
+ ObjectStore::Transaction t;
+ t.remove_collection(cid);
+ cerr << "remove collection" << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ cerr << "add collection" << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ {
+ ObjectStore::Transaction t;
+ t.remove_collection(cid);
+ cerr << "remove collection" << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+}
+
+TEST_P(StoreTest, SimpleObjectTest) {
+ int r;
+ coll_t cid = coll_t("coll");
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ cerr << "Creating collection " << cid << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
+ {
+ ObjectStore::Transaction t;
+ t.touch(cid, hoid);
+ cerr << "Creating object " << hoid << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ {
+ ObjectStore::Transaction t;
+ t.remove(cid, hoid);
+ t.remove_collection(cid);
+ cerr << "Cleaning" << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+}
+
+TEST_P(StoreTest, SimpleObjectLongnameTest) {
+ int r;
+ coll_t cid = coll_t("coll");
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ cerr << "Creating collection " << cid << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ ghobject_t hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP)));
+ {
+ ObjectStore::Transaction t;
+ t.touch(cid, hoid);
+ cerr << "Creating object " << hoid << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ {
+ ObjectStore::Transaction t;
+ t.remove(cid, hoid);
+ t.remove_collection(cid);
+ cerr << "Cleaning" << std::endl;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+}
+
+TEST_P(StoreTest, ManyObjectTest) {
+ int NUM_OBJS = 2000;
+ int r = 0;
+ coll_t cid("blah");
+ string base = "";
+ for (int i = 0; i < 100; ++i) base.append("aaaaa");
+ set<ghobject_t> created;
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ for (int i = 0; i < NUM_OBJS; ++i) {
+ if (!(i % 5)) {
+ cerr << "Object " << i << std::endl;
+ }
+ ObjectStore::Transaction t;
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%d", i);
+ ghobject_t hoid(hobject_t(sobject_t(string(buf) + base, CEPH_NOSNAP)));
+ t.touch(cid, hoid);
+ created.insert(hoid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+
+ for (set<ghobject_t>::iterator i = created.begin();
+ i != created.end();
+ ++i) {
+ struct stat buf;
+ ASSERT_TRUE(!store->stat(cid, *i, &buf));
+ }
+
+ set<ghobject_t> listed;
+ vector<ghobject_t> objects;
+ r = store->collection_list(cid, objects);
+ ASSERT_EQ(r, 0);
+
+ cerr << "objects.size() is " << objects.size() << std::endl;
+ for (vector<ghobject_t> ::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ listed.insert(*i);
+ ASSERT_TRUE(created.count(*i));
+ }
+ ASSERT_TRUE(listed.size() == created.size());
+
+ ghobject_t start, next;
+ objects.clear();
+ r = store->collection_list_partial(
+ cid,
+ ghobject_t::get_max(),
+ 50,
+ 60,
+ 0,
+ &objects,
+ &next
+ );
+ ASSERT_EQ(r, 0);
+ ASSERT_TRUE(objects.empty());
+
+ objects.clear();
+ listed.clear();
+ while (1) {
+ r = store->collection_list_partial(cid, start,
+ 50,
+ 60,
+ 0,
+ &objects,
+ &next);
+ ASSERT_TRUE(sorted(objects));
+ ASSERT_EQ(r, 0);
+ listed.insert(objects.begin(), objects.end());
+ if (objects.size() < 50) {
+ ASSERT_TRUE(next.is_max());
+ break;
+ }
+ objects.clear();
+ start = next;
+ }
+ cerr << "listed.size() is " << listed.size() << std::endl;
+ ASSERT_TRUE(listed.size() == created.size());
+ for (set<ghobject_t>::iterator i = listed.begin();
+ i != listed.end();
+ ++i) {
+ ASSERT_TRUE(created.count(*i));
+ }
+
+ for (set<ghobject_t>::iterator i = created.begin();
+ i != created.end();
+ ++i) {
+ ObjectStore::Transaction t;
+ t.remove(cid, *i);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ cerr << "cleaning up" << std::endl;
+ {
+ ObjectStore::Transaction t;
+ t.remove_collection(cid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+}
+
+class ObjectGenerator {
+public:
+ virtual ghobject_t create_object(gen_type *gen) = 0;
+ virtual ~ObjectGenerator() {}
+};
+
+class MixedGenerator : public ObjectGenerator {
+public:
+ unsigned seq;
+ MixedGenerator() : seq(0) {}
+ ghobject_t create_object(gen_type *gen) {
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%u", seq);
+
+ boost::uniform_int<> true_false(0, 1);
+ string name(buf);
+ if (true_false(*gen)) {
+ // long
+ for (int i = 0; i < 100; ++i) name.append("aaaaa");
+ } else if (true_false(*gen)) {
+ name = "DIR_" + name;
+ }
+
+ // hash
+ //boost::binomial_distribution<uint32_t> bin(0xFFFFFF, 0.5);
+ ++seq;
+ return ghobject_t(hobject_t(name, string(), rand() & 2 ? CEPH_NOSNAP : rand(), rand() & 0xFF, 0, ""));
+ }
+};
+
+class SyntheticWorkloadState {
+public:
+ static const unsigned max_in_flight = 16;
+ static const unsigned max_objects = 3000;
+ coll_t cid;
+ unsigned in_flight;
+ set<ghobject_t> available_objects;
+ set<ghobject_t> in_use_objects;
+ ObjectGenerator *object_gen;
+ gen_type *rng;
+ ObjectStore *store;
+ ObjectStore::Sequencer *osr;
+
+ Mutex lock;
+ Cond cond;
+
+ class C_SyntheticOnReadable : public Context {
+ public:
+ SyntheticWorkloadState *state;
+ ObjectStore::Transaction *t;
+ ghobject_t hoid;
+ C_SyntheticOnReadable(SyntheticWorkloadState *state,
+ ObjectStore::Transaction *t, ghobject_t hoid)
+ : state(state), t(t), hoid(hoid) {}
+
+ void finish(int r) {
+ ASSERT_TRUE(r >= 0);
+ Mutex::Locker locker(state->lock);
+ if (state->in_use_objects.count(hoid)) {
+ state->available_objects.insert(hoid);
+ state->in_use_objects.erase(hoid);
+ }
+ --(state->in_flight);
+ state->cond.Signal();
+ }
+ };
+
+
+ SyntheticWorkloadState(ObjectStore *store,
+ ObjectGenerator *gen,
+ gen_type *rng,
+ ObjectStore::Sequencer *osr,
+ coll_t cid)
+ : cid(cid), in_flight(0), object_gen(gen), rng(rng), store(store), osr(osr),
+ lock("State lock") {}
+
+ int init() {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ return store->apply_transaction(t);
+ }
+
+ ghobject_t get_uniform_random_object() {
+ while (in_flight >= max_in_flight || available_objects.empty())
+ cond.Wait(lock);
+ boost::uniform_int<> choose(0, available_objects.size() - 1);
+ int index = choose(*rng);
+ set<ghobject_t>::iterator i = available_objects.begin();
+ for ( ; index > 0; --index, ++i) ;
+ ghobject_t ret = *i;
+ available_objects.erase(i);
+ return ret;
+ }
+
+ void wait_for_ready() {
+ while (in_flight >= max_in_flight)
+ cond.Wait(lock);
+ }
+
+ void wait_for_done() {
+ Mutex::Locker locker(lock);
+ while (in_flight)
+ cond.Wait(lock);
+ }
+
+ bool can_create() {
+ return (available_objects.size() + in_use_objects.size()) < max_objects;
+ }
+
+ bool can_unlink() {
+ return (available_objects.size() + in_use_objects.size()) > 0;
+ }
+
+ int touch() {
+ Mutex::Locker locker(lock);
+ if (!can_create())
+ return -ENOSPC;
+ wait_for_ready();
+ ghobject_t new_obj = object_gen->create_object(rng);
+ in_use_objects.insert(new_obj);
+ available_objects.erase(new_obj);
+ ObjectStore::Transaction *t = new ObjectStore::Transaction;
+ t->touch(cid, new_obj);
+ ++in_flight;
+ return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, new_obj));
+ }
+
+ void scan() {
+ Mutex::Locker locker(lock);
+ while (in_flight)
+ cond.Wait(lock);
+ vector<ghobject_t> objects;
+ set<ghobject_t> objects_set, objects_set2;
+ ghobject_t next, current;
+ while (1) {
+ cerr << "scanning..." << std::endl;
+ int r = store->collection_list_partial(cid, current, 50, 100,
+ 0, &objects, &next);
+ ASSERT_EQ(r, 0);
+ ASSERT_TRUE(sorted(objects));
+ objects_set.insert(objects.begin(), objects.end());
+ objects.clear();
+ if (next.is_max()) break;
+ current = next;
+ }
+ ASSERT_EQ(objects_set.size(), available_objects.size());
+ for (set<ghobject_t>::iterator i = objects_set.begin();
+ i != objects_set.end();
+ ++i) {
+ ASSERT_GT(available_objects.count(*i), (unsigned)0);
+ }
+
+ int r = store->collection_list(cid, objects);
+ ASSERT_EQ(r, 0);
+ objects_set2.insert(objects.begin(), objects.end());
+ ASSERT_EQ(objects_set2.size(), available_objects.size());
+ for (set<ghobject_t>::iterator i = objects_set2.begin();
+ i != objects_set2.end();
+ ++i) {
+ ASSERT_GT(available_objects.count(*i), (unsigned)0);
+ }
+ }
+
+ int stat() {
+ ghobject_t hoid;
+ {
+ Mutex::Locker locker(lock);
+ if (!can_unlink())
+ return -ENOENT;
+ hoid = get_uniform_random_object();
+ in_use_objects.insert(hoid);
+ ++in_flight;
+ }
+ struct stat buf;
+ int r = store->stat(cid, hoid, &buf);
+ {
+ Mutex::Locker locker(lock);
+ --in_flight;
+ cond.Signal();
+ in_use_objects.erase(hoid);
+ available_objects.insert(hoid);
+ }
+ return r;
+ }
+
+ int unlink() {
+ Mutex::Locker locker(lock);
+ if (!can_unlink())
+ return -ENOENT;
+ ghobject_t to_remove = get_uniform_random_object();
+ ObjectStore::Transaction *t = new ObjectStore::Transaction;
+ t->remove(cid, to_remove);
+ ++in_flight;
+ return store->queue_transaction(osr, t, new C_SyntheticOnReadable(this, t, to_remove));
+ }
+
+ void print_internal_state() {
+ Mutex::Locker locker(lock);
+ cerr << "available_objects: " << available_objects.size()
+ << " in_use_objects: " << in_use_objects.size()
+ << " total objects: " << in_use_objects.size() + available_objects.size()
+ << " in_flight " << in_flight << std::endl;
+ }
+};
+
+TEST_P(StoreTest, Synthetic) {
+ ObjectStore::Sequencer osr("test");
+ MixedGenerator gen;
+ gen_type rng(time(NULL));
+ coll_t cid("synthetic_1");
+
+ SyntheticWorkloadState test_obj(store.get(), &gen, &rng, &osr, cid);
+ test_obj.init();
+ for (int i = 0; i < 1000; ++i) {
+ if (!(i % 10)) cerr << "seeding object " << i << std::endl;
+ test_obj.touch();
+ }
+ for (int i = 0; i < 10000; ++i) {
+ if (!(i % 10)) {
+ cerr << "Op " << i << std::endl;
+ test_obj.print_internal_state();
+ }
+ boost::uniform_int<> true_false(0, 99);
+ int val = true_false(rng);
+ if (val > 97) {
+ test_obj.scan();
+ } else if (val > 50) {
+ test_obj.stat();
+ } else if (val > 30) {
+ test_obj.unlink();
+ } else {
+ test_obj.touch();
+ }
+ }
+ test_obj.wait_for_done();
+}
+
+TEST_P(StoreTest, HashCollisionTest) {
+ coll_t cid("blah");
+ int r;
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ string base = "";
+ for (int i = 0; i < 100; ++i) base.append("aaaaa");
+ set<ghobject_t> created;
+ for (int n = 0; n < 10; ++n) {
+ char nbuf[100];
+ sprintf(nbuf, "n%d", n);
+ for (int i = 0; i < 1000; ++i) {
+ char buf[100];
+ sprintf(buf, "%d", i);
+ if (!(i % 5)) {
+ cerr << "Object n" << n << " "<< i << std::endl;
+ }
+ ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, 0, 0, string(nbuf)));
+ {
+ ObjectStore::Transaction t;
+ t.touch(cid, hoid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ created.insert(hoid);
+ }
+ }
+ vector<ghobject_t> objects;
+ r = store->collection_list(cid, objects);
+ ASSERT_EQ(r, 0);
+ set<ghobject_t> listed(objects.begin(), objects.end());
+ cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl;
+ ASSERT_TRUE(listed.size() == created.size());
+ objects.clear();
+ listed.clear();
+ ghobject_t current, next;
+ while (1) {
+ r = store->collection_list_partial(cid, current, 50, 60,
+ 0, &objects, &next);
+ ASSERT_EQ(r, 0);
+ ASSERT_TRUE(sorted(objects));
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ if (listed.count(*i))
+ cerr << *i << " repeated" << std::endl;
+ listed.insert(*i);
+ }
+ if (objects.size() < 50) {
+ ASSERT_TRUE(next.is_max());
+ break;
+ }
+ objects.clear();
+ current = next;
+ }
+ cerr << "listed.size() is " << listed.size() << std::endl;
+ ASSERT_TRUE(listed.size() == created.size());
+ for (set<ghobject_t>::iterator i = listed.begin();
+ i != listed.end();
+ ++i) {
+ ASSERT_TRUE(created.count(*i));
+ }
+
+ for (set<ghobject_t>::iterator i = created.begin();
+ i != created.end();
+ ++i) {
+ ObjectStore::Transaction t;
+ t.collection_remove(cid, *i);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ ObjectStore::Transaction t;
+ t.remove_collection(cid);
+ store->apply_transaction(t);
+}
+
+TEST_P(StoreTest, OMapTest) {
+ coll_t cid("blah");
+ ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
+ int r;
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+
+ map<string, bufferlist> attrs;
+ {
+ ObjectStore::Transaction t;
+ t.touch(cid, hoid);
+ t.omap_clear(cid, hoid);
+ map<string, bufferlist> start_set;
+ t.omap_setkeys(cid, hoid, start_set);
+ store->apply_transaction(t);
+ }
+
+ for (int i = 0; i < 100; i++) {
+ if (!(i%5)) {
+ std::cout << "On iteration " << i << std::endl;
+ }
+ ObjectStore::Transaction t;
+ bufferlist bl;
+ map<string, bufferlist> cur_attrs;
+ r = store->omap_get(cid, hoid, &bl, &cur_attrs);
+ ASSERT_EQ(r, 0);
+ for (map<string, bufferlist>::iterator j = attrs.begin();
+ j != attrs.end();
+ ++j) {
+ bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str());
+ if (!correct) {
+ std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl;
+ if (cur_attrs.count(j->first) > 0) {
+ std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl;
+ }
+ }
+ ASSERT_EQ(correct, true);
+ }
+ ASSERT_EQ(attrs.size(), cur_attrs.size());
+
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%d", i);
+ bl.clear();
+ bufferptr bp(buf, strlen(buf) + 1);
+ bl.append(bp);
+ map<string, bufferlist> to_add;
+ to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl));
+ attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl));
+ t.omap_setkeys(cid, hoid, to_add);
+ store->apply_transaction(t);
+ }
+
+ int i = 0;
+ while (attrs.size()) {
+ if (!(i%5)) {
+ std::cout << "removal: On iteration " << i << std::endl;
+ }
+ ObjectStore::Transaction t;
+ bufferlist bl;
+ map<string, bufferlist> cur_attrs;
+ r = store->omap_get(cid, hoid, &bl, &cur_attrs);
+ ASSERT_EQ(r, 0);
+ for (map<string, bufferlist>::iterator j = attrs.begin();
+ j != attrs.end();
+ ++j) {
+ bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str());
+ if (!correct) {
+ std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl;
+ if (cur_attrs.count(j->first) > 0) {
+ std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl;
+ }
+ }
+ ASSERT_EQ(correct, true);
+ }
+
+ string to_remove = attrs.begin()->first;
+ set<string> keys_to_remove;
+ keys_to_remove.insert(to_remove);
+ t.omap_rmkeys(cid, hoid, keys_to_remove);
+ store->apply_transaction(t);
+
+ attrs.erase(to_remove);
+
+ ++i;
+ }
+
+ ObjectStore::Transaction t;
+ t.remove(cid, hoid);
+ t.remove_collection(cid);
+ store->apply_transaction(t);
+}
+
+TEST_P(StoreTest, XattrTest) {
+ coll_t cid("blah");
+ ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
+ bufferlist big;
+ for (unsigned i = 0; i < 10000; ++i) {
+ big.append('\0');
+ }
+ bufferlist small;
+ for (unsigned i = 0; i < 10; ++i) {
+ small.append('\0');
+ }
+ int r;
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ t.touch(cid, hoid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+
+ map<string, bufferlist> attrs;
+ {
+ ObjectStore::Transaction t;
+ t.setattr(cid, hoid, "attr1", small);
+ attrs["attr1"] = small;
+ t.setattr(cid, hoid, "attr2", big);
+ attrs["attr2"] = big;
+ t.setattr(cid, hoid, "attr3", small);
+ attrs["attr3"] = small;
+ t.setattr(cid, hoid, "attr1", small);
+ attrs["attr1"] = small;
+ t.setattr(cid, hoid, "attr4", big);
+ attrs["attr4"] = big;
+ t.setattr(cid, hoid, "attr3", big);
+ attrs["attr3"] = big;
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+
+ map<string, bufferptr> aset;
+ store->getattrs(cid, hoid, aset);
+ ASSERT_EQ(aset.size(), attrs.size());
+ for (map<string, bufferptr>::iterator i = aset.begin();
+ i != aset.end();
+ ++i) {
+ bufferlist bl;
+ bl.push_back(i->second);
+ ASSERT_TRUE(attrs[i->first] == bl);
+ }
+
+ {
+ ObjectStore::Transaction t;
+ t.rmattr(cid, hoid, "attr2");
+ attrs.erase("attr2");
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+
+ aset.clear();
+ store->getattrs(cid, hoid, aset);
+ ASSERT_EQ(aset.size(), attrs.size());
+ for (map<string, bufferptr>::iterator i = aset.begin();
+ i != aset.end();
+ ++i) {
+ bufferlist bl;
+ bl.push_back(i->second);
+ ASSERT_TRUE(attrs[i->first] == bl);
+ }
+
+ bufferptr bp;
+ r = store->getattr(cid, hoid, "attr2", bp);
+ ASSERT_EQ(r, -ENODATA);
+
+ r = store->getattr(cid, hoid, "attr3", bp);
+ ASSERT_GE(r, 0);
+ bufferlist bl2;
+ bl2.push_back(bp);
+ ASSERT_TRUE(bl2 == attrs["attr3"]);
+}
+
+void colsplittest(
+ ObjectStore *store,
+ unsigned num_objects,
+ unsigned common_suffix_size
+ ) {
+ coll_t cid("from");
+ coll_t tid("to");
+ int r = 0;
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ {
+ ObjectStore::Transaction t;
+ for (uint32_t i = 0; i < 2*num_objects; ++i) {
+ stringstream objname;
+ objname << "obj" << i;
+ t.touch(cid, ghobject_t(hobject_t(
+ objname.str(),
+ "",
+ CEPH_NOSNAP,
+ i<<common_suffix_size,
+ 0, "")));
+ }
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(tid);
+ t.split_collection(cid, common_suffix_size+1, 0, tid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+
+ ObjectStore::Transaction t;
+ vector<ghobject_t> objects;
+ r = store->collection_list(cid, objects);
+ ASSERT_EQ(r, 0);
+ ASSERT_EQ(objects.size(), num_objects);
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ ASSERT_EQ(!(i->hobj.hash & (1<<common_suffix_size)), 0u);
+ t.remove(cid, *i);
+ }
+
+ objects.clear();
+ r = store->collection_list(tid, objects);
+ ASSERT_EQ(r, 0);
+ ASSERT_EQ(objects.size(), num_objects);
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ ASSERT_EQ(i->hobj.hash & (1<<common_suffix_size), 0u);
+ t.remove(tid, *i);
+ }
+
+ t.remove_collection(cid);
+ t.remove_collection(tid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+}
+
+TEST_P(StoreTest, ColSplitTest1) {
+ colsplittest(store.get(), 10000, 11);
+}
+TEST_P(StoreTest, ColSplitTest2) {
+ colsplittest(store.get(), 100, 7);
+}
+
+#if 0
+TEST_P(StoreTest, ColSplitTest3) {
+ colsplittest(store.get(), 100000, 25);
+}
+#endif
+
+/**
+ * This test tests adding two different groups
+ * of objects, each with 1 common prefix and 1
+ * different prefix. We then remove half
+ * in order to verify that the merging correctly
+ * stops at the common prefix subdir. See bug
+ * #5273 */
+TEST_P(StoreTest, TwoHash) {
+ coll_t cid("asdf");
+ int r;
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ std::cout << "Making objects" << std::endl;
+ for (int i = 0; i < 360; ++i) {
+ ObjectStore::Transaction t;
+ ghobject_t o;
+ if (i < 8) {
+ o.hobj.hash = (i << 16) | 0xA1;
+ t.touch(cid, o);
+ }
+ o.hobj.hash = (i << 16) | 0xB1;
+ t.touch(cid, o);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ std::cout << "Removing half" << std::endl;
+ for (int i = 1; i < 8; ++i) {
+ ObjectStore::Transaction t;
+ ghobject_t o;
+ o.hobj.hash = (i << 16) | 0xA1;
+ t.remove(cid, o);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ std::cout << "Checking" << std::endl;
+ for (int i = 1; i < 8; ++i) {
+ ObjectStore::Transaction t;
+ ghobject_t o;
+ o.hobj.hash = (i << 16) | 0xA1;
+ bool exists = store->exists(cid, o);
+ ASSERT_EQ(exists, false);
+ }
+ {
+ ghobject_t o;
+ o.hobj.hash = 0xA1;
+ bool exists = store->exists(cid, o);
+ ASSERT_EQ(exists, true);
+ }
+ std::cout << "Cleanup" << std::endl;
+ for (int i = 0; i < 360; ++i) {
+ ObjectStore::Transaction t;
+ ghobject_t o;
+ o.hobj.hash = (i << 16) | 0xA1;
+ t.remove(cid, o);
+ o.hobj.hash = (i << 16) | 0xB1;
+ t.remove(cid, o);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ ObjectStore::Transaction t;
+ t.remove_collection(cid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+}
+
+TEST_P(StoreTest, MoveRename) {
+ coll_t temp_cid("mytemp");
+ hobject_t temp_oid("tmp_oid", "", CEPH_NOSNAP, 0, 0, "");
+ coll_t cid("dest");
+ hobject_t oid("dest_oid", "", CEPH_NOSNAP, 0, 0, "");
+ int r;
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(cid);
+ t.touch(cid, oid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ ASSERT_TRUE(store->exists(cid, oid));
+ bufferlist data, attr;
+ map<string, bufferlist> omap;
+ data.append("data payload");
+ attr.append("attr value");
+ omap["omap_key"].append("omap value");
+ {
+ ObjectStore::Transaction t;
+ t.create_collection(temp_cid);
+ t.touch(temp_cid, temp_oid);
+ t.write(temp_cid, temp_oid, 0, data.length(), data);
+ t.setattr(temp_cid, temp_oid, "attr", attr);
+ t.omap_setkeys(temp_cid, temp_oid, omap);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ ASSERT_TRUE(store->exists(temp_cid, temp_oid));
+ {
+ ObjectStore::Transaction t;
+ t.remove(cid, oid);
+ t.collection_move_rename(temp_cid, temp_oid, cid, oid);
+ r = store->apply_transaction(t);
+ ASSERT_EQ(r, 0);
+ }
+ ASSERT_TRUE(store->exists(cid, oid));
+ ASSERT_FALSE(store->exists(temp_cid, temp_oid));
+ {
+ bufferlist newdata;
+ r = store->read(cid, oid, 0, 1000, newdata);
+ ASSERT_GE(r, 0);
+ ASSERT_TRUE(newdata.contents_equal(data));
+ bufferlist newattr;
+ r = store->getattr(cid, oid, "attr", newattr);
+ ASSERT_GE(r, 0);
+ ASSERT_TRUE(newattr.contents_equal(attr));
+ set<string> keys;
+ keys.insert("omap_key");
+ map<string, bufferlist> newomap;
+ r = store->omap_get_values(cid, oid, keys, &newomap);
+ ASSERT_GE(r, 0);
+ ASSERT_EQ(1u, newomap.size());
+ ASSERT_TRUE(newomap.count("omap_key"));
+ ASSERT_TRUE(newomap["omap_key"].contents_equal(omap["omap_key"]));
+ }
+}
+
+INSTANTIATE_TEST_CASE_P(
+ ObjectStore,
+ StoreTest,
+ ::testing::Values("filestore", "keyvaluestore-dev"));
+
+#else
+
+// Google Test may not support value-parameterized tests with some
+// compilers. If we use conditional compilation to compile out all
+// code referring to the gtest_main library, MSVC linker will not link
+// that library at all and consequently complain about missing entry
+// point defined in that library (fatal error LNK1561: entry point
+// must be defined). This dummy test keeps gtest_main linked in.
+TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {}
+
+#endif
+
+
+//
+// support tests for qa/workunits/filestore/filestore.sh
+//
+TEST(EXT4StoreTest, _detect_fs) {
+ if (::getenv("DISK") == NULL || ::getenv("MOUNTPOINT") == NULL) {
+ cerr << "SKIP because DISK and MOUNTPOINT environment variables are not set. It is meant to run from qa/workunits/filestore/filestore.sh " << std::endl;
+ return;
+ }
+ const string disk(::getenv("DISK"));
+ EXPECT_LT((unsigned)0, disk.size());
+ const string mnt(::getenv("MOUNTPOINT"));
+ EXPECT_LT((unsigned)0, mnt.size());
+ ::umount(mnt.c_str());
+
+ const string dir("store_test_temp_dir");
+ const string journal("store_test_temp_journal");
+
+ //
+ // without user_xattr, ext4 fails
+ //
+ {
+ g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "true");
+ EXPECT_EQ(::system((string("mount -o loop,nouser_xattr ") + disk + " " + mnt).c_str()), 0);
+ EXPECT_EQ(::chdir(mnt.c_str()), 0);
+ EXPECT_EQ(::mkdir(dir.c_str(), 0755), 0);
+ FileStore store(dir, journal);
+ EXPECT_EQ(store._detect_fs(), -ENOTSUP);
+ EXPECT_EQ(::chdir(".."), 0);
+ EXPECT_EQ(::umount(mnt.c_str()), 0);
+ }
+ //
+ // mounted with user_xattr, ext4 fails if filestore_xattr_use_omap is false
+ //
+ {
+ g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "false");
+ EXPECT_EQ(::system((string("mount -o loop,user_xattr ") + disk + " " + mnt).c_str()), 0);
+ EXPECT_EQ(::chdir(mnt.c_str()), 0);
+ FileStore store(dir, journal);
+ EXPECT_EQ(store._detect_fs(), -ENOTSUP);
+ EXPECT_EQ(::chdir(".."), 0);
+ EXPECT_EQ(::umount(mnt.c_str()), 0);
+ }
+ //
+ // mounted with user_xattr, ext4 succeeds if filestore_xattr_use_omap is true
+ //
+ {
+ g_ceph_context->_conf->set_val("filestore_xattr_use_omap", "true");
+ EXPECT_EQ(::system((string("mount -o loop,user_xattr ") + disk + " " + mnt).c_str()), 0);
+ EXPECT_EQ(::chdir(mnt.c_str()), 0);
+ FileStore store(dir, journal);
+ EXPECT_EQ(store._detect_fs(), 0);
+ EXPECT_EQ(::chdir(".."), 0);
+ EXPECT_EQ(::umount(mnt.c_str()), 0);
+ }
+}
+
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->set_val("osd_journal_size", "400");
+ g_ceph_context->_conf->set_val("filestore_index_retry_probability", "0.5");
+ g_ceph_context->_conf->set_val("filestore_op_thread_timeout", "1000");
+ g_ceph_context->_conf->set_val("filestore_op_thread_suicide_timeout", "10000");
+ g_ceph_context->_conf->apply_changes(NULL);
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make ceph_test_filestore ; ./ceph_test_filestore --gtest_filter=StoreTest.* --log-to-stderr=true --debug-filestore=20"
+// End:
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <iostream>
+#include <sstream>
+#include <boost/scoped_ptr.hpp>
+#include "os/FileStore.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+#include "test/common/ObjectContents.h"
+#include "FileStoreTracker.h"
+#include "os/LevelDBStore.h"
+#include "os/KeyValueDB.h"
+#include "os/ObjectStore.h"
+
+void usage(const string &name) {
+ std::cerr << "Usage: " << name << " [new|continue] store_path store_journal db_path"
+ << std::endl;
+}
+
+template <typename T>
+typename T::iterator rand_choose(T &cont) {
+ if (cont.size() == 0) {
+ return cont.end();
+ }
+ int index = rand() % cont.size();
+ typename T::iterator retval = cont.begin();
+
+ for (; index > 0; --index) ++retval;
+ return retval;
+}
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+
+ std::cerr << "args: " << args << std::endl;
+ if (args.size() < 4) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ string store_path(args[1]);
+ string store_dev(args[2]);
+ string db_path(args[3]);
+
+ bool start_new = false;
+ if (string(args[0]) == string("new")) start_new = true;
+
+ LevelDBStore *_db = new LevelDBStore(g_ceph_context, db_path);
+ assert(!_db->create_and_open(std::cerr));
+ boost::scoped_ptr<KeyValueDB> db(_db);
+ boost::scoped_ptr<ObjectStore> store(new FileStore(store_path, store_dev));
+
+
+ if (start_new) {
+ std::cerr << "mkfs" << std::endl;
+ assert(!store->mkfs());
+ ObjectStore::Transaction t;
+ assert(!store->mount());
+ t.create_collection(coll_t("coll"));
+ store->apply_transaction(t);
+ } else {
+ assert(!store->mount());
+ }
+
+ FileStoreTracker tracker(store.get(), db.get());
+
+ set<string> objects;
+ for (unsigned i = 0; i < 10; ++i) {
+ stringstream stream;
+ stream << "Object_" << i;
+ tracker.verify("coll", stream.str(), true);
+ objects.insert(stream.str());
+ }
+
+ while (1) {
+ FileStoreTracker::Transaction t;
+ for (unsigned j = 0; j < 100; ++j) {
+ int val = rand() % 100;
+ if (val < 30) {
+ t.write("coll", *rand_choose(objects));
+ } else if (val < 60) {
+ t.clone("coll", *rand_choose(objects),
+ *rand_choose(objects));
+ } else if (val < 70) {
+ t.remove("coll", *rand_choose(objects));
+ } else {
+ t.clone_range("coll", *rand_choose(objects),
+ *rand_choose(objects));
+ }
+ }
+ tracker.submit_transaction(t);
+ tracker.verify("coll", *rand_choose(objects));
+ }
+ return 0;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <iostream>
+#include <sstream>
+#include <time.h>
+#include <stdlib.h>
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include "os/FileStore.h"
+
+#include "DeterministicOpSequence.h"
+#include "FileStoreDiff.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_
+#undef dout_prefix
+#define dout_prefix *_dout << "test_idempotent_sequence "
+
+void usage(const char *name, std::string command = "") {
+ assert(name != NULL);
+
+ std::string more = "cmd <args...>";
+ std::string diff = "diff <filestoreA> <journalA> <filestoreB> <journalB>";
+ std::string get_last_op = "get-last-op <filestore> <journal>";
+ std::string run_seq_to = "run-sequence-to <num-ops> <filestore> <journal>";
+
+ if (!command.empty()) {
+ if (command == "diff")
+ more = diff;
+ else if (command == "get-last-op")
+ more = get_last_op;
+ else if (command == "run-sequence-to")
+ more = run_seq_to;
+ }
+ std::cout << "usage: " << name << " " << more << " [options]" << std::endl;
+
+ std::cout << "\n\
+Commands:\n\
+ " << diff << "\n\
+ " << get_last_op << "\n\
+ " << run_seq_to << "\n\
+\n\
+Global Options:\n\
+ -c FILE Read configuration from FILE\n\
+ --osd-data PATH Set OSD Data path\n\
+ --osd-journal PATH Set OSD Journal path\n\
+ --osd-journal-size VAL Set Journal size\n\
+ --help This message\n\
+\n\
+Test-specific Options:\n\
+ --test-seed VAL Seed to run the test\n\
+ --test-status-file PATH Path to keep the status file\n\
+ --test-num-colls VAL Number of collections to create on init\n\
+ --test-num-objs VAL Number of objects to create on init\n\
+" << std::endl;
+}
+
+const char *our_name = NULL;
+int seed = 0, num_txs = 100, num_colls = 30, num_objs = 0;
+bool is_seed_set = false;
+int verify_at = 0;
+std::string status_file;
+
+int run_diff(std::string& a_path, std::string& a_journal,
+ std::string& b_path, std::string& b_journal)
+{
+ FileStore *a = new FileStore(a_path, a_journal, "a");
+ FileStore *b = new FileStore(b_path, b_journal, "b");
+
+ int ret = 0;
+ {
+ FileStoreDiff fsd(a, b);
+ if (fsd.diff()) {
+ dout(0) << "diff found an difference" << dendl;
+ ret = -1;
+ } else {
+ dout(0) << "no diff" << dendl;
+ }
+ }
+
+ delete a;
+ delete b;
+ return ret;
+}
+
+int run_get_last_op(std::string& filestore_path, std::string& journal_path)
+{
+ FileStore *store = new FileStore(filestore_path, journal_path);
+
+ int err = store->mount();
+ if (err) {
+ store->umount();
+ delete store;
+ return err;
+ }
+
+ coll_t txn_coll("meta");
+ hobject_t txn_object(sobject_t("txn", CEPH_NOSNAP));
+ bufferlist bl;
+ store->read(txn_coll, txn_object, 0, 100, bl);
+ int32_t txn = 0;
+ if (bl.length()) {
+ bufferlist::iterator p = bl.begin();
+ ::decode(txn, p);
+ }
+
+ store->umount();
+ delete store;
+
+ cout << txn << std::endl;
+ return 0;
+}
+
+int run_sequence_to(int val, std::string& filestore_path,
+ std::string& journal_path)
+{
+ num_txs = val;
+
+ if (!is_seed_set)
+ seed = (int) time(NULL);
+
+ FileStore *store = new FileStore(filestore_path, journal_path);
+
+ int err;
+
+ // mkfs iff directory dne
+ err = ::mkdir(filestore_path.c_str(), 0755);
+ if (err) {
+ cerr << filestore_path << " already exists" << std::endl;
+ store->umount();
+ delete store;
+ return err;
+ }
+
+ err = store->mkfs();
+ ceph_assert(err == 0);
+
+ err = store->mount();
+ ceph_assert(err == 0);
+
+ DeterministicOpSequence op_sequence(store, status_file);
+ op_sequence.init(num_colls, num_objs);
+ op_sequence.generate(seed, num_txs);
+ store->umount();
+ return 0;
+}
+
+int run_command(std::string& command, std::vector<std::string>& args)
+{
+ if (command.empty()) {
+ usage(our_name);
+ exit(0);
+ }
+
+ /* We'll have a class that will handle the options, the command
+ * and its arguments. For the time being, and so we can move on, let's
+ * tolerate this big, ugly code.
+ */
+ if (command == "diff") {
+ /* expect 4 arguments: (filestore path + journal path)*2 */
+ if (args.size() == 4) {
+ return run_diff(args[0], args[1], args[2], args[3]);
+ }
+ } else if (command == "get-last-op") {
+ /* expect 2 arguments: a filestore path + journal */
+ if (args.size() == 2) {
+ return run_get_last_op(args[0], args[1]);
+ }
+ } else if (command == "run-sequence-to") {
+ /* expect 3 arguments: # of operations and a filestore path + journal. */
+ if (args.size() == 3) {
+ return run_sequence_to(strtoll(args[0].c_str(), NULL, 10), args[1], args[2]);
+ }
+ } else {
+ std::cout << "unknown command " << command << std::endl;
+ usage(our_name);
+ exit(1);
+ }
+
+ usage(our_name, command);
+ exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+ vector<const char*> def_args;
+ vector<const char*> args;
+ our_name = argv[0];
+ argv_to_vec(argc, argv, args);
+
+ global_init(&def_args, args,
+ CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+
+ std::string command;
+ std::vector<std::string> command_args;
+
+ for (std::vector<const char*>::iterator i = args.begin(); i != args.end();) {
+ string val;
+
+ if (ceph_argparse_double_dash(args, i)) {
+ break;
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-seed", (char*) NULL)) {
+ seed = strtoll(val.c_str(), NULL, 10);
+ is_seed_set = true;
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-num-colls", (char*) NULL)) {
+ num_colls = strtoll(val.c_str(), NULL, 10);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-num-objs", (char*) NULL)) {
+ num_objs = strtoll(val.c_str(), NULL, 10);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-status-file", (char*) NULL)) {
+ status_file = val;
+ } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) {
+ usage(our_name);
+ exit(0);
+ } else {
+ if (command.empty())
+ command = *i++;
+ else
+ command_args.push_back(string(*i++));
+ }
+ }
+
+ int ret = run_command(command, command_args);
+
+ return ret;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ */
+#include <stdio.h>
+#include <string.h>
+#include <iostream>
+#include <assert.h>
+#include <time.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <cctype>
+#include <errno.h>
+#include <sys/time.h>
+#include "os/ObjectStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include "workload_generator.h"
+#include "include/assert.h"
+
+#include "TestObjectStoreState.h"
+
+static const char *our_name = NULL;
+void usage();
+
+boost::scoped_ptr<WorkloadGenerator> wrkldgen;
+
+#define dout_subsys ceph_subsys_
+
+
+WorkloadGenerator::WorkloadGenerator(vector<const char*> args)
+ : TestObjectStoreState(NULL),
+ m_max_in_flight(def_max_in_flight),
+ m_num_ops(-1),
+ m_destroy_coll_every_nr_runs(def_destroy_coll_every_nr_runs),
+ m_num_colls(def_num_colls),
+ m_write_data_bytes(0), m_write_xattr_obj_bytes(0),
+ m_write_xattr_coll_bytes(0), m_write_pglog_bytes(0),
+ m_suppress_write_data(false), m_suppress_write_xattr_obj(false),
+ m_suppress_write_xattr_coll(false), m_suppress_write_log(false),
+ m_do_stats(false),
+ m_stats_finished_txs(0),
+ m_stats_lock("WorldloadGenerator::m_stats_lock"),
+ m_stats_show_secs(5),
+ m_stats_total_written(0),
+ m_stats_begin()
+{
+ int err = 0;
+
+ m_nr_runs.set(0);
+
+ init_args(args);
+ dout(0) << "data = " << g_conf->osd_data << dendl;
+ dout(0) << "journal = " << g_conf->osd_journal << dendl;
+ dout(0) << "journal size = " << g_conf->osd_journal_size << dendl;
+
+ err = ::mkdir(g_conf->osd_data.c_str(), 0755);
+ ceph_assert(err == 0 || (err < 0 && errno == EEXIST));
+ ObjectStore *store_ptr = ObjectStore::create(g_ceph_context,
+ g_conf->osd_objectstore,
+ g_conf->osd_data,
+ g_conf->osd_journal);
+ m_store.reset(store_ptr);
+ err = m_store->mkfs();
+ ceph_assert(err == 0);
+ err = m_store->mount();
+ ceph_assert(err == 0);
+
+ set_max_in_flight(m_max_in_flight);
+ set_num_objs_per_coll(def_num_obj_per_coll);
+
+ init(m_num_colls, 0);
+
+ dout(0) << "#colls = " << m_num_colls << dendl;
+ dout(0) << "#objs per coll = " << m_num_objs_per_coll << dendl;
+ dout(0) << "#txs per destr = " << m_destroy_coll_every_nr_runs << dendl;
+
+}
+
+size_t WorkloadGenerator::_parse_size_or_die(std::string& val)
+{
+ size_t s = 0;
+ int multiplier = 0;
+ size_t i = 0;
+
+ if (val.empty()) // this should never happen, but catch it anyway.
+ goto die;
+
+
+ for (i = 0; i < val.length(); i++) {
+ if (!isdigit(val[i])) {
+ if (isalpha(val[i])) {
+ val[i] = tolower(val[i]);
+ switch (val[i]) {
+ case 'b': break;
+ case 'k': multiplier = 10; break;
+ case 'm': multiplier = 20; break;
+ case 'g': multiplier = 30; break;
+ default:
+ goto die;
+ }
+ val[i] = '\0';
+ break;
+ } else {
+ goto die;
+ }
+ }
+ }
+
+ s = strtoll(val.c_str(), NULL, 10) * (1 << multiplier);
+ return s;
+
+die:
+ usage();
+ exit(1);
+}
+
+void WorkloadGenerator::_suppress_ops_or_die(std::string& val)
+{
+ for (size_t i = 0; i < val.length(); i++) {
+ switch (val[i]) {
+ case 'c': m_suppress_write_xattr_coll = true; break;
+ case 'o': m_suppress_write_xattr_obj = true; break;
+ case 'l': m_suppress_write_log = true; break;
+ case 'd': m_suppress_write_data = true; break;
+ default:
+ usage();
+ exit(1);
+ }
+ }
+}
+
+void WorkloadGenerator::init_args(vector<const char*> args)
+{
+ for (std::vector<const char*>::iterator i = args.begin(); i != args.end();) {
+ string val;
+
+ if (ceph_argparse_double_dash(args, i)) {
+ break;
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-num-colls", (char*) NULL)) {
+ m_num_colls = strtoll(val.c_str(), NULL, 10);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-objs-per-coll", (char*) NULL)) {
+ m_num_objs_per_coll = strtoll(val.c_str(), NULL, 10);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-destroy-coll-per-N-trans", (char*) NULL)) {
+ m_destroy_coll_every_nr_runs = strtoll(val.c_str(), NULL, 10);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-num-ops", (char*) NULL)) {
+ m_num_ops = strtoll(val.c_str(), NULL, 10);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-max-in-flight", (char*) NULL)) {
+ m_max_in_flight = strtoll(val.c_str(), NULL, 10);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-write-data-size", (char*) NULL)) {
+ m_write_data_bytes = _parse_size_or_die(val);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-write-xattr-obj-size", (char*) NULL)) {
+ m_write_xattr_obj_bytes = _parse_size_or_die(val);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-write-xattr-coll-size", (char*) NULL)) {
+ m_write_xattr_coll_bytes = _parse_size_or_die(val);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-write-pglog-size", (char*) NULL)) {
+ m_write_pglog_bytes = _parse_size_or_die(val);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-suppress-ops", (char*) NULL)) {
+ _suppress_ops_or_die(val);
+ } else if (ceph_argparse_witharg(args, i, &val,
+ "--test-show-stats-period", (char*) NULL)) {
+ m_stats_show_secs = strtoll(val.c_str(), NULL, 10);
+ } else if (ceph_argparse_flag(args, i, "--test-show-stats", (char*) NULL)) {
+ m_do_stats = true;
+ } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) {
+ usage();
+ exit(0);
+ }
+ }
+}
+
+int WorkloadGenerator::get_uniform_random_value(int min, int max)
+{
+ boost::uniform_int<> value(min, max);
+ return value(m_rng);
+}
+
+TestObjectStoreState::coll_entry_t *WorkloadGenerator::get_rnd_coll_entry(bool erase = false)
+{
+ int index = get_uniform_random_value(0, m_collections_ids.size()-1);
+ coll_entry_t *entry = get_coll_at(index, erase);
+ return entry;
+}
+
+hobject_t *WorkloadGenerator::get_rnd_obj(coll_entry_t *entry)
+{
+ assert(entry != NULL);
+
+ bool create =
+ (get_uniform_random_value(0,100) < 50 || !entry->m_objects.size());
+
+ if (create && ((int) entry->m_objects.size() < m_num_objs_per_coll)) {
+ return (entry->touch_obj(entry->m_next_object_id++));
+ }
+
+ int idx = get_uniform_random_value(0, entry->m_objects.size()-1);
+ return entry->get_obj_at(idx);
+}
+
+/**
+ * We'll generate a random amount of bytes, ranging from a single byte up to
+ * a couple of MB.
+ */
+size_t WorkloadGenerator::get_random_byte_amount(size_t min, size_t max)
+{
+ size_t diff = max - min;
+ return (size_t) (min + (rand() % diff));
+}
+
+void WorkloadGenerator::get_filled_byte_array(bufferlist& bl, size_t size)
+{
+ static const char alphanum[] = "0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz";
+
+ bufferptr bp(size);
+ if (false) {
+ for (unsigned int i = 0; i < size - 1; i++) {
+ bp[i] = alphanum[rand() % sizeof(alphanum)];
+ }
+ bp[size - 1] = '\0';
+ } else {
+ bp.zero();
+ }
+ bl.append(bp);
+}
+
+void WorkloadGenerator::do_write_object(ObjectStore::Transaction *t,
+ coll_t coll, hobject_t obj,
+ C_StatState *stat)
+{
+ if (m_suppress_write_data) {
+ dout(5) << __func__ << " suppressed" << dendl;
+ return;
+ }
+
+ size_t size = m_write_data_bytes;
+ if (!size)
+ size = get_random_byte_amount(min_write_bytes, max_write_bytes);
+
+ bufferlist bl;
+ get_filled_byte_array(bl, size);
+
+ dout(2) << __func__ << " " << coll << "/" << obj
+ << " size " << bl.length() << dendl;
+
+ if (m_do_stats && (stat != NULL))
+ stat->written_data += bl.length();
+
+ t->write(coll, obj, 0, bl.length(), bl);
+}
+
+void WorkloadGenerator::do_setattr_object(ObjectStore::Transaction *t,
+ coll_t coll, hobject_t obj,
+ C_StatState *stat)
+{
+ if (m_suppress_write_xattr_obj) {
+ dout(5) << __func__ << " suppressed" << dendl;
+ return;
+ }
+
+ size_t size = m_write_xattr_obj_bytes;
+ if (!size)
+ size = get_random_byte_amount(min_xattr_obj_bytes, max_xattr_obj_bytes);
+
+ bufferlist bl;
+ get_filled_byte_array(bl, size);
+
+ dout(2) << __func__ << " " << coll << "/" << obj << " size " << size << dendl;
+
+ if (m_do_stats && (stat != NULL))
+ stat->written_data += bl.length();
+
+ t->setattr(coll, obj, "objxattr", bl);
+}
+
+void WorkloadGenerator::do_setattr_collection(ObjectStore::Transaction *t,
+ coll_t coll, C_StatState *stat)
+{
+ if (m_suppress_write_xattr_coll) {
+ dout(5) << __func__ << " suppressed" << dendl;
+ return;
+ }
+
+ size_t size = m_write_xattr_coll_bytes;
+ if (!size)
+ size = get_random_byte_amount(min_xattr_coll_bytes, max_xattr_coll_bytes);
+
+ bufferlist bl;
+ get_filled_byte_array(bl, size);
+ dout(2) << __func__ << " coll " << coll << " size " << size << dendl;
+
+ if (m_do_stats && (stat != NULL))
+ stat->written_data += bl.length();
+
+ t->collection_setattr(coll, "collxattr", bl);
+}
+
+void WorkloadGenerator::do_append_log(ObjectStore::Transaction *t,
+ coll_entry_t *entry, C_StatState *stat)
+{
+ if (m_suppress_write_log) {
+ dout(5) << __func__ << " suppressed" << dendl;
+ return;
+ }
+
+ size_t size = (m_write_pglog_bytes ? m_write_pglog_bytes : log_append_bytes);
+
+ bufferlist bl;
+ get_filled_byte_array(bl, size);
+ hobject_t log_obj = entry->m_meta_obj;
+
+ dout(2) << __func__ << " coll " << entry->m_coll << " "
+ << META_COLL << " /" << log_obj << " (" << bl.length() << ")" << dendl;
+
+ if (m_do_stats && (stat != NULL))
+ stat->written_data += bl.length();
+
+ uint64_t s = pg_log_size[entry->m_coll];
+ t->write(META_COLL, log_obj, s, bl.length(), bl);
+ pg_log_size[entry->m_coll] += bl.length();
+}
+
+void WorkloadGenerator::do_destroy_collection(ObjectStore::Transaction *t,
+ coll_entry_t *entry,
+ C_StatState *stat)
+{
+ m_nr_runs.set(0);
+ entry->m_osr.flush();
+ vector<ghobject_t> ls;
+ m_store->collection_list(entry->m_coll, ls);
+ dout(2) << __func__ << " coll " << entry->m_coll
+ << " (" << ls.size() << " objects)" << dendl;
+
+ for (vector<ghobject_t>::iterator it = ls.begin(); it < ls.end(); ++it) {
+ t->remove(entry->m_coll, *it);
+ }
+
+ t->remove_collection(entry->m_coll);
+ t->remove(META_COLL, entry->m_meta_obj);
+}
+
+TestObjectStoreState::coll_entry_t
+*WorkloadGenerator::do_create_collection(ObjectStore::Transaction *t,
+ C_StatState *stat)
+{
+ coll_entry_t *entry = coll_create(m_next_coll_nr++);
+ if (!entry) {
+ dout(0) << __func__ << " failed to create coll id "
+ << m_next_coll_nr << dendl;
+ return NULL;
+ }
+ m_collections.insert(make_pair(entry->m_id, entry));
+
+ dout(2) << __func__ << " id " << entry->m_id << " coll " << entry->m_coll << dendl;
+ t->create_collection(entry->m_coll);
+ dout(2) << __func__ << " meta " << META_COLL << "/" << entry->m_meta_obj << dendl;
+ t->touch(META_COLL, entry->m_meta_obj);
+ return entry;
+}
+
+void WorkloadGenerator::do_stats()
+{
+ utime_t now = ceph_clock_now(NULL);
+ m_stats_lock.Lock();
+
+ utime_t duration = (now - m_stats_begin);
+
+ // when cast to double, a utime_t behaves properly
+ double throughput = (m_stats_total_written / ((double) duration));
+ double tx_throughput (m_stats_finished_txs / ((double) duration));
+
+ dout(0) << __func__
+ << " written: " << m_stats_total_written
+ << " duration: " << duration << " sec"
+ << " bandwidth: " << prettybyte_t(throughput) << "/s"
+ << " iops: " << tx_throughput << "/s"
+ << dendl;
+
+ m_stats_lock.Unlock();
+}
+
+void WorkloadGenerator::run()
+{
+ bool create_coll = false;
+ int ops_run = 0;
+
+ utime_t stats_interval(m_stats_show_secs, 0);
+ utime_t now = ceph_clock_now(NULL);
+ utime_t stats_time = now;
+ m_stats_begin = now;
+
+ do {
+ C_StatState *stat_state = NULL;
+
+ if (m_num_ops && (ops_run == m_num_ops))
+ break;
+
+ if (!create_coll && !m_collections.size()) {
+ dout(0) << "We ran out of collections!" << dendl;
+ break;
+ }
+
+ dout(5) << __func__
+ << " m_finished_lock is-locked: " << m_finished_lock.is_locked()
+ << " in-flight: " << m_in_flight.read()
+ << dendl;
+
+ wait_for_ready();
+
+ ObjectStore::Transaction *t = new ObjectStore::Transaction;
+ Context *c;
+ bool destroy_collection = false;
+ TestObjectStoreState::coll_entry_t *entry = NULL;
+
+
+ if (m_do_stats) {
+ utime_t now = ceph_clock_now(NULL);
+ utime_t elapsed = now - stats_time;
+ if (elapsed >= stats_interval) {
+ do_stats();
+ stats_time = now;
+ }
+ stat_state = new C_StatState(this, now);
+ }
+
+ if (create_coll) {
+ create_coll = false;
+
+ entry = do_create_collection(t, stat_state);
+ if (!entry) {
+ dout(0) << __func__ << " something went terribly wrong creating coll" << dendl;
+ break;
+ }
+
+ c = new C_OnReadable(this, t);
+ goto queue_tx;
+ }
+
+ destroy_collection = should_destroy_collection();
+ entry = get_rnd_coll_entry(destroy_collection);
+ assert(entry != NULL);
+
+ if (destroy_collection) {
+ do_destroy_collection(t, entry, stat_state);
+ c = new C_OnDestroyed(this, t, entry);
+ if (!m_num_ops)
+ create_coll = true;
+ } else {
+ hobject_t *obj = get_rnd_obj(entry);
+
+ do_write_object(t, entry->m_coll, *obj, stat_state);
+ do_setattr_object(t, entry->m_coll, *obj, stat_state);
+ do_setattr_collection(t, entry->m_coll, stat_state);
+ do_append_log(t, entry, stat_state);
+
+ c = new C_OnReadable(this, t);
+ }
+
+queue_tx:
+
+ if (m_do_stats) {
+ Context *tmp = c;
+ c = new C_StatWrapper(stat_state, tmp);
+ }
+
+ m_store->queue_transaction(&(entry->m_osr), t, c);
+
+ inc_in_flight();
+
+ ops_run ++;
+
+ } while (true);
+
+ dout(2) << __func__ << " waiting for "
+ << m_in_flight.read() << " in-flight transactions" << dendl;
+
+ wait_for_done();
+
+ do_stats();
+
+ dout(0) << __func__ << " finishing" << dendl;
+}
+
+void usage()
+{
+ cout << "usage: " << our_name << "[options]" << std::endl;
+
+ cout << "\
+\n\
+Global Options:\n\
+ -c FILE Read configuration from FILE\n\
+ --osd-objectstore TYPE Set OSD ObjectStore type\n\
+ --osd-data PATH Set OSD Data path\n\
+ --osd-journal PATH Set OSD Journal path\n\
+ --osd-journal-size VAL Set Journal size\n\
+ --help This message\n\
+\n\
+Test-specific Options:\n\
+ --test-num-colls VAL Set the number of collections\n\
+ --test-num-objs-per-coll VAL Set the number of objects per collection\n\
+ --test-destroy-coll-per-N-trans VAL Set how many transactions to run before\n\
+ destroying a collection.\n\
+ --test-num-ops VAL Run a certain number of operations\n\
+ (a VAL of 0 runs the test forever)\n\
+ --test-max-in-flight VAL Maximum number of in-flight transactions\n\
+ (default: 50)\n\
+ --test-suppress-ops OPS Suppress ops specified in OPS\n\
+ --test-write-data-size SIZE Specify SIZE for all data writes\n\
+ --test-write-xattr-obj-size SIZE Specify SIZE for all xattrs on objects\n\
+ --test-write-xattr-coll-size SIZE Specify SIZE for all xattrs on colls\n\
+ --test-write-pglog-size SIZE Specify SIZE for all pglog writes\n\
+ --test-show-stats Show stats as we go\n\
+ --test-show-stats-period SECS Show stats every SECS (default: 5)\n\
+\n\
+ SIZE is a numeric value that can be assumed as being bytes, or may be any\n\
+ other unit if specified: B or b, K or k, M or m, G or g.\n\
+ e.g., 1G = 1024M = 1048576k = 1073741824\n\
+\n\
+ OPS can be one or more of the following options:\n\
+ c writes on collection's xattrs\n\
+ o writes on object's xattr\n\
+ l writes on pglog\n\
+ d data writes on objects\n\
+\n\
+" << std::endl;
+}
+
+int main(int argc, const char *argv[])
+{
+ vector<const char*> def_args;
+ vector<const char*> args;
+
+ our_name = argv[0];
+
+ def_args.push_back("--osd-journal-size");
+ def_args.push_back("400");
+// def_args.push_back("--osd-data");
+// def_args.push_back("workload_gen_dir");
+// def_args.push_back("--osd-journal");
+// def_args.push_back("workload_gen_dir/journal");
+ argv_to_vec(argc, argv, args);
+
+ global_init(&def_args, args,
+ CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+
+ WorkloadGenerator *wrkldgen_ptr = new WorkloadGenerator(args);
+ wrkldgen.reset(wrkldgen_ptr);
+ wrkldgen->run();
+ return 0;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2012 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ */
+#ifndef WORKLOAD_GENERATOR_H_
+#define WORKLOAD_GENERATOR_H_
+
+#include "os/ObjectStore.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/random/mersenne_twister.hpp>
+#include <boost/random/uniform_int.hpp>
+#include <map>
+#include <sys/time.h>
+
+#include "TestObjectStoreState.h"
+
+typedef boost::mt11213b rngen_t;
+
+class WorkloadGenerator : public TestObjectStoreState {
+ public:
+ static const int def_max_in_flight = 50;
+
+ static const int def_destroy_coll_every_nr_runs = 100;
+ static const int def_num_obj_per_coll = 6000;
+ static const int def_num_colls = 30;
+
+ static const size_t min_write_bytes = 1;
+ static const size_t max_write_mb = 5;
+ static const size_t max_write_bytes = (max_write_mb * 1024 * 1024);
+
+ static const size_t min_xattr_obj_bytes = 2;
+ static const size_t max_xattr_obj_bytes = 300;
+ static const size_t min_xattr_coll_bytes = 4;
+ static const size_t max_xattr_coll_bytes = 600;
+
+ static const size_t log_append_bytes = 1024;
+
+ struct C_StatState {
+ utime_t start;
+ unsigned int written_data;
+ WorkloadGenerator *wrkldgen;
+
+ C_StatState(WorkloadGenerator *state, utime_t s)
+ : start(s), written_data(0), wrkldgen(state) { }
+ };
+
+
+ protected:
+ int m_max_in_flight;
+ int m_num_ops;
+ int m_destroy_coll_every_nr_runs;
+ atomic_t m_nr_runs;
+
+ int m_num_colls;
+
+ rngen_t m_rng;
+
+ map<coll_t, uint64_t> pg_log_size;
+
+ size_t m_write_data_bytes;
+ size_t m_write_xattr_obj_bytes;
+ size_t m_write_xattr_coll_bytes;
+ size_t m_write_pglog_bytes;
+
+ bool m_suppress_write_data;
+ bool m_suppress_write_xattr_obj;
+ bool m_suppress_write_xattr_coll;
+ bool m_suppress_write_log;
+
+ bool m_do_stats;
+
+ int m_stats_finished_txs;
+ Mutex m_stats_lock;
+ int m_stats_show_secs;
+
+ size_t m_stats_total_written;
+ utime_t m_stats_begin;
+
+ private:
+
+ void _suppress_ops_or_die(std::string& val);
+ size_t _parse_size_or_die(std::string& val);
+ void init_args(vector<const char*> args);
+
+ int get_uniform_random_value(int min, int max);
+ coll_entry_t *get_rnd_coll_entry(bool erase);
+ hobject_t *get_rnd_obj(coll_entry_t *entry);
+ int get_random_collection_nr();
+ int get_random_object_nr(int coll_nr);
+
+ size_t get_random_byte_amount(size_t min, size_t max);
+ void get_filled_byte_array(bufferlist& bl, size_t size);
+
+ void do_write_object(ObjectStore::Transaction *t,
+ coll_t coll, hobject_t obj, C_StatState *stat);
+ void do_setattr_object(ObjectStore::Transaction *t,
+ coll_t coll, hobject_t obj, C_StatState *stat);
+ void do_setattr_collection(ObjectStore::Transaction *t, coll_t coll,
+ C_StatState *stat);
+ void do_append_log(ObjectStore::Transaction *t, coll_entry_t *entry,
+ C_StatState *stat);
+
+ bool should_destroy_collection() {
+ return ((m_destroy_coll_every_nr_runs > 0) &&
+ ((int)m_nr_runs.read() >= m_destroy_coll_every_nr_runs));
+ }
+ void do_destroy_collection(ObjectStore::Transaction *t, coll_entry_t *entry,
+ C_StatState *stat);
+ coll_entry_t *do_create_collection(ObjectStore::Transaction *t,
+ C_StatState *stat);
+
+ void do_stats();
+
+public:
+ WorkloadGenerator(vector<const char*> args);
+ ~WorkloadGenerator() {
+ m_store->umount();
+ }
+
+ class C_OnReadable: public TestObjectStoreState::C_OnFinished {
+ WorkloadGenerator *wrkldgen_state;
+
+ public:
+ C_OnReadable(WorkloadGenerator *state,
+ ObjectStore::Transaction *t)
+ :TestObjectStoreState::C_OnFinished(state, t), wrkldgen_state(state) { }
+
+ void finish(int r)
+ {
+ TestObjectStoreState::C_OnFinished::finish(r);
+ wrkldgen_state->m_nr_runs.inc();
+ }
+ };
+
+ class C_OnDestroyed: public C_OnReadable {
+ coll_entry_t *m_entry;
+
+ public:
+ C_OnDestroyed(WorkloadGenerator *state,
+ ObjectStore::Transaction *t, coll_entry_t *entry) :
+ C_OnReadable(state, t), m_entry(entry) {}
+
+ void finish(int r) {
+ C_OnReadable::finish(r);
+ delete m_entry;
+ }
+ };
+
+ class C_StatWrapper : public Context {
+ C_StatState *stat_state;
+ Context *ctx;
+
+ public:
+ C_StatWrapper(C_StatState *state, Context *context)
+ : stat_state(state), ctx(context) { }
+
+ void finish(int r) {
+ ctx->complete(r);
+
+ stat_state->wrkldgen->m_stats_lock.Lock();
+
+ stat_state->wrkldgen->m_stats_total_written += stat_state->written_data;
+ stat_state->wrkldgen->m_stats_finished_txs ++;
+ stat_state->wrkldgen->m_stats_lock.Unlock();
+ }
+ };
+
+ void run(void);
+};
+
+bool operator<(const WorkloadGenerator::coll_entry_t& l,
+ const WorkloadGenerator::coll_entry_t& r) {
+ return (l.m_id < r.m_id);
+}
+
+#endif /* WORKLOAD_GENERATOR_H_ */
projects / ceph.git / commitdiff