export LANG="$OLDLANG"
}
-function TEST_pool_create_rep_expected_num_objects() {
- local dir=$1
- setup $dir || return 1
-
- export CEPH_ARGS
- run_mon $dir a || return 1
- run_mgr $dir x || return 1
- # disable pg dir merge
- run_osd_filestore $dir 0 || return 1
-
- ceph osd pool create rep_expected_num_objects 64 64 replicated replicated_rule 100000 || return 1
- # wait for pg dir creating
- sleep 30
- ceph pg ls
- find ${dir}/0/current -ls
- ret=$(find ${dir}/0/current/1.0_head/ | grep DIR | wc -l)
- if [ "$ret" -le 2 ];
- then
- return 1
- else
- echo "TEST_pool_create_rep_expected_num_objects PASS"
- fi
-}
-
function check_pool_priority() {
local dir=$1
shift
done
}
-function TEST_filestore_to_bluestore() {
- local dir=$1
-
- local flimit=$(ulimit -n)
- if [ $flimit -lt 1536 ]; then
- echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens."
- fi
-
- run_mon $dir a || return 1
- run_mgr $dir x || return 1
- run_osd_filestore $dir 0 || return 1
- osd_pid=$(cat $dir/osd.0.pid)
- run_osd_filestore $dir 1 || return 1
- run_osd_filestore $dir 2 || return 1
-
- sleep 5
-
- create_pool foo 16
-
- # write some objects
- timeout 20 rados bench -p foo 10 write -b 4096 --no-cleanup || return 1
-
- # kill
- while kill $osd_pid; do sleep 1 ; done
- ceph osd down 0
-
- mv $dir/0 $dir/0.old || return 1
- mkdir $dir/0 || return 1
- ofsid=$(cat $dir/0.old/fsid)
- echo "osd fsid $ofsid"
- O=$CEPH_ARGS
- CEPH_ARGS+="--log-file $dir/cot.log --log-max-recent 0 "
- ceph-objectstore-tool --type bluestore --data-path $dir/0 --fsid $ofsid \
- --op mkfs --no-mon-config || return 1
- ceph-objectstore-tool --data-path $dir/0.old --target-data-path $dir/0 \
- --op dup || return 1
- CEPH_ARGS=$O
-
- activate_osd $dir 0 || return 1
-
- while ! ceph osd stat | grep '3 up' ; do sleep 1 ; done
- ceph osd metadata 0 | grep bluestore || return 1
-
- ceph osd scrub 0
-
- # give it some time
- sleep 15
- # and make sure mon is sync'ed
- flush_pg_stats
-
- wait_for_clean || return 1
-}
-
main osd-dup "$@"
# Local Variables:
delete_pool $poolname
}
-# This is a filestore only test because it requires data digest in object info
-function TEST_rep_read_unfound() {
- local dir=$1
- local objname=myobject
-
- setup_osds 3 _filestore || return 1
-
- ceph osd pool delete foo foo --yes-i-really-really-mean-it || return 1
- local poolname=test-pool
- create_pool $poolname 1 1 || return 1
- ceph osd pool set $poolname size 2
- wait_for_clean || return 1
-
- ceph pg dump pgs
-
- dd if=/dev/urandom bs=8k count=1 of=$dir/ORIGINAL
- rados -p $poolname put $objname $dir/ORIGINAL
-
- local primary=$(get_primary $poolname $objname)
- local other=$(get_not_primary $poolname $objname)
-
- dd if=/dev/urandom bs=8k count=1 of=$dir/CORRUPT
- objectstore_tool $dir $primary $objname set-bytes $dir/CORRUPT || return 1
- objectstore_tool $dir $other $objname set-bytes $dir/CORRUPT || return 1
-
- timeout 30 rados -p $poolname get $objname $dir/tmp &
-
- sleep 5
-
- flush_pg_stats
- ceph --format=json pg dump pgs | jq '.'
-
- if ! ceph --format=json pg dump pgs | jq '.pg_stats | .[0].state' | grep -q recovery_unfound
- then
- echo "Failure to get to recovery_unfound state"
- return 1
- fi
-
- objectstore_tool $dir $other $objname set-bytes $dir/ORIGINAL || return 1
-
- wait
-
- if ! cmp $dir/ORIGINAL $dir/tmp
- then
- echo "Bad data after primary repair"
- return 1
- fi
-}
-
main osd-rep-recov-eio.sh "$@"
# Local Variables:
--osd-scrub-min-interval=5 \
--osd-scrub-interval-randomize-ratio=0"
for id in $(seq 0 2) ; do
- if [ "$allow_overwrites" = "true" ]; then
- run_osd $dir $id $ceph_osd_args || return 1
- else
- run_osd_filestore $dir $id $ceph_osd_args || return 1
- fi
+ run_osd $dir $id $ceph_osd_args || return 1
done
create_rbd_pool || return 1
wait_for_clean || return 1
run_mon $dir a || return 1
run_mgr $dir x || return 1
for id in $(seq 0 3) ; do
- if [ "$allow_overwrites" = "true" ]; then
- run_osd $dir $id || return 1
- else
- run_osd_filestore $dir $id || return 1
- fi
+ run_osd $dir $id || return 1
done
create_rbd_pool || return 1
wait_for_clean || return 1
run_mon $dir a || return 1
run_mgr $dir x || return 1
for id in $(seq 0 9) ; do
- if [ "$allow_overwrites" = "true" ]; then
- run_osd $dir $id || return 1
- else
- run_osd_filestore $dir $id || return 1
- fi
+ run_osd $dir $id || return 1
done
create_rbd_pool || return 1
wait_for_clean || return 1
run_mon $dir a || return 1
run_mgr $dir x || return 1
for id in $(seq 0 3) ; do
- if [ "$allow_overwrites" = "true" ]; then
- run_osd $dir $id || return 1
- else
- run_osd_filestore $dir $id || return 1
- fi
+ run_osd $dir $id || return 1
done
create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
run_mon $dir a || return 1
run_mgr $dir x || return 1
for id in $(seq 0 2) ; do
- if [ "$allow_overwrites" = "true" ]; then
- run_osd $dir $id || return 1
- else
- run_osd_filestore $dir $id || return 1
- fi
+ run_osd $dir $id || return 1
done
create_rbd_pool || return 1
wait_for_clean || return 1
run_mon $dir a || return 1
run_mgr $dir x || return 1
for id in $(seq 0 2) ; do
- if [ "$allow_overwrites" = "true" ]; then
- run_osd $dir $id || return 1
- else
- run_osd_filestore $dir $id || return 1
- fi
+ run_osd $dir $id || return 1
done
create_rbd_pool || return 1
create_pool foo 1
add_subdirectory(journal)
add_subdirectory(erasure-code)
- add_subdirectory(filestore)
add_subdirectory(fs)
add_subdirectory(libcephfs)
add_subdirectory(libcephsqlite)
add_subdirectory(neorados)
add_subdirectory(objectstore)
add_subdirectory(ObjectMap)
- add_subdirectory(os)
add_subdirectory(osd)
add_subdirectory(osdc)
add_subdirectory(immutable_object_cache)
ceph_perf_local
DESTINATION bin)
-# ceph_test_filejournal
-add_executable(ceph_test_filejournal
- test_filejournal.cc
- )
-target_link_libraries(ceph_test_filejournal
- os
- ceph-common
- ${UNITTEST_LIBS}
- global
- ${EXTRALIBS}
- ${BLKID_LIBRARIES}
- ${CMAKE_DL_LIBS}
- ${EXTRALIBS}
- )
-install(TARGETS
- ceph_test_filejournal
- DESTINATION ${CMAKE_INSTALL_BINDIR})
-
# ceph_test_keys
add_executable(ceph_test_keys
testkeys.cc
+++ /dev/null
-#ceph_test_filestore
-add_executable(ceph_test_filestore
- TestFileStore.cc)
-target_link_libraries(ceph_test_filestore os global ${UNITTEST_LIBS})
-
-
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "os/filestore/FileStore.h"
-#include <gtest/gtest.h>
-
-using namespace std;
-
-class TestFileStore {
-public:
- static void create_backend(FileStore &fs, unsigned long f_type) {
- fs.create_backend(f_type);
- }
-};
-
-TEST(FileStore, create)
-{
- {
- map<string,string> pm;
- FileStore fs(g_ceph_context, "a", "b");
- TestFileStore::create_backend(fs, 0);
- fs.collect_metadata(&pm);
- ASSERT_EQ(pm["filestore_backend"], "generic");
- }
-#if defined(__linux__)
- {
- map<string,string> pm;
- FileStore fs(g_ceph_context, "a", "b");
- TestFileStore::create_backend(fs, BTRFS_SUPER_MAGIC);
- fs.collect_metadata(&pm);
- ASSERT_EQ(pm["filestore_backend"], "btrfs");
- }
-# ifdef HAVE_LIBXFS
- {
- map<string,string> pm;
- FileStore fs(g_ceph_context, "a", "b");
- TestFileStore::create_backend(fs, XFS_SUPER_MAGIC);
- fs.collect_metadata(&pm);
- ASSERT_EQ(pm["filestore_backend"], "xfs");
- }
-# endif
-#endif
-#ifdef HAVE_LIBZFS
- {
- map<string,string> pm;
- FileStore fs("a", "b");
- TestFileStore::create_backend(fs, ZFS_SUPER_MAGIC);
- fs.collect_metadata(&pm);
- ASSERT_EQ(pm["filestore_backend"], "zfs");
- }
-#endif
-}
-
-int main(int argc, char **argv) {
- auto args = argv_to_vec(argc, argv);
-
- auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
- CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf.set_val("osd_journal_size", "100");
- g_ceph_context->_conf.apply_changes(NULL);
-
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
-
-/*
- * Local Variables:
- * compile-command: "cd ../.. ; make ceph_test_filestore &&
- * ./ceph_test_filestore \
- * --gtest_filter=*.* --log-to-stderr=true --debug-filestore=20
- * "
- * End:
- */
install(TARGETS ceph_test_keyvaluedb
DESTINATION ${CMAKE_INSTALL_BINDIR})
-# ceph_test_filestore_idempotent
-add_executable(ceph_test_filestore_idempotent
- test_idempotent.cc
- FileStoreTracker.cc
- ${CMAKE_SOURCE_DIR}/src/test/common/ObjectContents.cc
- )
-target_link_libraries(ceph_test_filestore_idempotent
- os
- global
- ${EXTRALIBS}
- ${BLKID_LIBRARIES}
- ${CMAKE_DL_LIBS}
- )
-
-# ceph_test_filestore_idempotent_sequence
-add_executable(ceph_test_filestore_idempotent_sequence
- test_idempotent_sequence.cc
- DeterministicOpSequence.cc
- TestObjectStoreState.cc
- FileStoreDiff.cc
- )
-target_link_libraries(ceph_test_filestore_idempotent_sequence
- os
- global
- ${EXTRALIBS}
- ${BLKID_LIBRARIES}
- ${CMAKE_DL_LIBS}
- )
-install(TARGETS ceph_test_filestore_idempotent_sequence
- DESTINATION ${CMAKE_INSTALL_BINDIR})
-
-# unittest_chain_xattr
-add_executable(unittest_chain_xattr
- chain_xattr.cc
- )
-add_ceph_unittest(unittest_chain_xattr)
-target_link_libraries(unittest_chain_xattr os global)
-
# unittest_rocksdb_option
add_executable(unittest_rocksdb_option
TestRocksdbOptionParse.cc
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include <fstream>
-#include <time.h>
-#include <stdlib.h>
-#include <signal.h>
-#include <sstream>
-#include "os/ObjectStore.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "common/debug.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/lexical_cast.hpp>
-
-#include "DeterministicOpSequence.h"
-#include "common/config.h"
-#include "include/ceph_assert.h"
-
-#define dout_context g_ceph_context
-#define dout_subsys ceph_subsys_filestore
-#undef dout_prefix
-#define dout_prefix *_dout << "deterministic_seq "
-
-using namespace std;
-
-DeterministicOpSequence::DeterministicOpSequence(ObjectStore *store,
- std::string status)
- : TestObjectStoreState(store),
- txn(0)
-{
- if (!status.empty())
- m_status.open(status.c_str());
-}
-
-DeterministicOpSequence::~DeterministicOpSequence()
-{
- // TODO Auto-generated destructor stub
-}
-
-bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen)
-{
- bool ok = false;
- switch (op) {
- case DSOP_TOUCH:
- ok = do_touch(gen);
- break;
- case DSOP_WRITE:
- ok = do_write(gen);
- break;
- case DSOP_CLONE:
- ok = do_clone(gen);
- break;
- case DSOP_CLONE_RANGE:
- ok = do_clone_range(gen);
- break;
- case DSOP_OBJ_REMOVE:
- ok = do_remove(gen);
- break;
- case DSOP_COLL_MOVE:
- ok = do_coll_move(gen);
- break;
- case DSOP_SET_ATTRS:
- ok = do_set_attrs(gen);
- break;
- case DSOP_COLL_CREATE:
- ok = do_coll_create(gen);
- break;
-
- default:
- cout << "bad op " << op << std::endl;
- ceph_abort_msg("bad op");
- }
- return ok;
-}
-
-void DeterministicOpSequence::generate(int seed, int num_txs)
-{
- std::ostringstream ss;
- ss << "generate run " << num_txs << " --seed " << seed;
-
- if (m_status.is_open()) {
- m_status << ss.str() << std::endl;
- m_status.flush();
- }
-
- dout(0) << ss.str() << dendl;
-
- rngen_t gen(seed);
- boost::uniform_int<> op_rng(DSOP_FIRST, DSOP_LAST);
-
- for (txn = 1; txn <= num_txs; ) {
- int op = op_rng(gen);
- _print_status(txn, op);
- dout(0) << "generate seq " << txn << " op " << op << dendl;
- if (run_one_op(op, gen))
- txn++;
- }
-}
-
-void DeterministicOpSequence::_print_status(int seq, int op)
-{
- if (!m_status.is_open())
- return;
- m_status << seq << " " << op << std::endl;
- m_status.flush();
-}
-
-int DeterministicOpSequence::_gen_coll_id(rngen_t& gen)
-{
- boost::uniform_int<> coll_rng(0, m_collections_ids.size()-1);
- return coll_rng(gen);
-}
-
-int DeterministicOpSequence::_gen_obj_id(rngen_t& gen)
-{
- boost::uniform_int<> obj_rng(0, m_num_objects - 1);
- return obj_rng(gen);
-}
-
-void DeterministicOpSequence::note_txn(coll_entry_t *entry,
- ObjectStore::Transaction *t)
-{
- bufferlist bl;
- encode(txn, bl);
- ghobject_t oid = get_txn_object(entry->m_cid);
- t->truncate(entry->m_cid, oid, 0);
- t->write(entry->m_cid, oid, 0, bl.length(), bl);
- dout(10) << __func__ << " " << txn << dendl;
-}
-
-bool DeterministicOpSequence::do_touch(rngen_t& gen)
-{
- int coll_id = _gen_coll_id(gen);
- int obj_id = _gen_obj_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- // Don't care about other collections if already exists
- if (!entry->check_for_obj(obj_id)) {
- bool other_found = false;
- auto it = m_collections.begin();
- for (; it != m_collections.end(); ++it) {
- if (it->second->check_for_obj(obj_id)) {
- ceph_assert(it->first != entry->m_cid);
- other_found = true;
- }
- }
- if (other_found) {
- dout(0) << "do_touch new object in collection and exists in another" << dendl;
- return false;
- }
- }
- hobject_t *obj = entry->touch_obj(obj_id);
-
- dout(0) << "do_touch " << entry->m_cid << "/" << obj << dendl;
-
- _do_touch(entry, *obj);
- return true;
-}
-
-bool DeterministicOpSequence::do_remove(rngen_t& gen)
-{
- int coll_id = _gen_coll_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- if (entry->m_objects.size() == 0) {
- dout(0) << "do_remove no objects in collection" << dendl;
- return false;
- }
- int obj_id = entry->get_random_obj_id(gen);
- hobject_t *obj = entry->touch_obj(obj_id);
- ceph_assert(obj);
-
- dout(0) << "do_remove " << entry->m_cid << "/" << obj << dendl;
-
- _do_remove(entry, *obj);
- hobject_t *rmobj = entry->remove_obj(obj_id);
- ceph_assert(rmobj);
- delete rmobj;
- return true;
-}
-
-static void _gen_random(rngen_t& gen,
- size_t size, bufferlist& bl) {
-
- static const char alphanum[] = "0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
- "abcdefghijklmnopqrstuvwxyz";
-
- boost::uniform_int<> char_rng(0, sizeof(alphanum));
- bufferptr bp(size);
- for (unsigned int i = 0; i < size - 1; i++) {
- bp[i] = alphanum[char_rng(gen)];
- }
- bp[size - 1] = '\0';
- bl.append(bp);
-}
-
-static void gen_attrs(rngen_t &gen,
- map<string, bufferlist> *out) {
- boost::uniform_int<> num_rng(10, 30);
- boost::uniform_int<> key_size_rng(5, 10);
- boost::uniform_int<> val_size_rng(100, 1000);
- size_t num_attrs = static_cast<size_t>(num_rng(gen));
- for (size_t i = 0; i < num_attrs; ++i) {
- size_t key_size = static_cast<size_t>(num_rng(gen));
- size_t val_size = static_cast<size_t>(num_rng(gen));
- bufferlist keybl;
- _gen_random(gen, key_size, keybl);
- string key(keybl.c_str(), keybl.length());
- _gen_random(gen, val_size, (*out)[key]);
- }
-}
-
-bool DeterministicOpSequence::do_set_attrs(rngen_t& gen)
-{
- int coll_id = _gen_coll_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- if (entry->m_objects.size() == 0) {
- dout(0) << "do_set_attrs no objects in collection" << dendl;
- return false;
- }
- int obj_id = entry->get_random_obj_id(gen);
- hobject_t *obj = entry->touch_obj(obj_id);
- ceph_assert(obj);
-
- map<string, bufferlist> out;
- gen_attrs(gen, &out);
-
- dout(0) << "do_set_attrs " << out.size() << " entries" << dendl;
- _do_set_attrs(entry, *obj, out);
- return true;
-}
-
-bool DeterministicOpSequence::do_write(rngen_t& gen)
-{
- int coll_id = _gen_coll_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- if (entry->m_objects.size() == 0) {
- dout(0) << "do_write no objects in collection" << dendl;
- return false;
- }
- int obj_id = entry->get_random_obj_id(gen);
- hobject_t *obj = entry->touch_obj(obj_id);
- ceph_assert(obj);
-
- boost::uniform_int<> size_rng(100, (2 << 19));
- size_t size = (size_t) size_rng(gen);
- bufferlist bl;
- _gen_random(gen, size, bl);
-
- dout(0) << "do_write " << entry->m_cid << "/" << obj
- << " 0~" << size << dendl;
-
- _do_write(entry, *obj, 0, bl.length(), bl);
- return true;
-}
-
-bool DeterministicOpSequence::_prepare_clone(
- rngen_t& gen,
- coll_entry_t **entry_ret,
- int *orig_obj_id,
- hobject_t *orig_obj_ret,
- int *new_obj_id,
- hobject_t *new_obj_ret)
-{
- int coll_id = _gen_coll_id(gen);
-
- coll_entry_t *entry = get_coll_at(coll_id);
- ceph_assert(entry != NULL);
-
- if (entry->m_objects.size() < 2) {
- dout(0) << "_prepare_clone coll " << entry->m_cid
- << " doesn't have 2 or more objects" << dendl;
- return false;
- }
-
- *orig_obj_id = entry->get_random_obj_id(gen);
- hobject_t *orig_obj = entry->touch_obj(*orig_obj_id);
- ceph_assert(orig_obj);
-
- do {
- *new_obj_id = entry->get_random_obj_id(gen);
- } while (*new_obj_id == *orig_obj_id);
- hobject_t *new_obj = entry->touch_obj(*new_obj_id);
- ceph_assert(new_obj);
-
- *entry_ret = entry;
- *orig_obj_ret = *orig_obj;
- *new_obj_ret = *new_obj;
- return true;
-}
-
-bool DeterministicOpSequence::do_clone(rngen_t& gen)
-{
- coll_entry_t *entry;
- int orig_id, new_id;
- hobject_t orig_obj, new_obj;
- if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) {
- return false;
- }
-
- dout(0) << "do_clone " << entry->m_cid << "/" << orig_obj
- << " => " << entry->m_cid << "/" << new_obj << dendl;
-
- _do_clone(entry, orig_obj, new_obj);
- return true;
-}
-
-bool DeterministicOpSequence::do_clone_range(rngen_t& gen)
-{
- coll_entry_t *entry;
- int orig_id, new_id;
- hobject_t orig_obj, new_obj;
- if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) {
- return false;
- }
-
- /* Whenever we have to make a clone_range() operation, just write to the
- * object first, so we know we have something to clone in the said range.
- * This may not be the best solution ever, but currently we're not keeping
- * track of the written-to objects, and until we know for sure we really
- * need to, let's just focus on the task at hand.
- */
-
- boost::uniform_int<> write_size_rng(100, (2 << 19));
- size_t size = (size_t) write_size_rng(gen);
- bufferlist bl;
- _gen_random(gen, size, bl);
-
- boost::uniform_int<> clone_len(1, bl.length());
- size = (size_t) clone_len(gen);
-
- dout(0) << "do_clone_range " << entry->m_cid << "/" << orig_obj
- << " (0~" << size << ")"
- << " => " << entry->m_cid << "/" << new_obj
- << " (0)" << dendl;
- _do_write_and_clone_range(entry, orig_obj, new_obj, 0, size, 0, bl);
- return true;
-}
-
-bool DeterministicOpSequence::do_coll_move(rngen_t& gen)
-{
- coll_entry_t *entry;
- int orig_id, new_id;
- hobject_t orig_obj, new_obj;
- if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) {
- return false;
- }
-
- dout(0) << "do_coll_move " << entry->m_cid << "/" << orig_obj
- << " => " << entry->m_cid << "/" << new_obj << dendl;
- entry->remove_obj(orig_id);
-
- _do_coll_move(entry, orig_obj, new_obj);
-
- return true;
-}
-
-bool DeterministicOpSequence::do_coll_create(rngen_t& gen)
-{
- int i = m_collections.size();
- spg_t pgid(pg_t(i, 1), shard_id_t::NO_SHARD);
- coll_t cid(pgid);
- auto ch = m_store->create_new_collection(cid);
- coll_entry_t *entry = coll_create(pgid, ch);
- m_collections.insert(make_pair(cid, entry));
- rebuild_id_vec();
-
- _do_coll_create(entry, 10, 10);
-
- return true;
-}
-
-void DeterministicOpSequence::_do_coll_create(coll_entry_t *entry, uint32_t pg_num, uint64_t num_objs)
-{
- ObjectStore::Transaction t;
- t.create_collection(entry->m_cid, 32);
- note_txn(entry, &t);
- bufferlist hint;
- encode(pg_num, hint);
- encode(num_objs, hint);
- t.collection_hint(entry->m_cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint);
- dout(0) << "Give collection: " << entry->m_cid
- << " a hint, pg_num is: " << pg_num << ", num_objs is: "
- << num_objs << dendl;
-
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
-void DeterministicOpSequence::_do_touch(coll_entry_t *entry, hobject_t& obj)
-{
- ObjectStore::Transaction t;
- note_txn(entry, &t);
- t.touch(entry->m_cid, ghobject_t(obj));
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
-void DeterministicOpSequence::_do_remove(coll_entry_t *entry, hobject_t& obj)
-{
- ObjectStore::Transaction t;
- note_txn(entry, &t);
- t.remove(entry->m_cid, ghobject_t(obj));
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
-void DeterministicOpSequence::_do_set_attrs(coll_entry_t *entry,
- hobject_t &obj,
- const map<string, bufferlist> &attrs)
-{
- ObjectStore::Transaction t;
- note_txn(entry, &t);
- t.omap_setkeys(entry->m_cid, ghobject_t(obj), attrs);
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
-void DeterministicOpSequence::_do_write(coll_entry_t *entry, hobject_t& obj,
- uint64_t off, uint64_t len, const bufferlist& data)
-{
- ObjectStore::Transaction t;
- note_txn(entry, &t);
- t.write(entry->m_cid, ghobject_t(obj), off, len, data);
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
-void DeterministicOpSequence::_do_clone(coll_entry_t *entry, hobject_t& orig_obj,
- hobject_t& new_obj)
-{
- ObjectStore::Transaction t;
- note_txn(entry, &t);
- t.clone(entry->m_cid, ghobject_t(orig_obj), ghobject_t(new_obj));
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
-void DeterministicOpSequence::_do_clone_range(coll_entry_t *entry,
- hobject_t& orig_obj, hobject_t& new_obj, uint64_t srcoff,
- uint64_t srclen, uint64_t dstoff)
-{
- ObjectStore::Transaction t;
- note_txn(entry, &t);
- t.clone_range(entry->m_cid, ghobject_t(orig_obj), ghobject_t(new_obj),
- srcoff, srclen, dstoff);
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
-void DeterministicOpSequence::_do_write_and_clone_range(coll_entry_t *entry,
- hobject_t& orig_obj,
- hobject_t& new_obj,
- uint64_t srcoff,
- uint64_t srclen,
- uint64_t dstoff,
- bufferlist& bl)
-{
- ObjectStore::Transaction t;
- note_txn(entry, &t);
- t.write(entry->m_cid, ghobject_t(orig_obj), srcoff, bl.length(), bl);
- t.clone_range(entry->m_cid, ghobject_t(orig_obj), ghobject_t(new_obj),
- srcoff, srclen, dstoff);
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
-void DeterministicOpSequence::_do_coll_move(coll_entry_t *entry,
- hobject_t& orig_obj,
- hobject_t& new_obj)
-{
- ObjectStore::Transaction t;
- note_txn(entry, &t);
- t.remove(entry->m_cid, ghobject_t(new_obj));
- t.collection_move_rename(entry->m_cid, ghobject_t(orig_obj),
- entry->m_cid, ghobject_t(new_obj));
- m_store->queue_transaction(entry->m_ch, std::move(t));
-}
-
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#ifndef FILESTORE_DTRMNSTC_SEQ_OPS_H_
-#define FILESTORE_DTRMNSTC_SEQ_OPS_H_
-
-#include <iostream>
-#include <fstream>
-#include <set>
-#include "os/ObjectStore.h"
-#include <boost/scoped_ptr.hpp>
-#include <boost/random/mersenne_twister.hpp>
-#include <boost/random/uniform_int.hpp>
-
-#include "TestObjectStoreState.h"
-
-typedef boost::mt11213b rngen_t;
-
-class DeterministicOpSequence : public TestObjectStoreState {
- public:
- DeterministicOpSequence(ObjectStore *store,
- std::string status = std::string());
- virtual ~DeterministicOpSequence();
-
- virtual void generate(int seed, int num_txs);
-
- static ghobject_t get_txn_object(coll_t cid) {
- ghobject_t oid(hobject_t(sobject_t("txn", CEPH_NOSNAP)));
- spg_t pgid;
- bool r = cid.is_pg(&pgid);
- if (r) {
- oid.hobj.set_hash(pgid.ps());
- }
- return oid;
- }
-
-protected:
- enum {
- DSOP_TOUCH = 0,
- DSOP_WRITE = 1,
- DSOP_CLONE = 2,
- DSOP_CLONE_RANGE = 3,
- DSOP_OBJ_REMOVE = 4,
- DSOP_COLL_MOVE = 5,
- DSOP_SET_ATTRS = 6,
- DSOP_COLL_CREATE = 7,
-
- DSOP_FIRST = DSOP_TOUCH,
- DSOP_LAST = DSOP_COLL_CREATE,
- };
-
- int32_t txn;
-
- ObjectStore::CollectionHandle m_ch;
- std::ofstream m_status;
-
- bool run_one_op(int op, rngen_t& gen);
-
- void note_txn(coll_entry_t *entry, ObjectStore::Transaction *t);
- bool do_touch(rngen_t& gen);
- bool do_remove(rngen_t& gen);
- bool do_write(rngen_t& gen);
- bool do_clone(rngen_t& gen);
- bool do_clone_range(rngen_t& gen);
- bool do_coll_move(rngen_t& gen);
- bool do_set_attrs(rngen_t& gen);
- bool do_coll_create(rngen_t& gen);
-
- virtual void _do_touch(coll_entry_t *entry, hobject_t& obj);
- virtual void _do_remove(coll_entry_t *entry, hobject_t& obj);
- virtual void _do_write(coll_entry_t *entry, hobject_t& obj, uint64_t off,
- uint64_t len, const bufferlist& data);
- virtual void _do_set_attrs(coll_entry_t *entry,
- hobject_t &obj,
- const std::map<std::string, bufferlist> &attrs);
- virtual void _do_clone(coll_entry_t *entry, hobject_t& orig_obj, hobject_t& new_obj);
- virtual void _do_clone_range(coll_entry_t *entry, hobject_t& orig_obj,
- hobject_t& new_obj, uint64_t srcoff, uint64_t srclen, uint64_t dstoff);
- virtual void _do_write_and_clone_range(coll_entry_t *entry, hobject_t& orig_obj,
- hobject_t& new_obj, uint64_t srcoff, uint64_t srclen,
- uint64_t dstoff, bufferlist& bl);
- virtual void _do_coll_move(coll_entry_t *entry, hobject_t& orig_obj, hobject_t& new_obj);
- virtual void _do_coll_create(coll_entry_t *entry, uint32_t pg_num, uint64_t num_objs);
-
- int _gen_coll_id(rngen_t& gen);
- int _gen_obj_id(rngen_t& gen);
- void _print_status(int seq, int op);
-
- private:
- bool _prepare_clone(
- rngen_t& gen,
- coll_entry_t **entry_ret,
- int *orig_obj_id,
- hobject_t *orig_obj_ret,
- int *new_obj_id,
- hobject_t *new_obj_ret);
-};
-
-
-#endif /* FILESTORE_DTRMNSTC_SEQ_OPS_H_ */
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <stdio.h>
-#include <stdlib.h>
-#include <map>
-#include <boost/scoped_ptr.hpp>
-#include "common/debug.h"
-#include "os/filestore/FileStore.h"
-#include "common/config.h"
-
-#include "FileStoreDiff.h"
-
-#define dout_context g_ceph_context
-#define dout_subsys ceph_subsys_filestore
-#undef dout_prefix
-#define dout_prefix *_dout << "filestore_diff "
-
-using namespace std;
-
-FileStoreDiff::FileStoreDiff(FileStore *a, FileStore *b)
- : a_store(a), b_store(b)
-{
- int err;
- err = a_store->mount();
- ceph_assert(err == 0);
-
- err = b_store->mount();
- ceph_assert(err == 0);
-}
-
-FileStoreDiff::~FileStoreDiff()
-{
- a_store->umount();
- b_store->umount();
-}
-
-
-bool FileStoreDiff::diff_attrs(std::map<std::string,bufferptr,std::less<>>& b,
- std::map<std::string,bufferptr,std::less<>>& a)
-{
- bool ret = false;
- auto b_it = b.begin();
- auto a_it = a.begin();
- for (; b_it != b.end(); ++b_it, ++a_it) {
- if (b_it->first != a_it->first) {
- cout << "diff_attrs name mismatch (verify: " << b_it->first
- << ", store: " << a_it->first << ")" << std::endl;
- ret = true;
- continue;
- }
-
- if (!b_it->second.cmp(a_it->second)) {
- cout << "diff_attrs contents mismatch on attr " << b_it->first << std::endl;
- ret = true;
- continue;
- }
- }
- return ret;
-}
-
-static bool diff_omap(std::map<std::string,bufferlist>& b,
- std::map<std::string,bufferlist>& a)
-{
- bool ret = false;
- std::map<std::string, bufferlist>::iterator b_it = b.begin();
- std::map<std::string, bufferlist>::iterator a_it = a.begin();
- for (; b_it != b.end(); ++b_it, ++a_it) {
- if (a_it == a.end()) {
- cout << __func__ << " a reached end before b, a missing " << b_it->first
- << std::endl;
- ret = true;
- break;
- }
- if (b_it->first != a_it->first) {
- cout << "diff_attrs name mismatch (verify: " << b_it->first
- << ", store: " << a_it->first << ")" << std::endl;
- ret = true;
- continue;
- }
-
- if (!(b_it->second == a_it->second)) {
- cout << "diff_attrs contents mismatch on attr " << b_it->first << std::endl;
- ret = true;
- continue;
- }
- }
- return ret;
-}
-
-bool FileStoreDiff::diff_objects_stat(struct stat& a, struct stat& b)
-{
- bool ret = false;
-
- if (a.st_uid != b.st_uid) {
- cout << "diff_objects_stat uid mismatch (A: "
- << a.st_uid << " != B: " << b.st_uid << ")" << std::endl;
- ret = true;
- }
-
- if (a.st_gid != b.st_gid) {
- cout << "diff_objects_stat gid mismatch (A: "
- << a.st_gid << " != B: " << b.st_gid << ")" << std::endl;
- ret = true;
- }
-
- if (a.st_mode != b.st_mode) {
- cout << "diff_objects_stat mode mismatch (A: "
- << a.st_mode << " != B: " << b.st_mode << ")" << std::endl;
- ret = true;
- }
-
- if (a.st_nlink != b.st_nlink) {
- cout << "diff_objects_stat nlink mismatch (A: "
- << a.st_nlink << " != B: " << b.st_nlink << ")" << std::endl;
- ret = true;
- }
-
- if (a.st_size != b.st_size) {
- cout << "diff_objects_stat size mismatch (A: "
- << a.st_size << " != B: " << b.st_size << ")" << std::endl;
- ret = true;
- }
- return ret;
-}
-
-bool FileStoreDiff::diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll)
-{
- bool ret = false;
-
- int err;
- std::vector<ghobject_t> b_objects, a_objects;
- err = b_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(),
- INT_MAX, &b_objects, NULL);
- if (err < 0) {
- cout << "diff_objects list on verify coll " << coll.to_str()
- << " returns " << err << std::endl;
- return true;
- }
- err = a_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(),
- INT_MAX, &a_objects, NULL);
- if (err < 0) {
- cout << "diff_objects list on store coll " << coll.to_str()
- << " returns " << err << std::endl;
- return true;
- }
-
- if (b_objects.size() != a_objects.size()) {
- cout << "diff_objects " << coll << " num objs mismatch (A: " << a_objects.size()
- << ", B: " << b_objects.size() << ")" << std::endl;
- ret = true;
- cout << "a: " << a_objects << std::endl;
- cout << "b: " << b_objects << std::endl;
- }
-
- auto a_ch = a_store->open_collection(coll);
- auto b_ch = b_store->open_collection(coll);
- std::vector<ghobject_t>::iterator b_it = b_objects.begin();
- std::vector<ghobject_t>::iterator a_it = b_objects.begin();
- for (; b_it != b_objects.end(); ++b_it, ++a_it) {
- ghobject_t b_obj = *b_it, a_obj = *a_it;
- if (b_obj.hobj.oid.name != a_obj.hobj.oid.name) {
- cout << "diff_objects name mismatch on A object "
- << coll << "/" << a_obj << " and B object "
- << coll << "/" << b_obj << std::endl;
- ret = true;
- continue;
- }
-
- struct stat b_stat, a_stat;
- err = b_store->stat(b_ch, b_obj, &b_stat);
- if (err < 0) {
- cout << "diff_objects error stating B object "
- << coll.to_str() << "/" << b_obj.hobj.oid.name << std::endl;
- ret = true;
- }
- err = a_store->stat(a_ch, a_obj, &a_stat);
- if (err < 0) {
- cout << "diff_objects error stating A object "
- << coll << "/" << a_obj << std::endl;
- ret = true;
- }
-
- if (diff_objects_stat(a_stat, b_stat)) {
- cout << "diff_objects stat mismatch on "
- << coll << "/" << b_obj << std::endl;
- ret = true;
- }
-
- bufferlist a_obj_bl, b_obj_bl;
- b_store->read(b_ch, b_obj, 0, b_stat.st_size, b_obj_bl);
- a_store->read(a_ch, a_obj, 0, a_stat.st_size, a_obj_bl);
-
- if (!a_obj_bl.contents_equal(b_obj_bl)) {
- cout << "diff_objects content mismatch on "
- << coll << "/" << b_obj << std::endl;
- ret = true;
- }
-
- std::map<std::string, bufferptr, std::less<>> a_obj_attrs_map, b_obj_attrs_map;
- err = a_store->getattrs(a_ch, a_obj, a_obj_attrs_map);
- if (err < 0) {
- cout << "diff_objects getattrs on A object " << coll << "/" << a_obj
- << " returns " << err << std::endl;
- ret = true;
- }
- err = b_store->getattrs(b_ch, b_obj, b_obj_attrs_map);
- if (err < 0) {
- cout << "diff_objects getattrs on B object " << coll << "/" << b_obj
- << "returns " << err << std::endl;
- ret = true;
- }
-
- if (diff_attrs(b_obj_attrs_map, a_obj_attrs_map)) {
- cout << "diff_objects attrs mismatch on A object "
- << coll << "/" << a_obj << " and B object "
- << coll << "/" << b_obj << std::endl;
- ret = true;
- }
-
- std::map<std::string, bufferlist> a_obj_omap, b_obj_omap;
- std::set<std::string> a_omap_keys, b_omap_keys;
- err = a_store->omap_get_keys(a_ch, a_obj, &a_omap_keys);
- if (err < 0) {
- cout << "diff_objects getomap on A object " << coll << "/" << a_obj
- << " returns " << err << std::endl;
- ret = true;
- }
- err = a_store->omap_get_values(a_ch, a_obj, a_omap_keys, &a_obj_omap);
- if (err < 0) {
- cout << "diff_objects getomap on A object " << coll << "/" << a_obj
- << " returns " << err << std::endl;
- ret = true;
- }
- err = b_store->omap_get_keys(b_ch, b_obj, &b_omap_keys);
- if (err < 0) {
- cout << "diff_objects getomap on A object " << coll << "/" << b_obj
- << " returns " << err << std::endl;
- ret = true;
- }
- err = b_store->omap_get_values(b_ch, b_obj, b_omap_keys, &b_obj_omap);
- if (err < 0) {
- cout << "diff_objects getomap on A object " << coll << "/" << b_obj
- << " returns " << err << std::endl;
- ret = true;
- }
- if (diff_omap(a_obj_omap, b_obj_omap)) {
- cout << "diff_objects omap mismatch on A object "
- << coll << "/" << a_obj << " and B object "
- << coll << "/" << b_obj << std::endl;
- cout << "a: " << a_obj_omap << std::endl;
- cout << "b: " << b_obj_omap << std::endl;
- ret = true;
- }
- }
-
- return ret;
-}
-
-bool FileStoreDiff::diff()
-{
- bool ret = false;
-
- std::vector<coll_t> a_coll_list, b_coll_list;
- a_store->list_collections(a_coll_list);
- b_store->list_collections(b_coll_list);
-
- std::vector<coll_t>::iterator it = b_coll_list.begin();
- for (; it != b_coll_list.end(); ++it) {
- coll_t b_coll = *it;
- if (!a_store->collection_exists(b_coll)) {
- cout << "diff B coll " << b_coll.to_str() << " DNE on A" << std::endl;
- ret = true;
- continue;
- }
- for (std::vector<coll_t>::iterator j = a_coll_list.begin();
- j != a_coll_list.end(); ++j) {
- if (*j == *it) {
- a_coll_list.erase(j);
- break;
- }
- }
-
- if (diff_objects(a_store, b_store, b_coll))
- ret = true;
- }
- for (std::vector<coll_t>::iterator it = a_coll_list.begin();
- it != a_coll_list.end(); ++it) {
- cout << "diff A coll " << *it << " DNE on B" << std::endl;
- ret = true;
- }
-
- return ret;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#ifndef FILESTORE_DIFF_H_
-#define FILESTORE_DIFF_H_
-
-#include <iostream>
-#include <stdlib.h>
-#include <map>
-#include <boost/scoped_ptr.hpp>
-#include "common/debug.h"
-#include "os/filestore/FileStore.h"
-#include "common/config.h"
-
-class FileStoreDiff {
-
- private:
- FileStore *a_store;
- FileStore *b_store;
-
- bool diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll);
- bool diff_objects_stat(struct stat& a, struct stat& b);
- bool diff_attrs(std::map<std::string,bufferptr,std::less<>>& b,
- std::map<std::string,bufferptr,std::less<>>& a);
-
-public:
- FileStoreDiff(FileStore *a, FileStore *b);
- virtual ~FileStoreDiff();
-
- bool diff();
-};
-
-#endif /* FILESTOREDIFF_H_ */
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-#include "FileStoreTracker.h"
-#include <stdlib.h>
-#include <iostream>
-#include <boost/scoped_ptr.hpp>
-#include "include/Context.h"
-
-using namespace std;
-
-class OnApplied : public Context {
- FileStoreTracker *tracker;
- list<pair<pair<coll_t, string>, uint64_t> > in_flight;
-public:
- OnApplied(FileStoreTracker *tracker,
- list<pair<pair<coll_t, string>, uint64_t> > in_flight)
- : tracker(tracker), in_flight(in_flight) {}
-
- void finish(int r) override {
- for (list<pair<pair<coll_t, string>, uint64_t> >::iterator i =
- in_flight.begin();
- i != in_flight.end();
- ++i) {
- tracker->applied(i->first, i->second);
- }
- }
-};
-
-class OnCommitted : public Context {
- FileStoreTracker *tracker;
- list<pair<pair<coll_t, string>, uint64_t> > in_flight;
-public:
- OnCommitted(FileStoreTracker *tracker,
- list<pair<pair<coll_t, string>, uint64_t> > in_flight)
- : tracker(tracker), in_flight(in_flight) {}
-
- void finish(int r) override {
- for (list<pair<pair<coll_t, string>, uint64_t> >::iterator i =
- in_flight.begin();
- i != in_flight.end();
- ++i) {
- tracker->committed(i->first, i->second);
- }
- }
-};
-
-int FileStoreTracker::init()
-{
- set<string> to_get;
- to_get.insert("STATUS");
- map<string, bufferlist> got;
- db->get("STATUS", to_get, &got);
- restart_seq = 0;
- if (!got.empty()) {
- auto bp = got.begin()->second.cbegin();
- decode(restart_seq, bp);
- }
- ++restart_seq;
- KeyValueDB::Transaction t = db->get_transaction();
- got.clear();
- encode(restart_seq, got["STATUS"]);
- t->set("STATUS", got);
- db->submit_transaction(t);
- return 0;
-}
-
-void FileStoreTracker::submit_transaction(Transaction &t)
-{
- list<pair<pair<coll_t, string>, uint64_t> > in_flight;
- OutTransaction out;
- out.t = new ObjectStore::Transaction;
- out.in_flight = &in_flight;
- for (list<Transaction::Op*>::iterator i = t.ops.begin();
- i != t.ops.end();
- ++i) {
- (**i)(this, &out);
- }
- out.t->register_on_applied(new OnApplied(this, in_flight));
- out.t->register_on_commit(new OnCommitted(this, in_flight));
- auto ch = store->open_collection(coll_t());
- store->queue_transaction(ch, std::move(*out.t), nullptr);
- delete out.t;
-}
-
-void FileStoreTracker::write(const pair<coll_t, string> &obj,
- OutTransaction *out)
-{
- std::lock_guard l{lock};
- std::cerr << "Writing " << obj << std::endl;
- ObjectContents contents = get_current_content(obj);
-
- uint64_t offset = rand() % (SIZE/2);
- uint64_t len = rand() % (SIZE/2);
- if (!len) len = 10;
- contents.write(rand(), offset, len);
-
- bufferlist to_write;
- ObjectContents::Iterator iter = contents.get_iterator();
- iter.seek_to(offset);
- for (uint64_t i = offset;
- i < offset + len;
- ++i, ++iter) {
- ceph_assert(iter.valid());
- to_write.append(*iter);
- }
- out->t->write(coll_t(obj.first),
- ghobject_t(hobject_t(sobject_t(obj.second, CEPH_NOSNAP))),
- offset,
- len,
- to_write);
- out->in_flight->push_back(make_pair(obj, set_content(obj, contents)));
-}
-
-void FileStoreTracker::remove(const pair<coll_t, string> &obj,
- OutTransaction *out)
-{
- std::cerr << "Deleting " << obj << std::endl;
- std::lock_guard l{lock};
- ObjectContents old_contents = get_current_content(obj);
- if (!old_contents.exists())
- return;
- out->t->remove(coll_t(obj.first),
- ghobject_t(hobject_t(sobject_t(obj.second, CEPH_NOSNAP))));
- ObjectContents contents;
- out->in_flight->push_back(make_pair(obj, set_content(obj, contents)));
-}
-
-void FileStoreTracker::clone_range(const pair<coll_t, string> &from,
- const pair<coll_t, string> &to,
- OutTransaction *out) {
- std::lock_guard l{lock};
- std::cerr << "CloningRange " << from << " to " << to << std::endl;
- ceph_assert(from.first == to.first);
- ObjectContents from_contents = get_current_content(from);
- ObjectContents to_contents = get_current_content(to);
- if (!from_contents.exists()) {
- return;
- }
- if (from.second == to.second) {
- return;
- }
-
- uint64_t new_size = from_contents.size();
- interval_set<uint64_t> interval_to_clone;
- uint64_t offset = rand() % (new_size/2);
- uint64_t len = rand() % (new_size/2);
- if (!len) len = 10;
- interval_to_clone.insert(offset, len);
- to_contents.clone_range(from_contents, interval_to_clone);
- out->t->clone_range(coll_t(from.first),
- ghobject_t(hobject_t(sobject_t(from.second, CEPH_NOSNAP))),
- ghobject_t(hobject_t(sobject_t(to.second, CEPH_NOSNAP))),
- offset,
- len,
- offset);
- out->in_flight->push_back(make_pair(to, set_content(to, to_contents)));
-}
-
-void FileStoreTracker::clone(const pair<coll_t, string> &from,
- const pair<coll_t, string> &to,
- OutTransaction *out) {
- std::lock_guard l{lock};
- std::cerr << "Cloning " << from << " to " << to << std::endl;
- ceph_assert(from.first == to.first);
- if (from.second == to.second) {
- return;
- }
- ObjectContents from_contents = get_current_content(from);
- ObjectContents to_contents = get_current_content(to);
- if (!from_contents.exists()) {
- return;
- }
-
- if (to_contents.exists())
- out->t->remove(coll_t(to.first),
- ghobject_t(hobject_t(sobject_t(to.second, CEPH_NOSNAP))));
- out->t->clone(coll_t(from.first),
- ghobject_t(hobject_t(sobject_t(from.second, CEPH_NOSNAP))),
- ghobject_t(hobject_t(sobject_t(to.second, CEPH_NOSNAP))));
- out->in_flight->push_back(make_pair(to, set_content(to, from_contents)));
-}
-
-
-string obj_to_prefix(const pair<coll_t, string> &obj) {
- string sep;
- sep.push_back('^');
- return obj.first.to_str() + sep + obj.second + "_CONTENTS_";
-}
-
-string obj_to_meta_prefix(const pair<coll_t, string> &obj) {
- string sep;
- sep.push_back('^');
- return obj.first.to_str() + sep + obj.second;
-}
-
-string seq_to_key(uint64_t seq) {
- char buf[50];
- snprintf(buf, sizeof(buf), "%*llu", 20, (unsigned long long int)seq);
- return string(buf);
-}
-
-struct ObjStatus {
- uint64_t last_applied;
- uint64_t last_committed;
- uint64_t restart_seq;
- ObjStatus() : last_applied(0), last_committed(0), restart_seq(0) {}
-
- uint64_t get_last_applied(uint64_t seq) const {
- if (seq > restart_seq)
- return last_committed;
- else
- return last_applied;
- }
- void set_last_applied(uint64_t _last_applied, uint64_t seq) {
- last_applied = _last_applied;
- restart_seq = seq;
- }
- uint64_t trim_to() const {
- return last_applied < last_committed ?
- last_applied : last_committed;
- }
-};
-void encode(const ObjStatus &obj, bufferlist &bl) {
- encode(obj.last_applied, bl);
- encode(obj.last_committed, bl);
- encode(obj.restart_seq, bl);
-}
-void decode(ObjStatus &obj, bufferlist::const_iterator &bl) {
- decode(obj.last_applied, bl);
- decode(obj.last_committed, bl);
- decode(obj.restart_seq, bl);
-}
-
-
-ObjStatus get_obj_status(const pair<coll_t, string> &obj,
- KeyValueDB *db)
-{
- set<string> to_get;
- to_get.insert("META");
- map<string, bufferlist> got;
- db->get(obj_to_meta_prefix(obj), to_get, &got);
- ObjStatus retval;
- if (!got.empty()) {
- auto bp = got.begin()->second.cbegin();
- decode(retval, bp);
- }
- return retval;
-}
-
-void set_obj_status(const pair<coll_t, string> &obj,
- const ObjStatus &status,
- KeyValueDB::Transaction t)
-{
- map<string, bufferlist> to_set;
- encode(status, to_set["META"]);
- t->set(obj_to_meta_prefix(obj), to_set);
-}
-
-void _clean_forward(const pair<coll_t, string> &obj,
- uint64_t last_valid,
- KeyValueDB *db)
-{
- KeyValueDB::Transaction t = db->get_transaction();
- KeyValueDB::Iterator i = db->get_iterator(obj_to_prefix(obj));
- set<string> to_remove;
- i->upper_bound(seq_to_key(last_valid));
- for (; i->valid(); i->next()) {
- to_remove.insert(i->key());
- }
- t->rmkeys(obj_to_prefix(obj), to_remove);
- db->submit_transaction(t);
-}
-
-
-void FileStoreTracker::verify(const coll_t &coll, const string &obj,
- bool on_start) {
- std::lock_guard l{lock};
- std::cerr << "Verifying " << make_pair(coll, obj) << std::endl;
-
- pair<uint64_t, uint64_t> valid_reads = get_valid_reads(make_pair(coll, obj));
- std::cerr << "valid_reads is " << valid_reads << std::endl;
- bufferlist contents;
- auto ch = store->open_collection(coll_t(coll));
- int r = store->read(ch,
- ghobject_t(hobject_t(sobject_t(obj, CEPH_NOSNAP))),
- 0,
- 2*SIZE,
- contents);
- std::cerr << "exists: " << r << std::endl;
-
-
- for (uint64_t i = valid_reads.first;
- i < valid_reads.second;
- ++i) {
- ObjectContents old_contents = get_content(make_pair(coll, obj), i);
-
- std::cerr << "old_contents exists " << old_contents.exists() << std::endl;
- if (!old_contents.exists() && (r == -ENOENT))
- return;
-
- if (old_contents.exists() && (r == -ENOENT))
- continue;
-
- if (!old_contents.exists() && (r != -ENOENT))
- continue;
-
- if (contents.length() != old_contents.size()) {
- std::cerr << "old_contents.size() is "
- << old_contents.size() << std::endl;
- continue;
- }
-
- bufferlist::iterator bp = contents.begin();
- ObjectContents::Iterator iter = old_contents.get_iterator();
- iter.seek_to_first();
- bool matches = true;
- uint64_t pos = 0;
- for (; !bp.end() && iter.valid();
- ++iter, ++bp, ++pos) {
- if (*iter != *bp) {
- std::cerr << "does not match at pos " << pos << std::endl;
- matches = false;
- break;
- }
- }
- if (matches) {
- if (on_start)
- _clean_forward(make_pair(coll, obj), i, db);
- return;
- }
- }
- std::cerr << "Verifying " << make_pair(coll, obj) << " failed " << std::endl;
- ceph_abort();
-}
-
-ObjectContents FileStoreTracker::get_current_content(
- const pair<coll_t, string> &obj)
-{
- KeyValueDB::Iterator iter = db->get_iterator(
- obj_to_prefix(obj));
- iter->seek_to_last();
- if (iter->valid()) {
- bufferlist bl = iter->value();
- auto bp = bl.cbegin();
- pair<uint64_t, bufferlist> val;
- decode(val, bp);
- ceph_assert(seq_to_key(val.first) == iter->key());
- bp = val.second.begin();
- return ObjectContents(bp);
- }
- return ObjectContents();
-}
-
-ObjectContents FileStoreTracker::get_content(
- const pair<coll_t, string> &obj, uint64_t version)
-{
- set<string> to_get;
- map<string, bufferlist> got;
- to_get.insert(seq_to_key(version));
- db->get(obj_to_prefix(obj), to_get, &got);
- if (got.empty())
- return ObjectContents();
- pair<uint64_t, bufferlist> val;
- auto bp = got.begin()->second.cbegin();
- decode(val, bp);
- bp = val.second.begin();
- ceph_assert(val.first == version);
- return ObjectContents(bp);
-}
-
-pair<uint64_t, uint64_t> FileStoreTracker::get_valid_reads(
- const pair<coll_t, string> &obj)
-{
- pair<uint64_t, uint64_t> bounds = make_pair(0,1);
- KeyValueDB::Iterator iter = db->get_iterator(
- obj_to_prefix(obj));
- iter->seek_to_last();
- if (iter->valid()) {
- pair<uint64_t, bufferlist> val;
- bufferlist bl = iter->value();
- auto bp = bl.cbegin();
- decode(val, bp);
- bounds.second = val.first + 1;
- }
-
- ObjStatus obj_status = get_obj_status(obj, db);
- bounds.first = obj_status.get_last_applied(restart_seq);
- return bounds;
-}
-
-void clear_obsolete(const pair<coll_t, string> &obj,
- const ObjStatus &status,
- KeyValueDB *db,
- KeyValueDB::Transaction t)
-{
- KeyValueDB::Iterator iter = db->get_iterator(obj_to_prefix(obj));
- set<string> to_remove;
- iter->seek_to_first();
- for (; iter->valid() && iter->key() < seq_to_key(status.trim_to());
- iter->next())
- to_remove.insert(iter->key());
- t->rmkeys(obj_to_prefix(obj), to_remove);
-}
-
-void FileStoreTracker::committed(const pair<coll_t, string> &obj,
- uint64_t seq) {
- std::lock_guard l{lock};
- ObjStatus status = get_obj_status(obj, db);
- ceph_assert(status.last_committed < seq);
- status.last_committed = seq;
- KeyValueDB::Transaction t = db->get_transaction();
- clear_obsolete(obj, status, db, t);
- set_obj_status(obj, status, t);
- db->submit_transaction(t);
-}
-
-void FileStoreTracker::applied(const pair<coll_t, string> &obj,
- uint64_t seq) {
- std::lock_guard l{lock};
- std::cerr << "Applied " << obj << " version " << seq << std::endl;
- ObjStatus status = get_obj_status(obj, db);
- ceph_assert(status.last_applied < seq);
- status.set_last_applied(seq, restart_seq);
- KeyValueDB::Transaction t = db->get_transaction();
- clear_obsolete(obj, status, db, t);
- set_obj_status(obj, status, t);
- db->submit_transaction(t);
-}
-
-
-uint64_t FileStoreTracker::set_content(const pair<coll_t, string> &obj,
- ObjectContents &content) {
- KeyValueDB::Transaction t = db->get_transaction();
- KeyValueDB::Iterator iter = db->get_iterator(
- obj_to_prefix(obj));
- iter->seek_to_last();
- uint64_t most_recent = 0;
- if (iter->valid()) {
- pair<uint64_t, bufferlist> val;
- bufferlist bl = iter->value();
- auto bp = bl.cbegin();
- decode(val, bp);
- most_recent = val.first;
- }
- bufferlist buf_content;
- content.encode(buf_content);
- map<string, bufferlist> to_set;
- encode(make_pair(most_recent + 1, buf_content),
- to_set[seq_to_key(most_recent + 1)]);
- t->set(obj_to_prefix(obj), to_set);
- db->submit_transaction(t);
- return most_recent + 1;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-
-#ifndef FILESTORE_TRACKER_H
-#define FILESTORE_TRACKER_H
-#include "test/common/ObjectContents.h"
-#include "os/filestore/FileStore.h"
-#include "kv/KeyValueDB.h"
-#include <boost/scoped_ptr.hpp>
-#include <list>
-#include <map>
-#include "common/ceph_mutex.h"
-
-class FileStoreTracker {
- const static uint64_t SIZE = 4 * 1024;
- ObjectStore *store;
- KeyValueDB *db;
- ceph::mutex lock = ceph::make_mutex("Tracker Lock");
- uint64_t restart_seq;
-
- struct OutTransaction {
- std::list<std::pair<std::pair<coll_t, std::string>, uint64_t> > *in_flight;
- ObjectStore::Transaction *t;
- };
-public:
- FileStoreTracker(ObjectStore *store, KeyValueDB *db)
- : store(store), db(db),
- restart_seq(0) {}
-
- class Transaction {
- class Op {
- public:
- virtual void operator()(FileStoreTracker *harness,
- OutTransaction *out) = 0;
- virtual ~Op() {};
- };
- std::list<Op*> ops;
- class Write : public Op {
- public:
- coll_t coll;
- std::string oid;
- Write(const coll_t &coll,
- const std::string &oid)
- : coll(coll), oid(oid) {}
- void operator()(FileStoreTracker *harness,
- OutTransaction *out) override {
- harness->write(std::make_pair(coll, oid), out);
- }
- };
- class CloneRange : public Op {
- public:
- coll_t coll;
- std::string from;
- std::string to;
- CloneRange(const coll_t &coll,
- const std::string &from,
- const std::string &to)
- : coll(coll), from(from), to(to) {}
- void operator()(FileStoreTracker *harness,
- OutTransaction *out) override {
- harness->clone_range(std::make_pair(coll, from),
- std::make_pair(coll, to),
- out);
- }
- };
- class Clone : public Op {
- public:
- coll_t coll;
- std::string from;
- std::string to;
- Clone(const coll_t &coll,
- const std::string &from,
- const std::string &to)
- : coll(coll), from(from), to(to) {}
- void operator()(FileStoreTracker *harness,
- OutTransaction *out) override {
- harness->clone(std::make_pair(coll, from),
- std::make_pair(coll, to),
- out);
- }
- };
- class Remove: public Op {
- public:
- coll_t coll;
- std::string obj;
- Remove(const coll_t &coll,
- const std::string &obj)
- : coll(coll), obj(obj) {}
- void operator()(FileStoreTracker *harness,
- OutTransaction *out) override {
- harness->remove(std::make_pair(coll, obj),
- out);
- }
- };
- public:
- void write(const coll_t &coll, const std::string &oid) {
- ops.push_back(new Write(coll, oid));
- }
- void clone_range(const coll_t &coll, const std::string &from,
- const std::string &to) {
- ops.push_back(new CloneRange(coll, from, to));
- }
- void clone(const coll_t &coll, const std::string &from,
- const std::string &to) {
- ops.push_back(new Clone(coll, from, to));
- }
- void remove(const coll_t &coll, const std::string &oid) {
- ops.push_back(new Remove(coll, oid));
- }
- friend class FileStoreTracker;
- };
-
- int init();
- void submit_transaction(Transaction &t);
- void verify(const coll_t &coll,
- const std::string &from,
- bool on_start = false);
-
-private:
- ObjectContents get_current_content(const std::pair<coll_t, std::string> &obj);
- std::pair<uint64_t, uint64_t> get_valid_reads(const std::pair<coll_t, std::string> &obj);
- ObjectContents get_content(const std::pair<coll_t, std::string> &obj, uint64_t version);
-
- void committed(const std::pair<coll_t, std::string> &obj, uint64_t seq);
- void applied(const std::pair<coll_t, std::string> &obj, uint64_t seq);
- uint64_t set_content(const std::pair<coll_t, std::string> &obj, ObjectContents &content);
-
- // ObjectContents Operations
- void write(const std::pair<coll_t, std::string> &obj, OutTransaction *out);
- void remove(const std::pair<coll_t, std::string> &obj, OutTransaction *out);
- void clone_range(const std::pair<coll_t, std::string> &from,
- const std::pair<coll_t, std::string> &to,
- OutTransaction *out);
- void clone(const std::pair<coll_t, std::string> &from,
- const std::pair<coll_t, std::string> &to,
- OutTransaction *out);
- friend class OnApplied;
- friend class OnCommitted;
-};
-
-#endif
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Library Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Library Public License for more details.
- *
- */
-
-#include <stdio.h>
-#include <signal.h>
-#include "os/filestore/chain_xattr.h"
-#include "include/Context.h"
-#include "include/coredumpctl.h"
-#include "common/errno.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include <gtest/gtest.h>
-
-#define LARGE_BLOCK_LEN CHAIN_XATTR_MAX_BLOCK_LEN + 1024
-#define FILENAME "chain_xattr"
-
-using namespace std;
-
-TEST(chain_xattr, get_and_set) {
- const char* file = FILENAME;
- ::unlink(file);
- int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
- const string user("user.");
-
- {
- const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
- const string x(LARGE_BLOCK_LEN, 'X');
-
- {
- char y[LARGE_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0));
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN));
- ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
- ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN));
- }
-
- {
- char y[LARGE_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0));
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN));
- ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
- ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN));
- }
- }
-
- //
- // when chain_setxattr is used to store value that is
- // CHAIN_XATTR_MAX_BLOCK_LEN * 2 + 10 bytes long it
- //
- // add user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes
- // add user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes
- // add user.foo@2 => 10 bytes
- //
- // then ( no chain_removexattr in between ) when it is used to
- // override with a value that is exactly CHAIN_XATTR_MAX_BLOCK_LEN
- // bytes long it will
- //
- // replace user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes
- // remove user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes
- // leak user.foo@2 => 10 bytes
- //
- // see http://marc.info/?l=ceph-devel&m=136027076615853&w=4 for the
- // discussion
- //
- {
- const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
- const string x(LARGE_BLOCK_LEN, 'X');
-
- {
- char y[CHAIN_XATTR_MAX_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
- ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- }
-
- {
- char y[CHAIN_XATTR_MAX_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0));
- ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
- ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- }
- }
-
- {
- int x = 0;
- ASSERT_EQ(-ENOENT, chain_setxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x)));
- ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", 0, 0));
- ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x)));
- ASSERT_EQ(-ENOENT, chain_removexattr("UNLIKELY_TO_EXIST", "NAME"));
- int unlikely_to_be_a_valid_fd = 400;
- ASSERT_EQ(-EBADF, chain_fsetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x)));
- ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", 0, 0));
- ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x)));
- ASSERT_EQ(-EBADF, chain_fremovexattr(unlikely_to_be_a_valid_fd, "NAME"));
- }
-
- {
- int x;
- const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN * 2, '@');
- PrCtl unset_dumpable;
- ::testing::FLAGS_gtest_death_test_style = "threadsafe";
- ASSERT_DEATH(chain_setxattr(file, name.c_str(), &x, sizeof(x)), "");
- ASSERT_DEATH(chain_fsetxattr(fd, name.c_str(), &x, sizeof(x)), "");
- ::testing::FLAGS_gtest_death_test_style = "fast";
- }
-
- {
- const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
- const string x(LARGE_BLOCK_LEN, 'X');
- {
- char y[LARGE_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN - 1));
- ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
- }
-
- {
- char y[LARGE_BLOCK_LEN];
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN - 1));
- ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN));
- ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str()));
- }
- }
-
- ::close(fd);
- ::unlink(file);
-}
-
-TEST(chain_xattr, chunk_aligned) {
- const char* file = FILENAME;
- ::unlink(file);
- int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
- const string user("user.");
-
- // set N* chunk size
- const string name = "user.foo";
- const string name2 = "user.bar";
-
- for (int len = CHAIN_XATTR_MAX_BLOCK_LEN - 10;
- len < CHAIN_XATTR_MAX_BLOCK_LEN + 10;
- ++len) {
- cout << len << std::endl;
- const string x(len, 'x');
- char buf[len*2];
- ASSERT_EQ(len, chain_setxattr(file, name.c_str(), x.c_str(), len));
- char attrbuf[4096];
- int l = ceph_os_listxattr(file, attrbuf, sizeof(attrbuf));
- for (char *p = attrbuf; p - attrbuf < l; p += strlen(p) + 1) {
- cout << " attr " << p << std::endl;
- }
- ASSERT_EQ(len, chain_getxattr(file, name.c_str(), buf, len*2));
- ASSERT_EQ(0, chain_removexattr(file, name.c_str()));
-
- ASSERT_EQ(len, chain_fsetxattr(fd, name2.c_str(), x.c_str(), len));
- l = ceph_os_flistxattr(fd, attrbuf, sizeof(attrbuf));
- for (char *p = attrbuf; p - attrbuf < l; p += strlen(p) + 1) {
- cout << " attr " << p << std::endl;
- }
- ASSERT_EQ(len, chain_fgetxattr(fd, name2.c_str(), buf, len*2));
- ASSERT_EQ(0, chain_fremovexattr(fd, name2.c_str()));
- }
-
- for (int len = CHAIN_XATTR_SHORT_BLOCK_LEN - 10;
- len < CHAIN_XATTR_SHORT_BLOCK_LEN + 10;
- ++len) {
- cout << len << std::endl;
- const string x(len, 'x');
- char buf[len*2];
- ASSERT_EQ(len, chain_setxattr(file, name.c_str(), x.c_str(), len));
- char attrbuf[4096];
- int l = ceph_os_listxattr(file, attrbuf, sizeof(attrbuf));
- for (char *p = attrbuf; p - attrbuf < l; p += strlen(p) + 1) {
- cout << " attr " << p << std::endl;
- }
- ASSERT_EQ(len, chain_getxattr(file, name.c_str(), buf, len*2));
- }
-
- {
- // test tail path in chain_getxattr
- const char *aname = "user.baz";
- char buf[CHAIN_XATTR_SHORT_BLOCK_LEN*3];
- memset(buf, 'x', sizeof(buf));
- ASSERT_EQ((int)sizeof(buf), chain_setxattr(file, aname, buf, sizeof(buf)));
- ASSERT_EQ(-ERANGE, chain_getxattr(file, aname, buf,
- CHAIN_XATTR_SHORT_BLOCK_LEN*2));
- }
- {
- // test tail path in chain_fgetxattr
- const char *aname = "user.biz";
- char buf[CHAIN_XATTR_SHORT_BLOCK_LEN*3];
- memset(buf, 'x', sizeof(buf));
- ASSERT_EQ((int)sizeof(buf), chain_fsetxattr(fd, aname, buf, sizeof(buf)));
- ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, aname, buf,
- CHAIN_XATTR_SHORT_BLOCK_LEN*2));
- }
-
- ::close(fd);
- ::unlink(file);
-}
-
-void get_vector_from_xattr(vector<string> &xattrs, char* xattr, int size) {
- char *end = xattr + size;
- while (xattr < end) {
- if (*xattr == '\0' )
- break;
- xattrs.push_back(xattr);
- xattr += strlen(xattr) + 1;
- }
-}
-
-bool listxattr_cmp(char* xattr1, char* xattr2, int size) {
- vector<string> xattrs1;
- vector<string> xattrs2;
- get_vector_from_xattr(xattrs1, xattr1, size);
- get_vector_from_xattr(xattrs2, xattr2, size);
-
- if (xattrs1.size() != xattrs2.size())
- return false;
-
- std::sort(xattrs1.begin(), xattrs1.end());
- std::sort(xattrs2.begin(), xattrs2.end());
- std::vector<string> diff;
- std::set_difference(xattrs1.begin(), xattrs1.end(),
- xattrs2.begin(), xattrs2.end(),
- diff.begin());
-
- return diff.empty();
-}
-
-TEST(chain_xattr, listxattr) {
- const char* file = FILENAME;
- ::unlink(file);
- int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
- const string user("user.");
- const string name1 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '1');
- const string name2 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@');
- const string x(LARGE_BLOCK_LEN, 'X');
- const int y = 1234;
-
- int orig_size = chain_listxattr(file, NULL, 0);
- char *orig_buffer = NULL;
- string orig_str;
- if (orig_size) {
- orig_buffer = (char*)malloc(orig_size);
- chain_flistxattr(fd, orig_buffer, orig_size);
- orig_str = string(orig_buffer);
- orig_size = orig_str.size();
- }
-
- ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name1.c_str(), x.c_str(), LARGE_BLOCK_LEN));
- ASSERT_EQ((int)sizeof(y), chain_setxattr(file, name2.c_str(), &y, sizeof(y)));
-
- int buffer_size = 0;
- if (orig_size)
- buffer_size += orig_size + sizeof(char);
- buffer_size += name1.size() + sizeof(char) + name2.size() + sizeof(char);
-
- int index = 0;
- char* expected = (char*)malloc(buffer_size);
- ::memset(expected, '\0', buffer_size);
- if (orig_size) {
- ::strcpy(expected, orig_str.c_str());
- index = orig_size + 1;
- }
- ::strcpy(expected + index, name1.c_str());
- ::strcpy(expected + index + name1.size() + 1, name2.c_str());
- char* actual = (char*)malloc(buffer_size);
- ::memset(actual, '\0', buffer_size);
- ASSERT_LT(buffer_size, chain_listxattr(file, NULL, 0)); // size evaluation is conservative
- chain_listxattr(file, actual, buffer_size);
- ASSERT_TRUE(listxattr_cmp(expected, actual, buffer_size));
- ::memset(actual, '\0', buffer_size);
- chain_flistxattr(fd, actual, buffer_size);
- ASSERT_TRUE(listxattr_cmp(expected, actual, buffer_size));
-
- int unlikely_to_be_a_valid_fd = 400;
- ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, 0));
- ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, buffer_size));
- ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, 0));
- ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, buffer_size));
- ASSERT_EQ(-ERANGE, chain_listxattr(file, actual, 1));
- ASSERT_EQ(-ERANGE, chain_flistxattr(fd, actual, 1));
-
- ASSERT_EQ(0, chain_removexattr(file, name1.c_str()));
- ASSERT_EQ(0, chain_removexattr(file, name2.c_str()));
-
- free(orig_buffer);
- free(actual);
- free(expected);
- ::unlink(file);
-}
-
-list<string> get_xattrs(int fd)
-{
- char _buf[1024];
- char *buf = _buf;
- int len = sys_flistxattr(fd, _buf, sizeof(_buf));
- if (len < 0)
- return list<string>();
- list<string> ret;
- while (len > 0) {
- size_t next_len = strlen(buf);
- ret.push_back(string(buf, buf + next_len));
- ceph_assert(len >= (int)(next_len + 1));
- buf += (next_len + 1);
- len -= (next_len + 1);
- }
- return ret;
-}
-
-list<string> get_xattrs(string fn)
-{
- int fd = ::open(fn.c_str(), O_RDONLY);
- if (fd < 0)
- return list<string>();
- auto ret = get_xattrs(fd);
- ::close(fd);
- return ret;
-}
-
-TEST(chain_xattr, fskip_chain_cleanup_and_ensure_single_attr)
-{
- const char *name = "user.foo";
- const char *file = FILENAME;
- ::unlink(file);
- int fd = ::open(file, O_CREAT|O_RDWR|O_TRUNC, 0700);
- ceph_assert(fd >= 0);
-
- std::size_t existing_xattrs = get_xattrs(fd).size();
- char buf[800];
- memset(buf, 0x1F, sizeof(buf));
- // set chunked without either
- {
- std::size_t r = chain_fsetxattr(fd, name, buf, sizeof(buf));
- ASSERT_EQ(sizeof(buf), r);
- ASSERT_GT(get_xattrs(fd).size(), existing_xattrs + 1UL);
- }
-
- // verify
- {
- char buf2[sizeof(buf)*2];
- std::size_t r = chain_fgetxattr(fd, name, buf2, sizeof(buf2));
- ASSERT_EQ(sizeof(buf), r);
- ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
- }
-
- // overwrite
- {
- std::size_t r = chain_fsetxattr<false, true>(fd, name, buf, sizeof(buf));
- ASSERT_EQ(sizeof(buf), r);
- ASSERT_EQ(existing_xattrs + 1UL, get_xattrs(fd).size());
- }
-
- // verify
- {
- char buf2[sizeof(buf)*2];
- std::size_t r = chain_fgetxattr(fd, name, buf2, sizeof(buf2));
- ASSERT_EQ(sizeof(buf), r);
- ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
- }
-
- ::close(fd);
- ::unlink(file);
-}
-
-TEST(chain_xattr, skip_chain_cleanup_and_ensure_single_attr)
-{
- const char *name = "user.foo";
- const char *file = FILENAME;
- ::unlink(file);
- int fd = ::open(file, O_CREAT|O_RDWR|O_TRUNC, 0700);
- ceph_assert(fd >= 0);
- std::size_t existing_xattrs = get_xattrs(fd).size();
- ::close(fd);
-
- char buf[3000];
- memset(buf, 0x1F, sizeof(buf));
- // set chunked without either
- {
- std::size_t r = chain_setxattr(file, name, buf, sizeof(buf));
- ASSERT_EQ(sizeof(buf), r);
- ASSERT_GT(get_xattrs(file).size(), existing_xattrs + 1UL);
- }
-
- // verify
- {
- char buf2[sizeof(buf)*2];
- std::size_t r = chain_getxattr(file, name, buf2, sizeof(buf2));
- ASSERT_EQ(sizeof(buf), r);
- ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
- }
-
- // overwrite
- {
- std::size_t r = chain_setxattr<false, true>(file, name, buf, sizeof(buf));
- ASSERT_EQ(sizeof(buf), r);
- ASSERT_EQ(existing_xattrs + 1UL, get_xattrs(file).size());
- }
-
- // verify
- {
- char buf2[sizeof(buf)*2];
- std::size_t r = chain_getxattr(file, name, buf2, sizeof(buf2));
- ASSERT_EQ(sizeof(buf), r);
- ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
- }
-
- ::unlink(file);
-}
-
-int main(int argc, char **argv) {
- auto args = argv_to_vec(argc, argv);
-
- auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
- CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf.set_val("err_to_stderr", "false");
- g_ceph_context->_conf.set_val("log_to_stderr", "false");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- const char* file = FILENAME;
- int x = 1234;
- int y = 0;
- int tmpfd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700);
- int ret = ::ceph_os_fsetxattr(tmpfd, "user.test", &x, sizeof(x));
- if (ret >= 0)
- ret = ::ceph_os_fgetxattr(tmpfd, "user.test", &y, sizeof(y));
- ::close(tmpfd);
- ::unlink(file);
- if ((ret < 0) || (x != y)) {
- cerr << "SKIP all tests because extended attributes don't appear to work in the file system in which the tests are run: " << cpp_strerror(ret) << std::endl;
- } else {
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
- }
-}
-
-// Local Variables:
-// compile-command: "cd ../.. ; make unittest_chain_xattr ; valgrind --tool=memcheck ./unittest_chain_xattr # --gtest_filter=chain_xattr.get_and_set"
-// End:
#include <gtest/gtest.h>
#include "os/ObjectStore.h"
-#include "os/filestore/FileStore.h"
#if defined(WITH_BLUESTORE)
#include "os/bluestore/BlueStore.h"
#include "os/bluestore/BlueFS.h"
StoreTest,
::testing::Values(
"memstore",
- "filestore",
#if defined(WITH_BLUESTORE)
"bluestore",
#endif
StoreTestSpecificAUSize,
::testing::Values(
"memstore",
- "filestore",
#if defined(WITH_BLUESTORE)
"bluestore",
#endif
StoreTestOmapUpgrade,
::testing::Values(
"memstore",
- "filestore",
#if defined(WITH_BLUESTORE)
"bluestore",
#endif
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#include <iostream>
-#include <iterator>
-#include <sstream>
-#include <boost/scoped_ptr.hpp>
-#include "os/filestore/FileStore.h"
-#include "global/global_init.h"
-#include "common/ceph_argparse.h"
-#include "common/debug.h"
-#include "test/common/ObjectContents.h"
-#include "FileStoreTracker.h"
-#include "kv/KeyValueDB.h"
-#include "os/ObjectStore.h"
-
-using namespace std;
-
-void usage(const string &name) {
- std::cerr << "Usage: " << name << " [new|continue] store_path store_journal db_path"
- << std::endl;
-}
-
-template <typename T>
-typename T::iterator rand_choose(T &cont) {
- if (std::empty(cont)) {
- return std::end(cont);
- }
- return std::next(std::begin(cont), rand() % cont.size());
-}
-
-int main(int argc, char **argv) {
- auto args = argv_to_vec(argc, argv);
- auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
- CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- cct->_conf.apply_changes(nullptr);
-
- std::cerr << "args: " << args << std::endl;
- if (args.size() < 4) {
- usage(argv[0]);
- return 1;
- }
-
- string store_path(args[1]);
- string store_dev(args[2]);
- string db_path(args[3]);
-
- bool start_new = false;
- if (string(args[0]) == string("new")) start_new = true;
-
- KeyValueDB *_db = KeyValueDB::create(g_ceph_context, "leveldb", db_path);
- ceph_assert(!_db->create_and_open(std::cerr));
- boost::scoped_ptr<KeyValueDB> db(_db);
- boost::scoped_ptr<ObjectStore> store(new FileStore(cct.get(), store_path,
- store_dev));
-
- coll_t coll(spg_t(pg_t(0,12),shard_id_t::NO_SHARD));
- ObjectStore::CollectionHandle ch;
-
- if (start_new) {
- std::cerr << "mkfs" << std::endl;
- ceph_assert(!store->mkfs());
- ObjectStore::Transaction t;
- ceph_assert(!store->mount());
- ch = store->create_new_collection(coll);
- t.create_collection(coll, 0);
- store->queue_transaction(ch, std::move(t));
- } else {
- ceph_assert(!store->mount());
- ch = store->open_collection(coll);
- }
-
- FileStoreTracker tracker(store.get(), db.get());
-
- set<string> objects;
- for (unsigned i = 0; i < 10; ++i) {
- stringstream stream;
- stream << "Object_" << i;
- tracker.verify(coll, stream.str(), true);
- objects.insert(stream.str());
- }
-
- while (1) {
- FileStoreTracker::Transaction t;
- for (unsigned j = 0; j < 100; ++j) {
- int val = rand() % 100;
- if (val < 30) {
- t.write(coll, *rand_choose(objects));
- } else if (val < 60) {
- t.clone(coll, *rand_choose(objects),
- *rand_choose(objects));
- } else if (val < 70) {
- t.remove(coll, *rand_choose(objects));
- } else {
- t.clone_range(coll, *rand_choose(objects),
- *rand_choose(objects));
- }
- }
- tracker.submit_transaction(t);
- tracker.verify(coll, *rand_choose(objects));
- }
- return 0;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 New Dream Network
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include <sstream>
-#include <time.h>
-#include <stdlib.h>
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "common/debug.h"
-#include "os/filestore/FileStore.h"
-
-#include "DeterministicOpSequence.h"
-#include "FileStoreDiff.h"
-
-#include "common/config.h"
-#include "include/ceph_assert.h"
-
-#define dout_context g_ceph_context
-#define dout_subsys ceph_subsys_
-#undef dout_prefix
-#define dout_prefix *_dout << "test_idempotent_sequence "
-
-using namespace std;
-
-void usage(const char *name, std::string command = "") {
- ceph_assert(name != NULL);
-
- std::string more = "cmd <args...>";
- std::string diff = "diff <filestoreA> <journalA> <filestoreB> <journalB>";
- std::string get_last_op = "get-last-op <filestore> <journal>";
- std::string run_seq_to = "run-sequence-to <num-ops> <filestore> <journal>";
-
- if (!command.empty()) {
- if (command == "diff")
- more = diff;
- else if (command == "get-last-op")
- more = get_last_op;
- else if (command == "run-sequence-to")
- more = run_seq_to;
- }
- std::cout << "usage: " << name << " " << more << " [options]" << std::endl;
-
- std::cout << "\n\
-Commands:\n\
- " << diff << "\n\
- " << get_last_op << "\n\
- " << run_seq_to << "\n\
-\n\
-Global Options:\n\
- -c FILE Read configuration from FILE\n\
- --osd-data PATH Set OSD Data path\n\
- --osd-journal PATH Set OSD Journal path\n\
- --osd-journal-size VAL Set Journal size\n\
- --help This message\n\
-\n\
-Test-specific Options:\n\
- --test-seed VAL Seed to run the test\n\
- --test-status-file PATH Path to keep the status file\n\
- --test-num-colls VAL Number of collections to create on init\n\
- --test-num-objs VAL Number of objects to create on init\n\
-" << std::endl;
-}
-
-const char *our_name = NULL;
-int seed = 0, num_txs = 100, num_colls = 30, num_objs = 0;
-bool is_seed_set = false;
-int verify_at = 0;
-std::string status_file;
-
-int run_diff(std::string& a_path, std::string& a_journal,
- std::string& b_path, std::string& b_journal)
-{
- FileStore *a = new FileStore(g_ceph_context, a_path, a_journal, 0, "a");
- FileStore *b = new FileStore(g_ceph_context, b_path, b_journal, 0, "b");
-
- int ret = 0;
- {
- FileStoreDiff fsd(a, b);
- if (fsd.diff()) {
- dout(0) << "diff found an difference" << dendl;
- ret = -1;
- } else {
- dout(0) << "no diff" << dendl;
- }
- }
-
- delete a;
- delete b;
- return ret;
-}
-
-int run_get_last_op(std::string& filestore_path, std::string& journal_path)
-{
- FileStore *store = new FileStore(g_ceph_context, filestore_path,
- journal_path);
-
- int err = store->mount();
- if (err) {
- store->umount();
- delete store;
- return err;
- }
-
- vector<coll_t> cls;
- store->list_collections(cls);
-
- int32_t txn = 0;
- for (auto cid : cls) {
- ghobject_t txn_object = DeterministicOpSequence::get_txn_object(cid);
- bufferlist bl;
- auto ch = store->open_collection(cid);
- store->read(ch, txn_object, 0, 100, bl);
- int32_t t = 0;
- if (bl.length()) {
- auto p = bl.cbegin();
- decode(t, p);
- }
- if (t > txn) {
- txn = t;
- }
- }
-
- store->umount();
- delete store;
-
- cout << txn << std::endl;
- return 0;
-}
-
-int run_sequence_to(int val, std::string& filestore_path,
- std::string& journal_path)
-{
- num_txs = val;
-
- if (!is_seed_set)
- seed = (int) time(NULL);
-
- FileStore *store = new FileStore(g_ceph_context, filestore_path,
- journal_path);
-
- int err;
-
- // mkfs iff directory dne
- err = ::mkdir(filestore_path.c_str(), 0755);
- if (err) {
- cerr << filestore_path << " already exists" << std::endl;
- store->umount();
- delete store;
- return err;
- }
-
- err = store->mkfs();
- ceph_assert(err == 0);
-
- err = store->mount();
- ceph_assert(err == 0);
-
- DeterministicOpSequence op_sequence(store, status_file);
- op_sequence.init(num_colls, num_objs);
- op_sequence.generate(seed, num_txs);
- store->umount();
- return 0;
-}
-
-int run_command(std::string& command, std::vector<std::string>& args)
-{
- if (command.empty()) {
- usage(our_name);
- exit(0);
- }
-
- /* We'll have a class that will handle the options, the command
- * and its arguments. For the time being, and so we can move on, let's
- * tolerate this big, ugly code.
- */
- if (command == "diff") {
- /* expect 4 arguments: (filestore path + journal path)*2 */
- if (args.size() == 4) {
- return run_diff(args[0], args[1], args[2], args[3]);
- }
- } else if (command == "get-last-op") {
- /* expect 2 arguments: a filestore path + journal */
- if (args.size() == 2) {
- return run_get_last_op(args[0], args[1]);
- }
- } else if (command == "run-sequence-to") {
- /* expect 3 arguments: # of operations and a filestore path + journal. */
- if (args.size() == 3) {
- return run_sequence_to(strtoll(args[0].c_str(), NULL, 10), args[1], args[2]);
- }
- } else {
- std::cout << "unknown command " << command << std::endl;
- usage(our_name);
- exit(1);
- }
-
- usage(our_name, command);
- exit(1);
-}
-
-int main(int argc, const char *argv[])
-{
- our_name = argv[0];
- auto args = argv_to_vec(argc, argv);
-
- auto cct = global_init(NULL, args,
- CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf.apply_changes(nullptr);
-
- std::string command;
- std::vector<std::string> command_args;
-
- for (std::vector<const char*>::iterator i = args.begin(); i != args.end();) {
- string val;
-
- if (ceph_argparse_double_dash(args, i)) {
- break;
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-seed", (char*) NULL)) {
- seed = strtoll(val.c_str(), NULL, 10);
- is_seed_set = true;
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-num-colls", (char*) NULL)) {
- num_colls = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-num-objs", (char*) NULL)) {
- num_objs = strtoll(val.c_str(), NULL, 10);
- } else if (ceph_argparse_witharg(args, i, &val,
- "--test-status-file", (char*) NULL)) {
- status_file = val;
- } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) {
- usage(our_name);
- exit(0);
- } else {
- if (command.empty())
- command = *i++;
- else
- command_args.push_back(string(*i++));
- }
- }
-
- int ret = run_command(command, command_args);
-
- return ret;
-}
+++ /dev/null
-# unittest_lfnindex
-add_executable(unittest_lfnindex
- TestLFNIndex.cc
- )
-add_ceph_unittest(unittest_lfnindex)
-target_link_libraries(unittest_lfnindex os global)
-
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Library Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Library Public License for more details.
- *
- */
-
-#include <stdio.h>
-#include <signal.h>
-#include "os/filestore/LFNIndex.h"
-#include "os/filestore/chain_xattr.h"
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include <gtest/gtest.h>
-
-using namespace std;
-
-class TestWrapLFNIndex : public LFNIndex {
-public:
- TestWrapLFNIndex(CephContext* cct,
- coll_t collection,
- const char *base_path,
- uint32_t index_version)
- : LFNIndex(cct, collection, base_path, index_version) {}
-
- uint32_t collection_version() override {
- return index_version;
- }
-
- int cleanup() override { return 0; }
-
- int _split(
- uint32_t match,
- uint32_t bits,
- CollectionIndex* dest
- ) override { return 0; }
- int _merge(
- uint32_t bits,
- CollectionIndex* dest
- ) override { return 0; }
-
- void test_generate_and_parse(const ghobject_t &hoid, const std::string &mangled_expected) {
- const std::string mangled_name = lfn_generate_object_name(hoid);
- EXPECT_EQ(mangled_expected, mangled_name);
- ghobject_t hoid_parsed;
- EXPECT_EQ(0, lfn_parse_object_name(mangled_name, &hoid_parsed));
- EXPECT_EQ(hoid, hoid_parsed);
- }
-
-protected:
- int _init() override { return 0; }
-
- int _created(
- const vector<string> &path,
- const ghobject_t &hoid,
- const string &mangled_name
- ) override { return 0; }
-
- int _remove(
- const vector<string> &path,
- const ghobject_t &hoid,
- const string &mangled_name
- ) override { return 0; }
-
- int _lookup(
- const ghobject_t &hoid,
- vector<string> *path,
- string *mangled_name,
- int *exists
- ) override { return 0; }
-
- int _collection_list_partial(
- const ghobject_t &start,
- const ghobject_t &end,
- int max_count,
- vector<ghobject_t> *ls,
- ghobject_t *next
- ) override { return 0; }
- int _pre_hash_collection(
- uint32_t pg_num,
- uint64_t expected_num_objs
- ) override { return 0; }
-
-};
-
-class TestHASH_INDEX_TAG : public TestWrapLFNIndex, public ::testing::Test {
-public:
- TestHASH_INDEX_TAG()
- : TestWrapLFNIndex(g_ceph_context, coll_t(), "PATH_1",
- CollectionIndex::HASH_INDEX_TAG) {
- }
-};
-
-TEST_F(TestHASH_INDEX_TAG, generate_and_parse_name) {
- const vector<string> path;
- const std::string key;
- uint64_t hash = 0xABABABAB;
- uint64_t pool = -1;
-
- test_generate_and_parse(ghobject_t(hobject_t(object_t(".A/B_\\C.D"), key, CEPH_NOSNAP, hash, pool, "")),
- "\\.A\\sB_\\\\C.D_head_ABABABAB");
- test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")),
- "\\dA_head_ABABABAB");
-}
-
-class TestHASH_INDEX_TAG_2 : public TestWrapLFNIndex, public ::testing::Test {
-public:
- TestHASH_INDEX_TAG_2()
- : TestWrapLFNIndex(g_ceph_context,
- coll_t(), "PATH_1", CollectionIndex::HASH_INDEX_TAG_2) {
- }
-};
-
-TEST_F(TestHASH_INDEX_TAG_2, generate_and_parse_name) {
- const vector<string> path;
- const std::string key("KEY");
- uint64_t hash = 0xABABABAB;
- uint64_t pool = -1;
-
- {
- std::string name(".XA/B_\\C.D");
- name[1] = '\0';
- ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""));
-
- test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB");
- }
- test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")),
- "\\dA_KEY_head_ABABABAB");
-}
-
-class TestHOBJECT_WITH_POOL : public TestWrapLFNIndex, public ::testing::Test {
-public:
- TestHOBJECT_WITH_POOL()
- : TestWrapLFNIndex(g_ceph_context, coll_t(),
- "PATH_1", CollectionIndex::HOBJECT_WITH_POOL) {
- }
-};
-
-TEST_F(TestHOBJECT_WITH_POOL, generate_and_parse_name) {
- const vector<string> path;
- const std::string key("KEY");
- uint64_t hash = 0xABABABAB;
- uint64_t pool = 0xCDCDCDCD;
- int64_t gen = 0xefefefefef;
- shard_id_t shard_id(0xb);
-
- {
- std::string name(".XA/B_\\C.D");
- name[1] = '\0';
- ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""));
- hoid.hobj.nspace = "NSPACE";
-
- test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd");
- }
- {
- ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""));
- hoid.hobj.nspace = "NSPACE";
-
- test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd");
- }
- {
- std::string name(".XA/B_\\C.D");
- name[1] = '\0';
- ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id);
- hoid.hobj.nspace = "NSPACE";
-
- test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b");
- }
- {
- ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id);
- hoid.hobj.nspace = "NSPACE";
-
- test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b");
- }
-}
-
-class TestLFNIndex : public TestWrapLFNIndex, public ::testing::Test {
-public:
- TestLFNIndex()
- : TestWrapLFNIndex(g_ceph_context, coll_t(), "PATH_1",
- CollectionIndex::HOBJECT_WITH_POOL) {
- }
-
- void SetUp() override {
- ::chmod("PATH_1", 0700);
- ASSERT_EQ(0, ::system("rm -fr PATH_1"));
- ASSERT_EQ(0, ::mkdir("PATH_1", 0700));
- }
-
- void TearDown() override {
- ASSERT_EQ(0, ::system("rm -fr PATH_1"));
- }
-};
-
-TEST_F(TestLFNIndex, remove_object) {
- const vector<string> path;
-
- //
- // small object name removal
- //
- {
- std::string mangled_name;
- int exists = 666;
- ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP)));
-
- EXPECT_EQ(0, ::chmod("PATH_1", 0000));
- if (getuid() != 0) {
- EXPECT_EQ(-EACCES, remove_object(path, hoid));
- }
- EXPECT_EQ(0, ::chmod("PATH_1", 0700));
- EXPECT_EQ(-ENOENT, remove_object(path, hoid));
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- const std::string pathname("PATH_1/" + mangled_name);
- EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
- EXPECT_EQ(0, remove_object(path, hoid));
- EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
- EXPECT_EQ(ENOENT, errno);
- }
- //
- // long object name removal of a single file
- //
- {
- std::string mangled_name;
- int exists;
- const std::string object_name(1024, 'A');
- ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
-
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- EXPECT_EQ(0, exists);
- EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
- std::string pathname("PATH_1/" + mangled_name);
- EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
- EXPECT_EQ(0, created(hoid, pathname.c_str()));
-
- EXPECT_EQ(0, remove_object(path, hoid));
- EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
- EXPECT_EQ(ENOENT, errno);
- }
-
- //
- // long object name removal of the last file
- //
- {
- std::string mangled_name;
- int exists;
- const std::string object_name(1024, 'A');
- ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
-
- //
- // PATH_1/AAA..._0_long => does not match long object name
- //
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- EXPECT_EQ(0, exists);
- EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
- std::string pathname("PATH_1/" + mangled_name);
- EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
- EXPECT_EQ(0, created(hoid, pathname.c_str()));
- string LFN_ATTR = "user.cephos.lfn";
- if (index_version != HASH_INDEX_TAG) {
- char buf[100];
- snprintf(buf, sizeof(buf), "%d", index_version);
- LFN_ATTR += string(buf);
- }
- const std::string object_name_1 = object_name + "SUFFIX";
- EXPECT_EQ(object_name_1.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_1.c_str(), object_name_1.size()));
-
- //
- // PATH_1/AAA..._1_long => matches long object name
- //
- std::string mangled_name_1;
- exists = 666;
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_1, &exists));
- EXPECT_NE(std::string::npos, mangled_name_1.find("1_long"));
- EXPECT_EQ(0, exists);
- std::string pathname_1("PATH_1/" + mangled_name_1);
- auto retvalue = ::creat(pathname_1.c_str(), 0600);
- ceph_assert(retvalue > 2);
- EXPECT_EQ(0, ::close(retvalue));
- EXPECT_EQ(0, created(hoid, pathname_1.c_str()));
-
- //
- // remove_object skips PATH_1/AAA..._0_long and removes PATH_1/AAA..._1_long
- //
- EXPECT_EQ(0, remove_object(path, hoid));
- EXPECT_EQ(0, ::access(pathname.c_str(), 0));
- EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0));
- EXPECT_EQ(ENOENT, errno);
- EXPECT_EQ(0, ::unlink(pathname.c_str()));
- }
-
- //
- // long object name removal of a file in the middle of the list
- //
- {
- std::string mangled_name;
- int exists;
- const std::string object_name(1024, 'A');
- ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
-
- //
- // PATH_1/AAA..._0_long => matches long object name
- //
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- EXPECT_EQ(0, exists);
- EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
- std::string pathname("PATH_1/" + mangled_name);
- EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
- EXPECT_EQ(0, created(hoid, pathname.c_str()));
- //
- // PATH_1/AAA..._1_long => matches long object name
- //
- std::string mangled_name_1 = mangled_name;
- mangled_name_1.replace(mangled_name_1.find("0_long"), 6, "1_long");
- const std::string pathname_1("PATH_1/" + mangled_name_1);
- const std::string cmd("cp -a " + pathname + " " + pathname_1);
- EXPECT_EQ(0, ::system(cmd.c_str()));
- const string ATTR = "user.MARK";
- EXPECT_EQ((unsigned)1, (unsigned)chain_setxattr(pathname_1.c_str(), ATTR.c_str(), "Y", 1));
-
- //
- // remove_object replaces the file to be removed with the last from the
- // collision list. In this case it replaces
- // PATH_1/AAA..._0_long
- // with
- // PATH_1/AAA..._1_long
- //
- EXPECT_EQ(0, remove_object(path, hoid));
- EXPECT_EQ(0, ::access(pathname.c_str(), 0));
- char buffer[1] = { 0, };
- EXPECT_EQ((unsigned)1, (unsigned)chain_getxattr(pathname.c_str(), ATTR.c_str(), buffer, 1));
- EXPECT_EQ('Y', buffer[0]);
- EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0));
- EXPECT_EQ(ENOENT, errno);
- }
-}
-
-TEST_F(TestLFNIndex, get_mangled_name) {
- const vector<string> path;
-
- //
- // small object name
- //
- {
- std::string mangled_name;
- int exists = 666;
- ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP)));
-
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- EXPECT_NE(std::string::npos, mangled_name.find("ABC__head"));
- EXPECT_EQ(std::string::npos, mangled_name.find("0_long"));
- EXPECT_EQ(0, exists);
- const std::string pathname("PATH_1/" + mangled_name);
- EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- EXPECT_NE(std::string::npos, mangled_name.find("ABC__head"));
- EXPECT_EQ(1, exists);
- EXPECT_EQ(0, ::unlink(pathname.c_str()));
- }
- //
- // long object name
- //
- {
- std::string mangled_name;
- int exists;
- const std::string object_name(1024, 'A');
- ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
-
- //
- // long version of the mangled name and no matching
- // file exists
- //
- mangled_name.clear();
- exists = 666;
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
- EXPECT_EQ(0, exists);
-
- const std::string pathname("PATH_1/" + mangled_name);
-
- //
- // if a file by the same name exists but does not have the
- // expected extended attribute, it is silently removed
- //
- mangled_name.clear();
- exists = 666;
- EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
- EXPECT_EQ(0, exists);
- EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
- EXPECT_EQ(ENOENT, errno);
-
- //
- // if a file by the same name exists but does not have the
- // expected extended attribute, and cannot be removed,
- // return on error
- //
- mangled_name.clear();
- exists = 666;
- EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
- EXPECT_EQ(0, ::chmod("PATH_1", 0500));
- if (getuid() != 0) {
- EXPECT_EQ(-EACCES, get_mangled_name(path, hoid, &mangled_name, &exists));
- }
- EXPECT_EQ("", mangled_name);
- EXPECT_EQ(666, exists);
- EXPECT_EQ(0, ::chmod("PATH_1", 0700));
- EXPECT_EQ(0, ::unlink(pathname.c_str()));
-
- //
- // long version of the mangled name and a file
- // exists by that name and contains the long object name
- //
- mangled_name.clear();
- exists = 666;
- auto retvalue = ::creat(pathname.c_str(), 0600);
- ceph_assert(retvalue > 2);
- EXPECT_EQ(0, ::close(retvalue));
- EXPECT_EQ(0, created(hoid, pathname.c_str()));
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
- EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
- EXPECT_EQ(1, exists);
- EXPECT_EQ(0, ::access(pathname.c_str(), 0));
-
- //
- // long version of the mangled name and a file exists by that name
- // and contains a long object name with the same prefix but they
- // are not identical and it so happens that their SHA1 is
- // identical : a collision number is used to differentiate them
- //
- string LFN_ATTR = "user.cephos.lfn";
- if (index_version != HASH_INDEX_TAG) {
- char buf[100];
- snprintf(buf, sizeof(buf), "%d", index_version);
- LFN_ATTR += string(buf);
- }
- const std::string object_name_same_prefix = object_name + "SUFFIX";
- EXPECT_EQ(object_name_same_prefix.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_same_prefix.c_str(), object_name_same_prefix.size()));
- std::string mangled_name_same_prefix;
- exists = 666;
- EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_same_prefix, &exists));
- EXPECT_NE(std::string::npos, mangled_name_same_prefix.find("1_long"));
- EXPECT_EQ(0, exists);
-
- EXPECT_EQ(0, ::unlink(pathname.c_str()));
- }
-}
-
-int main(int argc, char **argv) {
- int fd = ::creat("detect", 0600);
- if (fd < 0){
- cerr << "failed to create file detect" << std::endl;
- return EXIT_FAILURE;
- }
- int ret = chain_fsetxattr(fd, "user.test", "A", 1);
- ::close(fd);
- ::unlink("detect");
- if (ret < 0) {
- cerr << "SKIP LFNIndex because unable to test for xattr" << std::endl;
- } else {
- auto args = argv_to_vec(argc, argv);
-
- auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
- CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
-
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
- }
-}
-
-/*
- * Local Variables:
- * compile-command: "cd ../.. ;
- * make unittest_lfnindex &&
- * valgrind --tool=memcheck ./unittest_lfnindex \
- * # --gtest_filter=TestLFNIndex.* --log-to-stderr=true --debug-filestore=20"
- * End:
- */
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-#include <gtest/gtest.h>
-#include <stdlib.h>
-#include <limits.h>
-
-#include "common/ceph_argparse.h"
-#include "common/common_init.h"
-#include "global/global_init.h"
-#include "common/config.h"
-#include "common/Finisher.h"
-#include "os/filestore/FileJournal.h"
-#include "include/Context.h"
-#include "common/ceph_mutex.h"
-#include "common/safe_io.h"
-#include "os/filestore/JournalingObjectStore.h"
-
-using namespace std;
-
-Finisher *finisher;
-ceph::condition_variable sync_cond;
-char path[200];
-uuid_d fsid;
-struct test_info {
- bool directio, aio, faio;
- const char *description;
-} subtests[3] = {
- { false, false, false, "DIRECTIO OFF AIO OFF" },
- { true, false, false, "DIRECTIO ON AIO OFF" },
- { true, true, true, "DIRECTIO ON AIO ON"}
-};
-
-// ----
-ceph::condition_variable cond;
-ceph::mutex wait_lock = ceph::make_mutex("lock");
-bool done;
-
-void wait()
-{
- std::unique_lock l{wait_lock};
- cond.wait(l, [] { return done; });
-}
-
-// ----
-class C_Sync {
-public:
- ceph::condition_variable cond;
- ceph::mutex lock = ceph::make_mutex("C_Sync::lock");
- bool done = false;
- C_SafeCond *c;
-
- C_Sync() {
- c = new C_SafeCond(lock, cond, &done);
- }
- ~C_Sync() {
- std::unique_lock l{lock};
- //cout << "wait" << std::endl;
- cond.wait(l, [this] { return done; });
- //cout << "waited" << std::endl;
- }
-};
-
-unsigned size_mb = 200;
-//Gtest argument prefix
-const char GTEST_PRFIX[] = "--gtest_";
-
-int main(int argc, char **argv) {
- auto args = argv_to_vec(argc, argv);
-
- auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
- CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
-
- char mb[10];
- sprintf(mb, "%u", size_mb);
- g_ceph_context->_conf.set_val("osd_journal_size", mb);
- g_ceph_context->_conf.apply_changes(nullptr);
-
- finisher = new Finisher(g_ceph_context);
-
- path[0] = '\0';
- if (!args.empty()) {
- for ( unsigned int i = 0; i < args.size(); ++i) {
- if (strncmp(args[i], GTEST_PRFIX, sizeof(GTEST_PRFIX) - 1)) {
- //Non gtest argument, set to path.
- size_t copy_len = std::min(sizeof(path) - 1, strlen(args[i]));
- strncpy(path, args[i], copy_len);
- path[copy_len] = '\0';
- break;
- }
- }
- }
- if ( path[0] == '\0') {
- srand(getpid() + time(0));
- snprintf(path, sizeof(path), "/var/tmp/ceph_test_filejournal.tmp.%d", rand());
- }
- cout << "path " << path << std::endl;
-
- ::testing::InitGoogleTest(&argc, argv);
-
- finisher->start();
-
- int r = RUN_ALL_TESTS();
-
- finisher->stop();
-
- unlink(path);
-
- return r;
-}
-
-TEST(TestFileJournal, Create) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "false");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- }
-}
-
-TEST(TestFileJournal, WriteSmall) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "false");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- vector<ObjectStore::Transaction> tls;
- bufferlist bl;
- bl.append("small");
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(1, bl, orig_len, new C_SafeCond(wait_lock, cond, &done));
- wait();
-
- fj.close();
- }
-}
-
-TEST(TestFileJournal, WriteBig) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "false");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- bufferlist bl;
- while (bl.length() < size_mb*1000/2) {
- char foo[1024*1024];
- memset(foo, 1, sizeof(foo));
- bl.append(foo, sizeof(foo));
- }
- vector<ObjectStore::Transaction> tls;
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(1, bl, orig_len, new C_SafeCond(wait_lock, cond, &done));
- wait();
- fj.close();
- }
-}
-
-TEST(TestFileJournal, WriteMany) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "false");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done));
-
- vector<ObjectStore::Transaction> tls;
- bufferlist bl;
- bl.append("small");
- uint64_t seq = 1;
- for (int i=0; i<100; i++) {
- bl.append("small");
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(seq++, bl, orig_len, gb.new_sub());
- }
- gb.activate();
-
- wait();
-
- fj.close();
- }
-}
-
-TEST(TestFileJournal, WriteManyVecs) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "false");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done));
-
- bufferlist first;
- first.append("small");
- vector<ObjectStore::Transaction> tls;
- int orig_len = fj.prepare_entry(tls, &first);
- fj.reserve_throttle_and_backoff(first.length());
- fj.submit_entry(1, first, orig_len, gb.new_sub());
-
- bufferlist bl;
- for (int i=0; i<IOV_MAX * 2; i++) {
- bufferptr bp = buffer::create_page_aligned(4096);
- memset(bp.c_str(), (char)i, 4096);
- bl.append(bp);
- }
- bufferlist origbl = bl;
- orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(2, bl, orig_len, gb.new_sub());
- gb.activate();
- wait();
-
- fj.close();
-
- fj.open(1);
- bufferlist inbl;
- string v;
- uint64_t seq = 0;
- ASSERT_EQ(true, fj.read_entry(inbl, seq));
- ASSERT_EQ(seq, 2ull);
- ASSERT_TRUE(inbl.contents_equal(origbl));
- ASSERT_EQ(0, fj.make_writeable());
- fj.close();
-
- }
-}
-
-TEST(TestFileJournal, ReplaySmall) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "false");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- vector<ObjectStore::Transaction> tls;
-
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done));
-
- bufferlist bl;
- bl.append("small");
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(1, bl, orig_len, gb.new_sub());
- bl.append("small");
- orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(2, bl, orig_len, gb.new_sub());
- bl.append("small");
- orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(3, bl, orig_len, gb.new_sub());
- gb.activate();
- wait();
-
- fj.close();
-
- fj.open(1);
-
- bufferlist inbl;
- string v;
- uint64_t seq = 0;
- ASSERT_EQ(true, fj.read_entry(inbl, seq));
- ASSERT_EQ(seq, 2ull);
- inbl.cbegin().copy(inbl.length(), v);
- ASSERT_EQ("small", v);
- inbl.clear();
- v.clear();
-
- ASSERT_EQ(true, fj.read_entry(inbl, seq));
- ASSERT_EQ(seq, 3ull);
- inbl.cbegin().copy(inbl.length(), v);
- ASSERT_EQ("small", v);
- inbl.clear();
- v.clear();
-
- ASSERT_TRUE(!fj.read_entry(inbl, seq));
-
- ASSERT_EQ(0, fj.make_writeable());
- fj.close();
- }
-}
-
-TEST(TestFileJournal, ReplayCorrupt) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "true");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- vector<ObjectStore::Transaction> tls;
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done));
-
- const char *needle = "i am a needle";
- const char *newneedle = "in a haystack";
- bufferlist bl;
- bl.append(needle);
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(1, bl, orig_len, gb.new_sub());
- bl.append(needle);
- orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(2, bl, orig_len, gb.new_sub());
- bl.append(needle);
- orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(3, bl, orig_len, gb.new_sub());
- bl.append(needle);
- orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(4, bl, orig_len, gb.new_sub());
- gb.activate();
- wait();
-
- fj.close();
-
- cout << "corrupting journal" << std::endl;
- char buf[1024*128];
- int fd = open(path, O_RDONLY);
- ASSERT_GE(fd, 0);
- int r = safe_read_exact(fd, buf, sizeof(buf));
- ASSERT_EQ(0, r);
- int n = 0;
- for (unsigned o=0; o < sizeof(buf) - strlen(needle); o++) {
- if (memcmp(buf+o, needle, strlen(needle)) == 0) {
- if (n >= 2) {
- cout << "replacing at offset " << o << std::endl;
- memcpy(buf+o, newneedle, strlen(newneedle));
- } else {
- cout << "leaving at offset " << o << std::endl;
- }
- n++;
- }
- }
- ASSERT_EQ(n, 4);
- close(fd);
- fd = open(path, O_WRONLY);
- ASSERT_GE(fd, 0);
- r = safe_write(fd, buf, sizeof(buf));
- ASSERT_EQ(r, 0);
- close(fd);
-
- fj.open(1);
-
- bufferlist inbl;
- string v;
- uint64_t seq = 0;
- ASSERT_EQ(true, fj.read_entry(inbl, seq));
- ASSERT_EQ(seq, 2ull);
- inbl.cbegin().copy(inbl.length(), v);
- ASSERT_EQ(needle, v);
- inbl.clear();
- v.clear();
- bool corrupt;
- ASSERT_FALSE(fj.read_entry(inbl, seq, &corrupt));
- ASSERT_TRUE(corrupt);
-
- ASSERT_EQ(0, fj.make_writeable());
- fj.close();
- }
-}
-
-TEST(TestFileJournal, WriteTrim) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "false");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- list<C_Sync*> ls;
-
- bufferlist bl;
- char foo[1024*1024];
- memset(foo, 1, sizeof(foo));
-
- uint64_t seq = 1, committed = 0;
- vector<ObjectStore::Transaction> tls;
-
- for (unsigned i=0; i<size_mb*2; i++) {
- bl.clear();
- bl.push_back(buffer::copy(foo, sizeof(foo)));
- bl.zero();
- ls.push_back(new C_Sync);
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(seq++, bl, orig_len, ls.back()->c);
-
- while (ls.size() > size_mb/2) {
- delete ls.front();
- ls.pop_front();
- committed++;
- fj.committed_thru(committed);
- }
- }
-
- while (ls.size()) {
- delete ls.front();
- ls.pop_front();
- fj.committed_thru(++committed);
- }
-
- ASSERT_TRUE(fj.journalq_empty());
-
- fj.close();
- }
-}
-
-TEST(TestFileJournal, WriteTrimSmall) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "false");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "0");
- g_ceph_context->_conf.apply_changes(nullptr);
- vector<ObjectStore::Transaction> tls;
-
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- list<C_Sync*> ls;
-
- bufferlist bl;
- char foo[1024*1024];
- memset(foo, 1, sizeof(foo));
-
- uint64_t seq = 1, committed = 0;
-
- for (unsigned i=0; i<size_mb*2; i++) {
- bl.clear();
- for (int k=0; k<128; k++)
- bl.push_back(buffer::copy(foo, sizeof(foo) / 128));
- bl.zero();
- ls.push_back(new C_Sync);
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(seq++, bl, orig_len, ls.back()->c);
-
- while (ls.size() > size_mb/2) {
- delete ls.front();
- ls.pop_front();
- committed++;
- fj.committed_thru(committed);
- }
- }
-
- while (ls.size()) {
- delete ls.front();
- ls.pop_front();
- fj.committed_thru(committed);
- }
-
- fj.close();
- }
-}
-
-TEST(TestFileJournal, ReplayDetectCorruptFooterMagic) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "true");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "1");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- vector<ObjectStore::Transaction> tls;
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done));
-
- const char *needle = "i am a needle";
- for (unsigned i = 1; i <= 4; ++i) {
- bufferlist bl;
- bl.append(needle);
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(i, bl, orig_len, gb.new_sub());
- }
- gb.activate();
- wait();
-
- bufferlist bl;
- bl.append("needle");
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(5, bl, orig_len, new C_SafeCond(wait_lock, cond, &done));
- wait();
-
- fj.close();
- int fd = open(path, O_WRONLY);
-
- cout << "corrupting journal" << std::endl;
- fj.open(0);
- fj.corrupt_footer_magic(fd, 2);
-
- uint64_t seq = 0;
- bl.clear();
- bool corrupt = false;
- bool result = fj.read_entry(bl, seq, &corrupt);
- ASSERT_TRUE(result);
- ASSERT_EQ(seq, 1UL);
- ASSERT_FALSE(corrupt);
-
- result = fj.read_entry(bl, seq, &corrupt);
- ASSERT_FALSE(result);
- ASSERT_TRUE(corrupt);
-
- ASSERT_EQ(0, fj.make_writeable());
- fj.close();
- ::close(fd);
- }
-}
-
-TEST(TestFileJournal, ReplayDetectCorruptPayload) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "true");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "1");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- vector<ObjectStore::Transaction> tls;
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done));
-
- const char *needle = "i am a needle";
- for (unsigned i = 1; i <= 4; ++i) {
- bufferlist bl;
- bl.append(needle);
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(i, bl, orig_len, gb.new_sub());
- }
- gb.activate();
- wait();
-
- bufferlist bl;
- bl.append("needle");
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(5, bl, orig_len, new C_SafeCond(wait_lock, cond, &done));
- wait();
-
- fj.close();
- int fd = open(path, O_WRONLY);
-
- cout << "corrupting journal" << std::endl;
- fj.open(0);
- fj.corrupt_payload(fd, 2);
-
- uint64_t seq = 0;
- bl.clear();
- bool corrupt = false;
- bool result = fj.read_entry(bl, seq, &corrupt);
- ASSERT_TRUE(result);
- ASSERT_EQ(seq, 1UL);
- ASSERT_FALSE(corrupt);
-
- result = fj.read_entry(bl, seq, &corrupt);
- ASSERT_FALSE(result);
- ASSERT_TRUE(corrupt);
-
- ASSERT_EQ(0, fj.make_writeable());
- fj.close();
- ::close(fd);
- }
-}
-
-TEST(TestFileJournal, ReplayDetectCorruptHeader) {
- g_ceph_context->_conf.set_val("journal_ignore_corruption", "true");
- g_ceph_context->_conf.set_val("journal_write_header_frequency", "1");
- g_ceph_context->_conf.apply_changes(nullptr);
-
- vector<ObjectStore::Transaction> tls;
- for (unsigned i = 0 ; i < 3; ++i) {
- SCOPED_TRACE(subtests[i].description);
- fsid.generate_random();
- FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path,
- subtests[i].directio, subtests[i].aio, subtests[i].faio);
- ASSERT_EQ(0, fj.create());
- ASSERT_EQ(0, fj.make_writeable());
-
- C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done));
-
- const char *needle = "i am a needle";
- for (unsigned i = 1; i <= 4; ++i) {
- bufferlist bl;
- bl.append(needle);
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(i, bl, orig_len, gb.new_sub());
- }
- gb.activate();
- wait();
-
- bufferlist bl;
- bl.append("needle");
- int orig_len = fj.prepare_entry(tls, &bl);
- fj.reserve_throttle_and_backoff(bl.length());
- fj.submit_entry(5, bl, orig_len, new C_SafeCond(wait_lock, cond, &done));
- wait();
-
- fj.close();
- int fd = open(path, O_WRONLY);
-
- cout << "corrupting journal" << std::endl;
- fj.open(0);
- fj.corrupt_header_magic(fd, 2);
-
- uint64_t seq = 0;
- bl.clear();
- bool corrupt = false;
- bool result = fj.read_entry(bl, seq, &corrupt);
- ASSERT_TRUE(result);
- ASSERT_EQ(seq, 1UL);
- ASSERT_FALSE(corrupt);
-
- result = fj.read_entry(bl, seq, &corrupt);
- ASSERT_FALSE(result);
- ASSERT_TRUE(corrupt);
-
- ASSERT_EQ(0, fj.make_writeable());
- fj.close();
- ::close(fd);
- }
-}
#include <iostream>
#include "common/ceph_argparse.h"
#include "common/debug.h"
-#include "os/filestore/FileStore.h"
+#include "os/bluestore/BlueStore.h"
#include "global/global_init.h"
#include "include/ceph_assert.h"
cout << "#dev " << filename << std::endl;
cout << "#mb " << mb << std::endl;
- ObjectStore *fs = new FileStore(cct.get(), filename, NULL);
+ ObjectStore *fs = new BlueStore(cct.get(), filename);
if (fs->mount() < 0) {
cout << "mount failed" << std::endl;
return -1;
#include <iostream>
#include <iterator>
#include <sstream>
-#include "os/filestore/FileStore.h"
+#include "os/bluestore/BlueStore.h"
#include "include/Context.h"
#include "common/ceph_argparse.h"
#include "common/ceph_mutex.h"
#include "include/unordered_map.h"
void usage(const string &name) {
- std::cerr << "Usage: " << name << " [xattr|omap] store_path store_journal"
+ std::cerr << "Usage: " << name << " [xattr|omap] store_path"
<< std::endl;
}
common_init_finish(g_ceph_context);
std::cerr << "args: " << args << std::endl;
- if (args.size() < 3) {
+ if (args.size() < 2) {
usage(argv[0]);
return 1;
}
string store_path(args[1]);
- string store_dev(args[2]);
- boost::scoped_ptr<ObjectStore> store(new FileStore(cct.get(), store_path,
- store_dev));
+ boost::scoped_ptr<ObjectStore> store(new BlueStore(cct.get(), store_path));
std::cerr << "mkfs starting" << std::endl;
ceph_assert(!store->mkfs());
projects / ceph.git / commitdiff