From: Nitzan Mordechai Date: Thu, 9 Feb 2023 05:54:42 +0000 (+0000) Subject: test: Remove all filestore tests and use X-Git-Tag: v18.1.0~304^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e682b60617e91370c98fe527e8cd6f3ce1ea7865;p=ceph.git test: Remove all filestore tests and use - test_trans convert FileStore to BlueStore test - xattr_bench convert FileStore to BlueStore usage - remove test_idempotent_sequence tests - remove test_idempotent - remove test_filejournal - removing filestore tests from store_test - remove rep_read_unfound test for filestore only - remove osd-dup convert filestore to bluestore - osd-scrub-repair start only bluestore osd - osd-pool-create remove filestore expected_num_object test - Remove chain_xattr and LFNIndex uneeded test Signed-off-by: Nitzan Mordechai --- diff --git a/qa/standalone/mon/osd-pool-create.sh b/qa/standalone/mon/osd-pool-create.sh index c7e1f66bf9b3..6d2c5ad3e9e9 100755 --- a/qa/standalone/mon/osd-pool-create.sh +++ b/qa/standalone/mon/osd-pool-create.sh @@ -211,30 +211,6 @@ function TEST_utf8_cli() { export LANG="$OLDLANG" } -function TEST_pool_create_rep_expected_num_objects() { - local dir=$1 - setup $dir || return 1 - - export CEPH_ARGS - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - # disable pg dir merge - run_osd_filestore $dir 0 || return 1 - - ceph osd pool create rep_expected_num_objects 64 64 replicated replicated_rule 100000 || return 1 - # wait for pg dir creating - sleep 30 - ceph pg ls - find ${dir}/0/current -ls - ret=$(find ${dir}/0/current/1.0_head/ | grep DIR | wc -l) - if [ "$ret" -le 2 ]; - then - return 1 - else - echo "TEST_pool_create_rep_expected_num_objects PASS" - fi -} - function check_pool_priority() { local dir=$1 shift diff --git a/qa/standalone/osd/osd-dup.sh b/qa/standalone/osd/osd-dup.sh index fdb2649ca055..ab442c53812b 100755 --- a/qa/standalone/osd/osd-dup.sh +++ b/qa/standalone/osd/osd-dup.sh @@ -23,59 +23,6 @@ function run() { done } -function TEST_filestore_to_bluestore() { - local dir=$1 - - local flimit=$(ulimit -n) - if [ $flimit -lt 1536 ]; then - echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." - fi - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd_filestore $dir 0 || return 1 - osd_pid=$(cat $dir/osd.0.pid) - run_osd_filestore $dir 1 || return 1 - run_osd_filestore $dir 2 || return 1 - - sleep 5 - - create_pool foo 16 - - # write some objects - timeout 20 rados bench -p foo 10 write -b 4096 --no-cleanup || return 1 - - # kill - while kill $osd_pid; do sleep 1 ; done - ceph osd down 0 - - mv $dir/0 $dir/0.old || return 1 - mkdir $dir/0 || return 1 - ofsid=$(cat $dir/0.old/fsid) - echo "osd fsid $ofsid" - O=$CEPH_ARGS - CEPH_ARGS+="--log-file $dir/cot.log --log-max-recent 0 " - ceph-objectstore-tool --type bluestore --data-path $dir/0 --fsid $ofsid \ - --op mkfs --no-mon-config || return 1 - ceph-objectstore-tool --data-path $dir/0.old --target-data-path $dir/0 \ - --op dup || return 1 - CEPH_ARGS=$O - - activate_osd $dir 0 || return 1 - - while ! ceph osd stat | grep '3 up' ; do sleep 1 ; done - ceph osd metadata 0 | grep bluestore || return 1 - - ceph osd scrub 0 - - # give it some time - sleep 15 - # and make sure mon is sync'ed - flush_pg_stats - - wait_for_clean || return 1 -} - main osd-dup "$@" # Local Variables: diff --git a/qa/standalone/osd/osd-rep-recov-eio.sh b/qa/standalone/osd/osd-rep-recov-eio.sh index 6f871ef8fa8c..c2423e06713e 100755 --- a/qa/standalone/osd/osd-rep-recov-eio.sh +++ b/qa/standalone/osd/osd-rep-recov-eio.sh @@ -416,55 +416,6 @@ function TEST_rep_recovery_unfound() { delete_pool $poolname } -# This is a filestore only test because it requires data digest in object info -function TEST_rep_read_unfound() { - local dir=$1 - local objname=myobject - - setup_osds 3 _filestore || return 1 - - ceph osd pool delete foo foo --yes-i-really-really-mean-it || return 1 - local poolname=test-pool - create_pool $poolname 1 1 || return 1 - ceph osd pool set $poolname size 2 - wait_for_clean || return 1 - - ceph pg dump pgs - - dd if=/dev/urandom bs=8k count=1 of=$dir/ORIGINAL - rados -p $poolname put $objname $dir/ORIGINAL - - local primary=$(get_primary $poolname $objname) - local other=$(get_not_primary $poolname $objname) - - dd if=/dev/urandom bs=8k count=1 of=$dir/CORRUPT - objectstore_tool $dir $primary $objname set-bytes $dir/CORRUPT || return 1 - objectstore_tool $dir $other $objname set-bytes $dir/CORRUPT || return 1 - - timeout 30 rados -p $poolname get $objname $dir/tmp & - - sleep 5 - - flush_pg_stats - ceph --format=json pg dump pgs | jq '.' - - if ! ceph --format=json pg dump pgs | jq '.pg_stats | .[0].state' | grep -q recovery_unfound - then - echo "Failure to get to recovery_unfound state" - return 1 - fi - - objectstore_tool $dir $other $objname set-bytes $dir/ORIGINAL || return 1 - - wait - - if ! cmp $dir/ORIGINAL $dir/tmp - then - echo "Bad data after primary repair" - return 1 - fi -} - main osd-rep-recov-eio.sh "$@" # Local Variables: diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh index e4e55cd229f9..13b30360c4e0 100755 --- a/qa/standalone/scrub/osd-scrub-repair.sh +++ b/qa/standalone/scrub/osd-scrub-repair.sh @@ -277,11 +277,7 @@ function auto_repair_erasure_coded() { --osd-scrub-min-interval=5 \ --osd-scrub-interval-randomize-ratio=0" for id in $(seq 0 2) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd $dir $id $ceph_osd_args || return 1 - else - run_osd_filestore $dir $id $ceph_osd_args || return 1 - fi + run_osd $dir $id $ceph_osd_args || return 1 done create_rbd_pool || return 1 wait_for_clean || return 1 @@ -803,11 +799,7 @@ function corrupt_and_repair_jerasure() { run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 3) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd $dir $id || return 1 - else - run_osd_filestore $dir $id || return 1 - fi + run_osd $dir $id || return 1 done create_rbd_pool || return 1 wait_for_clean || return 1 @@ -834,11 +826,7 @@ function corrupt_and_repair_lrc() { run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 9) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd $dir $id || return 1 - else - run_osd_filestore $dir $id || return 1 - fi + run_osd $dir $id || return 1 done create_rbd_pool || return 1 wait_for_clean || return 1 @@ -866,11 +854,7 @@ function unfound_erasure_coded() { run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 3) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd $dir $id || return 1 - else - run_osd_filestore $dir $id || return 1 - fi + run_osd $dir $id || return 1 done create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 @@ -933,11 +917,7 @@ function list_missing_erasure_coded() { run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 2) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd $dir $id || return 1 - else - run_osd_filestore $dir $id || return 1 - fi + run_osd $dir $id || return 1 done create_rbd_pool || return 1 wait_for_clean || return 1 @@ -3618,11 +3598,7 @@ function corrupt_scrub_erasure() { run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 2) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd $dir $id || return 1 - else - run_osd_filestore $dir $id || return 1 - fi + run_osd $dir $id || return 1 done create_rbd_pool || return 1 create_pool foo 1 diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index c9fa36e38215..9ef6ff363f57 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -58,7 +58,6 @@ if(NOT WIN32) add_subdirectory(journal) add_subdirectory(erasure-code) - add_subdirectory(filestore) add_subdirectory(fs) add_subdirectory(libcephfs) add_subdirectory(libcephsqlite) @@ -71,7 +70,6 @@ if(NOT WIN32) add_subdirectory(neorados) add_subdirectory(objectstore) add_subdirectory(ObjectMap) - add_subdirectory(os) add_subdirectory(osd) add_subdirectory(osdc) add_subdirectory(immutable_object_cache) @@ -443,24 +441,6 @@ install(TARGETS ceph_perf_local DESTINATION bin) -# ceph_test_filejournal -add_executable(ceph_test_filejournal - test_filejournal.cc - ) -target_link_libraries(ceph_test_filejournal - os - ceph-common - ${UNITTEST_LIBS} - global - ${EXTRALIBS} - ${BLKID_LIBRARIES} - ${CMAKE_DL_LIBS} - ${EXTRALIBS} - ) -install(TARGETS - ceph_test_filejournal - DESTINATION ${CMAKE_INSTALL_BINDIR}) - # ceph_test_keys add_executable(ceph_test_keys testkeys.cc diff --git a/src/test/filestore/CMakeLists.txt b/src/test/filestore/CMakeLists.txt deleted file mode 100644 index ae29faacf863..000000000000 --- a/src/test/filestore/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -#ceph_test_filestore -add_executable(ceph_test_filestore - TestFileStore.cc) -target_link_libraries(ceph_test_filestore os global ${UNITTEST_LIBS}) - - diff --git a/src/test/filestore/TestFileStore.cc b/src/test/filestore/TestFileStore.cc deleted file mode 100644 index 02bc549e1084..000000000000 --- a/src/test/filestore/TestFileStore.cc +++ /dev/null @@ -1,90 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2014 Cloudwatt - * - * Author: Loic Dachary - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#include "common/ceph_argparse.h" -#include "global/global_init.h" -#include "os/filestore/FileStore.h" -#include - -using namespace std; - -class TestFileStore { -public: - static void create_backend(FileStore &fs, unsigned long f_type) { - fs.create_backend(f_type); - } -}; - -TEST(FileStore, create) -{ - { - map pm; - FileStore fs(g_ceph_context, "a", "b"); - TestFileStore::create_backend(fs, 0); - fs.collect_metadata(&pm); - ASSERT_EQ(pm["filestore_backend"], "generic"); - } -#if defined(__linux__) - { - map pm; - FileStore fs(g_ceph_context, "a", "b"); - TestFileStore::create_backend(fs, BTRFS_SUPER_MAGIC); - fs.collect_metadata(&pm); - ASSERT_EQ(pm["filestore_backend"], "btrfs"); - } -# ifdef HAVE_LIBXFS - { - map pm; - FileStore fs(g_ceph_context, "a", "b"); - TestFileStore::create_backend(fs, XFS_SUPER_MAGIC); - fs.collect_metadata(&pm); - ASSERT_EQ(pm["filestore_backend"], "xfs"); - } -# endif -#endif -#ifdef HAVE_LIBZFS - { - map pm; - FileStore fs("a", "b"); - TestFileStore::create_backend(fs, ZFS_SUPER_MAGIC); - fs.collect_metadata(&pm); - ASSERT_EQ(pm["filestore_backend"], "zfs"); - } -#endif -} - -int main(int argc, char **argv) { - auto args = argv_to_vec(argc, argv); - - auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT, - CODE_ENVIRONMENT_UTILITY, - CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - g_ceph_context->_conf.set_val("osd_journal_size", "100"); - g_ceph_context->_conf.apply_changes(NULL); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -/* - * Local Variables: - * compile-command: "cd ../.. ; make ceph_test_filestore && - * ./ceph_test_filestore \ - * --gtest_filter=*.* --log-to-stderr=true --debug-filestore=20 - * " - * End: - */ diff --git a/src/test/objectstore/CMakeLists.txt b/src/test/objectstore/CMakeLists.txt index 3294616b9b26..a012264e8d37 100644 --- a/src/test/objectstore/CMakeLists.txt +++ b/src/test/objectstore/CMakeLists.txt @@ -37,44 +37,6 @@ target_link_libraries(ceph_test_keyvaluedb install(TARGETS ceph_test_keyvaluedb DESTINATION ${CMAKE_INSTALL_BINDIR}) -# ceph_test_filestore_idempotent -add_executable(ceph_test_filestore_idempotent - test_idempotent.cc - FileStoreTracker.cc - ${CMAKE_SOURCE_DIR}/src/test/common/ObjectContents.cc - ) -target_link_libraries(ceph_test_filestore_idempotent - os - global - ${EXTRALIBS} - ${BLKID_LIBRARIES} - ${CMAKE_DL_LIBS} - ) - -# ceph_test_filestore_idempotent_sequence -add_executable(ceph_test_filestore_idempotent_sequence - test_idempotent_sequence.cc - DeterministicOpSequence.cc - TestObjectStoreState.cc - FileStoreDiff.cc - ) -target_link_libraries(ceph_test_filestore_idempotent_sequence - os - global - ${EXTRALIBS} - ${BLKID_LIBRARIES} - ${CMAKE_DL_LIBS} - ) -install(TARGETS ceph_test_filestore_idempotent_sequence - DESTINATION ${CMAKE_INSTALL_BINDIR}) - -# unittest_chain_xattr -add_executable(unittest_chain_xattr - chain_xattr.cc - ) -add_ceph_unittest(unittest_chain_xattr) -target_link_libraries(unittest_chain_xattr os global) - # unittest_rocksdb_option add_executable(unittest_rocksdb_option TestRocksdbOptionParse.cc diff --git a/src/test/objectstore/DeterministicOpSequence.cc b/src/test/objectstore/DeterministicOpSequence.cc deleted file mode 100644 index a51c5cfc6c6c..000000000000 --- a/src/test/objectstore/DeterministicOpSequence.cc +++ /dev/null @@ -1,495 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* -* Ceph - scalable distributed file system -* -* Copyright (C) 2012 New Dream Network -* -* This is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License version 2.1, as published by the Free Software -* Foundation. See file COPYING. -*/ -#include -#include -#include -#include -#include -#include -#include -#include -#include "os/ObjectStore.h" -#include "common/ceph_argparse.h" -#include "global/global_init.h" -#include "common/debug.h" -#include -#include - -#include "DeterministicOpSequence.h" -#include "common/config.h" -#include "include/ceph_assert.h" - -#define dout_context g_ceph_context -#define dout_subsys ceph_subsys_filestore -#undef dout_prefix -#define dout_prefix *_dout << "deterministic_seq " - -using namespace std; - -DeterministicOpSequence::DeterministicOpSequence(ObjectStore *store, - std::string status) - : TestObjectStoreState(store), - txn(0) -{ - if (!status.empty()) - m_status.open(status.c_str()); -} - -DeterministicOpSequence::~DeterministicOpSequence() -{ - // TODO Auto-generated destructor stub -} - -bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen) -{ - bool ok = false; - switch (op) { - case DSOP_TOUCH: - ok = do_touch(gen); - break; - case DSOP_WRITE: - ok = do_write(gen); - break; - case DSOP_CLONE: - ok = do_clone(gen); - break; - case DSOP_CLONE_RANGE: - ok = do_clone_range(gen); - break; - case DSOP_OBJ_REMOVE: - ok = do_remove(gen); - break; - case DSOP_COLL_MOVE: - ok = do_coll_move(gen); - break; - case DSOP_SET_ATTRS: - ok = do_set_attrs(gen); - break; - case DSOP_COLL_CREATE: - ok = do_coll_create(gen); - break; - - default: - cout << "bad op " << op << std::endl; - ceph_abort_msg("bad op"); - } - return ok; -} - -void DeterministicOpSequence::generate(int seed, int num_txs) -{ - std::ostringstream ss; - ss << "generate run " << num_txs << " --seed " << seed; - - if (m_status.is_open()) { - m_status << ss.str() << std::endl; - m_status.flush(); - } - - dout(0) << ss.str() << dendl; - - rngen_t gen(seed); - boost::uniform_int<> op_rng(DSOP_FIRST, DSOP_LAST); - - for (txn = 1; txn <= num_txs; ) { - int op = op_rng(gen); - _print_status(txn, op); - dout(0) << "generate seq " << txn << " op " << op << dendl; - if (run_one_op(op, gen)) - txn++; - } -} - -void DeterministicOpSequence::_print_status(int seq, int op) -{ - if (!m_status.is_open()) - return; - m_status << seq << " " << op << std::endl; - m_status.flush(); -} - -int DeterministicOpSequence::_gen_coll_id(rngen_t& gen) -{ - boost::uniform_int<> coll_rng(0, m_collections_ids.size()-1); - return coll_rng(gen); -} - -int DeterministicOpSequence::_gen_obj_id(rngen_t& gen) -{ - boost::uniform_int<> obj_rng(0, m_num_objects - 1); - return obj_rng(gen); -} - -void DeterministicOpSequence::note_txn(coll_entry_t *entry, - ObjectStore::Transaction *t) -{ - bufferlist bl; - encode(txn, bl); - ghobject_t oid = get_txn_object(entry->m_cid); - t->truncate(entry->m_cid, oid, 0); - t->write(entry->m_cid, oid, 0, bl.length(), bl); - dout(10) << __func__ << " " << txn << dendl; -} - -bool DeterministicOpSequence::do_touch(rngen_t& gen) -{ - int coll_id = _gen_coll_id(gen); - int obj_id = _gen_obj_id(gen); - - coll_entry_t *entry = get_coll_at(coll_id); - ceph_assert(entry != NULL); - - // Don't care about other collections if already exists - if (!entry->check_for_obj(obj_id)) { - bool other_found = false; - auto it = m_collections.begin(); - for (; it != m_collections.end(); ++it) { - if (it->second->check_for_obj(obj_id)) { - ceph_assert(it->first != entry->m_cid); - other_found = true; - } - } - if (other_found) { - dout(0) << "do_touch new object in collection and exists in another" << dendl; - return false; - } - } - hobject_t *obj = entry->touch_obj(obj_id); - - dout(0) << "do_touch " << entry->m_cid << "/" << obj << dendl; - - _do_touch(entry, *obj); - return true; -} - -bool DeterministicOpSequence::do_remove(rngen_t& gen) -{ - int coll_id = _gen_coll_id(gen); - - coll_entry_t *entry = get_coll_at(coll_id); - ceph_assert(entry != NULL); - - if (entry->m_objects.size() == 0) { - dout(0) << "do_remove no objects in collection" << dendl; - return false; - } - int obj_id = entry->get_random_obj_id(gen); - hobject_t *obj = entry->touch_obj(obj_id); - ceph_assert(obj); - - dout(0) << "do_remove " << entry->m_cid << "/" << obj << dendl; - - _do_remove(entry, *obj); - hobject_t *rmobj = entry->remove_obj(obj_id); - ceph_assert(rmobj); - delete rmobj; - return true; -} - -static void _gen_random(rngen_t& gen, - size_t size, bufferlist& bl) { - - static const char alphanum[] = "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz"; - - boost::uniform_int<> char_rng(0, sizeof(alphanum)); - bufferptr bp(size); - for (unsigned int i = 0; i < size - 1; i++) { - bp[i] = alphanum[char_rng(gen)]; - } - bp[size - 1] = '\0'; - bl.append(bp); -} - -static void gen_attrs(rngen_t &gen, - map *out) { - boost::uniform_int<> num_rng(10, 30); - boost::uniform_int<> key_size_rng(5, 10); - boost::uniform_int<> val_size_rng(100, 1000); - size_t num_attrs = static_cast(num_rng(gen)); - for (size_t i = 0; i < num_attrs; ++i) { - size_t key_size = static_cast(num_rng(gen)); - size_t val_size = static_cast(num_rng(gen)); - bufferlist keybl; - _gen_random(gen, key_size, keybl); - string key(keybl.c_str(), keybl.length()); - _gen_random(gen, val_size, (*out)[key]); - } -} - -bool DeterministicOpSequence::do_set_attrs(rngen_t& gen) -{ - int coll_id = _gen_coll_id(gen); - - coll_entry_t *entry = get_coll_at(coll_id); - ceph_assert(entry != NULL); - - if (entry->m_objects.size() == 0) { - dout(0) << "do_set_attrs no objects in collection" << dendl; - return false; - } - int obj_id = entry->get_random_obj_id(gen); - hobject_t *obj = entry->touch_obj(obj_id); - ceph_assert(obj); - - map out; - gen_attrs(gen, &out); - - dout(0) << "do_set_attrs " << out.size() << " entries" << dendl; - _do_set_attrs(entry, *obj, out); - return true; -} - -bool DeterministicOpSequence::do_write(rngen_t& gen) -{ - int coll_id = _gen_coll_id(gen); - - coll_entry_t *entry = get_coll_at(coll_id); - ceph_assert(entry != NULL); - - if (entry->m_objects.size() == 0) { - dout(0) << "do_write no objects in collection" << dendl; - return false; - } - int obj_id = entry->get_random_obj_id(gen); - hobject_t *obj = entry->touch_obj(obj_id); - ceph_assert(obj); - - boost::uniform_int<> size_rng(100, (2 << 19)); - size_t size = (size_t) size_rng(gen); - bufferlist bl; - _gen_random(gen, size, bl); - - dout(0) << "do_write " << entry->m_cid << "/" << obj - << " 0~" << size << dendl; - - _do_write(entry, *obj, 0, bl.length(), bl); - return true; -} - -bool DeterministicOpSequence::_prepare_clone( - rngen_t& gen, - coll_entry_t **entry_ret, - int *orig_obj_id, - hobject_t *orig_obj_ret, - int *new_obj_id, - hobject_t *new_obj_ret) -{ - int coll_id = _gen_coll_id(gen); - - coll_entry_t *entry = get_coll_at(coll_id); - ceph_assert(entry != NULL); - - if (entry->m_objects.size() < 2) { - dout(0) << "_prepare_clone coll " << entry->m_cid - << " doesn't have 2 or more objects" << dendl; - return false; - } - - *orig_obj_id = entry->get_random_obj_id(gen); - hobject_t *orig_obj = entry->touch_obj(*orig_obj_id); - ceph_assert(orig_obj); - - do { - *new_obj_id = entry->get_random_obj_id(gen); - } while (*new_obj_id == *orig_obj_id); - hobject_t *new_obj = entry->touch_obj(*new_obj_id); - ceph_assert(new_obj); - - *entry_ret = entry; - *orig_obj_ret = *orig_obj; - *new_obj_ret = *new_obj; - return true; -} - -bool DeterministicOpSequence::do_clone(rngen_t& gen) -{ - coll_entry_t *entry; - int orig_id, new_id; - hobject_t orig_obj, new_obj; - if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) { - return false; - } - - dout(0) << "do_clone " << entry->m_cid << "/" << orig_obj - << " => " << entry->m_cid << "/" << new_obj << dendl; - - _do_clone(entry, orig_obj, new_obj); - return true; -} - -bool DeterministicOpSequence::do_clone_range(rngen_t& gen) -{ - coll_entry_t *entry; - int orig_id, new_id; - hobject_t orig_obj, new_obj; - if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) { - return false; - } - - /* Whenever we have to make a clone_range() operation, just write to the - * object first, so we know we have something to clone in the said range. - * This may not be the best solution ever, but currently we're not keeping - * track of the written-to objects, and until we know for sure we really - * need to, let's just focus on the task at hand. - */ - - boost::uniform_int<> write_size_rng(100, (2 << 19)); - size_t size = (size_t) write_size_rng(gen); - bufferlist bl; - _gen_random(gen, size, bl); - - boost::uniform_int<> clone_len(1, bl.length()); - size = (size_t) clone_len(gen); - - dout(0) << "do_clone_range " << entry->m_cid << "/" << orig_obj - << " (0~" << size << ")" - << " => " << entry->m_cid << "/" << new_obj - << " (0)" << dendl; - _do_write_and_clone_range(entry, orig_obj, new_obj, 0, size, 0, bl); - return true; -} - -bool DeterministicOpSequence::do_coll_move(rngen_t& gen) -{ - coll_entry_t *entry; - int orig_id, new_id; - hobject_t orig_obj, new_obj; - if (!_prepare_clone(gen, &entry, &orig_id, &orig_obj, &new_id, &new_obj)) { - return false; - } - - dout(0) << "do_coll_move " << entry->m_cid << "/" << orig_obj - << " => " << entry->m_cid << "/" << new_obj << dendl; - entry->remove_obj(orig_id); - - _do_coll_move(entry, orig_obj, new_obj); - - return true; -} - -bool DeterministicOpSequence::do_coll_create(rngen_t& gen) -{ - int i = m_collections.size(); - spg_t pgid(pg_t(i, 1), shard_id_t::NO_SHARD); - coll_t cid(pgid); - auto ch = m_store->create_new_collection(cid); - coll_entry_t *entry = coll_create(pgid, ch); - m_collections.insert(make_pair(cid, entry)); - rebuild_id_vec(); - - _do_coll_create(entry, 10, 10); - - return true; -} - -void DeterministicOpSequence::_do_coll_create(coll_entry_t *entry, uint32_t pg_num, uint64_t num_objs) -{ - ObjectStore::Transaction t; - t.create_collection(entry->m_cid, 32); - note_txn(entry, &t); - bufferlist hint; - encode(pg_num, hint); - encode(num_objs, hint); - t.collection_hint(entry->m_cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); - dout(0) << "Give collection: " << entry->m_cid - << " a hint, pg_num is: " << pg_num << ", num_objs is: " - << num_objs << dendl; - - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - -void DeterministicOpSequence::_do_touch(coll_entry_t *entry, hobject_t& obj) -{ - ObjectStore::Transaction t; - note_txn(entry, &t); - t.touch(entry->m_cid, ghobject_t(obj)); - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - -void DeterministicOpSequence::_do_remove(coll_entry_t *entry, hobject_t& obj) -{ - ObjectStore::Transaction t; - note_txn(entry, &t); - t.remove(entry->m_cid, ghobject_t(obj)); - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - -void DeterministicOpSequence::_do_set_attrs(coll_entry_t *entry, - hobject_t &obj, - const map &attrs) -{ - ObjectStore::Transaction t; - note_txn(entry, &t); - t.omap_setkeys(entry->m_cid, ghobject_t(obj), attrs); - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - -void DeterministicOpSequence::_do_write(coll_entry_t *entry, hobject_t& obj, - uint64_t off, uint64_t len, const bufferlist& data) -{ - ObjectStore::Transaction t; - note_txn(entry, &t); - t.write(entry->m_cid, ghobject_t(obj), off, len, data); - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - -void DeterministicOpSequence::_do_clone(coll_entry_t *entry, hobject_t& orig_obj, - hobject_t& new_obj) -{ - ObjectStore::Transaction t; - note_txn(entry, &t); - t.clone(entry->m_cid, ghobject_t(orig_obj), ghobject_t(new_obj)); - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - -void DeterministicOpSequence::_do_clone_range(coll_entry_t *entry, - hobject_t& orig_obj, hobject_t& new_obj, uint64_t srcoff, - uint64_t srclen, uint64_t dstoff) -{ - ObjectStore::Transaction t; - note_txn(entry, &t); - t.clone_range(entry->m_cid, ghobject_t(orig_obj), ghobject_t(new_obj), - srcoff, srclen, dstoff); - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - -void DeterministicOpSequence::_do_write_and_clone_range(coll_entry_t *entry, - hobject_t& orig_obj, - hobject_t& new_obj, - uint64_t srcoff, - uint64_t srclen, - uint64_t dstoff, - bufferlist& bl) -{ - ObjectStore::Transaction t; - note_txn(entry, &t); - t.write(entry->m_cid, ghobject_t(orig_obj), srcoff, bl.length(), bl); - t.clone_range(entry->m_cid, ghobject_t(orig_obj), ghobject_t(new_obj), - srcoff, srclen, dstoff); - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - -void DeterministicOpSequence::_do_coll_move(coll_entry_t *entry, - hobject_t& orig_obj, - hobject_t& new_obj) -{ - ObjectStore::Transaction t; - note_txn(entry, &t); - t.remove(entry->m_cid, ghobject_t(new_obj)); - t.collection_move_rename(entry->m_cid, ghobject_t(orig_obj), - entry->m_cid, ghobject_t(new_obj)); - m_store->queue_transaction(entry->m_ch, std::move(t)); -} - diff --git a/src/test/objectstore/DeterministicOpSequence.h b/src/test/objectstore/DeterministicOpSequence.h deleted file mode 100644 index 9c085d9a414d..000000000000 --- a/src/test/objectstore/DeterministicOpSequence.h +++ /dev/null @@ -1,109 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* -* Ceph - scalable distributed file system -* -* Copyright (C) 2012 New Dream Network -* -* This is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License version 2.1, as published by the Free Software -* Foundation. See file COPYING. -*/ -#ifndef FILESTORE_DTRMNSTC_SEQ_OPS_H_ -#define FILESTORE_DTRMNSTC_SEQ_OPS_H_ - -#include -#include -#include -#include "os/ObjectStore.h" -#include -#include -#include - -#include "TestObjectStoreState.h" - -typedef boost::mt11213b rngen_t; - -class DeterministicOpSequence : public TestObjectStoreState { - public: - DeterministicOpSequence(ObjectStore *store, - std::string status = std::string()); - virtual ~DeterministicOpSequence(); - - virtual void generate(int seed, int num_txs); - - static ghobject_t get_txn_object(coll_t cid) { - ghobject_t oid(hobject_t(sobject_t("txn", CEPH_NOSNAP))); - spg_t pgid; - bool r = cid.is_pg(&pgid); - if (r) { - oid.hobj.set_hash(pgid.ps()); - } - return oid; - } - -protected: - enum { - DSOP_TOUCH = 0, - DSOP_WRITE = 1, - DSOP_CLONE = 2, - DSOP_CLONE_RANGE = 3, - DSOP_OBJ_REMOVE = 4, - DSOP_COLL_MOVE = 5, - DSOP_SET_ATTRS = 6, - DSOP_COLL_CREATE = 7, - - DSOP_FIRST = DSOP_TOUCH, - DSOP_LAST = DSOP_COLL_CREATE, - }; - - int32_t txn; - - ObjectStore::CollectionHandle m_ch; - std::ofstream m_status; - - bool run_one_op(int op, rngen_t& gen); - - void note_txn(coll_entry_t *entry, ObjectStore::Transaction *t); - bool do_touch(rngen_t& gen); - bool do_remove(rngen_t& gen); - bool do_write(rngen_t& gen); - bool do_clone(rngen_t& gen); - bool do_clone_range(rngen_t& gen); - bool do_coll_move(rngen_t& gen); - bool do_set_attrs(rngen_t& gen); - bool do_coll_create(rngen_t& gen); - - virtual void _do_touch(coll_entry_t *entry, hobject_t& obj); - virtual void _do_remove(coll_entry_t *entry, hobject_t& obj); - virtual void _do_write(coll_entry_t *entry, hobject_t& obj, uint64_t off, - uint64_t len, const bufferlist& data); - virtual void _do_set_attrs(coll_entry_t *entry, - hobject_t &obj, - const std::map &attrs); - virtual void _do_clone(coll_entry_t *entry, hobject_t& orig_obj, hobject_t& new_obj); - virtual void _do_clone_range(coll_entry_t *entry, hobject_t& orig_obj, - hobject_t& new_obj, uint64_t srcoff, uint64_t srclen, uint64_t dstoff); - virtual void _do_write_and_clone_range(coll_entry_t *entry, hobject_t& orig_obj, - hobject_t& new_obj, uint64_t srcoff, uint64_t srclen, - uint64_t dstoff, bufferlist& bl); - virtual void _do_coll_move(coll_entry_t *entry, hobject_t& orig_obj, hobject_t& new_obj); - virtual void _do_coll_create(coll_entry_t *entry, uint32_t pg_num, uint64_t num_objs); - - int _gen_coll_id(rngen_t& gen); - int _gen_obj_id(rngen_t& gen); - void _print_status(int seq, int op); - - private: - bool _prepare_clone( - rngen_t& gen, - coll_entry_t **entry_ret, - int *orig_obj_id, - hobject_t *orig_obj_ret, - int *new_obj_id, - hobject_t *new_obj_ret); -}; - - -#endif /* FILESTORE_DTRMNSTC_SEQ_OPS_H_ */ diff --git a/src/test/objectstore/FileStoreDiff.cc b/src/test/objectstore/FileStoreDiff.cc deleted file mode 100644 index c356f29b31bb..000000000000 --- a/src/test/objectstore/FileStoreDiff.cc +++ /dev/null @@ -1,303 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* -* Ceph - scalable distributed file system -* -* Copyright (C) 2012 New Dream Network -* -* This is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License version 2.1, as published by the Free Software -* Foundation. See file COPYING. -*/ -#include -#include -#include -#include -#include "common/debug.h" -#include "os/filestore/FileStore.h" -#include "common/config.h" - -#include "FileStoreDiff.h" - -#define dout_context g_ceph_context -#define dout_subsys ceph_subsys_filestore -#undef dout_prefix -#define dout_prefix *_dout << "filestore_diff " - -using namespace std; - -FileStoreDiff::FileStoreDiff(FileStore *a, FileStore *b) - : a_store(a), b_store(b) -{ - int err; - err = a_store->mount(); - ceph_assert(err == 0); - - err = b_store->mount(); - ceph_assert(err == 0); -} - -FileStoreDiff::~FileStoreDiff() -{ - a_store->umount(); - b_store->umount(); -} - - -bool FileStoreDiff::diff_attrs(std::map>& b, - std::map>& a) -{ - bool ret = false; - auto b_it = b.begin(); - auto a_it = a.begin(); - for (; b_it != b.end(); ++b_it, ++a_it) { - if (b_it->first != a_it->first) { - cout << "diff_attrs name mismatch (verify: " << b_it->first - << ", store: " << a_it->first << ")" << std::endl; - ret = true; - continue; - } - - if (!b_it->second.cmp(a_it->second)) { - cout << "diff_attrs contents mismatch on attr " << b_it->first << std::endl; - ret = true; - continue; - } - } - return ret; -} - -static bool diff_omap(std::map& b, - std::map& a) -{ - bool ret = false; - std::map::iterator b_it = b.begin(); - std::map::iterator a_it = a.begin(); - for (; b_it != b.end(); ++b_it, ++a_it) { - if (a_it == a.end()) { - cout << __func__ << " a reached end before b, a missing " << b_it->first - << std::endl; - ret = true; - break; - } - if (b_it->first != a_it->first) { - cout << "diff_attrs name mismatch (verify: " << b_it->first - << ", store: " << a_it->first << ")" << std::endl; - ret = true; - continue; - } - - if (!(b_it->second == a_it->second)) { - cout << "diff_attrs contents mismatch on attr " << b_it->first << std::endl; - ret = true; - continue; - } - } - return ret; -} - -bool FileStoreDiff::diff_objects_stat(struct stat& a, struct stat& b) -{ - bool ret = false; - - if (a.st_uid != b.st_uid) { - cout << "diff_objects_stat uid mismatch (A: " - << a.st_uid << " != B: " << b.st_uid << ")" << std::endl; - ret = true; - } - - if (a.st_gid != b.st_gid) { - cout << "diff_objects_stat gid mismatch (A: " - << a.st_gid << " != B: " << b.st_gid << ")" << std::endl; - ret = true; - } - - if (a.st_mode != b.st_mode) { - cout << "diff_objects_stat mode mismatch (A: " - << a.st_mode << " != B: " << b.st_mode << ")" << std::endl; - ret = true; - } - - if (a.st_nlink != b.st_nlink) { - cout << "diff_objects_stat nlink mismatch (A: " - << a.st_nlink << " != B: " << b.st_nlink << ")" << std::endl; - ret = true; - } - - if (a.st_size != b.st_size) { - cout << "diff_objects_stat size mismatch (A: " - << a.st_size << " != B: " << b.st_size << ")" << std::endl; - ret = true; - } - return ret; -} - -bool FileStoreDiff::diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll) -{ - bool ret = false; - - int err; - std::vector b_objects, a_objects; - err = b_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(), - INT_MAX, &b_objects, NULL); - if (err < 0) { - cout << "diff_objects list on verify coll " << coll.to_str() - << " returns " << err << std::endl; - return true; - } - err = a_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(), - INT_MAX, &a_objects, NULL); - if (err < 0) { - cout << "diff_objects list on store coll " << coll.to_str() - << " returns " << err << std::endl; - return true; - } - - if (b_objects.size() != a_objects.size()) { - cout << "diff_objects " << coll << " num objs mismatch (A: " << a_objects.size() - << ", B: " << b_objects.size() << ")" << std::endl; - ret = true; - cout << "a: " << a_objects << std::endl; - cout << "b: " << b_objects << std::endl; - } - - auto a_ch = a_store->open_collection(coll); - auto b_ch = b_store->open_collection(coll); - std::vector::iterator b_it = b_objects.begin(); - std::vector::iterator a_it = b_objects.begin(); - for (; b_it != b_objects.end(); ++b_it, ++a_it) { - ghobject_t b_obj = *b_it, a_obj = *a_it; - if (b_obj.hobj.oid.name != a_obj.hobj.oid.name) { - cout << "diff_objects name mismatch on A object " - << coll << "/" << a_obj << " and B object " - << coll << "/" << b_obj << std::endl; - ret = true; - continue; - } - - struct stat b_stat, a_stat; - err = b_store->stat(b_ch, b_obj, &b_stat); - if (err < 0) { - cout << "diff_objects error stating B object " - << coll.to_str() << "/" << b_obj.hobj.oid.name << std::endl; - ret = true; - } - err = a_store->stat(a_ch, a_obj, &a_stat); - if (err < 0) { - cout << "diff_objects error stating A object " - << coll << "/" << a_obj << std::endl; - ret = true; - } - - if (diff_objects_stat(a_stat, b_stat)) { - cout << "diff_objects stat mismatch on " - << coll << "/" << b_obj << std::endl; - ret = true; - } - - bufferlist a_obj_bl, b_obj_bl; - b_store->read(b_ch, b_obj, 0, b_stat.st_size, b_obj_bl); - a_store->read(a_ch, a_obj, 0, a_stat.st_size, a_obj_bl); - - if (!a_obj_bl.contents_equal(b_obj_bl)) { - cout << "diff_objects content mismatch on " - << coll << "/" << b_obj << std::endl; - ret = true; - } - - std::map> a_obj_attrs_map, b_obj_attrs_map; - err = a_store->getattrs(a_ch, a_obj, a_obj_attrs_map); - if (err < 0) { - cout << "diff_objects getattrs on A object " << coll << "/" << a_obj - << " returns " << err << std::endl; - ret = true; - } - err = b_store->getattrs(b_ch, b_obj, b_obj_attrs_map); - if (err < 0) { - cout << "diff_objects getattrs on B object " << coll << "/" << b_obj - << "returns " << err << std::endl; - ret = true; - } - - if (diff_attrs(b_obj_attrs_map, a_obj_attrs_map)) { - cout << "diff_objects attrs mismatch on A object " - << coll << "/" << a_obj << " and B object " - << coll << "/" << b_obj << std::endl; - ret = true; - } - - std::map a_obj_omap, b_obj_omap; - std::set a_omap_keys, b_omap_keys; - err = a_store->omap_get_keys(a_ch, a_obj, &a_omap_keys); - if (err < 0) { - cout << "diff_objects getomap on A object " << coll << "/" << a_obj - << " returns " << err << std::endl; - ret = true; - } - err = a_store->omap_get_values(a_ch, a_obj, a_omap_keys, &a_obj_omap); - if (err < 0) { - cout << "diff_objects getomap on A object " << coll << "/" << a_obj - << " returns " << err << std::endl; - ret = true; - } - err = b_store->omap_get_keys(b_ch, b_obj, &b_omap_keys); - if (err < 0) { - cout << "diff_objects getomap on A object " << coll << "/" << b_obj - << " returns " << err << std::endl; - ret = true; - } - err = b_store->omap_get_values(b_ch, b_obj, b_omap_keys, &b_obj_omap); - if (err < 0) { - cout << "diff_objects getomap on A object " << coll << "/" << b_obj - << " returns " << err << std::endl; - ret = true; - } - if (diff_omap(a_obj_omap, b_obj_omap)) { - cout << "diff_objects omap mismatch on A object " - << coll << "/" << a_obj << " and B object " - << coll << "/" << b_obj << std::endl; - cout << "a: " << a_obj_omap << std::endl; - cout << "b: " << b_obj_omap << std::endl; - ret = true; - } - } - - return ret; -} - -bool FileStoreDiff::diff() -{ - bool ret = false; - - std::vector a_coll_list, b_coll_list; - a_store->list_collections(a_coll_list); - b_store->list_collections(b_coll_list); - - std::vector::iterator it = b_coll_list.begin(); - for (; it != b_coll_list.end(); ++it) { - coll_t b_coll = *it; - if (!a_store->collection_exists(b_coll)) { - cout << "diff B coll " << b_coll.to_str() << " DNE on A" << std::endl; - ret = true; - continue; - } - for (std::vector::iterator j = a_coll_list.begin(); - j != a_coll_list.end(); ++j) { - if (*j == *it) { - a_coll_list.erase(j); - break; - } - } - - if (diff_objects(a_store, b_store, b_coll)) - ret = true; - } - for (std::vector::iterator it = a_coll_list.begin(); - it != a_coll_list.end(); ++it) { - cout << "diff A coll " << *it << " DNE on B" << std::endl; - ret = true; - } - - return ret; -} diff --git a/src/test/objectstore/FileStoreDiff.h b/src/test/objectstore/FileStoreDiff.h deleted file mode 100644 index 56b53dddb1f9..000000000000 --- a/src/test/objectstore/FileStoreDiff.h +++ /dev/null @@ -1,42 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* -* Ceph - scalable distributed file system -* -* Copyright (C) 2012 New Dream Network -* -* This is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License version 2.1, as published by the Free Software -* Foundation. See file COPYING. -*/ -#ifndef FILESTORE_DIFF_H_ -#define FILESTORE_DIFF_H_ - -#include -#include -#include -#include -#include "common/debug.h" -#include "os/filestore/FileStore.h" -#include "common/config.h" - -class FileStoreDiff { - - private: - FileStore *a_store; - FileStore *b_store; - - bool diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll); - bool diff_objects_stat(struct stat& a, struct stat& b); - bool diff_attrs(std::map>& b, - std::map>& a); - -public: - FileStoreDiff(FileStore *a, FileStore *b); - virtual ~FileStoreDiff(); - - bool diff(); -}; - -#endif /* FILESTOREDIFF_H_ */ diff --git a/src/test/objectstore/FileStoreTracker.cc b/src/test/objectstore/FileStoreTracker.cc deleted file mode 100644 index 7d091cf3ee71..000000000000 --- a/src/test/objectstore/FileStoreTracker.cc +++ /dev/null @@ -1,452 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -#include "FileStoreTracker.h" -#include -#include -#include -#include "include/Context.h" - -using namespace std; - -class OnApplied : public Context { - FileStoreTracker *tracker; - list, uint64_t> > in_flight; -public: - OnApplied(FileStoreTracker *tracker, - list, uint64_t> > in_flight) - : tracker(tracker), in_flight(in_flight) {} - - void finish(int r) override { - for (list, uint64_t> >::iterator i = - in_flight.begin(); - i != in_flight.end(); - ++i) { - tracker->applied(i->first, i->second); - } - } -}; - -class OnCommitted : public Context { - FileStoreTracker *tracker; - list, uint64_t> > in_flight; -public: - OnCommitted(FileStoreTracker *tracker, - list, uint64_t> > in_flight) - : tracker(tracker), in_flight(in_flight) {} - - void finish(int r) override { - for (list, uint64_t> >::iterator i = - in_flight.begin(); - i != in_flight.end(); - ++i) { - tracker->committed(i->first, i->second); - } - } -}; - -int FileStoreTracker::init() -{ - set to_get; - to_get.insert("STATUS"); - map got; - db->get("STATUS", to_get, &got); - restart_seq = 0; - if (!got.empty()) { - auto bp = got.begin()->second.cbegin(); - decode(restart_seq, bp); - } - ++restart_seq; - KeyValueDB::Transaction t = db->get_transaction(); - got.clear(); - encode(restart_seq, got["STATUS"]); - t->set("STATUS", got); - db->submit_transaction(t); - return 0; -} - -void FileStoreTracker::submit_transaction(Transaction &t) -{ - list, uint64_t> > in_flight; - OutTransaction out; - out.t = new ObjectStore::Transaction; - out.in_flight = &in_flight; - for (list::iterator i = t.ops.begin(); - i != t.ops.end(); - ++i) { - (**i)(this, &out); - } - out.t->register_on_applied(new OnApplied(this, in_flight)); - out.t->register_on_commit(new OnCommitted(this, in_flight)); - auto ch = store->open_collection(coll_t()); - store->queue_transaction(ch, std::move(*out.t), nullptr); - delete out.t; -} - -void FileStoreTracker::write(const pair &obj, - OutTransaction *out) -{ - std::lock_guard l{lock}; - std::cerr << "Writing " << obj << std::endl; - ObjectContents contents = get_current_content(obj); - - uint64_t offset = rand() % (SIZE/2); - uint64_t len = rand() % (SIZE/2); - if (!len) len = 10; - contents.write(rand(), offset, len); - - bufferlist to_write; - ObjectContents::Iterator iter = contents.get_iterator(); - iter.seek_to(offset); - for (uint64_t i = offset; - i < offset + len; - ++i, ++iter) { - ceph_assert(iter.valid()); - to_write.append(*iter); - } - out->t->write(coll_t(obj.first), - ghobject_t(hobject_t(sobject_t(obj.second, CEPH_NOSNAP))), - offset, - len, - to_write); - out->in_flight->push_back(make_pair(obj, set_content(obj, contents))); -} - -void FileStoreTracker::remove(const pair &obj, - OutTransaction *out) -{ - std::cerr << "Deleting " << obj << std::endl; - std::lock_guard l{lock}; - ObjectContents old_contents = get_current_content(obj); - if (!old_contents.exists()) - return; - out->t->remove(coll_t(obj.first), - ghobject_t(hobject_t(sobject_t(obj.second, CEPH_NOSNAP)))); - ObjectContents contents; - out->in_flight->push_back(make_pair(obj, set_content(obj, contents))); -} - -void FileStoreTracker::clone_range(const pair &from, - const pair &to, - OutTransaction *out) { - std::lock_guard l{lock}; - std::cerr << "CloningRange " << from << " to " << to << std::endl; - ceph_assert(from.first == to.first); - ObjectContents from_contents = get_current_content(from); - ObjectContents to_contents = get_current_content(to); - if (!from_contents.exists()) { - return; - } - if (from.second == to.second) { - return; - } - - uint64_t new_size = from_contents.size(); - interval_set interval_to_clone; - uint64_t offset = rand() % (new_size/2); - uint64_t len = rand() % (new_size/2); - if (!len) len = 10; - interval_to_clone.insert(offset, len); - to_contents.clone_range(from_contents, interval_to_clone); - out->t->clone_range(coll_t(from.first), - ghobject_t(hobject_t(sobject_t(from.second, CEPH_NOSNAP))), - ghobject_t(hobject_t(sobject_t(to.second, CEPH_NOSNAP))), - offset, - len, - offset); - out->in_flight->push_back(make_pair(to, set_content(to, to_contents))); -} - -void FileStoreTracker::clone(const pair &from, - const pair &to, - OutTransaction *out) { - std::lock_guard l{lock}; - std::cerr << "Cloning " << from << " to " << to << std::endl; - ceph_assert(from.first == to.first); - if (from.second == to.second) { - return; - } - ObjectContents from_contents = get_current_content(from); - ObjectContents to_contents = get_current_content(to); - if (!from_contents.exists()) { - return; - } - - if (to_contents.exists()) - out->t->remove(coll_t(to.first), - ghobject_t(hobject_t(sobject_t(to.second, CEPH_NOSNAP)))); - out->t->clone(coll_t(from.first), - ghobject_t(hobject_t(sobject_t(from.second, CEPH_NOSNAP))), - ghobject_t(hobject_t(sobject_t(to.second, CEPH_NOSNAP)))); - out->in_flight->push_back(make_pair(to, set_content(to, from_contents))); -} - - -string obj_to_prefix(const pair &obj) { - string sep; - sep.push_back('^'); - return obj.first.to_str() + sep + obj.second + "_CONTENTS_"; -} - -string obj_to_meta_prefix(const pair &obj) { - string sep; - sep.push_back('^'); - return obj.first.to_str() + sep + obj.second; -} - -string seq_to_key(uint64_t seq) { - char buf[50]; - snprintf(buf, sizeof(buf), "%*llu", 20, (unsigned long long int)seq); - return string(buf); -} - -struct ObjStatus { - uint64_t last_applied; - uint64_t last_committed; - uint64_t restart_seq; - ObjStatus() : last_applied(0), last_committed(0), restart_seq(0) {} - - uint64_t get_last_applied(uint64_t seq) const { - if (seq > restart_seq) - return last_committed; - else - return last_applied; - } - void set_last_applied(uint64_t _last_applied, uint64_t seq) { - last_applied = _last_applied; - restart_seq = seq; - } - uint64_t trim_to() const { - return last_applied < last_committed ? - last_applied : last_committed; - } -}; -void encode(const ObjStatus &obj, bufferlist &bl) { - encode(obj.last_applied, bl); - encode(obj.last_committed, bl); - encode(obj.restart_seq, bl); -} -void decode(ObjStatus &obj, bufferlist::const_iterator &bl) { - decode(obj.last_applied, bl); - decode(obj.last_committed, bl); - decode(obj.restart_seq, bl); -} - - -ObjStatus get_obj_status(const pair &obj, - KeyValueDB *db) -{ - set to_get; - to_get.insert("META"); - map got; - db->get(obj_to_meta_prefix(obj), to_get, &got); - ObjStatus retval; - if (!got.empty()) { - auto bp = got.begin()->second.cbegin(); - decode(retval, bp); - } - return retval; -} - -void set_obj_status(const pair &obj, - const ObjStatus &status, - KeyValueDB::Transaction t) -{ - map to_set; - encode(status, to_set["META"]); - t->set(obj_to_meta_prefix(obj), to_set); -} - -void _clean_forward(const pair &obj, - uint64_t last_valid, - KeyValueDB *db) -{ - KeyValueDB::Transaction t = db->get_transaction(); - KeyValueDB::Iterator i = db->get_iterator(obj_to_prefix(obj)); - set to_remove; - i->upper_bound(seq_to_key(last_valid)); - for (; i->valid(); i->next()) { - to_remove.insert(i->key()); - } - t->rmkeys(obj_to_prefix(obj), to_remove); - db->submit_transaction(t); -} - - -void FileStoreTracker::verify(const coll_t &coll, const string &obj, - bool on_start) { - std::lock_guard l{lock}; - std::cerr << "Verifying " << make_pair(coll, obj) << std::endl; - - pair valid_reads = get_valid_reads(make_pair(coll, obj)); - std::cerr << "valid_reads is " << valid_reads << std::endl; - bufferlist contents; - auto ch = store->open_collection(coll_t(coll)); - int r = store->read(ch, - ghobject_t(hobject_t(sobject_t(obj, CEPH_NOSNAP))), - 0, - 2*SIZE, - contents); - std::cerr << "exists: " << r << std::endl; - - - for (uint64_t i = valid_reads.first; - i < valid_reads.second; - ++i) { - ObjectContents old_contents = get_content(make_pair(coll, obj), i); - - std::cerr << "old_contents exists " << old_contents.exists() << std::endl; - if (!old_contents.exists() && (r == -ENOENT)) - return; - - if (old_contents.exists() && (r == -ENOENT)) - continue; - - if (!old_contents.exists() && (r != -ENOENT)) - continue; - - if (contents.length() != old_contents.size()) { - std::cerr << "old_contents.size() is " - << old_contents.size() << std::endl; - continue; - } - - bufferlist::iterator bp = contents.begin(); - ObjectContents::Iterator iter = old_contents.get_iterator(); - iter.seek_to_first(); - bool matches = true; - uint64_t pos = 0; - for (; !bp.end() && iter.valid(); - ++iter, ++bp, ++pos) { - if (*iter != *bp) { - std::cerr << "does not match at pos " << pos << std::endl; - matches = false; - break; - } - } - if (matches) { - if (on_start) - _clean_forward(make_pair(coll, obj), i, db); - return; - } - } - std::cerr << "Verifying " << make_pair(coll, obj) << " failed " << std::endl; - ceph_abort(); -} - -ObjectContents FileStoreTracker::get_current_content( - const pair &obj) -{ - KeyValueDB::Iterator iter = db->get_iterator( - obj_to_prefix(obj)); - iter->seek_to_last(); - if (iter->valid()) { - bufferlist bl = iter->value(); - auto bp = bl.cbegin(); - pair val; - decode(val, bp); - ceph_assert(seq_to_key(val.first) == iter->key()); - bp = val.second.begin(); - return ObjectContents(bp); - } - return ObjectContents(); -} - -ObjectContents FileStoreTracker::get_content( - const pair &obj, uint64_t version) -{ - set to_get; - map got; - to_get.insert(seq_to_key(version)); - db->get(obj_to_prefix(obj), to_get, &got); - if (got.empty()) - return ObjectContents(); - pair val; - auto bp = got.begin()->second.cbegin(); - decode(val, bp); - bp = val.second.begin(); - ceph_assert(val.first == version); - return ObjectContents(bp); -} - -pair FileStoreTracker::get_valid_reads( - const pair &obj) -{ - pair bounds = make_pair(0,1); - KeyValueDB::Iterator iter = db->get_iterator( - obj_to_prefix(obj)); - iter->seek_to_last(); - if (iter->valid()) { - pair val; - bufferlist bl = iter->value(); - auto bp = bl.cbegin(); - decode(val, bp); - bounds.second = val.first + 1; - } - - ObjStatus obj_status = get_obj_status(obj, db); - bounds.first = obj_status.get_last_applied(restart_seq); - return bounds; -} - -void clear_obsolete(const pair &obj, - const ObjStatus &status, - KeyValueDB *db, - KeyValueDB::Transaction t) -{ - KeyValueDB::Iterator iter = db->get_iterator(obj_to_prefix(obj)); - set to_remove; - iter->seek_to_first(); - for (; iter->valid() && iter->key() < seq_to_key(status.trim_to()); - iter->next()) - to_remove.insert(iter->key()); - t->rmkeys(obj_to_prefix(obj), to_remove); -} - -void FileStoreTracker::committed(const pair &obj, - uint64_t seq) { - std::lock_guard l{lock}; - ObjStatus status = get_obj_status(obj, db); - ceph_assert(status.last_committed < seq); - status.last_committed = seq; - KeyValueDB::Transaction t = db->get_transaction(); - clear_obsolete(obj, status, db, t); - set_obj_status(obj, status, t); - db->submit_transaction(t); -} - -void FileStoreTracker::applied(const pair &obj, - uint64_t seq) { - std::lock_guard l{lock}; - std::cerr << "Applied " << obj << " version " << seq << std::endl; - ObjStatus status = get_obj_status(obj, db); - ceph_assert(status.last_applied < seq); - status.set_last_applied(seq, restart_seq); - KeyValueDB::Transaction t = db->get_transaction(); - clear_obsolete(obj, status, db, t); - set_obj_status(obj, status, t); - db->submit_transaction(t); -} - - -uint64_t FileStoreTracker::set_content(const pair &obj, - ObjectContents &content) { - KeyValueDB::Transaction t = db->get_transaction(); - KeyValueDB::Iterator iter = db->get_iterator( - obj_to_prefix(obj)); - iter->seek_to_last(); - uint64_t most_recent = 0; - if (iter->valid()) { - pair val; - bufferlist bl = iter->value(); - auto bp = bl.cbegin(); - decode(val, bp); - most_recent = val.first; - } - bufferlist buf_content; - content.encode(buf_content); - map to_set; - encode(make_pair(most_recent + 1, buf_content), - to_set[seq_to_key(most_recent + 1)]); - t->set(obj_to_prefix(obj), to_set); - db->submit_transaction(t); - return most_recent + 1; -} diff --git a/src/test/objectstore/FileStoreTracker.h b/src/test/objectstore/FileStoreTracker.h deleted file mode 100644 index 555e48fe5c72..000000000000 --- a/src/test/objectstore/FileStoreTracker.h +++ /dev/null @@ -1,140 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- - -#ifndef FILESTORE_TRACKER_H -#define FILESTORE_TRACKER_H -#include "test/common/ObjectContents.h" -#include "os/filestore/FileStore.h" -#include "kv/KeyValueDB.h" -#include -#include -#include -#include "common/ceph_mutex.h" - -class FileStoreTracker { - const static uint64_t SIZE = 4 * 1024; - ObjectStore *store; - KeyValueDB *db; - ceph::mutex lock = ceph::make_mutex("Tracker Lock"); - uint64_t restart_seq; - - struct OutTransaction { - std::list, uint64_t> > *in_flight; - ObjectStore::Transaction *t; - }; -public: - FileStoreTracker(ObjectStore *store, KeyValueDB *db) - : store(store), db(db), - restart_seq(0) {} - - class Transaction { - class Op { - public: - virtual void operator()(FileStoreTracker *harness, - OutTransaction *out) = 0; - virtual ~Op() {}; - }; - std::list ops; - class Write : public Op { - public: - coll_t coll; - std::string oid; - Write(const coll_t &coll, - const std::string &oid) - : coll(coll), oid(oid) {} - void operator()(FileStoreTracker *harness, - OutTransaction *out) override { - harness->write(std::make_pair(coll, oid), out); - } - }; - class CloneRange : public Op { - public: - coll_t coll; - std::string from; - std::string to; - CloneRange(const coll_t &coll, - const std::string &from, - const std::string &to) - : coll(coll), from(from), to(to) {} - void operator()(FileStoreTracker *harness, - OutTransaction *out) override { - harness->clone_range(std::make_pair(coll, from), - std::make_pair(coll, to), - out); - } - }; - class Clone : public Op { - public: - coll_t coll; - std::string from; - std::string to; - Clone(const coll_t &coll, - const std::string &from, - const std::string &to) - : coll(coll), from(from), to(to) {} - void operator()(FileStoreTracker *harness, - OutTransaction *out) override { - harness->clone(std::make_pair(coll, from), - std::make_pair(coll, to), - out); - } - }; - class Remove: public Op { - public: - coll_t coll; - std::string obj; - Remove(const coll_t &coll, - const std::string &obj) - : coll(coll), obj(obj) {} - void operator()(FileStoreTracker *harness, - OutTransaction *out) override { - harness->remove(std::make_pair(coll, obj), - out); - } - }; - public: - void write(const coll_t &coll, const std::string &oid) { - ops.push_back(new Write(coll, oid)); - } - void clone_range(const coll_t &coll, const std::string &from, - const std::string &to) { - ops.push_back(new CloneRange(coll, from, to)); - } - void clone(const coll_t &coll, const std::string &from, - const std::string &to) { - ops.push_back(new Clone(coll, from, to)); - } - void remove(const coll_t &coll, const std::string &oid) { - ops.push_back(new Remove(coll, oid)); - } - friend class FileStoreTracker; - }; - - int init(); - void submit_transaction(Transaction &t); - void verify(const coll_t &coll, - const std::string &from, - bool on_start = false); - -private: - ObjectContents get_current_content(const std::pair &obj); - std::pair get_valid_reads(const std::pair &obj); - ObjectContents get_content(const std::pair &obj, uint64_t version); - - void committed(const std::pair &obj, uint64_t seq); - void applied(const std::pair &obj, uint64_t seq); - uint64_t set_content(const std::pair &obj, ObjectContents &content); - - // ObjectContents Operations - void write(const std::pair &obj, OutTransaction *out); - void remove(const std::pair &obj, OutTransaction *out); - void clone_range(const std::pair &from, - const std::pair &to, - OutTransaction *out); - void clone(const std::pair &from, - const std::pair &to, - OutTransaction *out); - friend class OnApplied; - friend class OnCommitted; -}; - -#endif diff --git a/src/test/objectstore/chain_xattr.cc b/src/test/objectstore/chain_xattr.cc deleted file mode 100644 index a0db4e153d58..000000000000 --- a/src/test/objectstore/chain_xattr.cc +++ /dev/null @@ -1,469 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU Library Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Library Public License for more details. - * - */ - -#include -#include -#include "os/filestore/chain_xattr.h" -#include "include/Context.h" -#include "include/coredumpctl.h" -#include "common/errno.h" -#include "common/ceph_argparse.h" -#include "global/global_init.h" -#include - -#define LARGE_BLOCK_LEN CHAIN_XATTR_MAX_BLOCK_LEN + 1024 -#define FILENAME "chain_xattr" - -using namespace std; - -TEST(chain_xattr, get_and_set) { - const char* file = FILENAME; - ::unlink(file); - int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700); - const string user("user."); - - { - const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@'); - const string x(LARGE_BLOCK_LEN, 'X'); - - { - char y[LARGE_BLOCK_LEN]; - ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN)); - ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0)); - ASSERT_EQ(LARGE_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN)); - ASSERT_EQ(0, chain_removexattr(file, name.c_str())); - ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN)); - } - - { - char y[LARGE_BLOCK_LEN]; - ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN)); - ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0)); - ASSERT_EQ(LARGE_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN)); - ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str())); - ASSERT_EQ(0, memcmp(x.c_str(), y, LARGE_BLOCK_LEN)); - } - } - - // - // when chain_setxattr is used to store value that is - // CHAIN_XATTR_MAX_BLOCK_LEN * 2 + 10 bytes long it - // - // add user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes - // add user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes - // add user.foo@2 => 10 bytes - // - // then ( no chain_removexattr in between ) when it is used to - // override with a value that is exactly CHAIN_XATTR_MAX_BLOCK_LEN - // bytes long it will - // - // replace user.foo => CHAIN_XATTR_MAX_BLOCK_LEN bytes - // remove user.foo@1 => CHAIN_XATTR_MAX_BLOCK_LEN bytes - // leak user.foo@2 => 10 bytes - // - // see http://marc.info/?l=ceph-devel&m=136027076615853&w=4 for the - // discussion - // - { - const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@'); - const string x(LARGE_BLOCK_LEN, 'X'); - - { - char y[CHAIN_XATTR_MAX_BLOCK_LEN]; - ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN)); - ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN)); - ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), 0, 0)); - ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN)); - ASSERT_EQ(0, chain_removexattr(file, name.c_str())); - ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN)); - } - - { - char y[CHAIN_XATTR_MAX_BLOCK_LEN]; - ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN)); - ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), CHAIN_XATTR_MAX_BLOCK_LEN)); - ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), 0, 0)); - ASSERT_EQ(CHAIN_XATTR_MAX_BLOCK_LEN, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN)); - ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str())); - ASSERT_EQ(0, memcmp(x.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN)); - } - } - - { - int x = 0; - ASSERT_EQ(-ENOENT, chain_setxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x))); - ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", 0, 0)); - ASSERT_EQ(-ENOENT, chain_getxattr("UNLIKELY_TO_EXIST", "NAME", &x, sizeof(x))); - ASSERT_EQ(-ENOENT, chain_removexattr("UNLIKELY_TO_EXIST", "NAME")); - int unlikely_to_be_a_valid_fd = 400; - ASSERT_EQ(-EBADF, chain_fsetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x))); - ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", 0, 0)); - ASSERT_EQ(-EBADF, chain_fgetxattr(unlikely_to_be_a_valid_fd, "NAME", &x, sizeof(x))); - ASSERT_EQ(-EBADF, chain_fremovexattr(unlikely_to_be_a_valid_fd, "NAME")); - } - - { - int x; - const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN * 2, '@'); - PrCtl unset_dumpable; - ::testing::FLAGS_gtest_death_test_style = "threadsafe"; - ASSERT_DEATH(chain_setxattr(file, name.c_str(), &x, sizeof(x)), ""); - ASSERT_DEATH(chain_fsetxattr(fd, name.c_str(), &x, sizeof(x)), ""); - ::testing::FLAGS_gtest_death_test_style = "fast"; - } - - { - const string name = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@'); - const string x(LARGE_BLOCK_LEN, 'X'); - { - char y[LARGE_BLOCK_LEN]; - ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name.c_str(), x.c_str(), LARGE_BLOCK_LEN)); - ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, LARGE_BLOCK_LEN - 1)); - ASSERT_EQ(-ERANGE, chain_getxattr(file, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN)); - ASSERT_EQ(0, chain_removexattr(file, name.c_str())); - } - - { - char y[LARGE_BLOCK_LEN]; - ASSERT_EQ(LARGE_BLOCK_LEN, chain_fsetxattr(fd, name.c_str(), x.c_str(), LARGE_BLOCK_LEN)); - ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, LARGE_BLOCK_LEN - 1)); - ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, name.c_str(), y, CHAIN_XATTR_MAX_BLOCK_LEN)); - ASSERT_EQ(0, chain_fremovexattr(fd, name.c_str())); - } - } - - ::close(fd); - ::unlink(file); -} - -TEST(chain_xattr, chunk_aligned) { - const char* file = FILENAME; - ::unlink(file); - int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700); - const string user("user."); - - // set N* chunk size - const string name = "user.foo"; - const string name2 = "user.bar"; - - for (int len = CHAIN_XATTR_MAX_BLOCK_LEN - 10; - len < CHAIN_XATTR_MAX_BLOCK_LEN + 10; - ++len) { - cout << len << std::endl; - const string x(len, 'x'); - char buf[len*2]; - ASSERT_EQ(len, chain_setxattr(file, name.c_str(), x.c_str(), len)); - char attrbuf[4096]; - int l = ceph_os_listxattr(file, attrbuf, sizeof(attrbuf)); - for (char *p = attrbuf; p - attrbuf < l; p += strlen(p) + 1) { - cout << " attr " << p << std::endl; - } - ASSERT_EQ(len, chain_getxattr(file, name.c_str(), buf, len*2)); - ASSERT_EQ(0, chain_removexattr(file, name.c_str())); - - ASSERT_EQ(len, chain_fsetxattr(fd, name2.c_str(), x.c_str(), len)); - l = ceph_os_flistxattr(fd, attrbuf, sizeof(attrbuf)); - for (char *p = attrbuf; p - attrbuf < l; p += strlen(p) + 1) { - cout << " attr " << p << std::endl; - } - ASSERT_EQ(len, chain_fgetxattr(fd, name2.c_str(), buf, len*2)); - ASSERT_EQ(0, chain_fremovexattr(fd, name2.c_str())); - } - - for (int len = CHAIN_XATTR_SHORT_BLOCK_LEN - 10; - len < CHAIN_XATTR_SHORT_BLOCK_LEN + 10; - ++len) { - cout << len << std::endl; - const string x(len, 'x'); - char buf[len*2]; - ASSERT_EQ(len, chain_setxattr(file, name.c_str(), x.c_str(), len)); - char attrbuf[4096]; - int l = ceph_os_listxattr(file, attrbuf, sizeof(attrbuf)); - for (char *p = attrbuf; p - attrbuf < l; p += strlen(p) + 1) { - cout << " attr " << p << std::endl; - } - ASSERT_EQ(len, chain_getxattr(file, name.c_str(), buf, len*2)); - } - - { - // test tail path in chain_getxattr - const char *aname = "user.baz"; - char buf[CHAIN_XATTR_SHORT_BLOCK_LEN*3]; - memset(buf, 'x', sizeof(buf)); - ASSERT_EQ((int)sizeof(buf), chain_setxattr(file, aname, buf, sizeof(buf))); - ASSERT_EQ(-ERANGE, chain_getxattr(file, aname, buf, - CHAIN_XATTR_SHORT_BLOCK_LEN*2)); - } - { - // test tail path in chain_fgetxattr - const char *aname = "user.biz"; - char buf[CHAIN_XATTR_SHORT_BLOCK_LEN*3]; - memset(buf, 'x', sizeof(buf)); - ASSERT_EQ((int)sizeof(buf), chain_fsetxattr(fd, aname, buf, sizeof(buf))); - ASSERT_EQ(-ERANGE, chain_fgetxattr(fd, aname, buf, - CHAIN_XATTR_SHORT_BLOCK_LEN*2)); - } - - ::close(fd); - ::unlink(file); -} - -void get_vector_from_xattr(vector &xattrs, char* xattr, int size) { - char *end = xattr + size; - while (xattr < end) { - if (*xattr == '\0' ) - break; - xattrs.push_back(xattr); - xattr += strlen(xattr) + 1; - } -} - -bool listxattr_cmp(char* xattr1, char* xattr2, int size) { - vector xattrs1; - vector xattrs2; - get_vector_from_xattr(xattrs1, xattr1, size); - get_vector_from_xattr(xattrs2, xattr2, size); - - if (xattrs1.size() != xattrs2.size()) - return false; - - std::sort(xattrs1.begin(), xattrs1.end()); - std::sort(xattrs2.begin(), xattrs2.end()); - std::vector diff; - std::set_difference(xattrs1.begin(), xattrs1.end(), - xattrs2.begin(), xattrs2.end(), - diff.begin()); - - return diff.empty(); -} - -TEST(chain_xattr, listxattr) { - const char* file = FILENAME; - ::unlink(file); - int fd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700); - const string user("user."); - const string name1 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '1'); - const string name2 = user + string(CHAIN_XATTR_MAX_NAME_LEN - user.size(), '@'); - const string x(LARGE_BLOCK_LEN, 'X'); - const int y = 1234; - - int orig_size = chain_listxattr(file, NULL, 0); - char *orig_buffer = NULL; - string orig_str; - if (orig_size) { - orig_buffer = (char*)malloc(orig_size); - chain_flistxattr(fd, orig_buffer, orig_size); - orig_str = string(orig_buffer); - orig_size = orig_str.size(); - } - - ASSERT_EQ(LARGE_BLOCK_LEN, chain_setxattr(file, name1.c_str(), x.c_str(), LARGE_BLOCK_LEN)); - ASSERT_EQ((int)sizeof(y), chain_setxattr(file, name2.c_str(), &y, sizeof(y))); - - int buffer_size = 0; - if (orig_size) - buffer_size += orig_size + sizeof(char); - buffer_size += name1.size() + sizeof(char) + name2.size() + sizeof(char); - - int index = 0; - char* expected = (char*)malloc(buffer_size); - ::memset(expected, '\0', buffer_size); - if (orig_size) { - ::strcpy(expected, orig_str.c_str()); - index = orig_size + 1; - } - ::strcpy(expected + index, name1.c_str()); - ::strcpy(expected + index + name1.size() + 1, name2.c_str()); - char* actual = (char*)malloc(buffer_size); - ::memset(actual, '\0', buffer_size); - ASSERT_LT(buffer_size, chain_listxattr(file, NULL, 0)); // size evaluation is conservative - chain_listxattr(file, actual, buffer_size); - ASSERT_TRUE(listxattr_cmp(expected, actual, buffer_size)); - ::memset(actual, '\0', buffer_size); - chain_flistxattr(fd, actual, buffer_size); - ASSERT_TRUE(listxattr_cmp(expected, actual, buffer_size)); - - int unlikely_to_be_a_valid_fd = 400; - ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, 0)); - ASSERT_GT(0, chain_listxattr("UNLIKELY_TO_EXIST", actual, buffer_size)); - ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, 0)); - ASSERT_GT(0, chain_flistxattr(unlikely_to_be_a_valid_fd, actual, buffer_size)); - ASSERT_EQ(-ERANGE, chain_listxattr(file, actual, 1)); - ASSERT_EQ(-ERANGE, chain_flistxattr(fd, actual, 1)); - - ASSERT_EQ(0, chain_removexattr(file, name1.c_str())); - ASSERT_EQ(0, chain_removexattr(file, name2.c_str())); - - free(orig_buffer); - free(actual); - free(expected); - ::unlink(file); -} - -list get_xattrs(int fd) -{ - char _buf[1024]; - char *buf = _buf; - int len = sys_flistxattr(fd, _buf, sizeof(_buf)); - if (len < 0) - return list(); - list ret; - while (len > 0) { - size_t next_len = strlen(buf); - ret.push_back(string(buf, buf + next_len)); - ceph_assert(len >= (int)(next_len + 1)); - buf += (next_len + 1); - len -= (next_len + 1); - } - return ret; -} - -list get_xattrs(string fn) -{ - int fd = ::open(fn.c_str(), O_RDONLY); - if (fd < 0) - return list(); - auto ret = get_xattrs(fd); - ::close(fd); - return ret; -} - -TEST(chain_xattr, fskip_chain_cleanup_and_ensure_single_attr) -{ - const char *name = "user.foo"; - const char *file = FILENAME; - ::unlink(file); - int fd = ::open(file, O_CREAT|O_RDWR|O_TRUNC, 0700); - ceph_assert(fd >= 0); - - std::size_t existing_xattrs = get_xattrs(fd).size(); - char buf[800]; - memset(buf, 0x1F, sizeof(buf)); - // set chunked without either - { - std::size_t r = chain_fsetxattr(fd, name, buf, sizeof(buf)); - ASSERT_EQ(sizeof(buf), r); - ASSERT_GT(get_xattrs(fd).size(), existing_xattrs + 1UL); - } - - // verify - { - char buf2[sizeof(buf)*2]; - std::size_t r = chain_fgetxattr(fd, name, buf2, sizeof(buf2)); - ASSERT_EQ(sizeof(buf), r); - ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf))); - } - - // overwrite - { - std::size_t r = chain_fsetxattr(fd, name, buf, sizeof(buf)); - ASSERT_EQ(sizeof(buf), r); - ASSERT_EQ(existing_xattrs + 1UL, get_xattrs(fd).size()); - } - - // verify - { - char buf2[sizeof(buf)*2]; - std::size_t r = chain_fgetxattr(fd, name, buf2, sizeof(buf2)); - ASSERT_EQ(sizeof(buf), r); - ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf))); - } - - ::close(fd); - ::unlink(file); -} - -TEST(chain_xattr, skip_chain_cleanup_and_ensure_single_attr) -{ - const char *name = "user.foo"; - const char *file = FILENAME; - ::unlink(file); - int fd = ::open(file, O_CREAT|O_RDWR|O_TRUNC, 0700); - ceph_assert(fd >= 0); - std::size_t existing_xattrs = get_xattrs(fd).size(); - ::close(fd); - - char buf[3000]; - memset(buf, 0x1F, sizeof(buf)); - // set chunked without either - { - std::size_t r = chain_setxattr(file, name, buf, sizeof(buf)); - ASSERT_EQ(sizeof(buf), r); - ASSERT_GT(get_xattrs(file).size(), existing_xattrs + 1UL); - } - - // verify - { - char buf2[sizeof(buf)*2]; - std::size_t r = chain_getxattr(file, name, buf2, sizeof(buf2)); - ASSERT_EQ(sizeof(buf), r); - ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf))); - } - - // overwrite - { - std::size_t r = chain_setxattr(file, name, buf, sizeof(buf)); - ASSERT_EQ(sizeof(buf), r); - ASSERT_EQ(existing_xattrs + 1UL, get_xattrs(file).size()); - } - - // verify - { - char buf2[sizeof(buf)*2]; - std::size_t r = chain_getxattr(file, name, buf2, sizeof(buf2)); - ASSERT_EQ(sizeof(buf), r); - ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf))); - } - - ::unlink(file); -} - -int main(int argc, char **argv) { - auto args = argv_to_vec(argc, argv); - - auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT, - CODE_ENVIRONMENT_UTILITY, - CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - g_ceph_context->_conf.set_val("err_to_stderr", "false"); - g_ceph_context->_conf.set_val("log_to_stderr", "false"); - g_ceph_context->_conf.apply_changes(nullptr); - - const char* file = FILENAME; - int x = 1234; - int y = 0; - int tmpfd = ::open(file, O_CREAT|O_WRONLY|O_TRUNC, 0700); - int ret = ::ceph_os_fsetxattr(tmpfd, "user.test", &x, sizeof(x)); - if (ret >= 0) - ret = ::ceph_os_fgetxattr(tmpfd, "user.test", &y, sizeof(y)); - ::close(tmpfd); - ::unlink(file); - if ((ret < 0) || (x != y)) { - cerr << "SKIP all tests because extended attributes don't appear to work in the file system in which the tests are run: " << cpp_strerror(ret) << std::endl; - } else { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); - } -} - -// Local Variables: -// compile-command: "cd ../.. ; make unittest_chain_xattr ; valgrind --tool=memcheck ./unittest_chain_xattr # --gtest_filter=chain_xattr.get_and_set" -// End: diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index 27bb9b26dbcc..32ddbfd9abd6 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -26,7 +26,6 @@ #include #include "os/ObjectStore.h" -#include "os/filestore/FileStore.h" #if defined(WITH_BLUESTORE) #include "os/bluestore/BlueStore.h" #include "os/bluestore/BlueFS.h" @@ -6839,7 +6838,6 @@ INSTANTIATE_TEST_SUITE_P( StoreTest, ::testing::Values( "memstore", - "filestore", #if defined(WITH_BLUESTORE) "bluestore", #endif @@ -6851,7 +6849,6 @@ INSTANTIATE_TEST_SUITE_P( StoreTestSpecificAUSize, ::testing::Values( "memstore", - "filestore", #if defined(WITH_BLUESTORE) "bluestore", #endif @@ -6863,7 +6860,6 @@ INSTANTIATE_TEST_SUITE_P( StoreTestOmapUpgrade, ::testing::Values( "memstore", - "filestore", #if defined(WITH_BLUESTORE) "bluestore", #endif diff --git a/src/test/objectstore/test_idempotent.cc b/src/test/objectstore/test_idempotent.cc deleted file mode 100644 index a2af4073ee51..000000000000 --- a/src/test/objectstore/test_idempotent.cc +++ /dev/null @@ -1,116 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2004-2006 Sage Weil - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#include -#include -#include -#include -#include "os/filestore/FileStore.h" -#include "global/global_init.h" -#include "common/ceph_argparse.h" -#include "common/debug.h" -#include "test/common/ObjectContents.h" -#include "FileStoreTracker.h" -#include "kv/KeyValueDB.h" -#include "os/ObjectStore.h" - -using namespace std; - -void usage(const string &name) { - std::cerr << "Usage: " << name << " [new|continue] store_path store_journal db_path" - << std::endl; -} - -template -typename T::iterator rand_choose(T &cont) { - if (std::empty(cont)) { - return std::end(cont); - } - return std::next(std::begin(cont), rand() % cont.size()); -} - -int main(int argc, char **argv) { - auto args = argv_to_vec(argc, argv); - auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, - CODE_ENVIRONMENT_UTILITY, - CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - cct->_conf.apply_changes(nullptr); - - std::cerr << "args: " << args << std::endl; - if (args.size() < 4) { - usage(argv[0]); - return 1; - } - - string store_path(args[1]); - string store_dev(args[2]); - string db_path(args[3]); - - bool start_new = false; - if (string(args[0]) == string("new")) start_new = true; - - KeyValueDB *_db = KeyValueDB::create(g_ceph_context, "leveldb", db_path); - ceph_assert(!_db->create_and_open(std::cerr)); - boost::scoped_ptr db(_db); - boost::scoped_ptr store(new FileStore(cct.get(), store_path, - store_dev)); - - coll_t coll(spg_t(pg_t(0,12),shard_id_t::NO_SHARD)); - ObjectStore::CollectionHandle ch; - - if (start_new) { - std::cerr << "mkfs" << std::endl; - ceph_assert(!store->mkfs()); - ObjectStore::Transaction t; - ceph_assert(!store->mount()); - ch = store->create_new_collection(coll); - t.create_collection(coll, 0); - store->queue_transaction(ch, std::move(t)); - } else { - ceph_assert(!store->mount()); - ch = store->open_collection(coll); - } - - FileStoreTracker tracker(store.get(), db.get()); - - set objects; - for (unsigned i = 0; i < 10; ++i) { - stringstream stream; - stream << "Object_" << i; - tracker.verify(coll, stream.str(), true); - objects.insert(stream.str()); - } - - while (1) { - FileStoreTracker::Transaction t; - for (unsigned j = 0; j < 100; ++j) { - int val = rand() % 100; - if (val < 30) { - t.write(coll, *rand_choose(objects)); - } else if (val < 60) { - t.clone(coll, *rand_choose(objects), - *rand_choose(objects)); - } else if (val < 70) { - t.remove(coll, *rand_choose(objects)); - } else { - t.clone_range(coll, *rand_choose(objects), - *rand_choose(objects)); - } - } - tracker.submit_transaction(t); - tracker.verify(coll, *rand_choose(objects)); - } - return 0; -} diff --git a/src/test/objectstore/test_idempotent_sequence.cc b/src/test/objectstore/test_idempotent_sequence.cc deleted file mode 100644 index cac395c36fc0..000000000000 --- a/src/test/objectstore/test_idempotent_sequence.cc +++ /dev/null @@ -1,259 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* -* Ceph - scalable distributed file system -* -* Copyright (C) 2012 New Dream Network -* -* This is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License version 2.1, as published by the Free Software -* Foundation. See file COPYING. -*/ -#include -#include -#include -#include -#include -#include -#include "common/ceph_argparse.h" -#include "global/global_init.h" -#include "common/debug.h" -#include "os/filestore/FileStore.h" - -#include "DeterministicOpSequence.h" -#include "FileStoreDiff.h" - -#include "common/config.h" -#include "include/ceph_assert.h" - -#define dout_context g_ceph_context -#define dout_subsys ceph_subsys_ -#undef dout_prefix -#define dout_prefix *_dout << "test_idempotent_sequence " - -using namespace std; - -void usage(const char *name, std::string command = "") { - ceph_assert(name != NULL); - - std::string more = "cmd "; - std::string diff = "diff "; - std::string get_last_op = "get-last-op "; - std::string run_seq_to = "run-sequence-to "; - - if (!command.empty()) { - if (command == "diff") - more = diff; - else if (command == "get-last-op") - more = get_last_op; - else if (command == "run-sequence-to") - more = run_seq_to; - } - std::cout << "usage: " << name << " " << more << " [options]" << std::endl; - - std::cout << "\n\ -Commands:\n\ - " << diff << "\n\ - " << get_last_op << "\n\ - " << run_seq_to << "\n\ -\n\ -Global Options:\n\ - -c FILE Read configuration from FILE\n\ - --osd-data PATH Set OSD Data path\n\ - --osd-journal PATH Set OSD Journal path\n\ - --osd-journal-size VAL Set Journal size\n\ - --help This message\n\ -\n\ -Test-specific Options:\n\ - --test-seed VAL Seed to run the test\n\ - --test-status-file PATH Path to keep the status file\n\ - --test-num-colls VAL Number of collections to create on init\n\ - --test-num-objs VAL Number of objects to create on init\n\ -" << std::endl; -} - -const char *our_name = NULL; -int seed = 0, num_txs = 100, num_colls = 30, num_objs = 0; -bool is_seed_set = false; -int verify_at = 0; -std::string status_file; - -int run_diff(std::string& a_path, std::string& a_journal, - std::string& b_path, std::string& b_journal) -{ - FileStore *a = new FileStore(g_ceph_context, a_path, a_journal, 0, "a"); - FileStore *b = new FileStore(g_ceph_context, b_path, b_journal, 0, "b"); - - int ret = 0; - { - FileStoreDiff fsd(a, b); - if (fsd.diff()) { - dout(0) << "diff found an difference" << dendl; - ret = -1; - } else { - dout(0) << "no diff" << dendl; - } - } - - delete a; - delete b; - return ret; -} - -int run_get_last_op(std::string& filestore_path, std::string& journal_path) -{ - FileStore *store = new FileStore(g_ceph_context, filestore_path, - journal_path); - - int err = store->mount(); - if (err) { - store->umount(); - delete store; - return err; - } - - vector cls; - store->list_collections(cls); - - int32_t txn = 0; - for (auto cid : cls) { - ghobject_t txn_object = DeterministicOpSequence::get_txn_object(cid); - bufferlist bl; - auto ch = store->open_collection(cid); - store->read(ch, txn_object, 0, 100, bl); - int32_t t = 0; - if (bl.length()) { - auto p = bl.cbegin(); - decode(t, p); - } - if (t > txn) { - txn = t; - } - } - - store->umount(); - delete store; - - cout << txn << std::endl; - return 0; -} - -int run_sequence_to(int val, std::string& filestore_path, - std::string& journal_path) -{ - num_txs = val; - - if (!is_seed_set) - seed = (int) time(NULL); - - FileStore *store = new FileStore(g_ceph_context, filestore_path, - journal_path); - - int err; - - // mkfs iff directory dne - err = ::mkdir(filestore_path.c_str(), 0755); - if (err) { - cerr << filestore_path << " already exists" << std::endl; - store->umount(); - delete store; - return err; - } - - err = store->mkfs(); - ceph_assert(err == 0); - - err = store->mount(); - ceph_assert(err == 0); - - DeterministicOpSequence op_sequence(store, status_file); - op_sequence.init(num_colls, num_objs); - op_sequence.generate(seed, num_txs); - store->umount(); - return 0; -} - -int run_command(std::string& command, std::vector& args) -{ - if (command.empty()) { - usage(our_name); - exit(0); - } - - /* We'll have a class that will handle the options, the command - * and its arguments. For the time being, and so we can move on, let's - * tolerate this big, ugly code. - */ - if (command == "diff") { - /* expect 4 arguments: (filestore path + journal path)*2 */ - if (args.size() == 4) { - return run_diff(args[0], args[1], args[2], args[3]); - } - } else if (command == "get-last-op") { - /* expect 2 arguments: a filestore path + journal */ - if (args.size() == 2) { - return run_get_last_op(args[0], args[1]); - } - } else if (command == "run-sequence-to") { - /* expect 3 arguments: # of operations and a filestore path + journal. */ - if (args.size() == 3) { - return run_sequence_to(strtoll(args[0].c_str(), NULL, 10), args[1], args[2]); - } - } else { - std::cout << "unknown command " << command << std::endl; - usage(our_name); - exit(1); - } - - usage(our_name, command); - exit(1); -} - -int main(int argc, const char *argv[]) -{ - our_name = argv[0]; - auto args = argv_to_vec(argc, argv); - - auto cct = global_init(NULL, args, - CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, - CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - g_ceph_context->_conf.apply_changes(nullptr); - - std::string command; - std::vector command_args; - - for (std::vector::iterator i = args.begin(); i != args.end();) { - string val; - - if (ceph_argparse_double_dash(args, i)) { - break; - } else if (ceph_argparse_witharg(args, i, &val, - "--test-seed", (char*) NULL)) { - seed = strtoll(val.c_str(), NULL, 10); - is_seed_set = true; - } else if (ceph_argparse_witharg(args, i, &val, - "--test-num-colls", (char*) NULL)) { - num_colls = strtoll(val.c_str(), NULL, 10); - } else if (ceph_argparse_witharg(args, i, &val, - "--test-num-objs", (char*) NULL)) { - num_objs = strtoll(val.c_str(), NULL, 10); - } else if (ceph_argparse_witharg(args, i, &val, - "--test-status-file", (char*) NULL)) { - status_file = val; - } else if (ceph_argparse_flag(args, i, "--help", (char*) NULL)) { - usage(our_name); - exit(0); - } else { - if (command.empty()) - command = *i++; - else - command_args.push_back(string(*i++)); - } - } - - int ret = run_command(command, command_args); - - return ret; -} diff --git a/src/test/os/CMakeLists.txt b/src/test/os/CMakeLists.txt deleted file mode 100644 index 35eb8f117802..000000000000 --- a/src/test/os/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# unittest_lfnindex -add_executable(unittest_lfnindex - TestLFNIndex.cc - ) -add_ceph_unittest(unittest_lfnindex) -target_link_libraries(unittest_lfnindex os global) - diff --git a/src/test/os/TestLFNIndex.cc b/src/test/os/TestLFNIndex.cc deleted file mode 100644 index c0976fcf1a9f..000000000000 --- a/src/test/os/TestLFNIndex.cc +++ /dev/null @@ -1,494 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU Library Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Library Public License for more details. - * - */ - -#include -#include -#include "os/filestore/LFNIndex.h" -#include "os/filestore/chain_xattr.h" -#include "common/ceph_argparse.h" -#include "global/global_init.h" -#include - -using namespace std; - -class TestWrapLFNIndex : public LFNIndex { -public: - TestWrapLFNIndex(CephContext* cct, - coll_t collection, - const char *base_path, - uint32_t index_version) - : LFNIndex(cct, collection, base_path, index_version) {} - - uint32_t collection_version() override { - return index_version; - } - - int cleanup() override { return 0; } - - int _split( - uint32_t match, - uint32_t bits, - CollectionIndex* dest - ) override { return 0; } - int _merge( - uint32_t bits, - CollectionIndex* dest - ) override { return 0; } - - void test_generate_and_parse(const ghobject_t &hoid, const std::string &mangled_expected) { - const std::string mangled_name = lfn_generate_object_name(hoid); - EXPECT_EQ(mangled_expected, mangled_name); - ghobject_t hoid_parsed; - EXPECT_EQ(0, lfn_parse_object_name(mangled_name, &hoid_parsed)); - EXPECT_EQ(hoid, hoid_parsed); - } - -protected: - int _init() override { return 0; } - - int _created( - const vector &path, - const ghobject_t &hoid, - const string &mangled_name - ) override { return 0; } - - int _remove( - const vector &path, - const ghobject_t &hoid, - const string &mangled_name - ) override { return 0; } - - int _lookup( - const ghobject_t &hoid, - vector *path, - string *mangled_name, - int *exists - ) override { return 0; } - - int _collection_list_partial( - const ghobject_t &start, - const ghobject_t &end, - int max_count, - vector *ls, - ghobject_t *next - ) override { return 0; } - int _pre_hash_collection( - uint32_t pg_num, - uint64_t expected_num_objs - ) override { return 0; } - -}; - -class TestHASH_INDEX_TAG : public TestWrapLFNIndex, public ::testing::Test { -public: - TestHASH_INDEX_TAG() - : TestWrapLFNIndex(g_ceph_context, coll_t(), "PATH_1", - CollectionIndex::HASH_INDEX_TAG) { - } -}; - -TEST_F(TestHASH_INDEX_TAG, generate_and_parse_name) { - const vector path; - const std::string key; - uint64_t hash = 0xABABABAB; - uint64_t pool = -1; - - test_generate_and_parse(ghobject_t(hobject_t(object_t(".A/B_\\C.D"), key, CEPH_NOSNAP, hash, pool, "")), - "\\.A\\sB_\\\\C.D_head_ABABABAB"); - test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")), - "\\dA_head_ABABABAB"); -} - -class TestHASH_INDEX_TAG_2 : public TestWrapLFNIndex, public ::testing::Test { -public: - TestHASH_INDEX_TAG_2() - : TestWrapLFNIndex(g_ceph_context, - coll_t(), "PATH_1", CollectionIndex::HASH_INDEX_TAG_2) { - } -}; - -TEST_F(TestHASH_INDEX_TAG_2, generate_and_parse_name) { - const vector path; - const std::string key("KEY"); - uint64_t hash = 0xABABABAB; - uint64_t pool = -1; - - { - std::string name(".XA/B_\\C.D"); - name[1] = '\0'; - ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, "")); - - test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB"); - } - test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")), - "\\dA_KEY_head_ABABABAB"); -} - -class TestHOBJECT_WITH_POOL : public TestWrapLFNIndex, public ::testing::Test { -public: - TestHOBJECT_WITH_POOL() - : TestWrapLFNIndex(g_ceph_context, coll_t(), - "PATH_1", CollectionIndex::HOBJECT_WITH_POOL) { - } -}; - -TEST_F(TestHOBJECT_WITH_POOL, generate_and_parse_name) { - const vector path; - const std::string key("KEY"); - uint64_t hash = 0xABABABAB; - uint64_t pool = 0xCDCDCDCD; - int64_t gen = 0xefefefefef; - shard_id_t shard_id(0xb); - - { - std::string name(".XA/B_\\C.D"); - name[1] = '\0'; - ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, "")); - hoid.hobj.nspace = "NSPACE"; - - test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd"); - } - { - ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")); - hoid.hobj.nspace = "NSPACE"; - - test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd"); - } - { - std::string name(".XA/B_\\C.D"); - name[1] = '\0'; - ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id); - hoid.hobj.nspace = "NSPACE"; - - test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b"); - } - { - ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id); - hoid.hobj.nspace = "NSPACE"; - - test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b"); - } -} - -class TestLFNIndex : public TestWrapLFNIndex, public ::testing::Test { -public: - TestLFNIndex() - : TestWrapLFNIndex(g_ceph_context, coll_t(), "PATH_1", - CollectionIndex::HOBJECT_WITH_POOL) { - } - - void SetUp() override { - ::chmod("PATH_1", 0700); - ASSERT_EQ(0, ::system("rm -fr PATH_1")); - ASSERT_EQ(0, ::mkdir("PATH_1", 0700)); - } - - void TearDown() override { - ASSERT_EQ(0, ::system("rm -fr PATH_1")); - } -}; - -TEST_F(TestLFNIndex, remove_object) { - const vector path; - - // - // small object name removal - // - { - std::string mangled_name; - int exists = 666; - ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP))); - - EXPECT_EQ(0, ::chmod("PATH_1", 0000)); - if (getuid() != 0) { - EXPECT_EQ(-EACCES, remove_object(path, hoid)); - } - EXPECT_EQ(0, ::chmod("PATH_1", 0700)); - EXPECT_EQ(-ENOENT, remove_object(path, hoid)); - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - const std::string pathname("PATH_1/" + mangled_name); - EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600))); - EXPECT_EQ(0, remove_object(path, hoid)); - EXPECT_EQ(-1, ::access(pathname.c_str(), 0)); - EXPECT_EQ(ENOENT, errno); - } - // - // long object name removal of a single file - // - { - std::string mangled_name; - int exists; - const std::string object_name(1024, 'A'); - ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP))); - - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - EXPECT_EQ(0, exists); - EXPECT_NE(std::string::npos, mangled_name.find("0_long")); - std::string pathname("PATH_1/" + mangled_name); - EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600))); - EXPECT_EQ(0, created(hoid, pathname.c_str())); - - EXPECT_EQ(0, remove_object(path, hoid)); - EXPECT_EQ(-1, ::access(pathname.c_str(), 0)); - EXPECT_EQ(ENOENT, errno); - } - - // - // long object name removal of the last file - // - { - std::string mangled_name; - int exists; - const std::string object_name(1024, 'A'); - ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP))); - - // - // PATH_1/AAA..._0_long => does not match long object name - // - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - EXPECT_EQ(0, exists); - EXPECT_NE(std::string::npos, mangled_name.find("0_long")); - std::string pathname("PATH_1/" + mangled_name); - EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600))); - EXPECT_EQ(0, created(hoid, pathname.c_str())); - string LFN_ATTR = "user.cephos.lfn"; - if (index_version != HASH_INDEX_TAG) { - char buf[100]; - snprintf(buf, sizeof(buf), "%d", index_version); - LFN_ATTR += string(buf); - } - const std::string object_name_1 = object_name + "SUFFIX"; - EXPECT_EQ(object_name_1.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_1.c_str(), object_name_1.size())); - - // - // PATH_1/AAA..._1_long => matches long object name - // - std::string mangled_name_1; - exists = 666; - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_1, &exists)); - EXPECT_NE(std::string::npos, mangled_name_1.find("1_long")); - EXPECT_EQ(0, exists); - std::string pathname_1("PATH_1/" + mangled_name_1); - auto retvalue = ::creat(pathname_1.c_str(), 0600); - ceph_assert(retvalue > 2); - EXPECT_EQ(0, ::close(retvalue)); - EXPECT_EQ(0, created(hoid, pathname_1.c_str())); - - // - // remove_object skips PATH_1/AAA..._0_long and removes PATH_1/AAA..._1_long - // - EXPECT_EQ(0, remove_object(path, hoid)); - EXPECT_EQ(0, ::access(pathname.c_str(), 0)); - EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0)); - EXPECT_EQ(ENOENT, errno); - EXPECT_EQ(0, ::unlink(pathname.c_str())); - } - - // - // long object name removal of a file in the middle of the list - // - { - std::string mangled_name; - int exists; - const std::string object_name(1024, 'A'); - ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP))); - - // - // PATH_1/AAA..._0_long => matches long object name - // - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - EXPECT_EQ(0, exists); - EXPECT_NE(std::string::npos, mangled_name.find("0_long")); - std::string pathname("PATH_1/" + mangled_name); - EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600))); - EXPECT_EQ(0, created(hoid, pathname.c_str())); - // - // PATH_1/AAA..._1_long => matches long object name - // - std::string mangled_name_1 = mangled_name; - mangled_name_1.replace(mangled_name_1.find("0_long"), 6, "1_long"); - const std::string pathname_1("PATH_1/" + mangled_name_1); - const std::string cmd("cp -a " + pathname + " " + pathname_1); - EXPECT_EQ(0, ::system(cmd.c_str())); - const string ATTR = "user.MARK"; - EXPECT_EQ((unsigned)1, (unsigned)chain_setxattr(pathname_1.c_str(), ATTR.c_str(), "Y", 1)); - - // - // remove_object replaces the file to be removed with the last from the - // collision list. In this case it replaces - // PATH_1/AAA..._0_long - // with - // PATH_1/AAA..._1_long - // - EXPECT_EQ(0, remove_object(path, hoid)); - EXPECT_EQ(0, ::access(pathname.c_str(), 0)); - char buffer[1] = { 0, }; - EXPECT_EQ((unsigned)1, (unsigned)chain_getxattr(pathname.c_str(), ATTR.c_str(), buffer, 1)); - EXPECT_EQ('Y', buffer[0]); - EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0)); - EXPECT_EQ(ENOENT, errno); - } -} - -TEST_F(TestLFNIndex, get_mangled_name) { - const vector path; - - // - // small object name - // - { - std::string mangled_name; - int exists = 666; - ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP))); - - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - EXPECT_NE(std::string::npos, mangled_name.find("ABC__head")); - EXPECT_EQ(std::string::npos, mangled_name.find("0_long")); - EXPECT_EQ(0, exists); - const std::string pathname("PATH_1/" + mangled_name); - EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600))); - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - EXPECT_NE(std::string::npos, mangled_name.find("ABC__head")); - EXPECT_EQ(1, exists); - EXPECT_EQ(0, ::unlink(pathname.c_str())); - } - // - // long object name - // - { - std::string mangled_name; - int exists; - const std::string object_name(1024, 'A'); - ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP))); - - // - // long version of the mangled name and no matching - // file exists - // - mangled_name.clear(); - exists = 666; - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - EXPECT_NE(std::string::npos, mangled_name.find("0_long")); - EXPECT_EQ(0, exists); - - const std::string pathname("PATH_1/" + mangled_name); - - // - // if a file by the same name exists but does not have the - // expected extended attribute, it is silently removed - // - mangled_name.clear(); - exists = 666; - EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600))); - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - EXPECT_NE(std::string::npos, mangled_name.find("0_long")); - EXPECT_EQ(0, exists); - EXPECT_EQ(-1, ::access(pathname.c_str(), 0)); - EXPECT_EQ(ENOENT, errno); - - // - // if a file by the same name exists but does not have the - // expected extended attribute, and cannot be removed, - // return on error - // - mangled_name.clear(); - exists = 666; - EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600))); - EXPECT_EQ(0, ::chmod("PATH_1", 0500)); - if (getuid() != 0) { - EXPECT_EQ(-EACCES, get_mangled_name(path, hoid, &mangled_name, &exists)); - } - EXPECT_EQ("", mangled_name); - EXPECT_EQ(666, exists); - EXPECT_EQ(0, ::chmod("PATH_1", 0700)); - EXPECT_EQ(0, ::unlink(pathname.c_str())); - - // - // long version of the mangled name and a file - // exists by that name and contains the long object name - // - mangled_name.clear(); - exists = 666; - auto retvalue = ::creat(pathname.c_str(), 0600); - ceph_assert(retvalue > 2); - EXPECT_EQ(0, ::close(retvalue)); - EXPECT_EQ(0, created(hoid, pathname.c_str())); - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists)); - EXPECT_NE(std::string::npos, mangled_name.find("0_long")); - EXPECT_EQ(1, exists); - EXPECT_EQ(0, ::access(pathname.c_str(), 0)); - - // - // long version of the mangled name and a file exists by that name - // and contains a long object name with the same prefix but they - // are not identical and it so happens that their SHA1 is - // identical : a collision number is used to differentiate them - // - string LFN_ATTR = "user.cephos.lfn"; - if (index_version != HASH_INDEX_TAG) { - char buf[100]; - snprintf(buf, sizeof(buf), "%d", index_version); - LFN_ATTR += string(buf); - } - const std::string object_name_same_prefix = object_name + "SUFFIX"; - EXPECT_EQ(object_name_same_prefix.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_same_prefix.c_str(), object_name_same_prefix.size())); - std::string mangled_name_same_prefix; - exists = 666; - EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_same_prefix, &exists)); - EXPECT_NE(std::string::npos, mangled_name_same_prefix.find("1_long")); - EXPECT_EQ(0, exists); - - EXPECT_EQ(0, ::unlink(pathname.c_str())); - } -} - -int main(int argc, char **argv) { - int fd = ::creat("detect", 0600); - if (fd < 0){ - cerr << "failed to create file detect" << std::endl; - return EXIT_FAILURE; - } - int ret = chain_fsetxattr(fd, "user.test", "A", 1); - ::close(fd); - ::unlink("detect"); - if (ret < 0) { - cerr << "SKIP LFNIndex because unable to test for xattr" << std::endl; - } else { - auto args = argv_to_vec(argc, argv); - - auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, - CODE_ENVIRONMENT_UTILITY, - CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); - } -} - -/* - * Local Variables: - * compile-command: "cd ../.. ; - * make unittest_lfnindex && - * valgrind --tool=memcheck ./unittest_lfnindex \ - * # --gtest_filter=TestLFNIndex.* --log-to-stderr=true --debug-filestore=20" - * End: - */ diff --git a/src/test/test_filejournal.cc b/src/test/test_filejournal.cc deleted file mode 100644 index f985b2c155a2..000000000000 --- a/src/test/test_filejournal.cc +++ /dev/null @@ -1,685 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -#include -#include -#include - -#include "common/ceph_argparse.h" -#include "common/common_init.h" -#include "global/global_init.h" -#include "common/config.h" -#include "common/Finisher.h" -#include "os/filestore/FileJournal.h" -#include "include/Context.h" -#include "common/ceph_mutex.h" -#include "common/safe_io.h" -#include "os/filestore/JournalingObjectStore.h" - -using namespace std; - -Finisher *finisher; -ceph::condition_variable sync_cond; -char path[200]; -uuid_d fsid; -struct test_info { - bool directio, aio, faio; - const char *description; -} subtests[3] = { - { false, false, false, "DIRECTIO OFF AIO OFF" }, - { true, false, false, "DIRECTIO ON AIO OFF" }, - { true, true, true, "DIRECTIO ON AIO ON"} -}; - -// ---- -ceph::condition_variable cond; -ceph::mutex wait_lock = ceph::make_mutex("lock"); -bool done; - -void wait() -{ - std::unique_lock l{wait_lock}; - cond.wait(l, [] { return done; }); -} - -// ---- -class C_Sync { -public: - ceph::condition_variable cond; - ceph::mutex lock = ceph::make_mutex("C_Sync::lock"); - bool done = false; - C_SafeCond *c; - - C_Sync() { - c = new C_SafeCond(lock, cond, &done); - } - ~C_Sync() { - std::unique_lock l{lock}; - //cout << "wait" << std::endl; - cond.wait(l, [this] { return done; }); - //cout << "waited" << std::endl; - } -}; - -unsigned size_mb = 200; -//Gtest argument prefix -const char GTEST_PRFIX[] = "--gtest_"; - -int main(int argc, char **argv) { - auto args = argv_to_vec(argc, argv); - - auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, - CODE_ENVIRONMENT_UTILITY, - CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - - char mb[10]; - sprintf(mb, "%u", size_mb); - g_ceph_context->_conf.set_val("osd_journal_size", mb); - g_ceph_context->_conf.apply_changes(nullptr); - - finisher = new Finisher(g_ceph_context); - - path[0] = '\0'; - if (!args.empty()) { - for ( unsigned int i = 0; i < args.size(); ++i) { - if (strncmp(args[i], GTEST_PRFIX, sizeof(GTEST_PRFIX) - 1)) { - //Non gtest argument, set to path. - size_t copy_len = std::min(sizeof(path) - 1, strlen(args[i])); - strncpy(path, args[i], copy_len); - path[copy_len] = '\0'; - break; - } - } - } - if ( path[0] == '\0') { - srand(getpid() + time(0)); - snprintf(path, sizeof(path), "/var/tmp/ceph_test_filejournal.tmp.%d", rand()); - } - cout << "path " << path << std::endl; - - ::testing::InitGoogleTest(&argc, argv); - - finisher->start(); - - int r = RUN_ALL_TESTS(); - - finisher->stop(); - - unlink(path); - - return r; -} - -TEST(TestFileJournal, Create) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "false"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - } -} - -TEST(TestFileJournal, WriteSmall) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "false"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - vector tls; - bufferlist bl; - bl.append("small"); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(1, bl, orig_len, new C_SafeCond(wait_lock, cond, &done)); - wait(); - - fj.close(); - } -} - -TEST(TestFileJournal, WriteBig) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "false"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - bufferlist bl; - while (bl.length() < size_mb*1000/2) { - char foo[1024*1024]; - memset(foo, 1, sizeof(foo)); - bl.append(foo, sizeof(foo)); - } - vector tls; - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(1, bl, orig_len, new C_SafeCond(wait_lock, cond, &done)); - wait(); - fj.close(); - } -} - -TEST(TestFileJournal, WriteMany) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "false"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done)); - - vector tls; - bufferlist bl; - bl.append("small"); - uint64_t seq = 1; - for (int i=0; i<100; i++) { - bl.append("small"); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(seq++, bl, orig_len, gb.new_sub()); - } - gb.activate(); - - wait(); - - fj.close(); - } -} - -TEST(TestFileJournal, WriteManyVecs) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "false"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done)); - - bufferlist first; - first.append("small"); - vector tls; - int orig_len = fj.prepare_entry(tls, &first); - fj.reserve_throttle_and_backoff(first.length()); - fj.submit_entry(1, first, orig_len, gb.new_sub()); - - bufferlist bl; - for (int i=0; i_conf.set_val("journal_ignore_corruption", "false"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - - vector tls; - - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done)); - - bufferlist bl; - bl.append("small"); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(1, bl, orig_len, gb.new_sub()); - bl.append("small"); - orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(2, bl, orig_len, gb.new_sub()); - bl.append("small"); - orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(3, bl, orig_len, gb.new_sub()); - gb.activate(); - wait(); - - fj.close(); - - fj.open(1); - - bufferlist inbl; - string v; - uint64_t seq = 0; - ASSERT_EQ(true, fj.read_entry(inbl, seq)); - ASSERT_EQ(seq, 2ull); - inbl.cbegin().copy(inbl.length(), v); - ASSERT_EQ("small", v); - inbl.clear(); - v.clear(); - - ASSERT_EQ(true, fj.read_entry(inbl, seq)); - ASSERT_EQ(seq, 3ull); - inbl.cbegin().copy(inbl.length(), v); - ASSERT_EQ("small", v); - inbl.clear(); - v.clear(); - - ASSERT_TRUE(!fj.read_entry(inbl, seq)); - - ASSERT_EQ(0, fj.make_writeable()); - fj.close(); - } -} - -TEST(TestFileJournal, ReplayCorrupt) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "true"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - - vector tls; - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done)); - - const char *needle = "i am a needle"; - const char *newneedle = "in a haystack"; - bufferlist bl; - bl.append(needle); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(1, bl, orig_len, gb.new_sub()); - bl.append(needle); - orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(2, bl, orig_len, gb.new_sub()); - bl.append(needle); - orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(3, bl, orig_len, gb.new_sub()); - bl.append(needle); - orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(4, bl, orig_len, gb.new_sub()); - gb.activate(); - wait(); - - fj.close(); - - cout << "corrupting journal" << std::endl; - char buf[1024*128]; - int fd = open(path, O_RDONLY); - ASSERT_GE(fd, 0); - int r = safe_read_exact(fd, buf, sizeof(buf)); - ASSERT_EQ(0, r); - int n = 0; - for (unsigned o=0; o < sizeof(buf) - strlen(needle); o++) { - if (memcmp(buf+o, needle, strlen(needle)) == 0) { - if (n >= 2) { - cout << "replacing at offset " << o << std::endl; - memcpy(buf+o, newneedle, strlen(newneedle)); - } else { - cout << "leaving at offset " << o << std::endl; - } - n++; - } - } - ASSERT_EQ(n, 4); - close(fd); - fd = open(path, O_WRONLY); - ASSERT_GE(fd, 0); - r = safe_write(fd, buf, sizeof(buf)); - ASSERT_EQ(r, 0); - close(fd); - - fj.open(1); - - bufferlist inbl; - string v; - uint64_t seq = 0; - ASSERT_EQ(true, fj.read_entry(inbl, seq)); - ASSERT_EQ(seq, 2ull); - inbl.cbegin().copy(inbl.length(), v); - ASSERT_EQ(needle, v); - inbl.clear(); - v.clear(); - bool corrupt; - ASSERT_FALSE(fj.read_entry(inbl, seq, &corrupt)); - ASSERT_TRUE(corrupt); - - ASSERT_EQ(0, fj.make_writeable()); - fj.close(); - } -} - -TEST(TestFileJournal, WriteTrim) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "false"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - list ls; - - bufferlist bl; - char foo[1024*1024]; - memset(foo, 1, sizeof(foo)); - - uint64_t seq = 1, committed = 0; - vector tls; - - for (unsigned i=0; ic); - - while (ls.size() > size_mb/2) { - delete ls.front(); - ls.pop_front(); - committed++; - fj.committed_thru(committed); - } - } - - while (ls.size()) { - delete ls.front(); - ls.pop_front(); - fj.committed_thru(++committed); - } - - ASSERT_TRUE(fj.journalq_empty()); - - fj.close(); - } -} - -TEST(TestFileJournal, WriteTrimSmall) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "false"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "0"); - g_ceph_context->_conf.apply_changes(nullptr); - vector tls; - - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - list ls; - - bufferlist bl; - char foo[1024*1024]; - memset(foo, 1, sizeof(foo)); - - uint64_t seq = 1, committed = 0; - - for (unsigned i=0; ic); - - while (ls.size() > size_mb/2) { - delete ls.front(); - ls.pop_front(); - committed++; - fj.committed_thru(committed); - } - } - - while (ls.size()) { - delete ls.front(); - ls.pop_front(); - fj.committed_thru(committed); - } - - fj.close(); - } -} - -TEST(TestFileJournal, ReplayDetectCorruptFooterMagic) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "true"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "1"); - g_ceph_context->_conf.apply_changes(nullptr); - - vector tls; - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done)); - - const char *needle = "i am a needle"; - for (unsigned i = 1; i <= 4; ++i) { - bufferlist bl; - bl.append(needle); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(i, bl, orig_len, gb.new_sub()); - } - gb.activate(); - wait(); - - bufferlist bl; - bl.append("needle"); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(5, bl, orig_len, new C_SafeCond(wait_lock, cond, &done)); - wait(); - - fj.close(); - int fd = open(path, O_WRONLY); - - cout << "corrupting journal" << std::endl; - fj.open(0); - fj.corrupt_footer_magic(fd, 2); - - uint64_t seq = 0; - bl.clear(); - bool corrupt = false; - bool result = fj.read_entry(bl, seq, &corrupt); - ASSERT_TRUE(result); - ASSERT_EQ(seq, 1UL); - ASSERT_FALSE(corrupt); - - result = fj.read_entry(bl, seq, &corrupt); - ASSERT_FALSE(result); - ASSERT_TRUE(corrupt); - - ASSERT_EQ(0, fj.make_writeable()); - fj.close(); - ::close(fd); - } -} - -TEST(TestFileJournal, ReplayDetectCorruptPayload) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "true"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "1"); - g_ceph_context->_conf.apply_changes(nullptr); - - vector tls; - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done)); - - const char *needle = "i am a needle"; - for (unsigned i = 1; i <= 4; ++i) { - bufferlist bl; - bl.append(needle); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(i, bl, orig_len, gb.new_sub()); - } - gb.activate(); - wait(); - - bufferlist bl; - bl.append("needle"); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(5, bl, orig_len, new C_SafeCond(wait_lock, cond, &done)); - wait(); - - fj.close(); - int fd = open(path, O_WRONLY); - - cout << "corrupting journal" << std::endl; - fj.open(0); - fj.corrupt_payload(fd, 2); - - uint64_t seq = 0; - bl.clear(); - bool corrupt = false; - bool result = fj.read_entry(bl, seq, &corrupt); - ASSERT_TRUE(result); - ASSERT_EQ(seq, 1UL); - ASSERT_FALSE(corrupt); - - result = fj.read_entry(bl, seq, &corrupt); - ASSERT_FALSE(result); - ASSERT_TRUE(corrupt); - - ASSERT_EQ(0, fj.make_writeable()); - fj.close(); - ::close(fd); - } -} - -TEST(TestFileJournal, ReplayDetectCorruptHeader) { - g_ceph_context->_conf.set_val("journal_ignore_corruption", "true"); - g_ceph_context->_conf.set_val("journal_write_header_frequency", "1"); - g_ceph_context->_conf.apply_changes(nullptr); - - vector tls; - for (unsigned i = 0 ; i < 3; ++i) { - SCOPED_TRACE(subtests[i].description); - fsid.generate_random(); - FileJournal fj(g_ceph_context, fsid, finisher, &sync_cond, path, - subtests[i].directio, subtests[i].aio, subtests[i].faio); - ASSERT_EQ(0, fj.create()); - ASSERT_EQ(0, fj.make_writeable()); - - C_GatherBuilder gb(g_ceph_context, new C_SafeCond(wait_lock, cond, &done)); - - const char *needle = "i am a needle"; - for (unsigned i = 1; i <= 4; ++i) { - bufferlist bl; - bl.append(needle); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(i, bl, orig_len, gb.new_sub()); - } - gb.activate(); - wait(); - - bufferlist bl; - bl.append("needle"); - int orig_len = fj.prepare_entry(tls, &bl); - fj.reserve_throttle_and_backoff(bl.length()); - fj.submit_entry(5, bl, orig_len, new C_SafeCond(wait_lock, cond, &done)); - wait(); - - fj.close(); - int fd = open(path, O_WRONLY); - - cout << "corrupting journal" << std::endl; - fj.open(0); - fj.corrupt_header_magic(fd, 2); - - uint64_t seq = 0; - bl.clear(); - bool corrupt = false; - bool result = fj.read_entry(bl, seq, &corrupt); - ASSERT_TRUE(result); - ASSERT_EQ(seq, 1UL); - ASSERT_FALSE(corrupt); - - result = fj.read_entry(bl, seq, &corrupt); - ASSERT_FALSE(result); - ASSERT_TRUE(corrupt); - - ASSERT_EQ(0, fj.make_writeable()); - fj.close(); - ::close(fd); - } -} diff --git a/src/test/test_trans.cc b/src/test/test_trans.cc index 2ac4463035d5..f843e0037f2d 100644 --- a/src/test/test_trans.cc +++ b/src/test/test_trans.cc @@ -15,7 +15,7 @@ #include #include "common/ceph_argparse.h" #include "common/debug.h" -#include "os/filestore/FileStore.h" +#include "os/bluestore/BlueStore.h" #include "global/global_init.h" #include "include/ceph_assert.h" @@ -52,7 +52,7 @@ int main(int argc, const char **argv) cout << "#dev " << filename << std::endl; cout << "#mb " << mb << std::endl; - ObjectStore *fs = new FileStore(cct.get(), filename, NULL); + ObjectStore *fs = new BlueStore(cct.get(), filename); if (fs->mount() < 0) { cout << "mount failed" << std::endl; return -1; diff --git a/src/test/xattr_bench.cc b/src/test/xattr_bench.cc index 423e6a8c8908..b117e16f26ac 100644 --- a/src/test/xattr_bench.cc +++ b/src/test/xattr_bench.cc @@ -18,7 +18,7 @@ #include #include #include -#include "os/filestore/FileStore.h" +#include "os/bluestore/BlueStore.h" #include "include/Context.h" #include "common/ceph_argparse.h" #include "common/ceph_mutex.h" @@ -33,7 +33,7 @@ #include "include/unordered_map.h" void usage(const string &name) { - std::cerr << "Usage: " << name << " [xattr|omap] store_path store_journal" + std::cerr << "Usage: " << name << " [xattr|omap] store_path" << std::endl; } @@ -160,16 +160,14 @@ int main(int argc, char **argv) { common_init_finish(g_ceph_context); std::cerr << "args: " << args << std::endl; - if (args.size() < 3) { + if (args.size() < 2) { usage(argv[0]); return 1; } string store_path(args[1]); - string store_dev(args[2]); - boost::scoped_ptr store(new FileStore(cct.get(), store_path, - store_dev)); + boost::scoped_ptr store(new BlueStore(cct.get(), store_path)); std::cerr << "mkfs starting" << std::endl; ceph_assert(!store->mkfs());