set(libos_srcs
ObjectStore.cc
Transaction.cc
- DBObjectMap.cc
memstore/MemStore.cc
kstore/KStore.cc
kstore/kstore_types.cc
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-
-#include "include/int_types.h"
-#include "include/buffer.h"
-
-#include <iostream>
-#include <set>
-#include <map>
-#include <string>
-#include <vector>
-
-#include "os/ObjectMap.h"
-#include "kv/KeyValueDB.h"
-#include "DBObjectMap.h"
-#include <errno.h>
-
-#include "common/debug.h"
-#include "common/config.h"
-#include "include/ceph_assert.h"
-
-#define dout_context cct
-#define dout_subsys ceph_subsys_filestore
-#undef dout_prefix
-#define dout_prefix *_dout << "filestore "
-
-using std::map;
-using std::ostream;
-using std::ostringstream;
-using std::set;
-using std::string;
-using std::stringstream;
-using std::vector;
-
-using ceph::bufferlist;
-
-const string DBObjectMap::USER_PREFIX = "_USER_";
-const string DBObjectMap::XATTR_PREFIX = "_AXATTR_";
-const string DBObjectMap::SYS_PREFIX = "_SYS_";
-const string DBObjectMap::COMPLETE_PREFIX = "_COMPLETE_";
-const string DBObjectMap::HEADER_KEY = "HEADER";
-const string DBObjectMap::USER_HEADER_KEY = "USER_HEADER";
-const string DBObjectMap::GLOBAL_STATE_KEY = "HEADER";
-const string DBObjectMap::HOBJECT_TO_SEQ = "_HOBJTOSEQ_";
-
-// Legacy
-const string DBObjectMap::LEAF_PREFIX = "_LEAF_";
-const string DBObjectMap::REVERSE_LEAF_PREFIX = "_REVLEAF_";
-
-static void append_escaped(const string &in, string *out)
-{
- for (string::const_iterator i = in.begin(); i != in.end(); ++i) {
- if (*i == '%') {
- out->push_back('%');
- out->push_back('p');
- } else if (*i == '.') {
- out->push_back('%');
- out->push_back('e');
- } else if (*i == '_') {
- out->push_back('%');
- out->push_back('u');
- } else {
- out->push_back(*i);
- }
- }
-}
-
-int DBObjectMap::check(std::ostream &out, bool repair, bool force)
-{
- int errors = 0, comp_errors = 0;
- bool repaired = false;
- map<uint64_t, uint64_t> parent_to_num_children;
- map<uint64_t, uint64_t> parent_to_actual_num_children;
- KeyValueDB::Iterator iter = db->get_iterator(HOBJECT_TO_SEQ);
- for (iter->seek_to_first(); iter->valid(); iter->next()) {
- _Header header;
- bufferlist bl = iter->value();
- while (true) {
- auto bliter = bl.cbegin();
- header.decode(bliter);
- if (header.seq != 0)
- parent_to_actual_num_children[header.seq] = header.num_children;
-
- if (state.v == 2 || force) {
- // Check complete table
- bool complete_error = false;
- boost::optional<string> prev;
- KeyValueDB::Iterator complete_iter = db->get_iterator(USER_PREFIX + header_key(header.seq) + COMPLETE_PREFIX);
- for (complete_iter->seek_to_first(); complete_iter->valid();
- complete_iter->next()) {
- if (prev && prev >= complete_iter->key()) {
- out << "Bad complete for " << header.oid << std::endl;
- complete_error = true;
- break;
- }
- prev = string(complete_iter->value().c_str(), complete_iter->value().length() - 1);
- }
- if (complete_error) {
- out << "Complete mapping for " << header.seq << " :" << std::endl;
- for (complete_iter->seek_to_first(); complete_iter->valid();
- complete_iter->next()) {
- out << complete_iter->key() << " -> " << string(complete_iter->value().c_str(), complete_iter->value().length() - 1) << std::endl;
- }
- if (repair) {
- repaired = true;
- KeyValueDB::Transaction t = db->get_transaction();
- t->rmkeys_by_prefix(USER_PREFIX + header_key(header.seq) + COMPLETE_PREFIX);
- db->submit_transaction(t);
- out << "Cleared complete mapping to repair" << std::endl;
- } else {
- errors++; // Only count when not repaired
- comp_errors++; // Track errors here for version update
- }
- }
- }
-
- if (header.parent == 0)
- break;
-
- if (!parent_to_num_children.count(header.parent))
- parent_to_num_children[header.parent] = 0;
- parent_to_num_children[header.parent]++;
- if (parent_to_actual_num_children.count(header.parent))
- break;
-
- set<string> to_get;
- map<string, bufferlist> got;
- to_get.insert(HEADER_KEY);
- db->get(sys_parent_prefix(header), to_get, &got);
- if (got.empty()) {
- out << "Missing: seq " << header.parent << std::endl;
- errors++;
- break;
- } else {
- bl = got.begin()->second;
- }
- }
- }
-
- for (map<uint64_t, uint64_t>::iterator i = parent_to_num_children.begin();
- i != parent_to_num_children.end();
- parent_to_num_children.erase(i++)) {
- if (!parent_to_actual_num_children.count(i->first))
- continue;
- if (parent_to_actual_num_children[i->first] != i->second) {
- out << "Invalid: seq " << i->first << " recorded children: "
- << parent_to_actual_num_children[i->first] << " found: "
- << i->second << std::endl;
- errors++;
- }
- parent_to_actual_num_children.erase(i->first);
- }
-
- // Only advance the version from 2 to 3 here
- // Mark as legacy because there are still older structures
- // we don't update. The value of legacy is only used
- // for internal assertions.
- if (comp_errors == 0 && state.v == 2 && repair) {
- state.v = 3;
- state.legacy = true;
- set_state();
- }
-
- if (errors == 0 && repaired)
- return -1;
- return errors;
-}
-
-string DBObjectMap::ghobject_key(const ghobject_t &oid)
-{
- string out;
- append_escaped(oid.hobj.oid.name, &out);
- out.push_back('.');
- append_escaped(oid.hobj.get_key(), &out);
- out.push_back('.');
- append_escaped(oid.hobj.nspace, &out);
- out.push_back('.');
-
- char snap_with_hash[1000];
- char *t = snap_with_hash;
- char *end = t + sizeof(snap_with_hash);
- if (oid.hobj.snap == CEPH_NOSNAP)
- t += snprintf(t, end - t, "head");
- else if (oid.hobj.snap == CEPH_SNAPDIR)
- t += snprintf(t, end - t, "snapdir");
- else
- t += snprintf(t, end - t, "%llx", (long long unsigned)oid.hobj.snap);
-
- if (oid.hobj.pool == -1)
- t += snprintf(t, end - t, ".none");
- else
- t += snprintf(t, end - t, ".%llx", (long long unsigned)oid.hobj.pool);
- t += snprintf(t, end - t, ".%.*X", (int)(sizeof(uint32_t)*2), oid.hobj.get_hash());
-
- if (oid.generation != ghobject_t::NO_GEN ||
- oid.shard_id != shard_id_t::NO_SHARD) {
- t += snprintf(t, end - t, ".%llx", (long long unsigned)oid.generation);
- t += snprintf(t, end - t, ".%x", (int)oid.shard_id);
- }
- out += string(snap_with_hash);
- return out;
-}
-
-// ok: pglog%u3%efs1...0.none.0017B237
-// bad: plana8923501-10...4c.3.ffffffffffffffff.2
-// fixed: plana8923501-10...4c.3.CB767F2D.ffffffffffffffff.2
-// returns 0 for false, 1 for true, negative for error
-int DBObjectMap::is_buggy_ghobject_key_v1(CephContext* cct,
- const string &in)
-{
- int dots = 5; // skip 5 .'s
- const char *s = in.c_str();
- do {
- while (*s && *s != '.')
- ++s;
- if (!*s) {
- derr << "unexpected null at " << (int)(s-in.c_str()) << dendl;
- return -EINVAL;
- }
- ++s;
- } while (*s && --dots);
- if (!*s) {
- derr << "unexpected null at " << (int)(s-in.c_str()) << dendl;
- return -EINVAL;
- }
- // we are now either at a hash value (32 bits, 8 chars) or a generation
- // value (64 bits) '.' and shard id. count the dots!
- int len = 0;
- while (*s && *s != '.') {
- ++s;
- ++len;
- }
- if (*s == '\0') {
- if (len != 8) {
- derr << "hash value is not 8 chars" << dendl;
- return -EINVAL; // the hash value is always 8 chars.
- }
- return 0;
- }
- if (*s != '.') { // the shard follows.
- derr << "missing final . and shard id at " << (int)(s-in.c_str()) << dendl;
- return -EINVAL;
- }
- return 1;
-}
-
-
-string DBObjectMap::map_header_key(const ghobject_t &oid)
-{
- return ghobject_key(oid);
-}
-
-string DBObjectMap::header_key(uint64_t seq)
-{
- char buf[100];
- snprintf(buf, sizeof(buf), "%.*" PRId64, (int)(2*sizeof(seq)), seq);
- return string(buf);
-}
-
-string DBObjectMap::complete_prefix(Header header)
-{
- return USER_PREFIX + header_key(header->seq) + COMPLETE_PREFIX;
-}
-
-string DBObjectMap::user_prefix(Header header)
-{
- return USER_PREFIX + header_key(header->seq) + USER_PREFIX;
-}
-
-string DBObjectMap::sys_prefix(Header header)
-{
- return USER_PREFIX + header_key(header->seq) + SYS_PREFIX;
-}
-
-string DBObjectMap::xattr_prefix(Header header)
-{
- return USER_PREFIX + header_key(header->seq) + XATTR_PREFIX;
-}
-
-string DBObjectMap::sys_parent_prefix(_Header header)
-{
- return USER_PREFIX + header_key(header.parent) + SYS_PREFIX;
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::init()
-{
- invalid = false;
- if (ready) {
- return 0;
- }
- ceph_assert(!parent_iter);
- if (header->parent) {
- Header parent = map->lookup_parent(header);
- if (!parent) {
- ceph_abort();
- return -EINVAL;
- }
- parent_iter = std::make_shared<DBObjectMapIteratorImpl>(map, parent);
- }
- key_iter = map->db->get_iterator(map->user_prefix(header));
- ceph_assert(key_iter);
- complete_iter = map->db->get_iterator(map->complete_prefix(header));
- ceph_assert(complete_iter);
- cur_iter = key_iter;
- ceph_assert(cur_iter);
- ready = true;
- return 0;
-}
-
-ObjectMap::ObjectMapIterator DBObjectMap::get_iterator(
- const ghobject_t &oid)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return ObjectMapIterator(new EmptyIteratorImpl());
- DBObjectMapIterator iter = _get_iterator(header);
- iter->hlock.swap(hl);
- return iter;
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::seek_to_first()
-{
- init();
- r = 0;
- if (parent_iter) {
- r = parent_iter->seek_to_first();
- if (r < 0)
- return r;
- }
- r = key_iter->seek_to_first();
- if (r < 0)
- return r;
- return adjust();
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::seek_to_last()
-{
- init();
- r = 0;
- if (parent_iter) {
- r = parent_iter->seek_to_last();
- if (r < 0)
- return r;
- if (parent_iter->valid())
- r = parent_iter->next();
- if (r < 0)
- return r;
- }
- r = key_iter->seek_to_last();
- if (r < 0)
- return r;
- if (key_iter->valid())
- r = key_iter->next();
- if (r < 0)
- return r;
- return adjust();
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::lower_bound(const string &to)
-{
- init();
- r = 0;
- if (parent_iter) {
- r = parent_iter->lower_bound(to);
- if (r < 0)
- return r;
- }
- r = key_iter->lower_bound(to);
- if (r < 0)
- return r;
- return adjust();
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::lower_bound_parent(const string &to)
-{
- int r = lower_bound(to);
- if (r < 0)
- return r;
- if (valid() && !on_parent())
- return next_parent();
- else
- return r;
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::upper_bound(const string &after)
-{
- init();
- r = 0;
- if (parent_iter) {
- r = parent_iter->upper_bound(after);
- if (r < 0)
- return r;
- }
- r = key_iter->upper_bound(after);
- if (r < 0)
- return r;
- return adjust();
-}
-
-bool DBObjectMap::DBObjectMapIteratorImpl::valid()
-{
- bool valid = !invalid && ready;
- ceph_assert(!valid || cur_iter->valid());
- return valid;
-}
-
-bool DBObjectMap::DBObjectMapIteratorImpl::valid_parent()
-{
- if (parent_iter && parent_iter->valid() &&
- (!key_iter->valid() || key_iter->key() > parent_iter->key()))
- return true;
- return false;
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::next()
-{
- ceph_assert(cur_iter->valid());
- ceph_assert(valid());
- cur_iter->next();
- return adjust();
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::next_parent()
-{
- r = next();
- if (r < 0)
- return r;
- while (parent_iter && parent_iter->valid() && !on_parent()) {
- ceph_assert(valid());
- r = lower_bound(parent_iter->key());
- if (r < 0)
- return r;
- }
-
- if (!parent_iter || !parent_iter->valid()) {
- invalid = true;
- }
- return 0;
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::in_complete_region(const string &to_test,
- string *begin,
- string *end)
-{
- /* This is clumsy because one cannot call prev() on end(), nor can one
- * test for == begin().
- */
- complete_iter->upper_bound(to_test);
- if (complete_iter->valid()) {
- complete_iter->prev();
- if (!complete_iter->valid()) {
- complete_iter->upper_bound(to_test);
- return false;
- }
- } else {
- complete_iter->seek_to_last();
- if (!complete_iter->valid())
- return false;
- }
-
- ceph_assert(complete_iter->key() <= to_test);
- ceph_assert(complete_iter->value().length() >= 1);
- string _end(complete_iter->value().c_str(),
- complete_iter->value().length() - 1);
- if (_end.empty() || _end > to_test) {
- if (begin)
- *begin = complete_iter->key();
- if (end)
- *end = _end;
- return true;
- } else {
- complete_iter->next();
- ceph_assert(!complete_iter->valid() || complete_iter->key() > to_test);
- return false;
- }
-}
-
-/**
- * Moves parent_iter to the next position both out of the complete_region and
- * not equal to key_iter. Then, we set cur_iter to parent_iter if valid and
- * less than key_iter and key_iter otherwise.
- */
-int DBObjectMap::DBObjectMapIteratorImpl::adjust()
-{
- string begin, end;
- while (parent_iter && parent_iter->valid()) {
- if (in_complete_region(parent_iter->key(), &begin, &end)) {
- if (end.size() == 0) {
- parent_iter->seek_to_last();
- if (parent_iter->valid())
- parent_iter->next();
- } else
- parent_iter->lower_bound(end);
- } else if (key_iter->valid() && key_iter->key() == parent_iter->key()) {
- parent_iter->next();
- } else {
- break;
- }
- }
- if (valid_parent()) {
- cur_iter = parent_iter;
- } else if (key_iter->valid()) {
- cur_iter = key_iter;
- } else {
- invalid = true;
- }
- ceph_assert(invalid || cur_iter->valid());
- return 0;
-}
-
-
-string DBObjectMap::DBObjectMapIteratorImpl::key()
-{
- return cur_iter->key();
-}
-
-bufferlist DBObjectMap::DBObjectMapIteratorImpl::value()
-{
- return cur_iter->value();
-}
-
-std::string_view DBObjectMap::DBObjectMapIteratorImpl::value_as_sv()
-{
- return cur_iter->value_as_sv();
-}
-
-int DBObjectMap::DBObjectMapIteratorImpl::status()
-{
- return r;
-}
-
-int DBObjectMap::set_keys(const ghobject_t &oid,
- const map<string, bufferlist> &set,
- const SequencerPosition *spos)
-{
- KeyValueDB::Transaction t = db->get_transaction();
- MapHeaderLock hl(this, oid);
- Header header = lookup_create_map_header(hl, oid, t);
- if (!header)
- return -EINVAL;
- if (check_spos(oid, header, spos))
- return 0;
-
- t->set(user_prefix(header), set);
-
- return db->submit_transaction(t);
-}
-
-int DBObjectMap::set_header(const ghobject_t &oid,
- const bufferlist &bl,
- const SequencerPosition *spos)
-{
- KeyValueDB::Transaction t = db->get_transaction();
- MapHeaderLock hl(this, oid);
- Header header = lookup_create_map_header(hl, oid, t);
- if (!header)
- return -EINVAL;
- if (check_spos(oid, header, spos))
- return 0;
- _set_header(header, bl, t);
- return db->submit_transaction(t);
-}
-
-void DBObjectMap::_set_header(Header header, const bufferlist &bl,
- KeyValueDB::Transaction t)
-{
- map<string, bufferlist> to_set;
- to_set[USER_HEADER_KEY] = bl;
- t->set(sys_prefix(header), to_set);
-}
-
-int DBObjectMap::get_header(const ghobject_t &oid,
- bufferlist *bl)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header) {
- return 0;
- }
- return _get_header(header, bl);
-}
-
-int DBObjectMap::_get_header(Header header,
- bufferlist *bl)
-{
- map<string, bufferlist> out;
- while (true) {
- out.clear();
- set<string> to_get;
- to_get.insert(USER_HEADER_KEY);
- int r = db->get(sys_prefix(header), to_get, &out);
- if (r == 0 && !out.empty())
- break;
- if (r < 0)
- return r;
- Header current(header);
- if (!current->parent)
- break;
- header = lookup_parent(current);
- }
-
- if (!out.empty())
- bl->swap(out.begin()->second);
- return 0;
-}
-
-int DBObjectMap::clear(const ghobject_t &oid,
- const SequencerPosition *spos)
-{
- KeyValueDB::Transaction t = db->get_transaction();
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- if (check_spos(oid, header, spos))
- return 0;
- remove_map_header(hl, oid, header, t);
- ceph_assert(header->num_children > 0);
- header->num_children--;
- int r = _clear(header, t);
- if (r < 0)
- return r;
- return db->submit_transaction(t);
-}
-
-int DBObjectMap::_clear(Header header,
- KeyValueDB::Transaction t)
-{
- while (1) {
- if (header->num_children) {
- set_header(header, t);
- break;
- }
- clear_header(header, t);
- if (!header->parent)
- break;
- Header parent = lookup_parent(header);
- if (!parent) {
- return -EINVAL;
- }
- ceph_assert(parent->num_children > 0);
- parent->num_children--;
- header.swap(parent);
- }
- return 0;
-}
-
-int DBObjectMap::copy_up_header(Header header,
- KeyValueDB::Transaction t)
-{
- bufferlist bl;
- int r = _get_header(header, &bl);
- if (r < 0)
- return r;
-
- _set_header(header, bl, t);
- return 0;
-}
-
-int DBObjectMap::rm_keys(const ghobject_t &oid,
- const set<string> &to_clear,
- const SequencerPosition *spos)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- KeyValueDB::Transaction t = db->get_transaction();
- if (check_spos(oid, header, spos))
- return 0;
- t->rmkeys(user_prefix(header), to_clear);
- if (!header->parent) {
- return db->submit_transaction(t);
- }
-
- ceph_assert(state.legacy);
-
- {
- // We only get here for legacy (v2) stores
- // Copy up all keys from parent excluding to_clear
- // and remove parent
- // This eliminates a v2 format use of complete for this oid only
- map<string, bufferlist> to_write;
- ObjectMapIterator iter = _get_iterator(header);
- for (iter->seek_to_first() ; iter->valid() ; iter->next()) {
- if (iter->status())
- return iter->status();
- if (!to_clear.count(iter->key()))
- to_write[iter->key()] = iter->value();
- }
- t->set(user_prefix(header), to_write);
- } // destruct iter which has parent in_use
-
- copy_up_header(header, t);
- Header parent = lookup_parent(header);
- if (!parent)
- return -EINVAL;
- parent->num_children--;
- _clear(parent, t);
- header->parent = 0;
- set_map_header(hl, oid, *header, t);
- t->rmkeys_by_prefix(complete_prefix(header));
- return db->submit_transaction(t);
-}
-
-int DBObjectMap::clear_keys_header(const ghobject_t &oid,
- const SequencerPosition *spos)
-{
- KeyValueDB::Transaction t = db->get_transaction();
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- if (check_spos(oid, header, spos))
- return 0;
-
- // save old attrs
- KeyValueDB::Iterator iter = db->get_iterator(xattr_prefix(header));
- if (!iter)
- return -EINVAL;
- map<string, bufferlist> attrs;
- for (iter->seek_to_first(); !iter->status() && iter->valid(); iter->next())
- attrs.insert(make_pair(iter->key(), iter->value()));
- if (iter->status())
- return iter->status();
-
- // remove current header
- remove_map_header(hl, oid, header, t);
- ceph_assert(header->num_children > 0);
- header->num_children--;
- int r = _clear(header, t);
- if (r < 0)
- return r;
-
- // create new header
- Header newheader = generate_new_header(oid, Header());
- set_map_header(hl, oid, *newheader, t);
- if (!attrs.empty())
- t->set(xattr_prefix(newheader), attrs);
- return db->submit_transaction(t);
-}
-
-int DBObjectMap::get(const ghobject_t &oid,
- bufferlist *_header,
- map<string, bufferlist> *out)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- _get_header(header, _header);
- ObjectMapIterator iter = _get_iterator(header);
- for (iter->seek_to_first(); iter->valid(); iter->next()) {
- if (iter->status())
- return iter->status();
- out->insert(make_pair(iter->key(), iter->value()));
- }
- return 0;
-}
-
-int DBObjectMap::get_keys(const ghobject_t &oid,
- set<string> *keys)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- ObjectMapIterator iter = _get_iterator(header);
- for (iter->seek_to_first(); iter->valid(); iter->next()) {
- if (iter->status())
- return iter->status();
- keys->insert(iter->key());
- }
- return 0;
-}
-
-int DBObjectMap::scan(Header header,
- const set<string> &in_keys,
- set<string> *out_keys,
- map<string, bufferlist> *out_values)
-{
- ObjectMapIterator db_iter = _get_iterator(header);
- for (set<string>::const_iterator key_iter = in_keys.begin();
- key_iter != in_keys.end();
- ++key_iter) {
- db_iter->lower_bound(*key_iter);
- if (db_iter->status())
- return db_iter->status();
- if (db_iter->valid() && db_iter->key() == *key_iter) {
- if (out_keys)
- out_keys->insert(*key_iter);
- if (out_values)
- out_values->insert(make_pair(db_iter->key(), db_iter->value()));
- }
- }
- return 0;
-}
-
-int DBObjectMap::get_values(const ghobject_t &oid,
- const set<string> &keys,
- map<string, bufferlist> *out)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- return scan(header, keys, 0, out);
-}
-
-int DBObjectMap::check_keys(const ghobject_t &oid,
- const set<string> &keys,
- set<string> *out)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- return scan(header, keys, out, 0);
-}
-
-int DBObjectMap::get_xattrs(const ghobject_t &oid,
- const set<string> &to_get,
- map<string, bufferlist> *out)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- return db->get(xattr_prefix(header), to_get, out);
-}
-
-int DBObjectMap::get_all_xattrs(const ghobject_t &oid,
- set<string> *out)
-{
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- KeyValueDB::Iterator iter = db->get_iterator(xattr_prefix(header));
- if (!iter)
- return -EINVAL;
- for (iter->seek_to_first(); !iter->status() && iter->valid(); iter->next())
- out->insert(iter->key());
- return iter->status();
-}
-
-int DBObjectMap::set_xattrs(const ghobject_t &oid,
- const map<string, bufferlist> &to_set,
- const SequencerPosition *spos)
-{
- KeyValueDB::Transaction t = db->get_transaction();
- MapHeaderLock hl(this, oid);
- Header header = lookup_create_map_header(hl, oid, t);
- if (!header)
- return -EINVAL;
- if (check_spos(oid, header, spos))
- return 0;
- t->set(xattr_prefix(header), to_set);
- return db->submit_transaction(t);
-}
-
-int DBObjectMap::remove_xattrs(const ghobject_t &oid,
- const set<string> &to_remove,
- const SequencerPosition *spos)
-{
- KeyValueDB::Transaction t = db->get_transaction();
- MapHeaderLock hl(this, oid);
- Header header = lookup_map_header(hl, oid);
- if (!header)
- return -ENOENT;
- if (check_spos(oid, header, spos))
- return 0;
- t->rmkeys(xattr_prefix(header), to_remove);
- return db->submit_transaction(t);
-}
-
-// ONLY USED FOR TESTING
-// Set version to 2 to avoid asserts
-int DBObjectMap::legacy_clone(const ghobject_t &oid,
- const ghobject_t &target,
- const SequencerPosition *spos)
-{
- state.legacy = true;
-
- if (oid == target)
- return 0;
-
- MapHeaderLock _l1(this, std::min(oid, target));
- MapHeaderLock _l2(this, std::max(oid, target));
- MapHeaderLock *lsource, *ltarget;
- if (oid > target) {
- lsource = &_l2;
- ltarget= &_l1;
- } else {
- lsource = &_l1;
- ltarget= &_l2;
- }
-
- KeyValueDB::Transaction t = db->get_transaction();
- {
- Header destination = lookup_map_header(*ltarget, target);
- if (destination) {
- if (check_spos(target, destination, spos))
- return 0;
- destination->num_children--;
- remove_map_header(*ltarget, target, destination, t);
- _clear(destination, t);
- }
- }
-
- Header parent = lookup_map_header(*lsource, oid);
- if (!parent)
- return db->submit_transaction(t);
-
- Header source = generate_new_header(oid, parent);
- Header destination = generate_new_header(target, parent);
- if (spos)
- destination->spos = *spos;
-
- parent->num_children = 2;
- set_header(parent, t);
- set_map_header(*lsource, oid, *source, t);
- set_map_header(*ltarget, target, *destination, t);
-
- map<string, bufferlist> to_set;
- KeyValueDB::Iterator xattr_iter = db->get_iterator(xattr_prefix(parent));
- for (xattr_iter->seek_to_first();
- xattr_iter->valid();
- xattr_iter->next())
- to_set.insert(make_pair(xattr_iter->key(), xattr_iter->value()));
- t->set(xattr_prefix(source), to_set);
- t->set(xattr_prefix(destination), to_set);
- t->rmkeys_by_prefix(xattr_prefix(parent));
- return db->submit_transaction(t);
-}
-
-int DBObjectMap::clone(const ghobject_t &oid,
- const ghobject_t &target,
- const SequencerPosition *spos)
-{
- if (oid == target)
- return 0;
-
- MapHeaderLock _l1(this, std::min(oid, target));
- MapHeaderLock _l2(this, std::max(oid, target));
- MapHeaderLock *lsource, *ltarget;
- if (oid > target) {
- lsource = &_l2;
- ltarget= &_l1;
- } else {
- lsource = &_l1;
- ltarget= &_l2;
- }
-
- KeyValueDB::Transaction t = db->get_transaction();
- {
- Header destination = lookup_map_header(*ltarget, target);
- if (destination) {
- if (check_spos(target, destination, spos))
- return 0;
- destination->num_children--;
- remove_map_header(*ltarget, target, destination, t);
- _clear(destination, t);
- }
- }
-
- Header source = lookup_map_header(*lsource, oid);
- if (!source)
- return db->submit_transaction(t);
-
- Header destination = generate_new_header(target, Header());
- if (spos)
- destination->spos = *spos;
-
- set_map_header(*ltarget, target, *destination, t);
-
- bufferlist bl;
- int r = _get_header(source, &bl);
- if (r < 0)
- return r;
- _set_header(destination, bl, t);
-
- map<string, bufferlist> to_set;
- KeyValueDB::Iterator xattr_iter = db->get_iterator(xattr_prefix(source));
- for (xattr_iter->seek_to_first();
- xattr_iter->valid();
- xattr_iter->next())
- to_set.insert(make_pair(xattr_iter->key(), xattr_iter->value()));
- t->set(xattr_prefix(destination), to_set);
-
- map<string, bufferlist> to_write;
- ObjectMapIterator iter = _get_iterator(source);
- for (iter->seek_to_first() ; iter->valid() ; iter->next()) {
- if (iter->status())
- return iter->status();
- to_write[iter->key()] = iter->value();
- }
- t->set(user_prefix(destination), to_write);
-
- return db->submit_transaction(t);
-}
-
-int DBObjectMap::upgrade_to_v2()
-{
- dout(1) << __func__ << " start" << dendl;
- KeyValueDB::Iterator iter = db->get_iterator(HOBJECT_TO_SEQ);
- iter->seek_to_first();
- while (iter->valid()) {
- unsigned count = 0;
- KeyValueDB::Transaction t = db->get_transaction();
- set<string> remove;
- map<string, bufferlist> add;
- for (;
- iter->valid() && count < 300;
- iter->next()) {
- dout(20) << __func__ << " key is " << iter->key() << dendl;
- int r = is_buggy_ghobject_key_v1(cct, iter->key());
- if (r < 0) {
- derr << __func__ << " bad key '" << iter->key() << "'" << dendl;
- return r;
- }
- if (!r) {
- dout(20) << __func__ << " " << iter->key() << " ok" << dendl;
- continue;
- }
-
- // decode header to get oid
- _Header hdr;
- bufferlist bl = iter->value();
- auto bliter = bl.cbegin();
- hdr.decode(bliter);
-
- string newkey(ghobject_key(hdr.oid));
- dout(20) << __func__ << " " << iter->key() << " -> " << newkey << dendl;
- add[newkey] = iter->value();
- remove.insert(iter->key());
- ++count;
- }
-
- if (!remove.empty()) {
- dout(20) << __func__ << " updating " << remove.size() << " keys" << dendl;
- t->rmkeys(HOBJECT_TO_SEQ, remove);
- t->set(HOBJECT_TO_SEQ, add);
- int r = db->submit_transaction(t);
- if (r < 0)
- return r;
- }
- }
-
- state.v = 2;
-
- set_state();
- return 0;
-}
-
-void DBObjectMap::set_state()
-{
- std::lock_guard l{header_lock};
- KeyValueDB::Transaction t = db->get_transaction();
- write_state(t);
- int ret = db->submit_transaction_sync(t);
- ceph_assert(ret == 0);
- dout(1) << __func__ << " done" << dendl;
- return;
-}
-
-int DBObjectMap::get_state()
-{
- map<string, bufferlist> result;
- set<string> to_get;
- to_get.insert(GLOBAL_STATE_KEY);
- int r = db->get(SYS_PREFIX, to_get, &result);
- if (r < 0)
- return r;
- if (!result.empty()) {
- auto bliter = result.begin()->second.cbegin();
- state.decode(bliter);
- } else {
- // New store
- state.v = State::CUR_VERSION;
- state.seq = 1;
- state.legacy = false;
- }
- return 0;
-}
-
-int DBObjectMap::init(bool do_upgrade)
-{
- int ret = get_state();
- if (ret < 0)
- return ret;
- if (state.v < 1) {
- dout(1) << "DBObjectMap is *very* old; upgrade to an older version first"
- << dendl;
- return -ENOTSUP;
- }
- if (state.v < 2) { // Needs upgrade
- if (!do_upgrade) {
- dout(1) << "DOBjbectMap requires an upgrade,"
- << " set filestore_update_to"
- << dendl;
- return -ENOTSUP;
- } else {
- int r = upgrade_to_v2();
- if (r < 0)
- return r;
- }
- }
- ostringstream ss;
- int errors = check(ss, true);
- if (errors) {
- derr << ss.str() << dendl;
- if (errors > 0)
- return -EINVAL;
- }
- dout(20) << "(init)dbobjectmap: seq is " << state.seq << dendl;
- return 0;
-}
-
-int DBObjectMap::sync(const ghobject_t *oid,
- const SequencerPosition *spos) {
- KeyValueDB::Transaction t = db->get_transaction();
- if (oid) {
- ceph_assert(spos);
- MapHeaderLock hl(this, *oid);
- Header header = lookup_map_header(hl, *oid);
- if (header) {
- dout(10) << "oid: " << *oid << " setting spos to "
- << *spos << dendl;
- header->spos = *spos;
- set_map_header(hl, *oid, *header, t);
- }
- /* It may appear that this and the identical portion of the else
- * block can combined below, but in this block, the transaction
- * must be submitted under *both* the MapHeaderLock and the full
- * header_lock.
- *
- * See 2b63dd25fc1c73fa42e52e9ea4ab5a45dd9422a0 and bug 9891.
- */
- std::lock_guard l{header_lock};
- write_state(t);
- return db->submit_transaction_sync(t);
- } else {
- std::lock_guard l{header_lock};
- write_state(t);
- return db->submit_transaction_sync(t);
- }
-}
-
-int DBObjectMap::write_state(KeyValueDB::Transaction _t) {
- ceph_assert(ceph_mutex_is_locked_by_me(header_lock));
- dout(20) << "dbobjectmap: seq is " << state.seq << dendl;
- KeyValueDB::Transaction t = _t ? _t : db->get_transaction();
- bufferlist bl;
- state.encode(bl);
- map<string, bufferlist> to_write;
- to_write[GLOBAL_STATE_KEY] = bl;
- t->set(SYS_PREFIX, to_write);
- return _t ? 0 : db->submit_transaction(t);
-}
-
-
-DBObjectMap::Header DBObjectMap::_lookup_map_header(
- const MapHeaderLock &l,
- const ghobject_t &oid)
-{
- ceph_assert(l.get_locked() == oid);
-
- _Header *header = new _Header();
- {
- std::lock_guard l{cache_lock};
- if (caches.lookup(oid, header)) {
- ceph_assert(!in_use.count(header->seq));
- in_use.insert(header->seq);
- return Header(header, RemoveOnDelete(this));
- }
- }
-
- bufferlist out;
- int r = db->get(HOBJECT_TO_SEQ, map_header_key(oid), &out);
- if (r < 0 || out.length()==0) {
- delete header;
- return Header();
- }
-
- Header ret(header, RemoveOnDelete(this));
- auto iter = out.cbegin();
- ret->decode(iter);
- {
- std::lock_guard l{cache_lock};
- caches.add(oid, *ret);
- }
-
- ceph_assert(!in_use.count(header->seq));
- in_use.insert(header->seq);
- return ret;
-}
-
-DBObjectMap::Header DBObjectMap::_generate_new_header(const ghobject_t &oid,
- Header parent)
-{
- Header header = Header(new _Header(), RemoveOnDelete(this));
- header->seq = state.seq++;
- if (parent) {
- header->parent = parent->seq;
- header->spos = parent->spos;
- }
- header->num_children = 1;
- header->oid = oid;
- ceph_assert(!in_use.count(header->seq));
- in_use.insert(header->seq);
-
- write_state();
- return header;
-}
-
-DBObjectMap::Header DBObjectMap::lookup_parent(Header input)
-{
- std::unique_lock l{header_lock};
- header_cond.wait(l, [&input, this] { return !in_use.count(input->parent); });
- map<string, bufferlist> out;
- set<string> keys;
- keys.insert(HEADER_KEY);
-
- dout(20) << "lookup_parent: parent " << input->parent
- << " for seq " << input->seq << dendl;
- int r = db->get(sys_parent_prefix(input), keys, &out);
- if (r < 0) {
- ceph_abort();
- return Header();
- }
- if (out.empty()) {
- ceph_abort();
- return Header();
- }
-
- Header header = Header(new _Header(), RemoveOnDelete(this));
- auto iter = out.begin()->second.cbegin();
- header->decode(iter);
- ceph_assert(header->seq == input->parent);
- dout(20) << "lookup_parent: parent seq is " << header->seq << " with parent "
- << header->parent << dendl;
- in_use.insert(header->seq);
- return header;
-}
-
-DBObjectMap::Header DBObjectMap::lookup_create_map_header(
- const MapHeaderLock &hl,
- const ghobject_t &oid,
- KeyValueDB::Transaction t)
-{
- std::lock_guard l{header_lock};
- Header header = _lookup_map_header(hl, oid);
- if (!header) {
- header = _generate_new_header(oid, Header());
- set_map_header(hl, oid, *header, t);
- }
- return header;
-}
-
-void DBObjectMap::clear_header(Header header, KeyValueDB::Transaction t)
-{
- dout(20) << "clear_header: clearing seq " << header->seq << dendl;
- t->rmkeys_by_prefix(user_prefix(header));
- t->rmkeys_by_prefix(sys_prefix(header));
- if (state.legacy)
- t->rmkeys_by_prefix(complete_prefix(header)); // Needed when header.parent != 0
- t->rmkeys_by_prefix(xattr_prefix(header));
- set<string> keys;
- keys.insert(header_key(header->seq));
- t->rmkeys(USER_PREFIX, keys);
-}
-
-void DBObjectMap::set_header(Header header, KeyValueDB::Transaction t)
-{
- dout(20) << "set_header: setting seq " << header->seq << dendl;
- map<string, bufferlist> to_write;
- header->encode(to_write[HEADER_KEY]);
- t->set(sys_prefix(header), to_write);
-}
-
-void DBObjectMap::remove_map_header(
- const MapHeaderLock &l,
- const ghobject_t &oid,
- Header header,
- KeyValueDB::Transaction t)
-{
- ceph_assert(l.get_locked() == oid);
- dout(20) << "remove_map_header: removing " << header->seq
- << " oid " << oid << dendl;
- set<string> to_remove;
- to_remove.insert(map_header_key(oid));
- t->rmkeys(HOBJECT_TO_SEQ, to_remove);
- {
- std::lock_guard l{cache_lock};
- caches.clear(oid);
- }
-}
-
-void DBObjectMap::set_map_header(
- const MapHeaderLock &l,
- const ghobject_t &oid, _Header header,
- KeyValueDB::Transaction t)
-{
- ceph_assert(l.get_locked() == oid);
- dout(20) << "set_map_header: setting " << header.seq
- << " oid " << oid << " parent seq "
- << header.parent << dendl;
- map<string, bufferlist> to_set;
- header.encode(to_set[map_header_key(oid)]);
- t->set(HOBJECT_TO_SEQ, to_set);
- {
- std::lock_guard l{cache_lock};
- caches.add(oid, header);
- }
-}
-
-bool DBObjectMap::check_spos(const ghobject_t &oid,
- Header header,
- const SequencerPosition *spos)
-{
- if (!spos || *spos > header->spos) {
- stringstream out;
- if (spos)
- dout(10) << "oid: " << oid << " not skipping op, *spos "
- << *spos << dendl;
- else
- dout(10) << "oid: " << oid << " not skipping op, *spos "
- << "empty" << dendl;
- dout(10) << " > header.spos " << header->spos << dendl;
- return false;
- } else {
- dout(10) << "oid: " << oid << " skipping op, *spos " << *spos
- << " <= header.spos " << header->spos << dendl;
- return true;
- }
-}
-
-int DBObjectMap::list_objects(vector<ghobject_t> *out)
-{
- KeyValueDB::Iterator iter = db->get_iterator(HOBJECT_TO_SEQ);
- for (iter->seek_to_first(); iter->valid(); iter->next()) {
- bufferlist bl = iter->value();
- auto bliter = bl.cbegin();
- _Header header;
- header.decode(bliter);
- out->push_back(header.oid);
- }
- return 0;
-}
-
-int DBObjectMap::list_object_headers(vector<_Header> *out)
-{
- int error = 0;
- KeyValueDB::Iterator iter = db->get_iterator(HOBJECT_TO_SEQ);
- for (iter->seek_to_first(); iter->valid(); iter->next()) {
- bufferlist bl = iter->value();
- auto bliter = bl.cbegin();
- _Header header;
- header.decode(bliter);
- out->push_back(header);
- while (header.parent) {
- set<string> to_get;
- map<string, bufferlist> got;
- to_get.insert(HEADER_KEY);
- db->get(sys_parent_prefix(header), to_get, &got);
- if (got.empty()) {
- dout(0) << "Missing: seq " << header.parent << dendl;
- error = -ENOENT;
- break;
- } else {
- bl = got.begin()->second;
- auto bliter = bl.cbegin();
- header.decode(bliter);
- out->push_back(header);
- }
- }
- }
- return error;
-}
-
-ostream& operator<<(ostream& out, const DBObjectMap::_Header& h)
-{
- out << "seq=" << h.seq << " parent=" << h.parent
- << " num_children=" << h.num_children
- << " ghobject=" << h.oid;
- return out;
-}
-
-int DBObjectMap::rename(const ghobject_t &from,
- const ghobject_t &to,
- const SequencerPosition *spos)
-{
- if (from == to)
- return 0;
-
- MapHeaderLock _l1(this, std::min(from, to));
- MapHeaderLock _l2(this, std::max(from, to));
- MapHeaderLock *lsource, *ltarget;
- if (from > to) {
- lsource = &_l2;
- ltarget= &_l1;
- } else {
- lsource = &_l1;
- ltarget= &_l2;
- }
-
- KeyValueDB::Transaction t = db->get_transaction();
- {
- Header destination = lookup_map_header(*ltarget, to);
- if (destination) {
- if (check_spos(to, destination, spos))
- return 0;
- destination->num_children--;
- remove_map_header(*ltarget, to, destination, t);
- _clear(destination, t);
- }
- }
-
- Header hdr = lookup_map_header(*lsource, from);
- if (!hdr)
- return db->submit_transaction(t);
-
- remove_map_header(*lsource, from, hdr, t);
- hdr->oid = to;
- set_map_header(*ltarget, to, *hdr, t);
-
- return db->submit_transaction(t);
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-#ifndef DBOBJECTMAP_DB_H
-#define DBOBJECTMAP_DB_H
-
-#include "include/buffer_fwd.h"
-#include <set>
-#include <map>
-#include <string>
-
-#include <vector>
-#include <boost/scoped_ptr.hpp>
-
-#include "os/ObjectMap.h"
-#include "kv/KeyValueDB.h"
-#include "osd/osd_types.h"
-#include "common/ceph_mutex.h"
-#include "common/simple_cache.hpp"
-#include <boost/optional/optional_io.hpp>
-
-#include "SequencerPosition.h"
-
-/**
- * DBObjectMap: Implements ObjectMap in terms of KeyValueDB
- *
- * Prefix space structure:
- *
- * @see complete_prefix
- * @see user_prefix
- * @see sys_prefix
- *
- * - HOBJECT_TO_SEQ: Contains leaf mapping from ghobject_t->header.seq and
- * corresponding omap header
- * - SYS_PREFIX: GLOBAL_STATE_KEY - contains next seq number
- * @see State
- * @see write_state
- * @see init
- * @see generate_new_header
- * - USER_PREFIX + header_key(header->seq) + USER_PREFIX
- * : key->value for header->seq
- * - USER_PREFIX + header_key(header->seq) + COMPLETE_PREFIX: see below
- * - USER_PREFIX + header_key(header->seq) + XATTR_PREFIX: xattrs
- * - USER_PREFIX + header_key(header->seq) + SYS_PREFIX
- * : USER_HEADER_KEY - omap header for header->seq
- * : HEADER_KEY - encoding of header for header->seq
- *
- * For each node (represented by a header), we
- * store three mappings: the key mapping, the complete mapping, and the parent.
- * The complete mapping (COMPLETE_PREFIX space) is key->key. Each x->y entry in
- * this mapping indicates that the key mapping contains all entries on [x,y).
- * Note, max std::string is represented by "", so ""->"" indicates that the parent
- * is unnecessary (@see rm_keys). When looking up a key not contained in the
- * the complete std::set, we have to check the parent if we don't find it in the
- * key std::set. During rm_keys, we copy keys from the parent and update the
- * complete std::set to reflect the change @see rm_keys.
- */
-class DBObjectMap : public ObjectMap {
-public:
-
- KeyValueDB *get_db() override { return db.get(); }
-
- /**
- * Serializes access to next_seq as well as the in_use std::set
- */
- ceph::mutex header_lock = ceph::make_mutex("DBOBjectMap");
- ceph::condition_variable header_cond;
- ceph::condition_variable map_header_cond;
-
- /**
- * Std::Set of headers currently in use
- */
- std::set<uint64_t> in_use;
- std::set<ghobject_t> map_header_in_use;
-
- /**
- * Takes the map_header_in_use entry in constructor, releases in
- * destructor
- */
- class MapHeaderLock {
- DBObjectMap *db;
- boost::optional<ghobject_t> locked;
-
- MapHeaderLock(const MapHeaderLock &);
- MapHeaderLock &operator=(const MapHeaderLock &);
- public:
- explicit MapHeaderLock(DBObjectMap *db) : db(db) {}
- MapHeaderLock(DBObjectMap *db, const ghobject_t &oid) : db(db), locked(oid) {
- std::unique_lock l{db->header_lock};
- db->map_header_cond.wait(l, [db, this] {
- return !db->map_header_in_use.count(*locked);
- });
- db->map_header_in_use.insert(*locked);
- }
-
- const ghobject_t &get_locked() const {
- ceph_assert(locked);
- return *locked;
- }
-
- void swap(MapHeaderLock &o) {
- ceph_assert(db == o.db);
-
- // centos6's boost optional doesn't seem to have swap :(
- boost::optional<ghobject_t> _locked = o.locked;
- o.locked = locked;
- locked = _locked;
- }
-
- ~MapHeaderLock() {
- if (locked) {
- std::lock_guard l{db->header_lock};
- ceph_assert(db->map_header_in_use.count(*locked));
- db->map_header_cond.notify_all();
- db->map_header_in_use.erase(*locked);
- }
- }
- };
-
- DBObjectMap(CephContext* cct, KeyValueDB *db)
- : ObjectMap(cct, db),
- caches(cct->_conf->filestore_omap_header_cache_size)
- {}
-
- int set_keys(
- const ghobject_t &oid,
- const std::map<std::string, ceph::buffer::list> &set,
- const SequencerPosition *spos=0
- ) override;
-
- int set_header(
- const ghobject_t &oid,
- const ceph::buffer::list &bl,
- const SequencerPosition *spos=0
- ) override;
-
- int get_header(
- const ghobject_t &oid,
- ceph::buffer::list *bl
- ) override;
-
- int clear(
- const ghobject_t &oid,
- const SequencerPosition *spos=0
- ) override;
-
- int clear_keys_header(
- const ghobject_t &oid,
- const SequencerPosition *spos=0
- ) override;
-
- int rm_keys(
- const ghobject_t &oid,
- const std::set<std::string> &to_clear,
- const SequencerPosition *spos=0
- ) override;
-
- int get(
- const ghobject_t &oid,
- ceph::buffer::list *header,
- std::map<std::string, ceph::buffer::list> *out
- ) override;
-
- int get_keys(
- const ghobject_t &oid,
- std::set<std::string> *keys
- ) override;
-
- int get_values(
- const ghobject_t &oid,
- const std::set<std::string> &keys,
- std::map<std::string, ceph::buffer::list> *out
- ) override;
-
- int check_keys(
- const ghobject_t &oid,
- const std::set<std::string> &keys,
- std::set<std::string> *out
- ) override;
-
- int get_xattrs(
- const ghobject_t &oid,
- const std::set<std::string> &to_get,
- std::map<std::string, ceph::buffer::list> *out
- ) override;
-
- int get_all_xattrs(
- const ghobject_t &oid,
- std::set<std::string> *out
- ) override;
-
- int set_xattrs(
- const ghobject_t &oid,
- const std::map<std::string, ceph::buffer::list> &to_set,
- const SequencerPosition *spos=0
- ) override;
-
- int remove_xattrs(
- const ghobject_t &oid,
- const std::set<std::string> &to_remove,
- const SequencerPosition *spos=0
- ) override;
-
- int clone(
- const ghobject_t &oid,
- const ghobject_t &target,
- const SequencerPosition *spos=0
- ) override;
-
- int rename(
- const ghobject_t &from,
- const ghobject_t &to,
- const SequencerPosition *spos=0
- ) override;
-
- int legacy_clone(
- const ghobject_t &oid,
- const ghobject_t &target,
- const SequencerPosition *spos=0
- ) override;
-
- /// Read initial state from backing store
- int get_state();
- /// Write current state settings to DB
- void set_state();
- /// Read initial state and upgrade or initialize state
- int init(bool upgrade = false);
-
- /// Upgrade store to current version
- int upgrade_to_v2();
-
- /// Consistency check, debug, there must be no parallel writes
- int check(std::ostream &out, bool repair = false, bool force = false) override;
-
- /// Ensure that all previous operations are durable
- int sync(const ghobject_t *oid=0, const SequencerPosition *spos=0) override;
-
- void compact() override {
- ceph_assert(db);
- db->compact();
- }
-
- /// Util, get all objects, there must be no other concurrent access
- int list_objects(std::vector<ghobject_t> *objs ///< [out] objects
- );
-
- struct _Header;
- // Util, get all object headers, there must be no other concurrent access
- int list_object_headers(std::vector<_Header> *out ///< [out] headers
- );
-
- ObjectMapIterator get_iterator(const ghobject_t &oid) override;
-
- static const std::string USER_PREFIX;
- static const std::string XATTR_PREFIX;
- static const std::string SYS_PREFIX;
- static const std::string COMPLETE_PREFIX;
- static const std::string HEADER_KEY;
- static const std::string USER_HEADER_KEY;
- static const std::string GLOBAL_STATE_KEY;
- static const std::string HOBJECT_TO_SEQ;
-
- /// Legacy
- static const std::string LEAF_PREFIX;
- static const std::string REVERSE_LEAF_PREFIX;
-
- /// persistent state for store @see generate_header
- struct State {
- static const __u8 CUR_VERSION = 3;
- __u8 v;
- uint64_t seq;
- // legacy is false when complete regions never used
- bool legacy;
- State() : v(0), seq(1), legacy(false) {}
- explicit State(uint64_t seq) : v(0), seq(seq), legacy(false) {}
-
- void encode(ceph::buffer::list &bl) const {
- ENCODE_START(3, 1, bl);
- encode(v, bl);
- encode(seq, bl);
- encode(legacy, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(ceph::buffer::list::const_iterator &bl) {
- DECODE_START(3, bl);
- if (struct_v >= 2)
- decode(v, bl);
- else
- v = 0;
- decode(seq, bl);
- if (struct_v >= 3)
- decode(legacy, bl);
- else
- legacy = false;
- DECODE_FINISH(bl);
- }
-
- void dump(ceph::Formatter *f) const {
- f->dump_unsigned("v", v);
- f->dump_unsigned("seq", seq);
- f->dump_bool("legacy", legacy);
- }
-
- static void generate_test_instances(std::list<State*> &o) {
- o.push_back(new State(0));
- o.push_back(new State(20));
- }
- } state;
-
- struct _Header {
- uint64_t seq;
- uint64_t parent;
- uint64_t num_children;
-
- ghobject_t oid;
-
- SequencerPosition spos;
-
- void encode(ceph::buffer::list &bl) const {
- coll_t unused;
- ENCODE_START(2, 1, bl);
- encode(seq, bl);
- encode(parent, bl);
- encode(num_children, bl);
- encode(unused, bl);
- encode(oid, bl);
- encode(spos, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(ceph::buffer::list::const_iterator &bl) {
- coll_t unused;
- DECODE_START(2, bl);
- decode(seq, bl);
- decode(parent, bl);
- decode(num_children, bl);
- decode(unused, bl);
- decode(oid, bl);
- if (struct_v >= 2)
- decode(spos, bl);
- DECODE_FINISH(bl);
- }
-
- void dump(ceph::Formatter *f) const {
- f->dump_unsigned("seq", seq);
- f->dump_unsigned("parent", parent);
- f->dump_unsigned("num_children", num_children);
- f->dump_stream("oid") << oid;
- }
-
- static void generate_test_instances(std::list<_Header*> &o) {
- o.push_back(new _Header);
- o.push_back(new _Header);
- o.back()->parent = 20;
- o.back()->seq = 30;
- }
-
- size_t length() {
- return sizeof(_Header);
- }
-
- _Header() : seq(0), parent(0), num_children(1) {}
- };
-
- /// Std::String munging (public for testing)
- static std::string ghobject_key(const ghobject_t &oid);
- static std::string ghobject_key_v0(coll_t c, const ghobject_t &oid);
- static int is_buggy_ghobject_key_v1(CephContext* cct,
- const std::string &in);
-private:
- /// Implicit lock on Header->seq
- typedef std::shared_ptr<_Header> Header;
- ceph::mutex cache_lock = ceph::make_mutex("DBObjectMap::CacheLock");
- SimpleLRU<ghobject_t, _Header> caches;
-
- std::string map_header_key(const ghobject_t &oid);
- std::string header_key(uint64_t seq);
- std::string complete_prefix(Header header);
- std::string user_prefix(Header header);
- std::string sys_prefix(Header header);
- std::string xattr_prefix(Header header);
- std::string sys_parent_prefix(_Header header);
- std::string sys_parent_prefix(Header header) {
- return sys_parent_prefix(*header);
- }
-
- class EmptyIteratorImpl : public ObjectMapIteratorImpl {
- public:
- int seek_to_first() override { return 0; }
- int seek_to_last() { return 0; }
- int upper_bound(const std::string &after) override { return 0; }
- int lower_bound(const std::string &to) override { return 0; }
- bool valid() override { return false; }
- int next() override { ceph_abort(); return 0; }
- std::string key() override { ceph_abort(); return ""; }
- ceph::buffer::list value() override { ceph_abort(); return ceph::buffer::list(); }
- std::string_view value_as_sv() override { ceph_abort(); return std::string_view(); }
- int status() override { return 0; }
- };
-
-
- /// Iterator
- class DBObjectMapIteratorImpl : public ObjectMapIteratorImpl {
- public:
- DBObjectMap *map;
-
- /// NOTE: implicit lock hlock->get_locked() when returned out of the class
- MapHeaderLock hlock;
- /// NOTE: implicit lock on header->seq AND for all ancestors
- Header header;
-
- /// parent_iter == NULL iff no parent
- std::shared_ptr<DBObjectMapIteratorImpl> parent_iter;
- KeyValueDB::Iterator key_iter;
- KeyValueDB::Iterator complete_iter;
-
- /// cur_iter points to currently valid iterator
- std::shared_ptr<ObjectMapIteratorImpl> cur_iter;
- int r;
-
- /// init() called, key_iter, complete_iter, parent_iter filled in
- bool ready;
- /// past end
- bool invalid;
-
- DBObjectMapIteratorImpl(DBObjectMap *map, Header header) :
- map(map), hlock(map), header(header), r(0), ready(false), invalid(true) {}
- int seek_to_first() override;
- int seek_to_last();
- int upper_bound(const std::string &after) override;
- int lower_bound(const std::string &to) override;
- bool valid() override;
- int next() override;
- std::string key() override;
- ceph::buffer::list value() override;
- std::string_view value_as_sv() override;
- int status() override;
-
- bool on_parent() {
- return cur_iter == parent_iter;
- }
-
- /// skips to next valid parent entry
- int next_parent();
-
- /// first parent() >= to
- int lower_bound_parent(const std::string &to);
-
- /**
- * Tests whether to_test is in complete region
- *
- * postcondition: complete_iter will be max s.t. complete_iter->value > to_test
- */
- int in_complete_region(const std::string &to_test, ///< [in] key to test
- std::string *begin, ///< [out] beginning of region
- std::string *end ///< [out] end of region
- ); ///< @returns true if to_test is in the complete region, else false
-
- private:
- int init();
- bool valid_parent();
- int adjust();
- };
-
- typedef std::shared_ptr<DBObjectMapIteratorImpl> DBObjectMapIterator;
- DBObjectMapIterator _get_iterator(Header header) {
- return std::make_shared<DBObjectMapIteratorImpl>(this, header);
- }
-
- /// sys
-
- /// Removes node corresponding to header
- void clear_header(Header header, KeyValueDB::Transaction t);
-
- /// Std::Set node containing input to new contents
- void set_header(Header input, KeyValueDB::Transaction t);
-
- /// Remove leaf node corresponding to oid in c
- void remove_map_header(
- const MapHeaderLock &l,
- const ghobject_t &oid,
- Header header,
- KeyValueDB::Transaction t);
-
- /// Std::Set leaf node for c and oid to the value of header
- void set_map_header(
- const MapHeaderLock &l,
- const ghobject_t &oid, _Header header,
- KeyValueDB::Transaction t);
-
- /// Std::Set leaf node for c and oid to the value of header
- bool check_spos(const ghobject_t &oid,
- Header header,
- const SequencerPosition *spos);
-
- /// Lookup or create header for c oid
- Header lookup_create_map_header(
- const MapHeaderLock &l,
- const ghobject_t &oid,
- KeyValueDB::Transaction t);
-
- /**
- * Generate new header for c oid with new seq number
- *
- * Has the side effect of synchronously saving the new DBObjectMap state
- */
- Header _generate_new_header(const ghobject_t &oid, Header parent);
- Header generate_new_header(const ghobject_t &oid, Header parent) {
- std::lock_guard l{header_lock};
- return _generate_new_header(oid, parent);
- }
-
- /// Lookup leaf header for c oid
- Header _lookup_map_header(
- const MapHeaderLock &l,
- const ghobject_t &oid);
- Header lookup_map_header(
- const MapHeaderLock &l2,
- const ghobject_t &oid) {
- std::lock_guard l{header_lock};
- return _lookup_map_header(l2, oid);
- }
-
- /// Lookup header node for input
- Header lookup_parent(Header input);
-
-
- /// Helpers
- int _get_header(Header header, ceph::buffer::list *bl);
-
- /// Scan keys in header into out_keys and out_values (if nonnull)
- int scan(Header header,
- const std::set<std::string> &in_keys,
- std::set<std::string> *out_keys,
- std::map<std::string, ceph::buffer::list> *out_values);
-
- /// Remove header and all related prefixes
- int _clear(Header header,
- KeyValueDB::Transaction t);
-
- /* Scan complete region bumping *begin to the beginning of any
- * containing region and adding all complete region keys between
- * the updated begin and end to the complete_keys_to_remove std::set */
- int merge_new_complete(DBObjectMapIterator &iter,
- std::string *begin,
- const std::string &end,
- std::set<std::string> *complete_keys_to_remove);
-
- /// Writes out State (mainly next_seq)
- int write_state(KeyValueDB::Transaction _t =
- KeyValueDB::Transaction());
-
- /// Copies header entry from parent @see rm_keys
- int copy_up_header(Header header,
- KeyValueDB::Transaction t);
-
- /// Sets header @see set_header
- void _set_header(Header header, const ceph::buffer::list &bl,
- KeyValueDB::Transaction t);
-
- /**
- * Removes header seq lock and possibly object lock
- * once Header is out of scope
- * @see lookup_parent
- * @see generate_new_header
- */
- class RemoveOnDelete {
- public:
- DBObjectMap *db;
- explicit RemoveOnDelete(DBObjectMap *db) :
- db(db) {}
- void operator() (_Header *header) {
- std::lock_guard l{db->header_lock};
- ceph_assert(db->in_use.count(header->seq));
- db->in_use.erase(header->seq);
- db->header_cond.notify_all();
- delete header;
- }
- };
- friend class RemoveOnDelete;
-};
-WRITE_CLASS_ENCODER(DBObjectMap::_Header)
-WRITE_CLASS_ENCODER(DBObjectMap::State)
-
-std::ostream& operator<<(std::ostream& out, const DBObjectMap::_Header& h);
-
-#endif
# ceph_test_object_map
add_executable(ceph_test_object_map
- test_object_map.cc
KeyValueDBMemory.cc
)
add_ceph_unittest(ceph_test_object_map)
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-#include <iterator>
-#include <map>
-#include <set>
-#include <boost/scoped_ptr.hpp>
-
-#include "include/buffer.h"
-#include "test/ObjectMap/KeyValueDBMemory.h"
-#include "kv/KeyValueDB.h"
-#include "os/DBObjectMap.h"
-#include <sys/types.h>
-#include "global/global_init.h"
-#include "common/ceph_argparse.h"
-#include <dirent.h>
-
-#include "gtest/gtest.h"
-#include "stdlib.h"
-
-using namespace std;
-
-template <typename T>
-typename T::iterator rand_choose(T &cont) {
- if (std::empty(cont)) {
- return std::end(cont);
- }
- return std::next(std::begin(cont), rand() % cont.size());
-}
-
-string num_str(unsigned i) {
- char buf[100];
- snprintf(buf, sizeof(buf), "%.10u", i);
- return string(buf);
-}
-
-class ObjectMapTester {
-public:
- ObjectMap *db;
- set<string> key_space;
- set<string> object_name_space;
- map<string, map<string, string> > omap;
- map<string, string > hmap;
- map<string, map<string, string> > xattrs;
- unsigned seq;
-
- ObjectMapTester() : db(0), seq(0) {}
-
- string val_from_key(const string &object, const string &key) {
- return object + "_" + key + "_" + num_str(seq++);
- }
-
- void set_key(const string &objname, const string &key, const string &value) {
- set_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- key, value);
- }
-
- void set_xattr(const string &objname, const string &key, const string &value) {
- set_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- key, value);
- }
-
- void set_key(ghobject_t hoid,
- string key, string value) {
- map<string, bufferlist> to_write;
- bufferptr bp(value.c_str(), value.size());
- bufferlist bl;
- bl.append(bp);
- to_write.insert(make_pair(key, bl));
- db->set_keys(hoid, to_write);
- }
-
- void set_keys(ghobject_t hoid, const map<string, string> &to_set) {
- map<string, bufferlist> to_write;
- for (auto &&i: to_set) {
- bufferptr bp(i.second.data(), i.second.size());
- bufferlist bl;
- bl.append(bp);
- to_write.insert(make_pair(i.first, bl));
- }
- db->set_keys(hoid, to_write);
- }
-
- void set_xattr(ghobject_t hoid,
- string key, string value) {
- map<string, bufferlist> to_write;
- bufferptr bp(value.c_str(), value.size());
- bufferlist bl;
- bl.append(bp);
- to_write.insert(make_pair(key, bl));
- db->set_xattrs(hoid, to_write);
- }
-
- void set_header(const string &objname, const string &value) {
- set_header(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- value);
- }
-
- void set_header(ghobject_t hoid,
- const string &value) {
- bufferlist header;
- header.append(bufferptr(value.c_str(), value.size() + 1));
- db->set_header(hoid, header);
- }
-
- int get_header(const string &objname, string *value) {
- return get_header(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- value);
- }
-
- int get_header(ghobject_t hoid,
- string *value) {
- bufferlist header;
- int r = db->get_header(hoid, &header);
- if (r < 0)
- return r;
- if (header.length())
- *value = string(header.c_str());
- else
- *value = string("");
- return 0;
- }
-
- int get_xattr(const string &objname, const string &key, string *value) {
- return get_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- key, value);
- }
-
- int get_xattr(ghobject_t hoid,
- string key, string *value) {
- set<string> to_get;
- to_get.insert(key);
- map<string, bufferlist> got;
- db->get_xattrs(hoid, to_get, &got);
- if (!got.empty()) {
- *value = string(got.begin()->second.c_str(),
- got.begin()->second.length());
- return 1;
- } else {
- return 0;
- }
- }
-
- int get_key(const string &objname, const string &key, string *value) {
- return get_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- key, value);
- }
-
- int get_key(ghobject_t hoid,
- string key, string *value) {
- set<string> to_get;
- to_get.insert(key);
- map<string, bufferlist> got;
- db->get_values(hoid, to_get, &got);
- if (!got.empty()) {
- if (value) {
- *value = string(got.begin()->second.c_str(),
- got.begin()->second.length());
- }
- return 1;
- } else {
- return 0;
- }
- }
-
- void remove_key(const string &objname, const string &key) {
- remove_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- key);
- }
-
- void remove_keys(const string &objname, const set<string> &to_remove) {
- remove_keys(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- to_remove);
- }
-
- void remove_key(ghobject_t hoid,
- string key) {
- set<string> to_remove;
- to_remove.insert(key);
- db->rm_keys(hoid, to_remove);
- }
-
- void remove_keys(ghobject_t hoid,
- const set<string> &to_remove) {
- db->rm_keys(hoid, to_remove);
- }
-
- void remove_xattr(const string &objname, const string &key) {
- remove_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- key);
- }
-
- void remove_xattr(ghobject_t hoid,
- string key) {
- set<string> to_remove;
- to_remove.insert(key);
- db->remove_xattrs(hoid, to_remove);
- }
-
- void clone(const string &objname, const string &target) {
- clone(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- ghobject_t(hobject_t(sobject_t(target, CEPH_NOSNAP))));
- }
-
- void clone(ghobject_t hoid,
- ghobject_t hoid2) {
- db->clone(hoid, hoid2);
- }
-
- void rename(const string &objname, const string &target) {
- rename(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- ghobject_t(hobject_t(sobject_t(target, CEPH_NOSNAP))));
- }
-
- void rename(ghobject_t hoid,
- ghobject_t hoid2) {
- db->rename(hoid, hoid2);
- }
-
- void clear(const string &objname) {
- clear(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))));
- }
-
- void legacy_clone(const string &objname, const string &target) {
- legacy_clone(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))),
- ghobject_t(hobject_t(sobject_t(target, CEPH_NOSNAP))));
- }
-
- void legacy_clone(ghobject_t hoid,
- ghobject_t hoid2) {
- db->legacy_clone(hoid, hoid2);
- }
-
- void clear(ghobject_t hoid) {
- db->clear(hoid);
- }
-
- void clear_omap(const string &objname) {
- clear_omap(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))));
- }
-
- void clear_omap(const ghobject_t &objname) {
- db->clear_keys_header(objname);
- }
-
- void def_init() {
- for (unsigned i = 0; i < 10000; ++i) {
- key_space.insert("key_" + num_str(i));
- }
- for (unsigned i = 0; i < 100; ++i) {
- object_name_space.insert("name_" + num_str(i));
- }
- }
-
- void init_key_set(const set<string> &keys) {
- key_space = keys;
- }
-
- void init_object_name_space(const set<string> &onamespace) {
- object_name_space = onamespace;
- }
-
- void auto_set_xattr(ostream &out) {
- set<string>::iterator key = rand_choose(key_space);
- set<string>::iterator object = rand_choose(object_name_space);
-
- string value = val_from_key(*object, *key);
-
- xattrs[*object][*key] = value;
- set_xattr(*object, *key, value);
-
- out << "auto_set_xattr " << *object << ": " << *key << " -> "
- << value << std::endl;
- }
-
- void test_set_key(const string &obj, const string &key, const string &val) {
- omap[obj][key] = val;
- set_key(obj, key, val);
- }
-
- void test_set_keys(const string &obj, const map<string, string> &to_set) {
- for (auto &&i: to_set) {
- omap[obj][i.first] = i.second;
- }
- set_keys(
- ghobject_t(hobject_t(sobject_t(obj, CEPH_NOSNAP))),
- to_set);
- }
-
- void auto_set_keys(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
-
- map<string, string> to_set;
- unsigned amount = (rand() % 10) + 1;
- for (unsigned i = 0; i < amount; ++i) {
- set<string>::iterator key = rand_choose(key_space);
- string value = val_from_key(*object, *key);
- out << "auto_set_key " << *object << ": " << *key << " -> "
- << value << std::endl;
- to_set.insert(make_pair(*key, value));
- }
-
-
- test_set_keys(*object, to_set);
- }
-
- void xattrs_on_object(const string &object, set<string> *out) {
- if (!xattrs.count(object))
- return;
- const map<string, string> &xmap = xattrs.find(object)->second;
- for (map<string, string>::const_iterator i = xmap.begin();
- i != xmap.end();
- ++i) {
- out->insert(i->first);
- }
- }
-
- void keys_on_object(const string &object, set<string> *out) {
- if (!omap.count(object))
- return;
- const map<string, string> &kmap = omap.find(object)->second;
- for (map<string, string>::const_iterator i = kmap.begin();
- i != kmap.end();
- ++i) {
- out->insert(i->first);
- }
- }
-
- void xattrs_off_object(const string &object, set<string> *out) {
- *out = key_space;
- set<string> xspace;
- xattrs_on_object(object, &xspace);
- for (set<string>::iterator i = xspace.begin();
- i != xspace.end();
- ++i) {
- out->erase(*i);
- }
- }
-
- void keys_off_object(const string &object, set<string> *out) {
- *out = key_space;
- set<string> kspace;
- keys_on_object(object, &kspace);
- for (set<string>::iterator i = kspace.begin();
- i != kspace.end();
- ++i) {
- out->erase(*i);
- }
- }
-
- int auto_check_present_xattr(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- set<string> xspace;
- xattrs_on_object(*object, &xspace);
- set<string>::iterator key = rand_choose(xspace);
- if (key == xspace.end()) {
- return 1;
- }
-
- string result;
- int r = get_xattr(*object, *key, &result);
- if (!r) {
- out << "auto_check_present_key: failed to find key "
- << *key << " on object " << *object << std::endl;
- return 0;
- }
-
- if (result != xattrs[*object][*key]) {
- out << "auto_check_present_key: for key "
- << *key << " on object " << *object
- << " found value " << result << " where we should have found "
- << xattrs[*object][*key] << std::endl;
- return 0;
- }
-
- out << "auto_check_present_key: for key "
- << *key << " on object " << *object
- << " found value " << result << " where we should have found "
- << xattrs[*object][*key] << std::endl;
- return 1;
- }
-
-
- int auto_check_present_key(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- set<string> kspace;
- keys_on_object(*object, &kspace);
- set<string>::iterator key = rand_choose(kspace);
- if (key == kspace.end()) {
- return 1;
- }
-
- string result;
- int r = get_key(*object, *key, &result);
- if (!r) {
- out << "auto_check_present_key: failed to find key "
- << *key << " on object " << *object << std::endl;
- return 0;
- }
-
- if (result != omap[*object][*key]) {
- out << "auto_check_present_key: for key "
- << *key << " on object " << *object
- << " found value " << result << " where we should have found "
- << omap[*object][*key] << std::endl;
- return 0;
- }
-
- out << "auto_check_present_key: for key "
- << *key << " on object " << *object
- << " found value " << result << " where we should have found "
- << omap[*object][*key] << std::endl;
- return 1;
- }
-
- int auto_check_absent_xattr(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- set<string> xspace;
- xattrs_off_object(*object, &xspace);
- set<string>::iterator key = rand_choose(xspace);
- if (key == xspace.end()) {
- return 1;
- }
-
- string result;
- int r = get_xattr(*object, *key, &result);
- if (!r) {
- out << "auto_check_absent_key: did not find key "
- << *key << " on object " << *object << std::endl;
- return 1;
- }
-
- out << "auto_check_basent_key: for key "
- << *key << " on object " << *object
- << " found value " << result << " where we should have found nothing"
- << std::endl;
- return 0;
- }
-
- int auto_check_absent_key(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- set<string> kspace;
- keys_off_object(*object, &kspace);
- set<string>::iterator key = rand_choose(kspace);
- if (key == kspace.end()) {
- return 1;
- }
-
- string result;
- int r = get_key(*object, *key, &result);
- if (!r) {
- out << "auto_check_absent_key: did not find key "
- << *key << " on object " << *object << std::endl;
- return 1;
- }
-
- out << "auto_check_basent_key: for key "
- << *key << " on object " << *object
- << " found value " << result << " where we should have found nothing"
- << std::endl;
- return 0;
- }
-
- void test_clone(const string &object, const string &target, ostream &out) {
- clone(object, target);
- if (!omap.count(object)) {
- out << " source missing.";
- omap.erase(target);
- } else {
- out << " source present.";
- omap[target] = omap[object];
- }
- if (!hmap.count(object)) {
- out << " hmap source missing." << std::endl;
- hmap.erase(target);
- } else {
- out << " hmap source present." << std::endl;
- hmap[target] = hmap[object];
- }
- if (!xattrs.count(object)) {
- out << " hmap source missing." << std::endl;
- xattrs.erase(target);
- } else {
- out << " hmap source present." << std::endl;
- xattrs[target] = xattrs[object];
- }
- }
-
- void auto_clone_key(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- set<string>::iterator target = rand_choose(object_name_space);
- while (target == object) {
- target = rand_choose(object_name_space);
- }
- out << "clone " << *object << " to " << *target;
- test_clone(*object, *target, out);
- }
-
- void test_remove_keys(const string &obj, const set<string> &to_remove) {
- for (auto &&k: to_remove)
- omap[obj].erase(k);
- remove_keys(obj, to_remove);
- }
-
- void test_remove_key(const string &obj, const string &key) {
- omap[obj].erase(key);
- remove_key(obj, key);
- }
-
- void auto_remove_keys(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- set<string> kspace;
- keys_on_object(*object, &kspace);
- set<string> to_remove;
- for (unsigned i = 0; i < 3; ++i) {
- set<string>::iterator key = rand_choose(kspace);
- if (key == kspace.end())
- continue;
- out << "removing " << *key << " from " << *object << std::endl;
- to_remove.insert(*key);
- }
- test_remove_keys(*object, to_remove);
- }
-
- void auto_remove_xattr(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- set<string> kspace;
- xattrs_on_object(*object, &kspace);
- set<string>::iterator key = rand_choose(kspace);
- if (key == kspace.end()) {
- return;
- }
- out << "removing xattr " << *key << " from " << *object << std::endl;
- xattrs[*object].erase(*key);
- remove_xattr(*object, *key);
- }
-
- void auto_delete_object(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- out << "auto_delete_object " << *object << std::endl;
- clear(*object);
- omap.erase(*object);
- hmap.erase(*object);
- xattrs.erase(*object);
- }
-
- void test_clear(const string &obj) {
- clear_omap(obj);
- omap.erase(obj);
- hmap.erase(obj);
- }
-
- void auto_clear_omap(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- out << "auto_clear_object " << *object << std::endl;
- test_clear(*object);
- }
-
- void auto_write_header(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- string header = val_from_key(*object, "HEADER");
- out << "auto_write_header: " << *object << " -> " << header << std::endl;
- set_header(*object, header);
- hmap[*object] = header;
- }
-
- int auto_verify_header(ostream &out) {
- set<string>::iterator object = rand_choose(object_name_space);
- out << "verify_header: " << *object << " ";
- string header;
- int r = get_header(*object, &header);
- if (r < 0) {
- ceph_abort();
- }
- if (header.size() == 0) {
- if (hmap.count(*object)) {
- out << " failed to find header " << hmap[*object] << std::endl;
- return 0;
- } else {
- out << " found no header" << std::endl;
- return 1;
- }
- }
-
- if (!hmap.count(*object)) {
- out << " found header " << header << " should have been empty"
- << std::endl;
- return 0;
- } else if (header == hmap[*object]) {
- out << " found correct header " << header << std::endl;
- return 1;
- } else {
- out << " found incorrect header " << header
- << " where we should have found " << hmap[*object] << std::endl;
- return 0;
- }
- }
-
- void verify_keys(const std::string &obj, ostream &out) {
- set<string> in_db;
- ObjectMap::ObjectMapIterator iter = db->get_iterator(
- ghobject_t(hobject_t(sobject_t(obj, CEPH_NOSNAP))));
- for (iter->seek_to_first(); iter->valid(); iter->next()) {
- in_db.insert(iter->key());
- }
- bool err = false;
- for (auto &&i: omap[obj]) {
- if (!in_db.count(i.first)) {
- out << __func__ << ": obj " << obj << " missing key "
- << i.first << std::endl;
- err = true;
- } else {
- in_db.erase(i.first);
- }
- }
- if (!in_db.empty()) {
- out << __func__ << ": obj " << obj << " found extra keys "
- << in_db << std::endl;
- err = true;
- }
- ASSERT_FALSE(err);
- }
-
- void auto_verify_objects(ostream &out) {
- for (auto &&i: omap) {
- verify_keys(i.first, out);
- }
- }
-};
-
-class ObjectMapTest : public ::testing::Test {
-public:
- boost::scoped_ptr< ObjectMap > db;
- ObjectMapTester tester;
- void SetUp() override {
- char *path = getenv("OBJECT_MAP_PATH");
- if (!path) {
- db.reset(new DBObjectMap(g_ceph_context, new KeyValueDBMemory()));
- tester.db = db.get();
- return;
- }
-
- string strpath(path);
-
- cerr << "using path " << strpath << std::endl;
- KeyValueDB *store = KeyValueDB::create(g_ceph_context, "rocksdb", strpath);
- ceph_assert(!store->create_and_open(cerr));
-
- db.reset(new DBObjectMap(g_ceph_context, store));
- tester.db = db.get();
- }
-
- void TearDown() override {
- std::cerr << "Checking..." << std::endl;
- ASSERT_EQ(0, db->check(std::cerr));
- }
-};
-
-
-int main(int argc, char **argv) {
- auto args = argv_to_vec(argc, argv);
-
- auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
- CODE_ENVIRONMENT_UTILITY,
- CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
-
-TEST_F(ObjectMapTest, CreateOneObject) {
- ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)), 100, shard_id_t(0));
- map<string, bufferlist> to_set;
- string key("test");
- string val("test_val");
- bufferptr bp(val.c_str(), val.size());
- bufferlist bl;
- bl.append(bp);
- to_set.insert(make_pair(key, bl));
- ASSERT_EQ(db->set_keys(hoid, to_set), 0);
-
- map<string, bufferlist> got;
- set<string> to_get;
- to_get.insert(key);
- to_get.insert("not there");
- db->get_values(hoid, to_get, &got);
- ASSERT_EQ(got.size(), (unsigned)1);
- ASSERT_EQ(string(got[key].c_str(), got[key].length()), val);
-
- bufferlist header;
- got.clear();
- db->get(hoid, &header, &got);
- ASSERT_EQ(got.size(), (unsigned)1);
- ASSERT_EQ(string(got[key].c_str(), got[key].length()), val);
- ASSERT_EQ(header.length(), (unsigned)0);
-
- db->rm_keys(hoid, to_get);
- got.clear();
- db->get(hoid, &header, &got);
- ASSERT_EQ(got.size(), (unsigned)0);
-
- map<string, bufferlist> attrs;
- attrs["attr1"] = bl;
- db->set_xattrs(hoid, attrs);
-
- db->set_header(hoid, bl);
-
- db->clear_keys_header(hoid);
- set<string> attrs_got;
- db->get_all_xattrs(hoid, &attrs_got);
- ASSERT_EQ(attrs_got.size(), 1U);
- ASSERT_EQ(*(attrs_got.begin()), "attr1");
- db->get(hoid, &header, &got);
- ASSERT_EQ(got.size(), (unsigned)0);
- ASSERT_EQ(header.length(), 0U);
- got.clear();
-
- db->clear(hoid);
- db->get(hoid, &header, &got);
- ASSERT_EQ(got.size(), (unsigned)0);
- attrs_got.clear();
- db->get_all_xattrs(hoid, &attrs_got);
- ASSERT_EQ(attrs_got.size(), 0U);
-}
-
-TEST_F(ObjectMapTest, CloneOneObject) {
- ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)), 200, shard_id_t(0));
- ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP)), 201, shard_id_t(1));
-
- tester.set_key(hoid, "foo", "bar");
- tester.set_key(hoid, "foo2", "bar2");
- string result;
- int r = tester.get_key(hoid, "foo", &result);
- ASSERT_EQ(r, 1);
- ASSERT_EQ(result, "bar");
-
- db->clone(hoid, hoid2);
- r = tester.get_key(hoid, "foo", &result);
- ASSERT_EQ(r, 1);
- ASSERT_EQ(result, "bar");
- r = tester.get_key(hoid2, "foo", &result);
- ASSERT_EQ(r, 1);
- ASSERT_EQ(result, "bar");
-
- tester.remove_key(hoid, "foo");
- r = tester.get_key(hoid2, "foo", &result);
- ASSERT_EQ(r, 1);
- ASSERT_EQ(result, "bar");
- r = tester.get_key(hoid, "foo", &result);
- ASSERT_EQ(r, 0);
- r = tester.get_key(hoid, "foo2", &result);
- ASSERT_EQ(r, 1);
- ASSERT_EQ(result, "bar2");
-
- tester.set_key(hoid, "foo", "baz");
- tester.remove_key(hoid, "foo");
- r = tester.get_key(hoid, "foo", &result);
- ASSERT_EQ(r, 0);
-
- tester.set_key(hoid, "foo2", "baz");
- tester.remove_key(hoid, "foo2");
- r = tester.get_key(hoid, "foo2", &result);
- ASSERT_EQ(r, 0);
-
- map<string, bufferlist> got;
- bufferlist header;
-
- got.clear();
- db->clear(hoid);
- db->get(hoid, &header, &got);
- ASSERT_EQ(got.size(), (unsigned)0);
-
- got.clear();
- r = db->clear(hoid2);
- ASSERT_EQ(0, r);
- db->get(hoid2, &header, &got);
- ASSERT_EQ(got.size(), (unsigned)0);
-
- tester.set_key(hoid, "baz", "bar");
- got.clear();
- db->get(hoid, &header, &got);
- ASSERT_EQ(got.size(), (unsigned)1);
- db->clear(hoid);
- db->clear(hoid2);
-}
-
-TEST_F(ObjectMapTest, OddEvenClone) {
- ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)));
- ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP)));
-
- for (unsigned i = 0; i < 1000; ++i) {
- tester.set_key(hoid, "foo" + num_str(i), "bar" + num_str(i));
- }
-
- db->clone(hoid, hoid2);
-
- int r = 0;
- for (unsigned i = 0; i < 1000; ++i) {
- string result;
- r = tester.get_key(hoid, "foo" + num_str(i), &result);
- ASSERT_EQ(1, r);
- ASSERT_EQ("bar" + num_str(i), result);
- r = tester.get_key(hoid2, "foo" + num_str(i), &result);
- ASSERT_EQ(1, r);
- ASSERT_EQ("bar" + num_str(i), result);
-
- if (i % 2) {
- tester.remove_key(hoid, "foo" + num_str(i));
- } else {
- tester.remove_key(hoid2, "foo" + num_str(i));
- }
- }
-
- for (unsigned i = 0; i < 1000; ++i) {
- string result;
- string result2;
- r = tester.get_key(hoid, "foo" + num_str(i), &result);
- int r2 = tester.get_key(hoid2, "foo" + num_str(i), &result2);
- if (i % 2) {
- ASSERT_EQ(0, r);
- ASSERT_EQ(1, r2);
- ASSERT_EQ("bar" + num_str(i), result2);
- } else {
- ASSERT_EQ(0, r2);
- ASSERT_EQ(1, r);
- ASSERT_EQ("bar" + num_str(i), result);
- }
- }
-
- {
- ObjectMap::ObjectMapIterator iter = db->get_iterator(hoid);
- iter->seek_to_first();
- for (unsigned i = 0; i < 1000; ++i) {
- if (!(i % 2)) {
- ASSERT_TRUE(iter->valid());
- ASSERT_EQ("foo" + num_str(i), iter->key());
- iter->next();
- }
- }
- }
-
- {
- ObjectMap::ObjectMapIterator iter2 = db->get_iterator(hoid2);
- iter2->seek_to_first();
- for (unsigned i = 0; i < 1000; ++i) {
- if (i % 2) {
- ASSERT_TRUE(iter2->valid());
- ASSERT_EQ("foo" + num_str(i), iter2->key());
- iter2->next();
- }
- }
- }
-
- db->clear(hoid);
- db->clear(hoid2);
-}
-
-TEST_F(ObjectMapTest, Rename) {
- ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)));
- ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP)));
-
- for (unsigned i = 0; i < 1000; ++i) {
- tester.set_key(hoid, "foo" + num_str(i), "bar" + num_str(i));
- }
-
- db->rename(hoid, hoid2);
- // Verify rename where target exists
- db->clone(hoid2, hoid);
- db->rename(hoid, hoid2);
-
- int r = 0;
- for (unsigned i = 0; i < 1000; ++i) {
- string result;
- r = tester.get_key(hoid2, "foo" + num_str(i), &result);
- ASSERT_EQ(1, r);
- ASSERT_EQ("bar" + num_str(i), result);
-
- if (i % 2) {
- tester.remove_key(hoid2, "foo" + num_str(i));
- }
- }
-
- for (unsigned i = 0; i < 1000; ++i) {
- string result;
- r = tester.get_key(hoid2, "foo" + num_str(i), &result);
- if (i % 2) {
- ASSERT_EQ(0, r);
- } else {
- ASSERT_EQ(1, r);
- ASSERT_EQ("bar" + num_str(i), result);
- }
- }
-
- {
- ObjectMap::ObjectMapIterator iter = db->get_iterator(hoid2);
- iter->seek_to_first();
- for (unsigned i = 0; i < 1000; ++i) {
- if (!(i % 2)) {
- ASSERT_TRUE(iter->valid());
- ASSERT_EQ("foo" + num_str(i), iter->key());
- iter->next();
- }
- }
- }
-
- db->clear(hoid2);
-}
-
-TEST_F(ObjectMapTest, OddEvenOldClone) {
- ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)));
- ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP)));
-
- for (unsigned i = 0; i < 1000; ++i) {
- tester.set_key(hoid, "foo" + num_str(i), "bar" + num_str(i));
- }
-
- db->legacy_clone(hoid, hoid2);
-
- int r = 0;
- for (unsigned i = 0; i < 1000; ++i) {
- string result;
- r = tester.get_key(hoid, "foo" + num_str(i), &result);
- ASSERT_EQ(1, r);
- ASSERT_EQ("bar" + num_str(i), result);
- r = tester.get_key(hoid2, "foo" + num_str(i), &result);
- ASSERT_EQ(1, r);
- ASSERT_EQ("bar" + num_str(i), result);
-
- if (i % 2) {
- tester.remove_key(hoid, "foo" + num_str(i));
- } else {
- tester.remove_key(hoid2, "foo" + num_str(i));
- }
- }
-
- for (unsigned i = 0; i < 1000; ++i) {
- string result;
- string result2;
- r = tester.get_key(hoid, "foo" + num_str(i), &result);
- int r2 = tester.get_key(hoid2, "foo" + num_str(i), &result2);
- if (i % 2) {
- ASSERT_EQ(0, r);
- ASSERT_EQ(1, r2);
- ASSERT_EQ("bar" + num_str(i), result2);
- } else {
- ASSERT_EQ(0, r2);
- ASSERT_EQ(1, r);
- ASSERT_EQ("bar" + num_str(i), result);
- }
- }
-
- {
- ObjectMap::ObjectMapIterator iter = db->get_iterator(hoid);
- iter->seek_to_first();
- for (unsigned i = 0; i < 1000; ++i) {
- if (!(i % 2)) {
- ASSERT_TRUE(iter->valid());
- ASSERT_EQ("foo" + num_str(i), iter->key());
- iter->next();
- }
- }
- }
-
- {
- ObjectMap::ObjectMapIterator iter2 = db->get_iterator(hoid2);
- iter2->seek_to_first();
- for (unsigned i = 0; i < 1000; ++i) {
- if (i % 2) {
- ASSERT_TRUE(iter2->valid());
- ASSERT_EQ("foo" + num_str(i), iter2->key());
- iter2->next();
- }
- }
- }
-
- db->clear(hoid);
- db->clear(hoid2);
-}
-
-TEST_F(ObjectMapTest, RandomTest) {
- tester.def_init();
- for (unsigned i = 0; i < 5000; ++i) {
- unsigned val = rand();
- val <<= 8;
- val %= 100;
- if (!(i%100))
- std::cout << "on op " << i
- << " val is " << val << std::endl;
-
- if (val < 7) {
- tester.auto_write_header(std::cerr);
- } else if (val < 14) {
- ASSERT_TRUE(tester.auto_verify_header(std::cerr));
- } else if (val < 30) {
- tester.auto_set_keys(std::cerr);
- } else if (val < 42) {
- tester.auto_set_xattr(std::cerr);
- } else if (val < 55) {
- ASSERT_TRUE(tester.auto_check_present_key(std::cerr));
- } else if (val < 62) {
- ASSERT_TRUE(tester.auto_check_present_xattr(std::cerr));
- } else if (val < 70) {
- ASSERT_TRUE(tester.auto_check_absent_key(std::cerr));
- } else if (val < 72) {
- ASSERT_TRUE(tester.auto_check_absent_xattr(std::cerr));
- } else if (val < 73) {
- tester.auto_clear_omap(std::cerr);
- } else if (val < 76) {
- tester.auto_delete_object(std::cerr);
- } else if (val < 85) {
- tester.auto_clone_key(std::cerr);
- } else if (val < 92) {
- tester.auto_remove_xattr(std::cerr);
- } else {
- tester.auto_remove_keys(std::cerr);
- }
-
- if (i % 500) {
- tester.auto_verify_objects(std::cerr);
- }
- }
-}
-
-TEST_F(ObjectMapTest, RandomTestNoDeletesXattrs) {
- tester.def_init();
- for (unsigned i = 0; i < 5000; ++i) {
- unsigned val = rand();
- val <<= 8;
- val %= 100;
- if (!(i%100))
- std::cout << "on op " << i
- << " val is " << val << std::endl;
-
- if (val < 45) {
- tester.auto_set_keys(std::cerr);
- } else if (val < 90) {
- tester.auto_remove_keys(std::cerr);
- } else {
- tester.auto_clone_key(std::cerr);
- }
-
- if (i % 500) {
- tester.auto_verify_objects(std::cerr);
- }
- }
-}
-
-string num_to_key(unsigned i) {
- char buf[100];
- int ret = snprintf(buf, sizeof(buf), "%010u", i);
- ceph_assert(ret > 0);
- return string(buf, ret);
-}
-
-TEST_F(ObjectMapTest, TestMergeNewCompleteContainBug) {
- /* This test exploits a bug in kraken and earlier where merge_new_complete
- * could miss complete entries fully contained by a new entry. To get this
- * to actually result in an incorrect return value, you need to remove at
- * least two values, one before a complete region, and one which occurs in
- * the parent after the complete region (but within 20 not yet completed
- * parent points of the first value).
- */
- for (unsigned i = 10; i < 160; i+=2) {
- tester.test_set_key("foo", num_to_key(i), "asdf");
- }
- tester.test_clone("foo", "foo2", std::cout);
- tester.test_clear("foo");
-
- tester.test_set_key("foo2", num_to_key(15), "asdf");
- tester.test_set_key("foo2", num_to_key(13), "asdf");
- tester.test_set_key("foo2", num_to_key(57), "asdf");
-
- tester.test_remove_key("foo2", num_to_key(15));
-
- set<string> to_remove;
- to_remove.insert(num_to_key(13));
- to_remove.insert(num_to_key(58));
- to_remove.insert(num_to_key(60));
- to_remove.insert(num_to_key(62));
- tester.test_remove_keys("foo2", to_remove);
-
- tester.verify_keys("foo2", std::cout);
- ASSERT_EQ(tester.get_key("foo2", num_to_key(10), nullptr), 1);
- ASSERT_EQ(tester.get_key("foo2", num_to_key(1), nullptr), 0);
- ASSERT_EQ(tester.get_key("foo2", num_to_key(56), nullptr), 1);
- // this one triggers the bug
- ASSERT_EQ(tester.get_key("foo2", num_to_key(58), nullptr), 0);
-}
-
-TEST_F(ObjectMapTest, TestIterateBug18533) {
- /* This test starts with the one immediately above to create a pair of
- * complete regions where one contains the other. Then, it deletes the
- * key at the start of the contained region. The logic in next_parent()
- * skips ahead to the end of the contained region, and we start copying
- * values down again from the parent into the child -- including some
- * that had actually been deleted. I think this works for any removal
- * within the outer complete region after the start of the contained
- * region.
- */
- for (unsigned i = 10; i < 160; i+=2) {
- tester.test_set_key("foo", num_to_key(i), "asdf");
- }
- tester.test_clone("foo", "foo2", std::cout);
- tester.test_clear("foo");
-
- tester.test_set_key("foo2", num_to_key(15), "asdf");
- tester.test_set_key("foo2", num_to_key(13), "asdf");
- tester.test_set_key("foo2", num_to_key(57), "asdf");
- tester.test_set_key("foo2", num_to_key(91), "asdf");
-
- tester.test_remove_key("foo2", num_to_key(15));
-
- set<string> to_remove;
- to_remove.insert(num_to_key(13));
- to_remove.insert(num_to_key(58));
- to_remove.insert(num_to_key(60));
- to_remove.insert(num_to_key(62));
- to_remove.insert(num_to_key(82));
- to_remove.insert(num_to_key(84));
- tester.test_remove_keys("foo2", to_remove);
-
- //tester.test_remove_key("foo2", num_to_key(15)); also does the trick
- tester.test_remove_key("foo2", num_to_key(80));
-
- // the iterator in verify_keys will return an extra value
- tester.verify_keys("foo2", std::cout);
-}
-
TYPE(health_check_t)
TYPE(health_check_map_t)
-#include "os/DBObjectMap.h"
-TYPE(DBObjectMap::_Header)
-TYPE(DBObjectMap::State)
-
#include "os/kstore/kstore_types.h"
TYPE(kstore_cnode_t)
TYPE(kstore_onode_t)
projects / ceph.git / commitdiff