eversion_t s,
set<eversion_t> *trimmed,
set<string>* trimmed_dups,
- bool* dirty_dups)
+ eversion_t *write_from_dups)
{
if (complete_to != log.end() &&
complete_to->version <= s) {
unindex(e); // remove from index,
// add to dup list
+ generic_dout(20) << "earliest_dup_version = " << earliest_dup_version << dendl;
if (e.version.version >= earliest_dup_version) {
- if (dirty_dups) *dirty_dups = true;
+ if (write_from_dups != nullptr && *write_from_dups > e.version) {
+ generic_dout(20) << "updating write_from_dups from " << *write_from_dups << " to " << e.version << dendl;
+ *write_from_dups = e.version;
+ }
dups.push_back(pg_log_dup_t(e));
index(dups.back());
for (const auto& extra : e.extra_reqids) {
assert(trim_to <= info.last_complete);
dout(10) << "trim " << log << " to " << trim_to << dendl;
- log.trim(cct, trim_to, &trimmed, &trimmed_dups, &dirty_dups);
+ log.trim(cct, trim_to, &trimmed, &trimmed_dups, &write_from_dups);
info.log_tail = log.tail;
}
}
// now handle dups
if (merge_log_dups(olog)) {
- dirty_dups = true;
changed = true;
}
olog.dups.front().version << " to " <<
olog.dups.back().version << dendl;
changed = true;
+ dirty_from_dups = eversion_t();
+ dirty_to_dups = eversion_t::max();
// since our log.dups is empty just copy them
for (const auto& i : olog.dups) {
log.dups.push_back(i);
auto log_tail_version = log.dups.back().version;
auto insert_cursor = log.dups.end();
+ eversion_t last_shared = eversion_t::max();
for (auto i = olog.dups.crbegin(); i != olog.dups.crend(); ++i) {
if (i->version <= log_tail_version) break;
log.dups.insert(insert_cursor, *i);
+ last_shared = i->version;
auto prev = insert_cursor;
--prev;
--insert_cursor; // make sure we insert in reverse order
}
+ mark_dirty_from_dups(last_shared);
}
if (olog.dups.front().version < log.dups.front().version) {
olog.dups.front().version << dendl;
changed = true;
+ eversion_t last;
auto insert_cursor = log.dups.begin();
for (auto i = olog.dups.cbegin(); i != olog.dups.cend(); ++i) {
if (i->version >= insert_cursor->version) break;
log.dups.insert(insert_cursor, *i);
+ last = i->version;
auto prev = insert_cursor;
--prev;
// be sure to pass address of copy in log.dups
log.index(*prev);
}
+ mark_dirty_to_dups(last);
}
}
}
while (!log.dups.empty() && log.dups.back().version >= log.tail) {
log.unindex(log.dups.back());
+ mark_dirty_from_dups(log.dups.back().version);
log.dups.pop_back();
}
}
!touched_log,
require_rollback,
clear_divergent_priors,
- dirty_dups,
+ dirty_to_dups,
+ dirty_from_dups,
+ write_from_dups,
&rebuilt_missing_with_deletes,
(pg_log_debug ? &log_keys_debug : nullptr));
undirty();
pg_log_t &log,
const coll_t& coll, const ghobject_t &log_oid,
map<eversion_t, hobject_t> &divergent_priors,
- bool require_rollback,
- bool dirty_dups)
+ bool require_rollback
+ )
{
_write_log_and_missing_wo_missing(
t, km, log, coll, log_oid,
divergent_priors, eversion_t::max(), eversion_t(), eversion_t(),
set<eversion_t>(),
set<string>(),
- true, true, require_rollback, dirty_dups, nullptr);
+ true, true, require_rollback,
+ eversion_t::max(), eversion_t(), eversion_t(), nullptr);
}
// static
const ghobject_t &log_oid,
const pg_missing_tracker_t &missing,
bool require_rollback,
- bool dirty_dups,
bool *rebuilt_missing_with_deletes)
{
_write_log_and_missing(
set<eversion_t>(),
set<string>(),
missing,
- true, require_rollback, false, dirty_dups, rebuilt_missing_with_deletes, nullptr);
+ true, require_rollback, false,
+ eversion_t::max(),
+ eversion_t(),
+ eversion_t(),
+ rebuilt_missing_with_deletes, nullptr);
}
// static
bool dirty_divergent_priors,
bool touch_log,
bool require_rollback,
- bool dirty_dups,
+ eversion_t dirty_to_dups,
+ eversion_t dirty_from_dups,
+ eversion_t write_from_dups,
set<string> *log_keys_debug
)
{
}
}
- // process dirty_dups after log_keys_debug is filled, so dups do not
+ // process dups after log_keys_debug is filled, so dups do not
// end up in that set
- if (dirty_dups) {
- pg_log_dup_t min;
+ if (dirty_to_dups != eversion_t()) {
+ pg_log_dup_t min, dirty_to_dup;
+ dirty_to_dup.version = dirty_to_dups;
t.omap_rmkeyrange(
coll, log_oid,
- min.get_key_name(), log.dups.begin()->get_key_name());
- for (const auto& entry : log.dups) {
- bufferlist bl;
- ::encode(entry, bl);
- (*km)[entry.get_key_name()].claim(bl);
- }
+ min.get_key_name(), dirty_to_dup.get_key_name());
+ }
+ if (dirty_to_dups != eversion_t::max() && dirty_from_dups != eversion_t::max()) {
+ pg_log_dup_t max, dirty_from_dup;
+ max.version = eversion_t::max();
+ dirty_from_dup.version = dirty_from_dups;
+ t.omap_rmkeyrange(
+ coll, log_oid,
+ dirty_from_dup.get_key_name(), max.get_key_name());
+ }
+
+ for (const auto& entry : log.dups) {
+ if (entry.version > dirty_to_dups)
+ break;
+ bufferlist bl;
+ ::encode(entry, bl);
+ (*km)[entry.get_key_name()].claim(bl);
+ }
+
+ for (list<pg_log_dup_t>::reverse_iterator p = log.dups.rbegin();
+ p != log.dups.rend() &&
+ (p->version >= dirty_from_dups || p->version >= write_from_dups) &&
+ p->version >= dirty_to_dups;
+ ++p) {
+ bufferlist bl;
+ ::encode(*p, bl);
+ (*km)[p->get_key_name()].claim(bl);
}
if (dirty_divergent_priors) {
bool touch_log,
bool require_rollback,
bool clear_divergent_priors,
- bool dirty_dups,
+ eversion_t dirty_to_dups,
+ eversion_t dirty_from_dups,
+ eversion_t write_from_dups,
bool *rebuilt_missing_with_deletes, // in/out param
set<string> *log_keys_debug
) {
}
}
- // process dirty_dups after log_keys_debug is filled, so dups do not
+ // process dups after log_keys_debug is filled, so dups do not
// end up in that set
- if (dirty_dups) {
- pg_log_dup_t min;
+ if (dirty_to_dups != eversion_t()) {
+ pg_log_dup_t min, dirty_to_dup;
+ dirty_to_dup.version = dirty_to_dups;
t.omap_rmkeyrange(
coll, log_oid,
- min.get_key_name(), log.dups.begin()->get_key_name());
- for (const auto& entry : log.dups) {
- bufferlist bl;
- ::encode(entry, bl);
- (*km)[entry.get_key_name()].claim(bl);
- }
+ min.get_key_name(), dirty_to_dup.get_key_name());
+ }
+ if (dirty_to_dups != eversion_t::max() && dirty_from_dups != eversion_t::max()) {
+ pg_log_dup_t max, dirty_from_dup;
+ max.version = eversion_t::max();
+ dirty_from_dup.version = dirty_from_dups;
+ t.omap_rmkeyrange(
+ coll, log_oid,
+ dirty_from_dup.get_key_name(), max.get_key_name());
+ }
+
+ for (const auto& entry : log.dups) {
+ if (entry.version > dirty_to_dups)
+ break;
+ bufferlist bl;
+ ::encode(entry, bl);
+ (*km)[entry.get_key_name()].claim(bl);
+ }
+
+ for (list<pg_log_dup_t>::reverse_iterator p = log.dups.rbegin();
+ p != log.dups.rend() &&
+ (p->version >= dirty_from_dups || p->version >= write_from_dups) &&
+ p->version >= dirty_to_dups;
+ ++p) {
+ bufferlist bl;
+ ::encode(*p, bl);
+ (*km)[p->get_key_name()].claim(bl);
}
if (clear_divergent_priors) {
eversion_t s,
set<eversion_t> *trimmed,
set<string>* trimmed_dups,
- bool* dirty_dups);
+ eversion_t *write_from_dups);
ostream& print(ostream& out) const;
}; // IndexedLog
eversion_t dirty_from; ///< must clear/writeout all keys >= dirty_from
eversion_t writeout_from; ///< must writout keys >= writeout_from
set<eversion_t> trimmed; ///< must clear keys in trimmed
+ eversion_t dirty_to_dups; ///< must clear/writeout all dups <= dirty_to_dups
+ eversion_t dirty_from_dups; ///< must clear/writeout all dups >= dirty_from_dups
+ eversion_t write_from_dups; ///< must write keys >= write_from_dups
set<string> trimmed_dups; ///< must clear keys in trimmed_dups
CephContext *cct;
bool pg_log_debug;
/// Log is clean on [dirty_to, dirty_from)
bool touched_log;
bool clear_divergent_priors;
- bool dirty_dups; /// log.dups is updated
bool rebuilt_missing_with_deletes = false;
void mark_dirty_to(eversion_t to) {
if (from < writeout_from)
writeout_from = from;
}
+ void mark_dirty_to_dups(eversion_t to) {
+ if (to > dirty_to_dups)
+ dirty_to_dups = to;
+ }
+ void mark_dirty_from_dups(eversion_t from) {
+ if (from < dirty_from_dups)
+ dirty_from_dups = from;
+ }
public:
bool is_dirty() const {
return !touched_log ||
!(trimmed.empty()) ||
!missing.is_clean() ||
!(trimmed_dups.empty()) ||
- dirty_dups ||
+ (dirty_to_dups != eversion_t()) ||
+ (dirty_from_dups != eversion_t::max()) ||
+ (write_from_dups != eversion_t::max()) ||
rebuilt_missing_with_deletes;
}
void mark_log_for_rewrite() {
mark_dirty_to(eversion_t::max());
mark_dirty_from(eversion_t());
+ mark_dirty_to_dups(eversion_t::max());
+ mark_dirty_from_dups(eversion_t());
touched_log = false;
}
bool get_rebuilt_missing_with_deletes() const {
writeout_from = eversion_t::max();
check();
missing.flush();
- dirty_dups = false;
+ dirty_to_dups = eversion_t();
+ dirty_from_dups = eversion_t::max();
+ write_from_dups = eversion_t::max();
}
public:
prefix_provider(dpp),
dirty_from(eversion_t::max()),
writeout_from(eversion_t::max()),
+ dirty_from_dups(eversion_t::max()),
+ write_from_dups(eversion_t::max()),
cct(cct),
pg_log_debug(!(cct && !(cct->_conf->osd_debug_pg_log_writeout))),
touched_log(false),
- clear_divergent_priors(false),
- dirty_dups(false)
+ clear_divergent_priors(false)
{ }
void reset_backfill();
log.claim_log_and_clear_rollback_info(o);
missing.clear();
mark_dirty_to(eversion_t::max());
+ mark_dirty_to_dups(eversion_t::max());
}
void split_into(
log.split_out_child(child_pgid, split_bits, &opg_log->log);
missing.split_into(child_pgid, split_bits, &(opg_log->missing));
opg_log->mark_dirty_to(eversion_t::max());
+ opg_log->mark_dirty_to_dups(eversion_t::max());
mark_dirty_to(eversion_t::max());
+ mark_dirty_to_dups(eversion_t::max());
if (missing.may_include_deletes)
opg_log->rebuilt_missing_with_deletes = true;
}
pg_log_t &log,
const coll_t& coll,
const ghobject_t &log_oid, map<eversion_t, hobject_t> &divergent_priors,
- bool require_rollback,
- bool dirty_dups);
+ bool require_rollback);
static void write_log_and_missing(
ObjectStore::Transaction& t,
const ghobject_t &log_oid,
const pg_missing_tracker_t &missing,
bool require_rollback,
- bool dirty_dups,
bool *rebuilt_missing_set_with_deletes);
static void _write_log_and_missing_wo_missing(
bool dirty_divergent_priors,
bool touch_log,
bool require_rollback,
- bool dirty_dups,
+ eversion_t dirty_to_dups,
+ eversion_t dirty_from_dups,
+ eversion_t write_from_dups,
set<string> *log_keys_debug
);
bool touch_log,
bool require_rollback,
bool clear_divergent_priors,
- bool dirty_dups,
+ eversion_t dirty_to_dups,
+ eversion_t dirty_from_dups,
+ eversion_t write_from_dups,
bool *rebuilt_missing_with_deletes,
set<string> *log_keys_debug
);
void dump(Formatter *f) const;
static void generate_test_instances(list<pg_log_dup_t*>& o);
+ bool operator==(const pg_log_dup_t &rhs) const {
+ return reqid == rhs.reqid &&
+ version == rhs.version &&
+ user_version == rhs.user_version &&
+ return_code == rhs.return_code;
+ }
+ bool operator!=(const pg_log_dup_t &rhs) const {
+ return !(*this == rhs);
+ }
+
friend std::ostream& operator<<(std::ostream& out, const pg_log_dup_t& e);
};
WRITE_CLASS_ENCODER(pg_log_dup_t)
}
-class PGLogMergeDupsTest : public ::testing::Test, protected PGLog {
+class PGLogMergeDupsTest : protected PGLog, public StoreTestFixture {
public:
- PGLogMergeDupsTest() : PGLog(g_ceph_context) { }
+ PGLogMergeDupsTest() : PGLog(g_ceph_context), StoreTestFixture("memstore") { }
- void SetUp() override { }
+ void SetUp() override {
+ StoreTestFixture::SetUp();
+ ObjectStore::Sequencer osr(__func__);
+ ObjectStore::Transaction t;
+ test_coll = coll_t(spg_t(pg_t(1, 1)));
+ t.create_collection(test_coll, 0);
+ store->apply_transaction(&osr, std::move(t));
+ }
void TearDown() override {
+ test_disk_roundtrip();
clear();
+ StoreTestFixture::TearDown();
}
static pg_log_dup_t create_dup_entry(uint a, uint b) {
void add_dups(uint a, uint b) {
log.dups.push_back(create_dup_entry(a, b));
+ write_from_dups = MIN(write_from_dups, log.dups.back().version);
}
void add_dups(const std::vector<pg_log_dup_t>& l) {
for (auto& i : l) {
log.dups.push_back(i);
+ write_from_dups = MIN(write_from_dups, log.dups.back().version);
}
}
EXPECT_EQ(1u, log.dup_index.count(i.reqid));
}
}
+
+ void test_disk_roundtrip() {
+ ObjectStore::Sequencer osr(__func__);
+ ObjectStore::Transaction t;
+ hobject_t hoid;
+ hoid.pool = 1;
+ hoid.oid = "log";
+ ghobject_t log_oid(hoid);
+ map<string, bufferlist> km;
+ write_log_and_missing(t, &km, test_coll, log_oid, false);
+ if (!km.empty()) {
+ t.omap_setkeys(test_coll, log_oid, km);
+ }
+ ASSERT_EQ(0u, store->apply_transaction(&osr, std::move(t)));
+
+ auto orig_dups = log.dups;
+ clear();
+ ostringstream err;
+ read_log_and_missing(store.get(), test_coll, test_coll, log_oid,
+ pg_info_t(), false, err, false);
+ ASSERT_EQ(orig_dups.size(), log.dups.size());
+ ASSERT_EQ(orig_dups, log.dups);
+ auto dups_it = log.dups.begin();
+ for (auto orig_dup : orig_dups) {
+ ASSERT_EQ(orig_dup, *dups_it);
+ ++dups_it;
+ }
+ }
+
+ coll_t test_coll;
};
TEST_F(PGLogMergeDupsTest, OtherEmpty) {
public PGLogTestBase,
public PGLog::IndexedLog
{
- std::list<hobject_t*> test_hobjects;
- CephContext *cct;
-
- void SetUp() override {
- cct = (new CephContext(CEPH_ENTITY_TYPE_OSD))->get();
-
- hobject_t::generate_test_instances(test_hobjects);
- }
+ CephContext *cct = g_ceph_context;
void SetUp(unsigned min_entries, unsigned max_entries, unsigned dup_track) {
constexpr size_t size = 10;
cct->_conf->set_val_or_die("osd_min_pg_log_entries", min_entries_s);
cct->_conf->set_val_or_die("osd_max_pg_log_entries", max_entries_s);
cct->_conf->set_val_or_die("osd_pg_log_dups_tracked", dup_track_s);
-}
-
- void TearDown() override {
- while (!test_hobjects.empty()) {
- delete test_hobjects.front();
- test_hobjects.pop_front();
- }
-
- cct->put();
}
}; // struct PGLogTrimTest
-# if 0
-TEST_F(PGLogTest, Trim1) {
- TestCase t;
-
- t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
- t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 150), mk_evt(10, 100)));
- t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 155), mk_evt(15, 150)));
- t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(20, 160), mk_evt(25, 152)));
- t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(21, 165), mk_evt(26, 160)));
- t.auth.push_back(mk_ple_mod(mk_obj(1), mk_evt(21, 165), mk_evt(31, 171)));
-
- t.setup();
-}
-#endif
-
-
TEST_F(PGLogTrimTest, TestMakingCephContext)
{
SetUp(1, 2, 5);
std::set<eversion_t> trimmed;
std::set<std::string> trimmed_dups;
- bool dirty_dups = false;
+ eversion_t write_from_dups = eversion_t::max();
- log.trim(cct, mk_evt(19, 157), &trimmed, &trimmed_dups, &dirty_dups);
+ log.trim(cct, mk_evt(19, 157), &trimmed, &trimmed_dups, &write_from_dups);
- EXPECT_EQ(true, dirty_dups);
+ EXPECT_EQ(eversion_t(15, 150), write_from_dups);
EXPECT_EQ(3u, log.log.size());
EXPECT_EQ(3u, trimmed.size());
EXPECT_EQ(2u, log.dups.size());
std::set<eversion_t> trimmed2;
std::set<std::string> trimmed_dups2;
- bool dirty_dups2 = false;
-
- log.trim(cct, mk_evt(20, 164), &trimmed2, &trimmed_dups2, &dirty_dups2);
+ eversion_t write_from_dups2 = eversion_t::max();
- EXPECT_EQ(true, dirty_dups2);
+ log.trim(cct, mk_evt(20, 164), &trimmed2, &trimmed_dups2, &write_from_dups2);
+
+ EXPECT_EQ(eversion_t(19, 160), write_from_dups2);
EXPECT_EQ(2u, log.log.size());
EXPECT_EQ(1u, trimmed2.size());
EXPECT_EQ(2u, log.dups.size());
log.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
log.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
- bool dirty_dups = false;
+ eversion_t write_from_dups = eversion_t::max();
- log.trim(cct, mk_evt(19, 157), nullptr, nullptr, &dirty_dups);
+ log.trim(cct, mk_evt(19, 157), nullptr, nullptr, &write_from_dups);
- EXPECT_EQ(true, dirty_dups);
+ EXPECT_EQ(eversion_t(15, 150), write_from_dups);
EXPECT_EQ(3u, log.log.size());
EXPECT_EQ(2u, log.dups.size());
}
std::set<eversion_t> trimmed;
std::set<std::string> trimmed_dups;
- bool dirty_dups = false;
+ eversion_t write_from_dups = eversion_t::max();
- log.trim(cct, mk_evt(19, 157), &trimmed, &trimmed_dups, &dirty_dups);
+ log.trim(cct, mk_evt(19, 157), &trimmed, &trimmed_dups, &write_from_dups);
- EXPECT_FALSE(dirty_dups);
+ EXPECT_EQ(eversion_t::max(), write_from_dups);
EXPECT_EQ(3u, log.log.size());
EXPECT_EQ(3u, trimmed.size());
EXPECT_EQ(0u, log.dups.size());
std::set<eversion_t> trimmed;
std::set<std::string> trimmed_dups;
- bool dirty_dups = false;
+ eversion_t write_from_dups = eversion_t::max();
- log.trim(cct, mk_evt(9, 99), &trimmed, &trimmed_dups, &dirty_dups);
+ log.trim(cct, mk_evt(9, 99), &trimmed, &trimmed_dups, &write_from_dups);
- EXPECT_FALSE(dirty_dups);
+ EXPECT_EQ(eversion_t::max(), write_from_dups);
EXPECT_EQ(6u, log.log.size());
EXPECT_EQ(0u, trimmed.size());
EXPECT_EQ(0u, log.dups.size());
std::set<eversion_t> trimmed;
std::set<std::string> trimmed_dups;
- bool dirty_dups = false;
+ eversion_t write_from_dups = eversion_t::max();
- log.trim(cct, mk_evt(22, 180), &trimmed, &trimmed_dups, &dirty_dups);
+ log.trim(cct, mk_evt(22, 180), &trimmed, &trimmed_dups, &write_from_dups);
- EXPECT_EQ(true, dirty_dups);
+ EXPECT_EQ(eversion_t(15, 150), write_from_dups);
EXPECT_EQ(0u, log.log.size());
EXPECT_EQ(6u, trimmed.size());
EXPECT_EQ(5u, log.dups.size());
log.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166),
osd_reqid_t(client, 8, 6)));
- bool dirty_dups = false;
+ eversion_t write_from_dups = eversion_t::max();
- log.trim(cct, mk_evt(19, 157), nullptr, nullptr, &dirty_dups);
+ log.trim(cct, mk_evt(19, 157), nullptr, nullptr, &write_from_dups);
- EXPECT_EQ(true, dirty_dups);
+ EXPECT_EQ(eversion_t(15, 150), write_from_dups);
EXPECT_EQ(3u, log.log.size());
EXPECT_EQ(2u, log.dups.size());
}
}
+
// Local Variables:
// compile-command: "cd ../.. ; make unittest_pglog ; ./unittest_pglog --log-to-stderr=true --debug-osd=20 # --gtest_filter=*.* "
// End:
if (!divergent.empty()) {
assert(missing.get_items().empty());
PGLog::write_log_and_missing_wo_missing(
- t, &km, log, coll, info.pgid.make_pgmeta_oid(), divergent, true, true);
+ t, &km, log, coll, info.pgid.make_pgmeta_oid(), divergent, true);
} else {
pg_missing_tracker_t tmissing(missing);
bool rebuilt_missing_set_with_deletes = missing.may_include_deletes;
PGLog::write_log_and_missing(
- t, &km, log, coll, info.pgid.make_pgmeta_oid(), tmissing, true, true,
+ t, &km, log, coll, info.pgid.make_pgmeta_oid(), tmissing, true,
&rebuilt_missing_set_with_deletes);
}
t.omap_setkeys(coll, info.pgid.make_pgmeta_oid(), km);
projects / ceph.git / commitdiff