VerifyFileStore: Check if two FileStore's match after applying a set of operations.

With DeterministicOpSequence we are able to reproduce exactly the same
sequence of operations, over and over. However, if the filestore fails
(e.g., because we injected a failure), we want to check if it is kept
consistent after replaying its journal.

With VerifyFileStore, which extends DeterministicOpSequence, we are able
to bring a brand new filestore to the state the failed filestore would
reach had it not failed. We can then compare to check if the failure
introduced inconsistencies after replaying the journal.

(This is still work in progress and not fully functional)

Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>

diff --git a/src/test/filestore_test/VerifyFileStore.cc b/src/test/filestore_test/VerifyFileStore.cc
new file mode 100644 (file)
index 0000000..f69e142
--- /dev/null
+++ b/src/test/filestore_test/VerifyFileStore.cc
@@ -0,0 +1,161 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <iostream>
+#include <fstream>
+#include <assert.h>
+#include <time.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <sstream>
+#include "os/FileStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/debug.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include "common/config.h"
+
+#include "DeterministicOpSequence.h"
+#include "VerifyFileStore.h"
+
+#define dout_subsys ceph_subsys_filestore
+#undef dout_prefix
+#define dout_prefix *_dout << "filestore_verify "
+
+void VerifyFileStore::generate(int seed, int verify_at) {
+
+  std::ostringstream ss;
+  ss << "generate run " << verify_at << " --seed " << seed;
+
+  if (m_status.is_open()) {
+    m_status << ss.str() << std::endl;
+    m_status.flush();
+  }
+
+  dout(0) << ss.str() << dendl;
+
+  rngen_t gen(seed);
+  boost::uniform_int<> op_rng(DSOP_FIRST, DSOP_LAST);
+
+  for (int i = 1; i <= verify_at; i++) {
+    if (i == verify_at)
+      m_do_verify = true;
+
+    int op = op_rng(gen);
+    _print_status(i, op);
+    dout(0) << "generate seq " << i << " op " << op << dendl;
+    run_one_op(op, gen);
+  }
+}
+
+void VerifyFileStore::_do_touch(coll_t coll, hobject_t& obj) {
+  dout(0) << "verify _do_touch entry" << dendl;
+  DeterministicOpSequence::_do_touch(coll, obj);
+
+  if (!m_do_verify)
+    return;
+
+  dout(0) << "verify _do_touch exit" << dendl;
+}
+
+void VerifyFileStore::_do_write(coll_t coll, hobject_t& obj,
+    uint64_t off, uint64_t len, const bufferlist& data)
+{
+  dout(0) << "verify _do_write entry" << dendl;
+  DeterministicOpSequence::_do_write(coll, obj, off, len, data);
+
+  if (!m_do_verify)
+    return;
+
+  int got;
+  bufferlist our_data;
+  got = m_store->read(coll, obj, off, len, our_data);
+
+  if (got < 0) {
+    dout(0) << "verify _do_write error: got " << got << dendl;
+    return;
+  }
+
+  if (got != (int) len) {
+    dout(0) << "verify _do_write error: got " << got
+        << " but expected " << len << dendl;
+    return;
+  }
+
+  m_store->umount();
+  m_store_to_verify->mount();
+
+  bufferlist their_data;
+  got = m_store_to_verify->read(coll, obj, off, len, their_data);
+
+  if (got < 0) {
+    dout(0) << "verify _do_write error: got " << got << dendl;
+    return;
+  }
+
+  if (got != (int) len) {
+    dout(0) << "verify _do_write error: got " << got
+        << " but expected " << len << dendl;
+    return;
+  }
+
+  bufferlist::iterator our_it = our_data.begin();
+  bufferlist::iterator their_it = their_data.begin();
+  bool matching = true;
+  for (int i = 0; !our_it.end() && !their_it.end(); ++our_it, ++their_it, ++i) {
+    if (*our_it != *their_it) {
+      dout(0) << "verify _do_write error: does not match at pos " << i << dendl;
+      matching = false;
+      break;
+    }
+  }
+
+  if (!(our_it.end() && their_it.end())) {
+    dout(0) << "verify _do_write error: no matching" << dendl;
+    matching = false;
+  }
+
+  m_store_to_verify->umount();
+  m_store->mount();
+
+  dout(0) << "verify _do_write "
+      << (matching ? "do" : "don't") << " match" << dendl;
+
+  dout(0) << "verify _do_write exit" << dendl;
+}
+
+void VerifyFileStore::_do_clone(coll_t coll, hobject_t& orig_obj,
+    hobject_t& new_obj)
+{
+  dout(0) << "verify _do_clone entry" << dendl;
+  DeterministicOpSequence::_do_clone(coll, orig_obj, new_obj);
+
+  if (!m_do_verify)
+    return;
+
+  dout(0) << "verify _do_clone exit" << dendl;
+}
+
+void VerifyFileStore::_do_coll_move(coll_t new_coll, coll_t old_coll,
+    hobject_t& obj)
+{
+  dout(0) << "verify _do_coll_move entry" << dendl;
+  DeterministicOpSequence::_do_coll_move(new_coll, old_coll, obj);
+
+  if (!m_do_verify)
+    return;
+
+  dout(0) << "verify _do_coll_move exit" << dendl;
+}

diff --git a/src/test/filestore_test/VerifyFileStore.h b/src/test/filestore_test/VerifyFileStore.h
new file mode 100644 (file)
index 0000000..852d748
--- /dev/null
+++ b/src/test/filestore_test/VerifyFileStore.h
@@ -0,0 +1,55 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 New Dream Network
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#ifndef VERIFY_FILESTORE_H_
+#define VERIFY_FILESTORE_H_
+
+#include <iostream>
+#include <fstream>
+#include <set>
+#include "os/FileStore.h"
+#include <boost/scoped_ptr.hpp>
+#include <boost/random/mersenne_twister.hpp>
+#include <boost/random/uniform_int.hpp>
+
+#include "TestFileStoreState.h"
+#include "DeterministicOpSequence.h"
+
+class VerifyFileStore : public DeterministicOpSequence {
+
+ private:
+  boost::shared_ptr<ObjectStore> m_store_to_verify;
+  bool m_do_verify;
+
+ public:
+  VerifyFileStore(FileStore *store_to_verify, FileStore *our_store,
+      std::string status_file) :
+        DeterministicOpSequence(our_store, status_file), m_do_verify(false) {
+    m_store_to_verify.reset(store_to_verify);
+
+//    int err;
+//    err = m_store_to_verify->mount();
+//    ceph_assert(err == 0);
+  }
+  ~VerifyFileStore() { }
+
+  void generate(int seed, int verify_at);
+
+ private:
+  virtual void _do_touch(coll_t coll, hobject_t& obj);
+  virtual void _do_write(coll_t coll, hobject_t& obj, uint64_t off,
+      uint64_t len, const bufferlist& data);
+  virtual void _do_clone(coll_t coll, hobject_t& orig_obj, hobject_t& new_obj);
+  virtual void _do_coll_move(coll_t new_coll, coll_t old_coll, hobject_t& obj);
+};
+
+#endif /* VERIFY_FILESTORE_H_ */