filestore: fix rollback safety check

There is a window in the old check between when current/commit_op_seq is
written and the snapshot is taken.  If ceph-osd crashes, we'll be unable to
start because we'll believe current/ was in use without proper checkpoints.

Instead, make the snapped/not snapped state of current/ explicit.

Fixes: #2118
Signed-off-by: Sage Weil <sage@newdream.net>
Reviewed-by: Samuel Just <samuel.just@dreamhost.com>
Reviewed-by: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 0c1632de3264e8b32e3fe636674602d1cccf927f..bf856480efc5adc62e72ea3fb2411ca0432e7439 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -1609,6 +1609,9 @@ int FileStore::mount()
     goto close_basedir_fd;
   }
 
+  char nosnapfn[200];
+  snprintf(nosnapfn, sizeof(nosnapfn), "%s/nosnap", current_fn.c_str());
+
   if (btrfs_stable_commits) {
     if (snaps.empty()) {
       dout(0) << "mount WARNING: no consistent snaps found, store may be in inconsistent state" << dendl;
@@ -1640,9 +1643,13 @@ int FileStore::mount()
 
        uint64_t cp = snaps.back();
        dout(10) << " most recent snap from " << snaps << " is " << cp << dendl;
-       
-       if (curr_seq && cp != curr_seq) {
-         if (!m_osd_use_stale_snap) { 
+
+       // if current/ is marked as non-snapshotted, refuse to roll
+       // back (without clear direction) to avoid throwing out new
+       // data.
+       struct stat st;
+       if (::stat(nosnapfn, &st) == 0) {
+         if (!m_osd_use_stale_snap) {
            derr << "ERROR: current/ volume data version is not equal to snapshotted version." << dendl;
            derr << "Current version " << curr_seq << ", last snap " << cp << dendl;
            derr << "Force rollback to snapshotted version with 'osd use stale snap = true'" << dendl;
@@ -1718,6 +1725,18 @@ int FileStore::mount()
 
   dout(5) << "mount op_seq is " << initial_op_seq << dendl;
 
+  if (!btrfs_stable_commits) {
+    // mark current/ as non-snapshotted so that we don't rollback away
+    // from it.
+    int r = ::creat(nosnapfn, 0644);
+    if (r < 0) {
+      derr << "FileStore::mount: failed to create current/nosnap" << dendl;
+      goto close_current_fd;
+    }
+  } else {
+    // clear nosnap marker, if present.
+    ::unlink(nosnapfn);
+  }
 
   {
     LevelDBStore *omap_store = new LevelDBStore(omap_dir);