librbd/cache/pwl: fix reorder issue between func process_writeback_dirty_entries

In fact, we not only make sure ops in order in func process_writeback_dirty_entries,
but also make sure ops in order between func process_writeback_dirty_entries.

Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>

diff --git a/src/librbd/cache/pwl/AbstractWriteLog.cc b/src/librbd/cache/pwl/AbstractWriteLog.cc
index 695b2f0041c9d09472e09e2e698ea0f387895fb0..c63df19c400b0fd2902652202e9455e506df8932 100644 (file)
--- a/src/librbd/cache/pwl/AbstractWriteLog.cc
+++ b/src/librbd/cache/pwl/AbstractWriteLog.cc
@@ -1639,6 +1639,7 @@ Context* AbstractWriteLog<I>::construct_flush_entry(std::shared_ptr<GenericLogEn
     m_lowest_flushing_sync_gen = log_entry->ram_entry.sync_gen_number;
   }
   m_flush_ops_in_flight += 1;
+  m_flush_ops_will_send += 1;
   /* For write same this is the bytes affected by the flush op, not the bytes transferred */
   m_flush_bytes_in_flight += log_entry->ram_entry.write_bytes;
 
@@ -1710,6 +1711,11 @@ void AbstractWriteLog<I>::process_writeback_dirty_entries() {
         all_clean = !m_flush_ops_in_flight;
         break;
       }
+
+      if (m_flush_ops_will_send) {
+       ldout(cct, 20) << "Previous flush-ops is still not sent" << dendl;
+       break;
+      }
       auto candidate = m_dirty_log_entries.front();
       bool flushable = can_flush_entry(candidate);
       if (flushable) {

diff --git a/src/librbd/cache/pwl/AbstractWriteLog.h b/src/librbd/cache/pwl/AbstractWriteLog.h
index 88e6470f12f7b4cd284b9490a2b6d40343dfd1d3..887d77010bb45605d94e37da103ae86d9acebadc 100644 (file)
--- a/src/librbd/cache/pwl/AbstractWriteLog.h
+++ b/src/librbd/cache/pwl/AbstractWriteLog.h
@@ -294,6 +294,7 @@ protected:
   std::atomic<int> m_async_flush_ops = {0};
   std::atomic<int> m_async_append_ops = {0};
 
+  std::atomic<int> m_flush_ops_will_send = {0};
   /* Acquire locks in order declared here */
 
   mutable ceph::mutex m_log_retire_lock;

diff --git a/src/librbd/cache/pwl/rwl/WriteLog.cc b/src/librbd/cache/pwl/rwl/WriteLog.cc
index f635015329a6c073f49a1ac8b0f744d3742bf059..55a22407dd8a03cefb0ed9f4d83e1e64699e4578 100644 (file)
--- a/src/librbd/cache/pwl/rwl/WriteLog.cc
+++ b/src/librbd/cache/pwl/rwl/WriteLog.cc
@@ -592,6 +592,7 @@ void WriteLog<I>::construct_flush_entries(pwl::GenericLogEntries entries_to_flus
              ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry
                                         << " " << *log_entry << dendl;
              log_entry->writeback(this->m_image_writeback, ctx);
+             this->m_flush_ops_will_send -= 1;
            }), 0);
         });
    }

diff --git a/src/librbd/cache/pwl/ssd/WriteLog.cc b/src/librbd/cache/pwl/ssd/WriteLog.cc
index fcced9d28742bbf3466ba192e6d3d7966d663df6..5c4be2fde0a497cc3eb67a35301f3c31e3635fdd 100644 (file)
--- a/src/librbd/cache/pwl/ssd/WriteLog.cc
+++ b/src/librbd/cache/pwl/ssd/WriteLog.cc
@@ -539,6 +539,7 @@ void WriteLog<I>::construct_flush_entries(pwl::GenericLogEntries entries_to_flus
              ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry
                                          << " " << *log_entry << dendl;
              log_entry->writeback(this->m_image_writeback, ctx);
+             this->m_flush_ops_will_send -= 1;
            }), 0);
        });
       }
@@ -589,6 +590,7 @@ void WriteLog<I>::construct_flush_entries(pwl::GenericLogEntries entries_to_flus
                                           << " " << *log_entry << dendl;
                log_entry->writeback_bl(this->m_image_writeback, ctx,
                                        std::move(captured_entry_bl));
+               this->m_flush_ops_will_send -= 1;
              }), 0);
          } else {
              m_image_ctx.op_work_queue->queue(new LambdaContext(
@@ -596,6 +598,7 @@ void WriteLog<I>::construct_flush_entries(pwl::GenericLogEntries entries_to_flus
                  ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry
                                              << " " << *log_entry << dendl;
                  log_entry->writeback(this->m_image_writeback, ctx);
+                 this->m_flush_ops_will_send -= 1;
                }), 0);
          }
        }