From: JonBailey1993 <jonathan.bailey1@ibm.com>
Date: Fri, 27 Sep 2024 11:30:53 +0000 (+0100)
Subject: common/io_exerciser: code enhancements to ceph_test_rados_io_sequence
X-Git-Tag: v20.0.0~890^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d31397457cc56fe105db6962b5b82d07e8340239;p=ceph.git

common/io_exerciser: code enhancements to ceph_test_rados_io_sequence

Removed dataGenerationSingleton.
Changed stringstream to use of fmt::format
Made enums into enum classes

Signed-off-by: Jon Bailey <jonathan.bailey1@ibm.com>
---

diff --git a/src/common/io_exerciser/DataGenerator.cc b/src/common/io_exerciser/DataGenerator.cc
index 276bdd7bd6c..9aa77eeb6e9 100644
--- a/src/common/io_exerciser/DataGenerator.cc
+++ b/src/common/io_exerciser/DataGenerator.cc
@@ -7,6 +7,9 @@
 #include "common/debug.h"
 #include "common/dout.h"
 
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+
 #include <chrono>
 #include <iostream>
 #include <stdexcept>
@@ -20,11 +23,6 @@ using SeededRandomGenerator = ceph::io_exerciser::data_generation
 using HeaderedSeededRandomGenerator = ceph::io_exerciser::data_generation
                                         ::HeaderedSeededRandomGenerator;
 
-std::mutex DataGenerator::DataGenerationSingleton::m_mutex;
-DataGenerator::DataGenerationSingleton
-  DataGenerator::DataGenerationSingleton::m_singletonInstance =
-    DataGenerator::DataGenerationSingleton();
-
 std::unique_ptr<DataGenerator> DataGenerator::create_generator(
     GenerationType generationType, const ObjectModel& model)
 {
@@ -41,78 +39,31 @@ std::unique_ptr<DataGenerator> DataGenerator::create_generator(
   return nullptr;
 }
 
-DataGenerator::DataGenerationSingleton::DataGenerationSingleton()
-{
-  m_created = false;
-}
-
-DataGenerator::DataGenerationSingleton
-  ::DataGenerationSingleton(uint64_t unique_id)
-{
-  m_uniqueId = unique_id;
-  m_created = true;
-}
-
-const DataGenerator::DataGenerationSingleton& 
-DataGenerator::DataGenerationSingleton::createSpecificInstance(uint64_t unique_id)
-{
-  std::scoped_lock lock(m_mutex);
-  ceph_assert(!m_singletonInstance.m_created);
-  m_singletonInstance = DataGenerationSingleton(unique_id);
-
-  return m_singletonInstance;
-}
-
-const DataGenerator::DataGenerationSingleton&
-  DataGenerator::DataGenerationSingleton::getInstance()
-{
-  if (!m_singletonInstance.m_created)
-  {
-    std::scoped_lock lock(m_mutex);
-    if (!m_singletonInstance.m_created)
-    {
-      std::mt19937_64 random_generator = 
-        std::mt19937_64(duration_cast<std::chrono::milliseconds>(
-          std::chrono::system_clock::now().time_since_epoch()).count());
-      m_singletonInstance = DataGenerator::DataGenerationSingleton(random_generator());
-    }
-  }
-  return m_singletonInstance;
-}
-
-const uint64_t DataGenerator::DataGenerationSingleton::getUniqueId()
-{
-  return getInstance().m_uniqueId;
-}
-
-void DataGenerator::generate_wrong_data(uint64_t offset, uint64_t length,
-                                        bufferlist& retlist)
+bufferlist DataGenerator::generate_wrong_data(uint64_t offset, uint64_t length)
 {
+  bufferlist retlist;
   uint64_t block_size = m_model.get_block_size();
   char buffer[block_size];
   for (uint64_t block_offset = offset;
        block_offset < offset + length;
        block_offset++)
-  { 
+  {
     std::memset(buffer, 0, block_size);
     retlist.append(ceph::bufferptr(buffer, block_size));
   }
+  return retlist;
 }
 
 bool DataGenerator::validate(bufferlist& bufferlist, uint64_t offset, uint64_t length)
 {
-  ceph::bufferlist comparison_list;
-  generate_data(offset, length, comparison_list);
-  return bufferlist.contents_equal(comparison_list);
+  return bufferlist.contents_equal(generate_data(offset, length));
 }
 
-#include <bitset>
-
 ceph::bufferptr SeededRandomGenerator::generate_block(uint64_t block_offset)
 {
   uint64_t block_size = m_model.get_block_size();
   char buffer[block_size];
-  
+
   std::mt19937_64 random_generator(m_model.get_seed(block_offset));
   uint64_t rand1 = random_generator();
   uint64_t rand2 = random_generator();
@@ -140,56 +91,98 @@ ceph::bufferptr SeededRandomGenerator::generate_block(uint64_t block_offset)
   return ceph::bufferptr(buffer, block_size);
 }
 
-void SeededRandomGenerator::generate_data(uint64_t offset, uint64_t length,
-                                          bufferlist& retlist)
+ceph::bufferptr SeededRandomGenerator::generate_wrong_block(uint64_t block_offset)
+{
+  uint64_t block_size = m_model.get_block_size();
+  char buffer[block_size];
+
+  std::mt19937_64 random_generator(m_model.get_seed(block_offset));
+  uint64_t rand1 = random_generator() - 1;
+  uint64_t rand2 = random_generator() + 1;
+
+  constexpr size_t generation_length = sizeof(uint64_t);
+
+  for (uint64_t i = 0; i < block_size; i+=(2*generation_length), rand1++, rand2--)
+  {
+    std::memcpy(buffer + i, &rand1, generation_length);
+    std::memcpy(buffer + i + generation_length, &rand2, generation_length);
+  }
+
+  size_t remainingBytes = block_size % (generation_length * 2);
+  if (remainingBytes > generation_length)
+  {
+    size_t remainingBytes2 = remainingBytes - generation_length;
+    std::memcpy(buffer + block_size - remainingBytes, &rand1, remainingBytes);
+    std::memcpy(buffer + block_size - remainingBytes2, &rand2, remainingBytes2);
+  }
+  else if (remainingBytes > 0)
+  {
+    std::memcpy(buffer + block_size - remainingBytes, &rand1, remainingBytes);
+  }
+
+  return ceph::bufferptr(buffer, block_size);
+}
+
+bufferlist SeededRandomGenerator::generate_data(uint64_t offset, uint64_t length)
 {
-  ceph_assert(retlist.length() == 0);
+  bufferlist retlist;
 
   for (uint64_t block_offset = offset; block_offset < offset + length; block_offset++)
   {
     retlist.append(generate_block(block_offset));
   }
+
+  return retlist;
+}
+
+bufferlist SeededRandomGenerator::generate_wrong_data(uint64_t offset, uint64_t length)
+{
+  bufferlist retlist;
+
+  for (uint64_t block_offset = offset; block_offset < offset + length; block_offset++)
+  {
+    retlist.append(generate_wrong_block(block_offset));
+  }
+
+  return retlist;
+}
+
+HeaderedSeededRandomGenerator
+  ::HeaderedSeededRandomGenerator(const ObjectModel& model,
+                                  std::optional<uint64_t> unique_run_id) :
+    SeededRandomGenerator(model),
+    unique_run_id(unique_run_id.value_or(generate_unique_run_id()))
+{
+
+}
+
+uint64_t HeaderedSeededRandomGenerator::generate_unique_run_id()
+{
+  std::mt19937_64 random_generator =
+        std::mt19937_64(duration_cast<std::chrono::milliseconds>(
+          std::chrono::system_clock::now().time_since_epoch()).count());
+
+      return random_generator();
 }
 
 ceph::bufferptr HeaderedSeededRandomGenerator::generate_block(uint64_t block_offset)
 {
-  UniqueIdBytes unique_run_id = DataGenerator::DataGenerationSingleton::getUniqueId();
   SeedBytes seed = m_model.get_seed(block_offset);
   TimeBytes current_time = duration_cast<std::chrono::milliseconds>(
       std::chrono::system_clock::now().time_since_epoch()).count();
 
   ceph::bufferptr bufferptr = SeededRandomGenerator::generate_block(block_offset);
 
-  std::memcpy(bufferptr.c_str(), &unique_run_id + uniqueIdStart(), uniqueIdLength());
+  std::memcpy(bufferptr.c_str() + uniqueIdStart(), &unique_run_id, uniqueIdLength());
   std::memcpy(bufferptr.c_str() + seedStart(), &seed, seedLength());
   std::memcpy(bufferptr.c_str() + timeStart(), &current_time, timeLength());
 
   return bufferptr;
 }
 
-void HeaderedSeededRandomGenerator::generate_wrong_data(uint64_t offset,
-                                                        uint64_t length,
-                                                        bufferlist& retlist)
+ceph::bufferptr HeaderedSeededRandomGenerator::generate_wrong_block(uint64_t block_offset)
 {
-  ceph_assert(retlist.length() == 0);
-  ceph_assert(m_model.get_block_size() >= headerLength());
-
-  ceph::bufferptr bufferptr;
-  for (uint64_t block_offset = offset; block_offset < offset + length; block_offset++)
-  {
-      UniqueIdBytes unique_run_id = DataGenerator::DataGenerationSingleton::getUniqueId();
-      SeedBytes seed = m_model.get_seed(block_offset-1);
-      TimeBytes current_time = duration_cast<std::chrono::milliseconds>(
-          std::chrono::system_clock::now().time_since_epoch()).count();
-
-      bufferptr = generate_block(block_offset-1);
-
-      std::memcpy(bufferptr.c_str(), &unique_run_id + uniqueIdStart(), uniqueIdLength());
-      std::memcpy(bufferptr.c_str() + seedStart(), &seed, seedLength());
-      std::memcpy(bufferptr.c_str() + timeStart(), &current_time, timeLength());
-
-      retlist.append(std::move(bufferptr));
-  }
+  return HeaderedSeededRandomGenerator::generate_block(block_offset % 8);
 }
 
 const HeaderedSeededRandomGenerator::UniqueIdBytes
@@ -253,7 +246,8 @@ bool HeaderedSeededRandomGenerator::validate(bufferlist& bufferlist,
 bool HeaderedSeededRandomGenerator::validate_block(uint64_t block_offset,
                                                    const char* buffer_start)
 {
-  // We validate the block matches what we generate byte for byte, however we ignore the time section of the header
+  // We validate the block matches what we generate byte for byte
+  // however we ignore the time section of the header
   ceph::bufferptr bufferptr = generate_block(block_offset);
   bool valid = strncmp(bufferptr.c_str(), buffer_start, timeStart()) == 0;
   valid = valid ? strncmp(bufferptr.c_str() + timeEnd(),
@@ -261,7 +255,7 @@ bool HeaderedSeededRandomGenerator::validate_block(uint64_t block_offset,
                           m_model.get_block_size() - timeEnd()) == 0 : valid;
   return valid;
 }
- 
+
 const HeaderedSeededRandomGenerator::ErrorType
   HeaderedSeededRandomGenerator::getErrorTypeForBlock(uint64_t read_offset,
                                                       uint64_t block_offset,
@@ -271,7 +265,7 @@ const HeaderedSeededRandomGenerator::ErrorType
   {
     UniqueIdBytes read_unique_run_id = readUniqueRunId(block_offset - read_offset,
                                                        bufferlist);
-    if (DataGenerationSingleton::getUniqueId() != read_unique_run_id)
+    if (unique_run_id != read_unique_run_id)
     {
       return ErrorType::RUN_ID_MISMATCH;
     }
@@ -312,7 +306,7 @@ void HeaderedSeededRandomGenerator
 
   if (blockError == ErrorType::DATA_MISMATCH || blockError == ErrorType::UNKNOWN)
   {
-    read_time = readDateTime(block_offset, bufferlist);
+    read_time = readDateTime(block_offset - read_offset, bufferlist);
     std::chrono::system_clock::time_point time_point{std::chrono::milliseconds{read_time}};
     ttp = std::chrono::system_clock::to_time_t(time_point);
 
@@ -323,76 +317,77 @@ void HeaderedSeededRandomGenerator
                 generate_block(block_offset).c_str(),
                 m_model.get_block_size() - bodyStart());
   }
-  std::stringstream ss;
+
+  std::string error_string;
   switch(blockError)
   {
     case ErrorType::RUN_ID_MISMATCH:
     {
       UniqueIdBytes read_unique_run_id = readUniqueRunId((block_offset - read_offset),
                                                           bufferlist);
-      ss << "Header (Run ID) mismatch detected at block " << block_offset 
-         << " (byte offset " << block_offset * m_model.get_block_size() << ")."
-         << " Header expected run id " << DataGenerationSingleton::getUniqueId()
-         << " but found id " << read_unique_run_id
-         << ". Block data corrupt or not written from this instance of this application.";
+      error_string = fmt::format("Header (Run ID) mismatch detected at block {} "
+        "(byte offset {}) Header expected run id {} but found id {}. "
+        "Block data corrupt or not written from this instance of this application.",
+      block_offset,
+      block_offset * m_model.get_block_size(),
+      unique_run_id,
+      read_unique_run_id);
     }
     break;
-    
+
     case ErrorType::SEED_MISMATCH:
     {
       SeedBytes read_seed = readSeed((block_offset - read_offset), bufferlist);
 
       if (m_model.get_seed_offsets(read_seed).size() == 0)
       {
-        ss << "Data (Seed) mismatch detected at block " << block_offset
-           << " (byte offset " << block_offset * m_model.get_block_size() << ")."
-           << " Header expected seed " << m_model.get_seed(block_offset)
-           << " but found seed " << read_seed
-           << ". Read data was not from any other recognised block in the object.";
+        error_string = fmt::format("Data (Seed) mismatch detected at block {}"
+          " (byte offset {}). Header expected seed {} but found seed {}. "
+          "Read data was not from any other recognised block in the object.",
+            block_offset,
+            block_offset * m_model.get_block_size(),
+            m_model.get_seed(block_offset),
+            read_seed);
       }
       else
       {
-        ss << "Data (Seed) mismatch detected at block " << block_offset
-           << " (byte offset " << block_offset * m_model.get_block_size() << ")."
-           << " Header expected seed " << m_model.get_seed(block_offset)
-           << " but found seed " << read_seed
-           << ". Read data was from a different block(s): "
-           << m_model.get_seed_offsets(read_seed);
+        std::vector<int> seed_offsets = m_model.get_seed_offsets(read_seed);
+        error_string = fmt::format("Data (Seed) mismatch detected at block {}"
+          " (byte offset {}). Header expected seed {} but found seed {}."
+          " Read data was from a different block(s): {}",
+            block_offset,
+            block_offset * m_model.get_block_size(),
+            m_model.get_seed(block_offset),
+            read_seed,
+            fmt::join(seed_offsets.begin(), seed_offsets.end(), ""));
       }
     }
     break;
 
     case ErrorType::DATA_MISMATCH:
     {
-      ss << "Data (Body) mismatch detected at block " << block_offset
-         << " (byte offset " << block_offset * m_model.get_block_size() << ")."
-         << " Header data matches, data body does not."
-         << " Data written at " << std::ctime(&ttp)
-         << "\nExpected data: " << std::endl;
-      ss << std::hex << std::setw(2) << std::setfill('0');
-      for (uint64_t i = headerLength();
-           i < m_model.get_block_size() - headerLength(); i++)
-      {
-        ss << static_cast<int>(generated_bytes[i]);
-      }
-      dout(0) << dendl;
-      ss << " Read data: " << std::endl;
-      for (uint64_t i = headerLength();
-           i < m_model.get_block_size() - headerLength(); i++)
-      {
-        ss << static_cast<int>(read_bytes[i]);
-      }
+      error_string = fmt::format("Data (Body) mismatch detected at block {}"
+        " (byte offset {}). Header data matches, data body does not."
+        " Data written at {}\nExpected data: \n{:02x}\nRead data:{:02x}",
+          block_offset,
+          block_offset * m_model.get_block_size(),
+          std::ctime(&ttp),
+          fmt::join(generated_bytes, generated_bytes + m_model.get_block_size(), ""),
+          fmt::join(read_bytes, read_bytes + m_model.get_block_size(), ""));
     }
     break;
 
     case ErrorType::DATA_NOT_FOUND:
     {
       uint64_t bufferlist_length = bufferlist.to_str().size();
-      ss << "Data (Body) could not be read at block " << block_offset
-         << " (byte offset " << block_offset * m_model.get_block_size() << ")"
-         << " offset in bufferlist returned from read: " << (block_offset - read_offset)
-         << " (" << (block_offset - read_offset) * m_model.get_block_size() << " bytes)."
-         << " Returned bufferlist length: " << bufferlist_length;
+      error_string = fmt::format("Data (Body) could not be read at block {}"
+        " (byte offset {}) offset in bufferlist returned from read: {}"
+        " ({} bytes). Returned bufferlist length: {}.",
+          block_offset,
+          block_offset * m_model.get_block_size(),
+          (block_offset - read_offset),
+          (block_offset - read_offset) * m_model.get_block_size(),
+          bufferlist_length);
     }
     break;
 
@@ -401,23 +396,16 @@ void HeaderedSeededRandomGenerator
 
     default:
     {
-      ss << "Data mismatch detected at block " << block_offset
-         << " (byte offset " << block_offset * m_model.get_block_size() << ")."
-         << "\nExpected data: " << std::endl;
-      ss << std::hex << std::setw(2) << std::setfill('0');
-      for (uint64_t i = 0; i < m_model.get_block_size(); i++)
-      {
-        ss << static_cast<int>(generated_bytes[i]);
-      }
-      ss << std::endl << "Read data: " << std::endl;
-      for (uint64_t i = 0; i < m_model.get_block_size(); i++)
-      {
-        ss << static_cast<int>(read_bytes[i]);
-      }
+      error_string = fmt::format("Data mismatch detected at block {}"
+        " (byte offset {}).\nExpected data:\n{:02x}\nRead data:\n{:02x}",
+          block_offset,
+          block_offset * m_model.get_block_size(),
+          fmt::join(generated_bytes, generated_bytes + m_model.get_block_size(), ""),
+          fmt::join(read_bytes, read_bytes + m_model.get_block_size(), ""));
     }
     break;
   }
-  dout(0) << ss.str() << dendl;
+  dout(0) << error_string << dendl;
 }
 
 void HeaderedSeededRandomGenerator
@@ -462,7 +450,7 @@ void HeaderedSeededRandomGenerator
 {
   uint64_t range_start = start_block_offset;
   uint64_t range_length = 0;
-  UniqueIdBytes initial_read_unique_run_id = readUniqueRunId(start_block_offset,
+  UniqueIdBytes initial_read_unique_run_id = readUniqueRunId(start_block_offset - read_offset,
                                                              bufferlist);
   for (uint64_t i = start_block_offset;
        i < start_block_offset + range_length_in_blocks; i++)
@@ -470,7 +458,7 @@ void HeaderedSeededRandomGenerator
     ceph_assert(getErrorTypeForBlock(read_offset, i, bufferlist)
                 == ErrorType::RUN_ID_MISMATCH);
 
-    UniqueIdBytes read_unique_run_id = readUniqueRunId(i, bufferlist);
+    UniqueIdBytes read_unique_run_id = readUniqueRunId(i - read_offset, bufferlist);
     if (initial_read_unique_run_id != read_unique_run_id ||
         i == (start_block_offset + range_length_in_blocks - 1))
     {
@@ -480,15 +468,17 @@ void HeaderedSeededRandomGenerator
       }
       else if (range_length > 1)
       {
-        dout(0) << "Data (Run ID) Mismatch detected from block " << range_start
-                << " (" << range_start * m_model.get_block_size() << " bytes)"
-                << " and spanning a range of " << range_length << " blocks"
-                << "(" << range_length * m_model.get_block_size() << " bytes). "
-                << "Expected run id " << DataGenerationSingleton::getUniqueId()
-                << " for range but found id " << initial_read_unique_run_id
-                << " for all blocks in range. "
-                << "Block data corrupt or not written from this instance of this application."
-                << dendl;
+        dout(0) << fmt::format("Data (Run ID) Mismatch detected from block {} ({} bytes)"
+                    " and spanning a range of {} blocks ({} bytes). "
+                    "Expected run id {} for range but found id {}"
+                    " for all blocks in range. "
+                    "Block data corrupt or not written from this instance of this application.",
+                      range_start,
+                      range_start * m_model.get_block_size(),
+                      range_length,
+                      range_length * m_model.get_block_size(),
+                      unique_run_id,
+                      initial_read_unique_run_id) << dendl;
       }
 
       range_start = i;
@@ -497,7 +487,7 @@ void HeaderedSeededRandomGenerator
     }
     else
     {
-      range_length++; 
+      range_length++;
     }
   }
 
@@ -509,14 +499,16 @@ void HeaderedSeededRandomGenerator
   }
   else if (range_length > 1)
   {
-    dout(0) << "Data (Run ID) Mismatch detected from block " << range_start
-            << " (" << range_start * m_model.get_block_size() << " bytes) "
-            << "and spanning a range of " << range_length << " blocks " 
-            << "(" << range_length * m_model.get_block_size() << " bytes). "
-            << "Expected run id " << DataGenerationSingleton::getUniqueId()
-            << " for range but found id " << initial_read_unique_run_id
-            << " for all blocks in range. "
-            << "Block data corrupt or not written from this instance of this application."
+    dout(0) << fmt::format("Data (Run ID) Mismatch detected from block {}"
+                " ({} bytes) and spanning a range of {} blocks ({} bytes). "
+                "Expected run id {} for range but found id for all blocks in range. "
+                "Block data corrupt or not written from this instance of this application.",
+                  range_start,
+                  range_start * m_model.get_block_size(),
+                  range_length,
+                  range_length * m_model.get_block_size(),
+                  unique_run_id,
+                  initial_read_unique_run_id)
             << dendl;
   }
 }
@@ -533,25 +525,32 @@ void HeaderedSeededRandomGenerator
   // Assert here if needed, as we can't support values
   // that can't be converted to a signed integer.
   ceph_assert(m_model.get_block_size() < (std::numeric_limits<uint64_t>::max() / 2));
-  int64_t range_offset = 0;
+  std::optional<int64_t> range_offset = 0;
 
   for (uint64_t i = start_block_offset;
        i < start_block_offset + range_length_in_blocks; i++)
   {
     ceph_assert(getErrorTypeForBlock(read_offset, i, bufferlist)
                 == ErrorType::SEED_MISMATCH);
-    SeedBytes read_seed = readSeed(i, bufferlist);
+    SeedBytes read_seed = readSeed(i - read_offset, bufferlist);
 
     std::vector<int> seed_found_offsets = m_model.get_seed_offsets(read_seed);
 
     if ((seed_found_offsets.size() == 1 &&
-        (static_cast<int64_t>(seed_found_offsets.front() - i) == range_offset)) || 
+        (static_cast<int64_t>(seed_found_offsets.front() - i) == range_offset)) ||
         range_length == 0)
     {
       if (range_length == 0)
       {
         range_start = i;
-        range_offset = seed_found_offsets.front() - i;
+        if (seed_found_offsets.size() > 0)
+        {
+          range_offset = seed_found_offsets.front() - i;
+        }
+        else
+        {
+          range_offset = std::nullopt;
+        }
       }
       range_length++;
     }
@@ -561,23 +560,44 @@ void HeaderedSeededRandomGenerator
       {
         printDebugInformationForBlock(read_offset, i - 1, bufferlist);
       }
-      else if (range_length > 1)
+      else if (range_length > 1 && range_offset.has_value())
       {
-        dout(0) << "Data (Seed) Mismatch detected from block " << range_start
-                << " (" << range_start * m_model.get_block_size() << " bytes) "
-                << "and spanning a range of " << range_length << " blocks "
-                << "(" << range_length * m_model.get_block_size() << " bytes). "
-                << "Returned data located starting from block "
-                << static_cast<uint64_t>(range_offset) + range_start
-                << " (" << (static_cast<uint64_t>(range_offset) + range_start)
-                            * m_model.get_block_size() << " bytes) "
-                << "and spanning a range of " << range_length << " blocks "
-                << "(" << range_length * m_model.get_block_size() << " bytes)."
+        dout(0) << fmt::format("Data (Seed) Mismatch detected from block {}"
+                    " ({} bytes) and spanning a range of {} blocks ({} bytes). "
+                    "Returned data located starting from block {} ({} bytes) "
+                    "and spanning a range of {} blocks ({} bytes).",
+                      range_start,
+                      range_start * m_model.get_block_size(),
+                      range_length, range_length * m_model.get_block_size(),
+                      static_cast<uint64_t>(*range_offset) + range_start,
+                      (static_cast<uint64_t>(*range_offset) + range_start)
+                        * m_model.get_block_size(),
+                      range_length,
+                      range_length * m_model.get_block_size())
+                << dendl;
+      }
+      else
+      {
+        dout(0) << fmt::format("Data (Seed) Mismatch detected from block {}"
+                    " ({} bytes) and spanning a range of {} blocks ({} bytes). "
+                    "Data seed mismatch spanning a range of {} blocks ({} bytes).",
+                      range_start,
+                      range_start * m_model.get_block_size(),
+                      range_length, range_length * m_model.get_block_size(),
+                      range_length,
+                      range_length * m_model.get_block_size())
                 << dendl;
       }
       range_length = 1;
       range_start = i;
-      range_offset = seed_found_offsets.front() - i;
+      if (seed_found_offsets.size() > 0)
+      {
+        range_offset = seed_found_offsets.front() - i;
+      }
+      else
+      {
+        range_offset = std::nullopt;
+      }
     }
   }
 
@@ -587,17 +607,33 @@ void HeaderedSeededRandomGenerator
                                   start_block_offset + range_length_in_blocks - 1,
                                   bufferlist);
   }
-  else if (range_length > 1)
+  else if (range_length > 1 && range_offset.has_value())
+  {
+    dout(0) << fmt::format("Data (Seed) Mismatch detected from block {} ({} bytes) "
+                "and spanning a range of {} blocks ({} bytes). "
+                "Returned data located starting from block {} ({} bytes) "
+                "and spanning a range of {} blocks ({} bytes).",
+                  range_start,
+                  range_start * m_model.get_block_size(),
+                  range_length,
+                  range_length * m_model.get_block_size(),
+                  *range_offset + range_start,
+                  (*range_offset + range_start) * m_model.get_block_size(),
+                  range_length,
+                  range_length * m_model.get_block_size())
+            << dendl;
+  }
+  else
   {
-    dout(0) << "Data (Seed) Mismatch detected from block " << range_start
-            << " (" << range_start * m_model.get_block_size() << " bytes) "
-            << "and spanning a range of " << range_length << " blocks "
-            << "(" << range_length * m_model.get_block_size() << " bytes). "
-            << "Returned data located starting from block "
-            << range_offset + range_start
-            << " (" << (range_offset + range_start) * m_model.get_block_size()
-            << " bytes) and spanning a range of " << range_length << " blocks "
-            << "(" << range_length * m_model.get_block_size() << " bytes)."
+    dout(0) << fmt::format("Data (Seed) Mismatch detected from block {} ({} bytes) "
+                "and spanning a range of {} blocks ({} bytes). "
+                "and spanning a range of {} blocks ({} bytes).",
+                  range_start,
+                  range_start * m_model.get_block_size(),
+                  range_length,
+                  range_length * m_model.get_block_size(),
+                  range_length,
+                  range_length * m_model.get_block_size())
             << dendl;
   }
 }
@@ -608,12 +644,13 @@ void HeaderedSeededRandomGenerator
                                              uint64_t range_length_in_blocks,
                                              const bufferlist& bufferlist)
 {
-  dout(0) << "Data Mismatch detected in blocks "
-          << "from " << start_block_offset
-          << " to " << start_block_offset + range_length_in_blocks - 1 << ". "
-          << "Headers look as expected for range, "
-          << "but generated data body does not match. "
-          << "More information given for individual blocks below." << dendl;
+  dout(0) << fmt::format("Data Mismatch detected in blocks from {} to {}. "
+              "Headers look as expected for range, "
+              "but generated data body does not match. "
+              "More information given for individual blocks below.",
+                start_block_offset,
+                start_block_offset + range_length_in_blocks - 1)
+          << dendl;
 
   for (uint64_t i = start_block_offset;
        i < start_block_offset + range_length_in_blocks; i++)
@@ -628,11 +665,13 @@ void HeaderedSeededRandomGenerator
                                       uint64_t range_length_in_blocks,
                                       const bufferlist& bufferlist)
 {
-  dout(0) << "Data Mismatch detected in blocks "
-  << "from " << start_block_offset 
-  << " to " << start_block_offset + range_length_in_blocks - 1 << ". "
-  << "Headers look as expected for range, but generated data body does not match."
-  << " More information given for individual blocks below." << dendl;
+  dout(0) << fmt::format("Data Mismatch detected in blocks from {} to {}. "
+              "Headers look as expected for range, "
+              "but generated data body does not match. "
+              "More information given for individual blocks below.",
+                start_block_offset,
+                start_block_offset + range_length_in_blocks - 1)
+          << dendl;
 
   for (uint64_t i = start_block_offset;
        i < start_block_offset + range_length_in_blocks; i++)
@@ -647,10 +686,11 @@ void HeaderedSeededRandomGenerator
                                            uint64_t range_length_in_blocks,
                                            const bufferlist& bufferlist)
 {
-  dout(0) << "Data not found for blocks "
-          << "from " << start_block_offset
-          << " to " << start_block_offset + range_length_in_blocks - 1 << ". "
-          << "More information given for individual blocks below." << dendl;
+  dout(0) << fmt::format("Data not found for blocks from {} to {}. "
+              "More information given for individual blocks below.",
+                start_block_offset,
+                start_block_offset + range_length_in_blocks - 1)
+          << dendl;
 
   for (uint64_t i = start_block_offset; i < start_block_offset + range_length_in_blocks; i++)
   {
diff --git a/src/common/io_exerciser/DataGenerator.h b/src/common/io_exerciser/DataGenerator.h
index 0f2a4be5b40..1e5784a54cc 100644
--- a/src/common/io_exerciser/DataGenerator.h
+++ b/src/common/io_exerciser/DataGenerator.h
@@ -16,18 +16,11 @@
  *   object, the buffer that was read and the expected buffer.
  *
  *
- * class DataGenerator::DataGenerationSingleton
- *   A singleton object created on first use to create and store
- *   a unique run identifier. Can either be created supplying an id
- *   or it can be generated at first use. Id cannot and should not be
- *   changed after being created.
- *
- * 
  * class SeededRandomGenerator
  *   Inherits from DataGenerator. Generates entirely random patterns
  *   based on the seed retrieved by the model.
- * 
- * 
+ *
+ *
  * class HeaderedSeededRandomGenerator
  *   Inherits from SeededDataGenerator. Generates entirely random patterns
  *   based on the seed retrieved by the model, however also appends a 
@@ -35,7 +28,7 @@
  *   a range of verbose debug options to help disagnose a miscompare
  *   whenever it detects unexpected data.
  */
- 
+
 namespace ceph {
   namespace io_exerciser {
     namespace data_generation {
@@ -45,44 +38,24 @@ namespace ceph {
         // CompressedGenerator
         // MixedGenerator
       };
-    
+
       class DataGenerator {
       public:
         virtual ~DataGenerator() = default;
         static std::unique_ptr<DataGenerator>
           create_generator(GenerationType generatorType,
                            const ObjectModel& model);
-        virtual void generate_data(uint64_t length, uint64_t offset,
-                                   bufferlist& retlist)=0;
+        virtual bufferlist generate_data(uint64_t length, uint64_t offset)=0;
         virtual bool validate(bufferlist& bufferlist, uint64_t offset,
                               uint64_t length);
 
         // Used for testing debug outputs from data generation
-        virtual void generate_wrong_data(uint64_t offset, uint64_t length,
-                                         bufferlist& retlist);
+        virtual bufferlist generate_wrong_data(uint64_t offset, uint64_t length);
 
       protected:
         const ObjectModel& m_model;
 
         DataGenerator(const ObjectModel& model) : m_model(model) {}
-
-        class DataGenerationSingleton {
-        private:
-          DataGenerationSingleton();
-          DataGenerationSingleton(uint64_t unique_id);
-
-          bool m_created = false;
-          uint64_t m_uniqueId = 0;
-
-          static DataGenerationSingleton m_singletonInstance;
-          static std::mutex m_mutex;
-
-        public:
-          static const DataGenerationSingleton&
-            createSpecificInstance(uint64_t unique_id);
-          static const DataGenerationSingleton& getInstance();
-          static const uint64_t getUniqueId();
-        };
       };
 
       class SeededRandomGenerator : public DataGenerator
@@ -92,19 +65,19 @@ namespace ceph {
             : DataGenerator(model) {}
 
           virtual bufferptr generate_block(uint64_t offset);
-          virtual void generate_data(uint64_t length, uint64_t offset,
-                                     bufferlist& retlist) override;
+          virtual bufferlist generate_data(uint64_t length, uint64_t offset);
+          virtual bufferptr generate_wrong_block(uint64_t offset);
+          virtual bufferlist generate_wrong_data(uint64_t offset, uint64_t length) override;
       };
 
       class HeaderedSeededRandomGenerator : public SeededRandomGenerator
       {
-        public:           
-          HeaderedSeededRandomGenerator(const ObjectModel& model)
-            : SeededRandomGenerator(model) {}
+        public:
+          HeaderedSeededRandomGenerator(const ObjectModel& model,
+                                        std::optional<uint64_t> unique_run_id = std::nullopt);
 
           bufferptr generate_block(uint64_t offset) override;
-          void generate_wrong_data(uint64_t offset, uint64_t length,
-                                   bufferlist& retlist) override;
+          bufferptr generate_wrong_block(uint64_t offset) override;
           bool validate(bufferlist& bufferlist, uint64_t offset,
                         uint64_t length) override;
 
@@ -113,6 +86,14 @@ namespace ceph {
           using SeedBytes = int;
           using TimeBytes = uint64_t;
 
+          enum class ErrorType {
+            RUN_ID_MISMATCH,
+            SEED_MISMATCH,
+            DATA_MISMATCH,
+            DATA_NOT_FOUND,
+            UNKNOWN
+          };
+
           constexpr uint8_t headerStart() const
             { return 0; };
           constexpr uint8_t uniqueIdStart() const
@@ -141,20 +122,16 @@ namespace ceph {
           const TimeBytes readDateTime(uint64_t block_offset,
                                        const bufferlist& bufferlist);
 
-          bool validate_block(uint64_t block_offset, const char* buffer_start);
+          const UniqueIdBytes unique_run_id;
 
-          enum class ErrorType {
-            RUN_ID_MISMATCH,
-            SEED_MISMATCH,
-            DATA_MISMATCH,
-            DATA_NOT_FOUND,
-            UNKNOWN
-          };
+          uint64_t generate_unique_run_id();
+
+          bool validate_block(uint64_t block_offset, const char* buffer_start);
 
           const ErrorType getErrorTypeForBlock(uint64_t read_offset,
                                                uint64_t block_offset,
                                                const bufferlist& bufferlist);
-          
+
           void printDebugInformationForBlock(uint64_t read_offset,
                                              uint64_t block_offset,
                                              const bufferlist& bufferlist);
diff --git a/src/common/io_exerciser/IoOp.cc b/src/common/io_exerciser/IoOp.cc
index 6b652e356b6..cd855ba6fff 100644
--- a/src/common/io_exerciser/IoOp.cc
+++ b/src/common/io_exerciser/IoOp.cc
@@ -11,7 +11,7 @@ IoOp::IoOp( OpType op,
   offset2(offset2), length2(length2),
   offset3(offset3), length3(length3)
 {
-  
+
 }
 
 std::string IoOp::value_to_string(uint64_t v) const
diff --git a/src/common/io_exerciser/IoOp.h b/src/common/io_exerciser/IoOp.h
index 93e1ce169ad..60c02a93d4e 100644
--- a/src/common/io_exerciser/IoOp.h
+++ b/src/common/io_exerciser/IoOp.h
@@ -17,7 +17,7 @@
 namespace ceph {
   namespace io_exerciser {
 
-    enum OpType {
+    enum class OpType {
       Done,       // End of I/O sequence
       BARRIER,    // Barrier - all prior I/Os must complete
       CREATE,     // Create object and pattern with data
diff --git a/src/common/io_exerciser/IoSequence.cc b/src/common/io_exerciser/IoSequence.cc
index 716752c53c2..4a7ca0593d1 100644
--- a/src/common/io_exerciser/IoSequence.cc
+++ b/src/common/io_exerciser/IoSequence.cc
@@ -1,7 +1,49 @@
 #include "IoSequence.h"
 
+using Sequence = ceph::io_exerciser::Sequence;
 using IoSequence = ceph::io_exerciser::IoSequence;
 
+std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, const Sequence& seq)
+{
+  switch (seq)
+  {
+    case Sequence::SEQUENCE_SEQ0:
+      os << "SEQUENCE_SEQ0";
+      break;
+    case Sequence::SEQUENCE_SEQ1:
+      os << "SEQUENCE_SEQ1";
+      break;
+    case Sequence::SEQUENCE_SEQ2:
+      os << "SEQUENCE_SEQ2";
+      break;
+    case Sequence::SEQUENCE_SEQ3:
+      os << "SEQUENCE_SEQ3";
+      break;
+    case Sequence::SEQUENCE_SEQ4:
+      os << "SEQUENCE_SEQ4";
+      break;
+    case Sequence::SEQUENCE_SEQ5:
+      os << "SEQUENCE_SEQ5";
+      break;
+    case Sequence::SEQUENCE_SEQ6:
+      os << "SEQUENCE_SEQ6";
+      break;
+    case Sequence::SEQUENCE_SEQ7:
+      os << "SEQUENCE_SEQ7";
+      break;
+    case Sequence::SEQUENCE_SEQ8:
+      os << "SEQUENCE_SEQ8";
+      break;
+    case Sequence::SEQUENCE_SEQ9:
+      os << "SEQUENCE_SEQ9";
+      break;
+    case Sequence::SEQUENCE_END:
+      os << "SEQUENCE_END";
+      break;
+  }
+  return os;
+}
+
 IoSequence::IoSequence(std::pair<int,int> obj_size_range,
                                            int seed) :
         min_obj_size(obj_size_range.first), max_obj_size(obj_size_range.second),
@@ -16,25 +58,25 @@ std::unique_ptr<IoSequence> IoSequence::generate_sequence(Sequence s,
                                                           int seed)
 {
   switch (s) {
-    case SEQUENCE_SEQ0:
+    case Sequence::SEQUENCE_SEQ0:
       return std::make_unique<Seq0>(obj_size_range, seed);
-    case SEQUENCE_SEQ1:
+    case Sequence::SEQUENCE_SEQ1:
       return std::make_unique<Seq1>(obj_size_range, seed);
-    case SEQUENCE_SEQ2:
+    case Sequence::SEQUENCE_SEQ2:
       return std::make_unique<Seq2>(obj_size_range, seed);
-    case SEQUENCE_SEQ3:
+    case Sequence::SEQUENCE_SEQ3:
       return std::make_unique<Seq3>(obj_size_range, seed);
-    case SEQUENCE_SEQ4:
+    case Sequence::SEQUENCE_SEQ4:
       return std::make_unique<Seq4>(obj_size_range, seed);
-    case SEQUENCE_SEQ5:
+    case Sequence::SEQUENCE_SEQ5:
       return std::make_unique<Seq5>(obj_size_range, seed);
-    case SEQUENCE_SEQ6:
+    case Sequence::SEQUENCE_SEQ6:
       return std::make_unique<Seq6>(obj_size_range, seed);
-    case SEQUENCE_SEQ7:
+    case Sequence::SEQUENCE_SEQ7:
       return std::make_unique<Seq7>(obj_size_range, seed);
-    case SEQUENCE_SEQ8:
+    case Sequence::SEQUENCE_SEQ8:
       return std::make_unique<Seq8>(obj_size_range, seed);
-    case SEQUENCE_SEQ9:
+    case Sequence::SEQUENCE_SEQ9:
       return std::make_unique<Seq9>(obj_size_range, seed);
     default:
       break;
diff --git a/src/common/io_exerciser/IoSequence.h b/src/common/io_exerciser/IoSequence.h
index 7dc7caa5df3..114ff76303f 100644
--- a/src/common/io_exerciser/IoSequence.h
+++ b/src/common/io_exerciser/IoSequence.h
@@ -31,7 +31,7 @@
 namespace ceph {
   namespace io_exerciser {
 
-    enum Sequence {
+    enum class Sequence {
       SEQUENCE_SEQ0,
       SEQUENCE_SEQ1,
       SEQUENCE_SEQ2,
@@ -46,11 +46,14 @@ namespace ceph {
       SEQUENCE_END,
       SEQUENCE_BEGIN = SEQUENCE_SEQ0
     };
+
     inline Sequence operator++( Sequence& s )
     {
       return s = (Sequence)(((int)(s) + 1));
     }
 
+    std::ostream& operator<<(std::ostream& os, const Sequence& seq);
+
     /* I/O Sequences */
 
     class IoSequence {
diff --git a/src/common/io_exerciser/ObjectModel.cc b/src/common/io_exerciser/ObjectModel.cc
index 4558ae78fa1..589f6434282 100644
--- a/src/common/io_exerciser/ObjectModel.cc
+++ b/src/common/io_exerciser/ObjectModel.cc
@@ -60,12 +60,12 @@ void ObjectModel::applyIoOp(IoOp& op)
   };
 
   switch (op.op) {
-  case BARRIER:
+  case OpType::BARRIER:
     reads.clear();
     writes.clear();
     break;
 
-  case CREATE:
+  case OpType::CREATE:
     ceph_assert(!created);
     ceph_assert(reads.empty());
     ceph_assert(writes.empty());
@@ -75,7 +75,7 @@ void ObjectModel::applyIoOp(IoOp& op)
                   generate_random);
     break;
 
-  case REMOVE:
+  case OpType::REMOVE:
     ceph_assert(created);
     ceph_assert(reads.empty());
     ceph_assert(writes.empty());
@@ -83,7 +83,7 @@ void ObjectModel::applyIoOp(IoOp& op)
     contents.resize(0);
     break;
 
-  case READ3:
+  case OpType::READ3:
     ceph_assert(created);
     ceph_assert(op.offset3 + op.length3 <= contents.size());
     // Not allowed: read overlapping with parallel write
@@ -91,7 +91,7 @@ void ObjectModel::applyIoOp(IoOp& op)
     reads.union_insert(op.offset3, op.length3);
     [[fallthrough]];
 
-  case READ2:
+  case OpType::READ2:
     ceph_assert(created);
     ceph_assert(op.offset2 + op.length2 <= contents.size());
     // Not allowed: read overlapping with parallel write
@@ -99,7 +99,7 @@ void ObjectModel::applyIoOp(IoOp& op)
     reads.union_insert(op.offset2, op.length2);
     [[fallthrough]];
 
-  case READ:
+  case OpType::READ:
     ceph_assert(created);
     ceph_assert(op.offset1 + op.length1 <= contents.size());
     // Not allowed: read overlapping with parallel write
@@ -108,7 +108,7 @@ void ObjectModel::applyIoOp(IoOp& op)
     num_io++;
     break;
 
-  case WRITE3:
+  case OpType::WRITE3:
     ceph_assert(created);
     // Not allowed: write overlapping with parallel read or write
     ceph_assert(!reads.intersects(op.offset3, op.length3));
@@ -121,7 +121,7 @@ void ObjectModel::applyIoOp(IoOp& op)
                   generate_random);
     [[fallthrough]];
 
-  case WRITE2:
+  case OpType::WRITE2:
     ceph_assert(created);
     // Not allowed: write overlapping with parallel read or write
     ceph_assert(!reads.intersects(op.offset2, op.length2));
@@ -134,7 +134,7 @@ void ObjectModel::applyIoOp(IoOp& op)
                   generate_random);
     [[fallthrough]];
 
-  case WRITE:
+  case OpType::WRITE:
     ceph_assert(created);
     // Not allowed: write overlapping with parallel read or write
     ceph_assert(!reads.intersects(op.offset1, op.length1));
diff --git a/src/common/io_exerciser/RadosIo.cc b/src/common/io_exerciser/RadosIo.cc
index 3a9a2c1c285..3f907ccf474 100644
--- a/src/common/io_exerciser/RadosIo.cc
+++ b/src/common/io_exerciser/RadosIo.cc
@@ -86,8 +86,8 @@ bool RadosIo::readyForIoOp(IoOp &op)
     return false;
   }
   switch (op.op) {
-  case Done:
-  case BARRIER:
+  case OpType::Done:
+  case OpType::BARRIER:
     return outstanding_io == 0;
   default:
     return outstanding_io < threads;
@@ -105,18 +105,18 @@ void RadosIo::applyIoOp(IoOp &op)
   wait_for_io(threads-1);
   
   switch (op.op) {
-  case Done:
+  case OpType::Done:
   [[ fallthrough ]];
-  case BARRIER:
+  case OpType::BARRIER:
     // Wait for all outstanding I/O to complete
     wait_for_io(0);
     break;    
 
-  case CREATE:
+  case OpType::CREATE:
     {
       start_io();
       op_info = std::make_shared<AsyncOpInfo>(0, op.length1);
-      db->generate_data(0, op.length1, op_info->bl1);
+      op_info->bl1 = db->generate_data(0, op.length1);
       op_info->wop.write_full(op_info->bl1);
       auto create_cb = [this] (boost::system::error_code ec) {
         ceph_assert(ec == boost::system::errc::success);
@@ -127,7 +127,7 @@ void RadosIo::applyIoOp(IoOp &op)
     }
     break;
 
-  case REMOVE:
+  case OpType::REMOVE:
     {
       start_io();
       op_info = std::make_shared<AsyncOpInfo>();
@@ -141,7 +141,7 @@ void RadosIo::applyIoOp(IoOp &op)
     }
     break;
 
-  case READ:
+  case OpType::READ:
     {
       start_io();
       op_info = std::make_shared<AsyncOpInfo>(op.offset1, op.length1);
@@ -159,7 +159,7 @@ void RadosIo::applyIoOp(IoOp &op)
     }
     break;
 
-  case READ2:
+  case OpType::READ2:
     {
       start_io();
       op_info = std::make_shared<AsyncOpInfo>(op.offset1,
@@ -186,7 +186,7 @@ void RadosIo::applyIoOp(IoOp &op)
     }
     break;
 
-  case READ3:
+  case OpType::READ3:
     {
       start_io();
       op_info = std::make_shared<AsyncOpInfo>(op.offset1, op.length1,
@@ -215,11 +215,11 @@ void RadosIo::applyIoOp(IoOp &op)
     }
     break;
 
-  case WRITE:
+  case OpType::WRITE:
     {
       start_io();
       op_info = std::make_shared<AsyncOpInfo>(op.offset1, op.length1);
-      db->generate_data(op.offset1, op.length1, op_info->bl1);
+      op_info->bl1 = db->generate_data(op.offset1, op.length1);
 
       op_info->wop.write(op.offset1 * block_size, op_info->bl1);
       auto write_cb = [this] (boost::system::error_code ec) {
@@ -232,13 +232,13 @@ void RadosIo::applyIoOp(IoOp &op)
     }
     break;
 
-  case WRITE2:
+  case OpType::WRITE2:
     {
       start_io();
       op_info = std::make_shared<AsyncOpInfo>(op.offset1, op.length1,
                                               op.offset2, op.length2);
-      db->generate_data(op.offset1, op.length1, op_info->bl1);
-      db->generate_data(op.offset2, op.length2, op_info->bl2);
+      op_info->bl1 = db->generate_data(op.offset1, op.length1);
+      op_info->bl2 = db->generate_data(op.offset2, op.length2);
       op_info->wop.write(op.offset1 * block_size, op_info->bl1);
       op_info->wop.write(op.offset2 * block_size, op_info->bl2);
       auto write2_cb = [this] (boost::system::error_code ec) {
@@ -251,15 +251,15 @@ void RadosIo::applyIoOp(IoOp &op)
     }
     break;
 
-  case WRITE3:
+  case OpType::WRITE3:
     {
       start_io();
       op_info = std::make_shared<AsyncOpInfo>(op.offset1, op.length1,
                                               op.offset2, op.length2,
                                               op.offset3, op.length3);
-      db->generate_data(op.offset1, op.length1, op_info->bl1);
-      db->generate_data(op.offset2, op.length2, op_info->bl2);
-      db->generate_data(op.offset3, op.length3, op_info->bl3);
+      op_info->bl1 = db->generate_data(op.offset1, op.length1);
+      op_info->bl2 = db->generate_data(op.offset2, op.length2);
+      op_info->bl3 = db->generate_data(op.offset3, op.length3);
       op_info->wop.write(op.offset1 * block_size, op_info->bl1);
       op_info->wop.write(op.offset2 * block_size, op_info->bl2);
       op_info->wop.write(op.offset3 * block_size, op_info->bl3);