From 19c04a8570c6da41844eaa63f33fcfbd8853375c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 28 Sep 2015 22:17:10 -0400 Subject: [PATCH] log reader --- db/db_impl.cc | 5 +++-- db/log_reader.cc | 13 +++++++++---- db/log_reader.h | 11 +++++++++-- db/repair.cc | 5 +++-- db/transaction_log_impl.cc | 6 ++++-- db/version_set.cc | 18 +++++++++++------- db/wal_manager.cc | 4 ++-- util/ldb_cmd.cc | 7 ++++++- 8 files changed, 47 insertions(+), 22 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index b5241582..af455bea 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -999,8 +999,9 @@ Status DBImpl::RecoverLogFiles(const std::vector& log_numbers, // paranoid_checks==false so that corruptions cause entire commits // to be skipped instead of propagating bad information (like overly // large sequence numbers). - log::Reader reader(std::move(file), &reporter, true /*checksum*/, - 0 /*initial_offset*/); + log::Reader reader(&db_options_, std::move(file), &reporter, + true /*checksum*/, + 0 /*initial_offset*/, log_number); Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, "Recovering log #%" PRIu64 "", log_number); diff --git a/db/log_reader.cc b/db/log_reader.cc index f6514cfd..d9f681f3 100644 --- a/db/log_reader.cc +++ b/db/log_reader.cc @@ -11,6 +11,7 @@ #include #include "rocksdb/env.h" +#include "rocksdb/options.h" #include "util/coding.h" #include "util/crc32c.h" @@ -20,9 +21,12 @@ namespace log { Reader::Reporter::~Reporter() { } -Reader::Reader(unique_ptr&& _file, Reporter* reporter, - bool checksum, uint64_t initial_offset) - : file_(std::move(_file)), +Reader::Reader(const DBOptions *opt, + unique_ptr&& _file, Reporter* reporter, + bool checksum, uint64_t initial_offset, + uint32_t log_num) + : db_options_(opt), + file_(std::move(_file)), reporter_(reporter), checksum_(checksum), backing_store_(new char[kBlockSize]), @@ -32,7 +36,8 @@ Reader::Reader(unique_ptr&& _file, Reporter* reporter, eof_offset_(0), last_record_offset_(0), end_of_buffer_offset_(0), - initial_offset_(initial_offset) {} + initial_offset_(initial_offset), + log_number_(log_num) {} Reader::~Reader() { delete[] backing_store_; diff --git a/db/log_reader.h b/db/log_reader.h index a7cf45b4..f2408313 100644 --- a/db/log_reader.h +++ b/db/log_reader.h @@ -18,6 +18,7 @@ namespace rocksdb { class SequentialFile; +class DBOptions; using std::unique_ptr; namespace log { @@ -51,8 +52,10 @@ class Reader { // // The Reader will start reading at the first record located at physical // position >= initial_offset within the file. - Reader(unique_ptr&& file, Reporter* reporter, - bool checksum, uint64_t initial_offset); + Reader(const DBOptions *opt, + unique_ptr&& file, Reporter* reporter, + bool checksum, uint64_t initial_offset, + uint32_t log_num); ~Reader(); @@ -83,6 +86,7 @@ class Reader { SequentialFile* file() { return file_.get(); } private: + const DBOptions *db_options_; const unique_ptr file_; Reporter* const reporter_; bool const checksum_; @@ -103,6 +107,9 @@ class Reader { // Offset at which to start looking for the first record to return uint64_t const initial_offset_; + // which log number this is + uint32_t const log_number_; + // Extend record types with the following special values enum { kEof = kMaxRecordType + 1, diff --git a/db/repair.cc b/db/repair.cc index 355150d9..83614c60 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -246,8 +246,9 @@ class Repairer { // corruptions cause entire commits to be skipped instead of // propagating bad information (like overly large sequence // numbers). - log::Reader reader(std::move(lfile), &reporter, true /*enable checksum*/, - 0/*initial_offset*/); + log::Reader reader(&options_, std::move(lfile), &reporter, + true /*enable checksum*/, + 0/*initial_offset*/, log); // Read all the records and add to a memtable std::string scratch; diff --git a/db/transaction_log_impl.cc b/db/transaction_log_impl.cc index b0bf6e4e..60a1fdeb 100644 --- a/db/transaction_log_impl.cc +++ b/db/transaction_log_impl.cc @@ -257,8 +257,10 @@ Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) { return s; } assert(file); - currentLogReader_.reset(new log::Reader(std::move(file), &reporter_, - read_options_.verify_checksums_, 0)); + currentLogReader_.reset(new log::Reader(options_, + std::move(file), &reporter_, + read_options_.verify_checksums_, 0, + logFile->LogNumber())); return Status::OK(); } } // namespace rocksdb diff --git a/db/version_set.cc b/db/version_set.cc index 03ccb795..46f7f54f 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -2155,8 +2155,10 @@ Status VersionSet::Recover( { VersionSet::LogReporter reporter; reporter.status = &s; - log::Reader reader(std::move(manifest_file), &reporter, true /*checksum*/, - 0 /*initial_offset*/); + log::Reader reader(db_options_, std::move(manifest_file), &reporter, + true /*checksum*/, + 0 /*initial_offset*/, + 0); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { @@ -2401,8 +2403,9 @@ Status VersionSet::ListColumnFamilies(std::vector* column_families, column_family_names.insert({0, kDefaultColumnFamilyName}); VersionSet::LogReporter reporter; reporter.status = &s; - log::Reader reader(std::move(file), &reporter, true /*checksum*/, - 0 /*initial_offset*/); + log::Reader reader(NULL, std::move(file), &reporter, + true /*checksum*/, + 0 /*initial_offset*/, 0); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { @@ -2554,8 +2557,8 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname, { VersionSet::LogReporter reporter; reporter.status = &s; - log::Reader reader(std::move(file), &reporter, true/*checksum*/, - 0/*initial_offset*/); + log::Reader reader(db_options_, std::move(file), &reporter, true/*checksum*/, + 0/*initial_offset*/, 0); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { @@ -2799,7 +2802,8 @@ bool VersionSet::ManifestContains(uint64_t manifest_file_num, fname.c_str()); return false; } - log::Reader reader(std::move(file), nullptr, true/*checksum*/, 0); + log::Reader reader(db_options_, std::move(file), nullptr, true/*checksum*/, + 0, 0); Slice r; std::string scratch; bool result = false; diff --git a/db/wal_manager.cc b/db/wal_manager.cc index 5651bae3..f3c2083e 100644 --- a/db/wal_manager.cc +++ b/db/wal_manager.cc @@ -441,8 +441,8 @@ Status WalManager::ReadFirstLine(const std::string& fname, reporter.fname = fname.c_str(); reporter.status = &status; reporter.ignore_error = !db_options_.paranoid_checks; - log::Reader reader(std::move(file), &reporter, true /*checksum*/, - 0 /*initial_offset*/); + log::Reader reader(&db_options_, std::move(file), &reporter, true /*checksum*/, + 0 /*initial_offset*/, *sequence); std::string scratch; Slice record; diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index e7b29d24..7ec55c36 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -1415,7 +1415,12 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values, } } else { StdErrReporter reporter; - log::Reader reader(move(file), &reporter, true, 0); + uint64_t log_number; + FileType type; + if (!ParseFileName(wal_file, &log_number, &type)) { + assert(0); + } + log::Reader reader(NULL, move(file), &reporter, true, 0, log_number); string scratch; WriteBatch batch; Slice record; -- 2.47.3