/*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
/*error_handler=*/nullptr, /*read_only=*/false));
compaction_job_stats_.Reset();
- ASSERT_OK(SetIdentityFile(WriteOptions(), env_, dbname_));
+ ASSERT_OK(
+ SetIdentityFile(WriteOptions(), env_, dbname_, Temperature::kUnknown));
VersionEdit new_db;
new_db.SetLogNumber(0);
}
ASSERT_OK(s);
// Make "CURRENT" file that points to the new manifest file.
- s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr);
+ s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
+ Temperature::kUnknown, nullptr);
ASSERT_OK(s);
}
// Persist it to IDENTITY file if allowed
if (!read_only) {
- s = SetIdentityFile(write_options, env_, dbname_, db_id_);
+ s = SetIdentityFile(write_options, env_, dbname_,
+ immutable_db_options_.metadata_write_temperature,
+ db_id_);
}
return s;
}
Status DBImpl::NewDB(std::vector<std::string>* new_filenames) {
VersionEdit new_db;
const WriteOptions write_options(Env::IOActivity::kDBOpen);
- Status s = SetIdentityFile(write_options, env_, dbname_);
+ Status s = SetIdentityFile(write_options, env_, dbname_,
+ immutable_db_options_.metadata_write_temperature);
if (!s.ok()) {
return s;
}
}
std::unique_ptr<FSWritableFile> file;
FileOptions file_options = fs_->OptimizeForManifestWrite(file_options_);
+ // DB option takes precedence when not kUnknown
+ if (immutable_db_options_.metadata_write_temperature !=
+ Temperature::kUnknown) {
+ file_options.temperature =
+ immutable_db_options_.metadata_write_temperature;
+ }
s = NewWritableFile(fs_.get(), manifest, &file, file_options);
if (!s.ok()) {
return s;
if (s.ok()) {
// Make "CURRENT" file that points to the new manifest file.
s = SetCurrentFile(write_options, fs_.get(), dbname_, 1,
+ immutable_db_options_.metadata_write_temperature,
directories_.GetDbDir());
if (new_filenames) {
new_filenames->emplace_back(
BuildDBOptions(immutable_db_options_, mutable_db_options_);
FileOptions opt_file_options =
fs_->OptimizeForLogWrite(file_options_, db_options);
+ // DB option takes precedence when not kUnknown
+ if (immutable_db_options_.wal_write_temperature != Temperature::kUnknown) {
+ opt_file_options.temperature = immutable_db_options_.wal_write_temperature;
+ }
std::string wal_dir = immutable_db_options_.GetWalDir();
std::string log_fname = LogFileName(wal_dir, log_file_num);
#include <atomic>
#include <cstdlib>
#include <functional>
+#include <iostream>
#include <memory>
#include "db/db_test_util.h"
#include "rocksdb/utilities/replayer.h"
#include "rocksdb/wal_filter.h"
#include "test_util/testutil.h"
+#include "util/defer.h"
#include "util/random.h"
#include "utilities/fault_injection_env.h"
ASSERT_EQ("d_value", Get("d"));
}
+TEST_F(DBTest2, VariousFileTemperatures) {
+ constexpr size_t kNumberFileTypes = static_cast<size_t>(kBlobFile) + 1U;
+
+ struct MyTestFS : public FileTemperatureTestFS {
+ explicit MyTestFS(const std::shared_ptr<FileSystem>& fs)
+ : FileTemperatureTestFS(fs) {
+ Reset();
+ }
+
+ IOStatus NewWritableFile(const std::string& fname, const FileOptions& opts,
+ std::unique_ptr<FSWritableFile>* result,
+ IODebugContext* dbg) override {
+ IOStatus ios =
+ FileTemperatureTestFS::NewWritableFile(fname, opts, result, dbg);
+ if (ios.ok()) {
+ uint64_t number;
+ FileType type;
+ if (ParseFileName(GetFileName(fname), &number, "LOG", &type)) {
+ if (type == kTableFile) {
+ // Not checked here
+ } else if (type == kWalFile) {
+ if (opts.temperature != expected_wal_temperature) {
+ std::cerr << "Attempt to open " << fname << " with temperature "
+ << temperature_to_string[opts.temperature]
+ << " rather than "
+ << temperature_to_string[expected_wal_temperature]
+ << std::endl;
+ assert(false);
+ }
+ } else if (type == kDescriptorFile) {
+ if (opts.temperature != expected_manifest_temperature) {
+ std::cerr << "Attempt to open " << fname << " with temperature "
+ << temperature_to_string[opts.temperature]
+ << " rather than "
+ << temperature_to_string[expected_wal_temperature]
+ << std::endl;
+ assert(false);
+ }
+ } else if (opts.temperature != expected_other_metadata_temperature) {
+ std::cerr << "Attempt to open " << fname << " with temperature "
+ << temperature_to_string[opts.temperature]
+ << " rather than "
+ << temperature_to_string[expected_wal_temperature]
+ << std::endl;
+ assert(false);
+ }
+ UpdateCount(type, 1);
+ }
+ }
+ return ios;
+ }
+
+ IOStatus RenameFile(const std::string& src, const std::string& dst,
+ const IOOptions& options,
+ IODebugContext* dbg) override {
+ IOStatus ios = FileTemperatureTestFS::RenameFile(src, dst, options, dbg);
+ if (ios.ok()) {
+ uint64_t number;
+ FileType src_type;
+ FileType dst_type;
+ assert(ParseFileName(GetFileName(src), &number, "LOG", &src_type));
+ assert(ParseFileName(GetFileName(dst), &number, "LOG", &dst_type));
+
+ UpdateCount(src_type, -1);
+ UpdateCount(dst_type, 1);
+ }
+ return ios;
+ }
+
+ void UpdateCount(FileType type, int delta) {
+ size_t i = static_cast<size_t>(type);
+ assert(i < kNumberFileTypes);
+ counts[i].FetchAddRelaxed(delta);
+ }
+
+ std::map<FileType, size_t> PopCounts() {
+ std::map<FileType, size_t> ret;
+ for (size_t i = 0; i < kNumberFileTypes; ++i) {
+ int c = counts[i].ExchangeRelaxed(0);
+ if (c > 0) {
+ ret[static_cast<FileType>(i)] = c;
+ }
+ }
+ return ret;
+ }
+
+ FileOptions OptimizeForLogWrite(
+ const FileOptions& file_options,
+ const DBOptions& /*db_options*/) const override {
+ FileOptions opts = file_options;
+ if (optimize_wal_temperature != Temperature::kUnknown) {
+ opts.temperature = optimize_wal_temperature;
+ }
+ return opts;
+ }
+
+ FileOptions OptimizeForManifestWrite(
+ const FileOptions& file_options) const override {
+ FileOptions opts = file_options;
+ if (optimize_manifest_temperature != Temperature::kUnknown) {
+ opts.temperature = optimize_manifest_temperature;
+ }
+ return opts;
+ }
+
+ void Reset() {
+ optimize_manifest_temperature = Temperature::kUnknown;
+ optimize_wal_temperature = Temperature::kUnknown;
+ expected_manifest_temperature = Temperature::kUnknown;
+ expected_other_metadata_temperature = Temperature::kUnknown;
+ expected_wal_temperature = Temperature::kUnknown;
+ for (auto& c : counts) {
+ c.StoreRelaxed(0);
+ }
+ }
+
+ Temperature optimize_manifest_temperature;
+ Temperature optimize_wal_temperature;
+ Temperature expected_manifest_temperature;
+ Temperature expected_other_metadata_temperature;
+ Temperature expected_wal_temperature;
+ std::array<RelaxedAtomic<int>, kNumberFileTypes> counts;
+ };
+
+ // We don't have enough non-unknown temps to confidently distinguish that
+ // a specific setting caused a specific outcome, in a single run. This is a
+ // reasonable work-around without blowing up test time. Only returns
+ // non-unknown temperatures.
+ auto RandomTemp = [] {
+ static std::vector<Temperature> temps = {
+ Temperature::kHot, Temperature::kWarm, Temperature::kCold};
+ return temps[Random::GetTLSInstance()->Uniform(
+ static_cast<int>(temps.size()))];
+ };
+
+ auto test_fs = std::make_shared<MyTestFS>(env_->GetFileSystem());
+ std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, test_fs));
+ for (bool use_optimize : {false, true}) {
+ std::cerr << "use_optimize: " << std::to_string(use_optimize) << std::endl;
+ for (bool use_temp_options : {false, true}) {
+ std::cerr << "use_temp_options: " << std::to_string(use_temp_options)
+ << std::endl;
+
+ Options options = CurrentOptions();
+ // Currently require for last level temperature
+ options.compaction_style = kCompactionStyleUniversal;
+ options.env = env.get();
+ test_fs->Reset();
+ if (use_optimize) {
+ test_fs->optimize_manifest_temperature = RandomTemp();
+ test_fs->expected_manifest_temperature =
+ test_fs->optimize_manifest_temperature;
+ test_fs->optimize_wal_temperature = RandomTemp();
+ test_fs->expected_wal_temperature = test_fs->optimize_wal_temperature;
+ }
+ if (use_temp_options) {
+ options.metadata_write_temperature = RandomTemp();
+ test_fs->expected_manifest_temperature =
+ options.metadata_write_temperature;
+ test_fs->expected_other_metadata_temperature =
+ options.metadata_write_temperature;
+ options.wal_write_temperature = RandomTemp();
+ test_fs->expected_wal_temperature = options.wal_write_temperature;
+ options.last_level_temperature = RandomTemp();
+ options.default_write_temperature = RandomTemp();
+ }
+
+ DestroyAndReopen(options);
+ Defer closer([&] { Close(); });
+
+ using FTC = std::map<FileType, size_t>;
+ // Files on DB startup
+ ASSERT_EQ(test_fs->PopCounts(), FTC({{kWalFile, 1},
+ {kDescriptorFile, 2},
+ {kCurrentFile, 2},
+ {kIdentityFile, 1},
+ {kOptionsFile, 1}}));
+
+ // Temperature count map
+ using TCM = std::map<Temperature, size_t>;
+ ASSERT_EQ(test_fs->CountCurrentSstFilesByTemp(), TCM({}));
+
+ ASSERT_OK(Put("foo", "1"));
+ ASSERT_OK(Put("bar", "1"));
+ ASSERT_OK(Flush());
+ ASSERT_OK(Put("foo", "2"));
+ ASSERT_OK(Put("bar", "2"));
+ ASSERT_OK(Flush());
+
+ ASSERT_EQ(test_fs->CountCurrentSstFilesByTemp(),
+ TCM({{options.default_write_temperature, 2}}));
+
+ ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
+
+ ASSERT_EQ(test_fs->CountCurrentSstFilesByTemp(),
+ TCM({{options.last_level_temperature, 1}}));
+
+ ASSERT_OK(Put("foo", "3"));
+ ASSERT_OK(Put("bar", "3"));
+ ASSERT_OK(Flush());
+
+ // Just in memtable/WAL
+ ASSERT_OK(Put("dog", "3"));
+
+ {
+ TCM expected;
+ expected[options.default_write_temperature] += 1;
+ expected[options.last_level_temperature] += 1;
+ ASSERT_EQ(test_fs->CountCurrentSstFilesByTemp(), expected);
+ }
+
+ // New files during operation
+ ASSERT_EQ(test_fs->PopCounts(), FTC({{kWalFile, 3}, {kTableFile, 4}}));
+
+ Reopen(options);
+
+ // New files during re-open/recovery
+ ASSERT_EQ(test_fs->PopCounts(), FTC({{kWalFile, 1},
+ {kTableFile, 1},
+ {kDescriptorFile, 1},
+ {kCurrentFile, 1},
+ {kOptionsFile, 1}}));
+
+ Destroy(options);
+ }
+ }
+}
+
TEST_F(DBTest2, LastLevelTemperature) {
class TestListener : public EventListener {
public:
return count;
}
+ std::map<Temperature, size_t> CountCurrentSstFilesByTemp() {
+ MutexLock lock(&mu_);
+ std::map<Temperature, size_t> ret;
+ for (const auto& e : current_sst_file_temperatures_) {
+ ret[e.second]++;
+ }
+ return ret;
+ }
+
void OverrideSstFileTemperature(uint64_t number, Temperature temp) {
MutexLock lock(&mu_);
current_sst_file_temperatures_[number] = temp;
requested_sst_file_temperatures_;
std::map<uint64_t, Temperature> current_sst_file_temperatures_;
- std::string GetFileName(const std::string& fname) {
+ static std::string GetFileName(const std::string& fname) {
auto filename = fname.substr(fname.find_last_of(kFilePathSeparator) + 1);
// workaround only for Windows that the file path could contain both Windows
// FilePathSeparator and '/'
}
void NewDB() {
- ASSERT_OK(SetIdentityFile(WriteOptions(), env_, dbname_));
+ ASSERT_OK(
+ SetIdentityFile(WriteOptions(), env_, dbname_, Temperature::kUnknown));
VersionEdit new_db;
new_db.SetLogNumber(0);
}
ASSERT_OK(s);
// Make "CURRENT" file that points to the new manifest file.
- s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr);
+ s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
+ Temperature::kUnknown, nullptr);
ASSERT_OK(s);
}
std::unique_ptr<log::Writer> new_desc_log_ptr;
{
FileOptions opt_file_opts = fs_->OptimizeForManifestWrite(file_options_);
+ // DB option (in file_options_) takes precedence when not kUnknown
+ if (file_options_.temperature != Temperature::kUnknown) {
+ opt_file_opts.temperature = file_options_.temperature;
+ }
mu->Unlock();
TEST_SYNC_POINT("VersionSet::LogAndApply:WriteManifestStart");
TEST_SYNC_POINT_CALLBACK("VersionSet::LogAndApply:WriteManifest", nullptr);
assert(manifest_io_status.ok());
}
if (s.ok() && new_descriptor_log) {
- io_s = SetCurrentFile(write_options, fs_.get(), dbname_,
- pending_manifest_file_number_,
- dir_contains_current_file);
+ io_s = SetCurrentFile(
+ write_options, fs_.get(), dbname_, pending_manifest_file_number_,
+ file_options_.temperature, dir_contains_current_file);
if (!io_s.ok()) {
s = io_s;
// Quarantine old manifest file in case new manifest file's CURRENT file
}
}
+ void CreateCurrentFile() {
+ // Make "CURRENT" file point to the new manifest file.
+ ASSERT_OK(SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
+ Temperature::kUnknown,
+ /* dir_contains_current_file */ nullptr));
+ }
+
// Create DB with 3 column families.
void NewDB() {
SequenceNumber last_seqno;
std::unique_ptr<log::Writer> log_writer;
- ASSERT_OK(SetIdentityFile(WriteOptions(), env_, dbname_));
+ ASSERT_OK(
+ SetIdentityFile(WriteOptions(), env_, dbname_, Temperature::kUnknown));
PrepareManifest(&column_families_, &last_seqno, &log_writer);
log_writer.reset();
- // Make "CURRENT" file point to the new manifest file.
- Status s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
EXPECT_OK(versions_->Recover(column_families_, false));
EXPECT_EQ(column_families_.size(),
edits_[i].MarkAtomicGroup(--remaining);
edits_[i].SetLastSequence(last_seqno_++);
}
- ASSERT_OK(SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr));
+ CreateCurrentFile();
}
void SetupIncompleteTrailingAtomicGroup(int atomic_group_size) {
edits_[i].MarkAtomicGroup(--remaining);
edits_[i].SetLastSequence(last_seqno_++);
}
- ASSERT_OK(SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr));
+ CreateCurrentFile();
}
void SetupCorruptedAtomicGroup(int atomic_group_size) {
}
edits_[i].SetLastSequence(last_seqno_++);
}
- ASSERT_OK(SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr));
+ CreateCurrentFile();
}
void SetupIncorrectAtomicGroup(int atomic_group_size) {
}
edits_[i].SetLastSequence(last_seqno_++);
}
- ASSERT_OK(SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr));
+ CreateCurrentFile();
}
void SetupTestSyncPoints() {
SequenceNumber last_seqno;
std::unique_ptr<log::Writer> log_writer;
PrepareManifest(&column_families, &last_seqno, &log_writer);
- Status s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
EXPECT_OK(versions_->Recover(column_families, false /* read_only */));
EXPECT_EQ(column_families.size(),
cfd_to_drop->Ref();
drop_cf_edit.SetColumnFamily(cfd_to_drop->GetID());
mutex_.Lock();
- s = versions_->LogAndApply(
+ Status s = versions_->LogAndApply(
cfd_to_drop, *cfd_to_drop->GetLatestMutableCFOptions(), read_options,
write_options, &drop_cf_edit, &mutex_, nullptr);
mutex_.Unlock();
TEST_F(EmptyDefaultCfNewManifest, Recover) {
PrepareManifest(nullptr, nullptr, &log_writer_);
log_writer_.reset();
- Status s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
- /* dir_contains_current_file */ nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
std::vector<ColumnFamilyDescriptor> column_families;
cf_options_);
std::string db_id;
bool has_missing_table_file = false;
- s = versions_->TryRecoverFromOneManifest(
+ Status s = versions_->TryRecoverFromOneManifest(
manifest_path, column_families, false, &db_id, &has_missing_table_file);
ASSERT_OK(s);
ASSERT_FALSE(has_missing_table_file);
assert(nullptr != log_writer);
VersionEdit new_db;
if (db_options_.write_dbid_to_manifest) {
- ASSERT_OK(SetIdentityFile(WriteOptions(), env_, dbname_));
+ ASSERT_OK(SetIdentityFile(WriteOptions(), env_, dbname_,
+ Temperature::kUnknown));
DBOptions tmp_db_options;
tmp_db_options.env = env_;
std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
PrepareManifest(nullptr, nullptr, &log_writer_);
log_writer_.reset();
- Status s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
- /* dir_contains_current_file */ nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
std::string db_id;
bool has_missing_table_file = false;
- s = versions_->TryRecoverFromOneManifest(manifest_path, column_families,
- read_only, &db_id,
- &has_missing_table_file);
+ Status s = versions_->TryRecoverFromOneManifest(
+ manifest_path, column_families, read_only, &db_id,
+ &has_missing_table_file);
auto iter =
std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName);
if (iter == cf_names.end()) {
ASSERT_OK(s);
}
log_writer_.reset();
- s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
- /* dir_contains_current_file */ nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
ASSERT_OK(s);
}
log_writer_.reset();
- s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
- /* dir_contains_current_file */ nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
ASSERT_OK(s);
}
log_writer_.reset();
- s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
- /* dir_contains_current_file */ nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
ASSERT_OK(s);
}
log_writer_.reset();
- s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1,
- /* dir_contains_current_file */ nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
WriteFileAdditionAndDeletionToManifest(
/*cf=*/0, std::vector<std::pair<int, FileMetaData>>(), deleted_files);
log_writer_.reset();
- Status s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
std::string db_id;
bool has_missing_table_file = false;
- s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_,
- /*read_only=*/false, &db_id,
- &has_missing_table_file);
+ Status s = versions_->TryRecoverFromOneManifest(
+ manifest_path, column_families_,
+ /*read_only=*/false, &db_id, &has_missing_table_file);
ASSERT_OK(s);
ASSERT_TRUE(has_missing_table_file);
for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) {
WriteFileAdditionAndDeletionToManifest(
/*cf=*/0, added_files, std::vector<std::pair<int, uint64_t>>());
log_writer_.reset();
- Status s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
std::string db_id;
bool has_missing_table_file = false;
- s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_,
- /*read_only=*/false, &db_id,
- &has_missing_table_file);
+ Status s = versions_->TryRecoverFromOneManifest(
+ manifest_path, column_families_,
+ /*read_only=*/false, &db_id, &has_missing_table_file);
ASSERT_OK(s);
ASSERT_TRUE(has_missing_table_file);
for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) {
WriteFileAdditionAndDeletionToManifest(
/*cf=*/0, std::vector<std::pair<int, FileMetaData>>(), deleted_files);
log_writer_.reset();
- Status s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
std::string db_id;
bool has_missing_table_file = false;
- s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_,
- /*read_only=*/false, &db_id,
- &has_missing_table_file);
+ Status s = versions_->TryRecoverFromOneManifest(
+ manifest_path, column_families_,
+ /*read_only=*/false, &db_id, &has_missing_table_file);
ASSERT_OK(s);
ASSERT_FALSE(has_missing_table_file);
for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) {
/*cf=*/0, added_files, std::vector<std::pair<int, uint64_t>>(),
blob_files);
log_writer_.reset();
- Status s = SetCurrentFile(WriteOptions(), fs_.get(), dbname_, 1, nullptr);
- ASSERT_OK(s);
+ CreateCurrentFile();
std::string manifest_path;
VerifyManifest(&manifest_path);
std::string db_id;
bool has_missing_table_file = false;
- s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_,
- /*read_only=*/false, &db_id,
- &has_missing_table_file);
+ Status s = versions_->TryRecoverFromOneManifest(
+ manifest_path, column_families_,
+ /*read_only=*/false, &db_id, &has_missing_table_file);
ASSERT_OK(s);
ASSERT_TRUE(has_missing_table_file);
}
IOStatus WriteStringToFile(FileSystem* fs, const Slice& data,
const std::string& fname, bool should_sync,
- const IOOptions& io_options) {
+ const IOOptions& io_options,
+ const FileOptions& file_options) {
std::unique_ptr<FSWritableFile> file;
- EnvOptions soptions;
- IOStatus s = fs->NewWritableFile(fname, soptions, &file, nullptr);
+ IOStatus s = fs->NewWritableFile(fname, file_options, &file, nullptr);
if (!s.ok()) {
return s;
}
IOStatus SetCurrentFile(const WriteOptions& write_options, FileSystem* fs,
const std::string& dbname, uint64_t descriptor_number,
+ Temperature temp,
FSDirectory* dir_contains_current_file) {
// Remove leading "dbname/" and add newline to manifest file name
std::string manifest = DescriptorFileName(dbname, descriptor_number);
std::string tmp = TempFileName(dbname, descriptor_number);
IOOptions opts;
IOStatus s = PrepareIOFromWriteOptions(write_options, opts);
+ FileOptions file_opts;
+ file_opts.temperature = temp;
if (s.ok()) {
- s = WriteStringToFile(fs, contents.ToString() + "\n", tmp, true, opts);
+ s = WriteStringToFile(fs, contents.ToString() + "\n", tmp, true, opts,
+ file_opts);
}
TEST_SYNC_POINT_CALLBACK("SetCurrentFile:BeforeRename", &s);
if (s.ok()) {
}
Status SetIdentityFile(const WriteOptions& write_options, Env* env,
- const std::string& dbname, const std::string& db_id) {
+ const std::string& dbname, Temperature temp,
+ const std::string& db_id) {
std::string id;
if (db_id.empty()) {
id = env->GenerateUniqueId();
Status s;
IOOptions opts;
s = PrepareIOFromWriteOptions(write_options, opts);
+ FileOptions file_opts;
+ file_opts.temperature = temp;
if (s.ok()) {
- s = WriteStringToFile(env, id, tmp, true, &opts);
+ s = WriteStringToFile(env->GetFileSystem().get(), id, tmp,
+ /*should_sync=*/true, opts, file_opts);
}
if (s.ok()) {
s = env->RenameFile(tmp, identify_file_name);
// when
IOStatus SetCurrentFile(const WriteOptions& write_options, FileSystem* fs,
const std::string& dbname, uint64_t descriptor_number,
+ Temperature temp,
FSDirectory* dir_contains_current_file);
// Make the IDENTITY file for the db
Status SetIdentityFile(const WriteOptions& write_options, Env* env,
- const std::string& dbname,
+ const std::string& dbname, Temperature temp,
const std::string& db_id = {});
// Sync manifest file `file`.
// If this option is set, when creating the last level files, pass this
// temperature to FileSystem used. Should be no-op for default FileSystem
// and users need to plug in their own FileSystem to take advantage of it.
- // When using FIFO compaction, this option is ignored.
+ // Currently only compatible with universal compaction.
//
// Dynamically changeable through the SetOptions() API
Temperature last_level_temperature = Temperature::kUnknown;
FileOptions() : EnvOptions(), handoff_checksum_type(ChecksumType::kCRC32c) {}
FileOptions(const DBOptions& opts)
- : EnvOptions(opts), handoff_checksum_type(ChecksumType::kCRC32c) {}
+ : EnvOptions(opts),
+ temperature(opts.metadata_write_temperature),
+ handoff_checksum_type(ChecksumType::kCRC32c) {}
FileOptions(const EnvOptions& opts)
: EnvOptions(opts), handoff_checksum_type(ChecksumType::kCRC32c) {}
// A utility routine: write "data" to the named file.
IOStatus WriteStringToFile(FileSystem* fs, const Slice& data,
const std::string& fname, bool should_sync = false,
- const IOOptions& io_options = IOOptions());
+ const IOOptions& io_options = IOOptions(),
+ const FileOptions& file_options = FileOptions());
// A utility routine: read contents of named file into *data
IOStatus ReadFileToString(FileSystem* fs, const std::string& fname,
// Default 100ms
uint64_t follower_catchup_retry_wait_ms = 100;
+ // When DB files other than SST, blob and WAL files are created, use this
+ // filesystem temperature. (See also `wal_write_temperature` and various
+ // `*_temperature` CF options.) When not `kUnknown`, this overrides any
+ // temperature set by OptimizeForManifestWrite functions.
+ Temperature metadata_write_temperature = Temperature::kUnknown;
+
+ // Use this filesystem temperature when creating WAL files. When not
+ // `kUnknown`, this overrides any temperature set by OptimizeForLogWrite
+ // functions.
+ Temperature wal_write_temperature = Temperature::kUnknown;
// End EXPERIMENTAL
};
{offsetof(struct ImmutableDBOptions, follower_catchup_retry_wait_ms),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
+ {"metadata_write_temperature",
+ {offsetof(struct ImmutableDBOptions, metadata_write_temperature),
+ OptionType::kTemperature, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"wal_write_temperature",
+ {offsetof(struct ImmutableDBOptions, wal_write_temperature),
+ OptionType::kTemperature, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
};
const std::string OptionsHelper::kDBOptionsName = "DBOptions";
follower_refresh_catchup_period_ms(
options.follower_refresh_catchup_period_ms),
follower_catchup_retry_count(options.follower_catchup_retry_count),
- follower_catchup_retry_wait_ms(options.follower_catchup_retry_wait_ms) {
+ follower_catchup_retry_wait_ms(options.follower_catchup_retry_wait_ms),
+ metadata_write_temperature(options.metadata_write_temperature),
+ wal_write_temperature(options.wal_write_temperature) {
fs = env->GetFileSystem();
clock = env->GetSystemClock().get();
logger = info_log.get();
db_host_id.c_str());
ROCKS_LOG_HEADER(log, " Options.enforce_single_del_contracts: %s",
enforce_single_del_contracts ? "true" : "false");
+ ROCKS_LOG_HEADER(log, " Options.metadata_write_temperature: %s",
+ temperature_to_string[metadata_write_temperature].c_str());
+ ROCKS_LOG_HEADER(log, " Options.wal_write_temperature: %s",
+ temperature_to_string[wal_write_temperature].c_str());
}
bool ImmutableDBOptions::IsWalDirSameAsDBPath() const {
uint64_t follower_refresh_catchup_period_ms;
uint64_t follower_catchup_retry_count;
uint64_t follower_catchup_retry_wait_ms;
+ Temperature metadata_write_temperature;
+ Temperature wal_write_temperature;
// Beginning convenience/helper objects that are not part of the base
// DBOptions
options.enforce_single_del_contracts =
immutable_db_options.enforce_single_del_contracts;
options.daily_offpeak_time_utc = mutable_db_options.daily_offpeak_time_utc;
+ options.follower_refresh_catchup_period_ms =
+ immutable_db_options.follower_refresh_catchup_period_ms;
+ options.follower_catchup_retry_count =
+ immutable_db_options.follower_catchup_retry_count;
+ options.follower_catchup_retry_wait_ms =
+ immutable_db_options.follower_catchup_retry_wait_ms;
+ options.metadata_write_temperature =
+ immutable_db_options.metadata_write_temperature;
+ options.wal_write_temperature = immutable_db_options.wal_write_temperature;
return options;
}
}
std::unique_ptr<FSWritableFile> wf;
- Status s =
- fs->NewWritableFile(file_name, FileOptions(), &wf, nullptr);
+ FileOptions file_options;
+ file_options.temperature = db_opt.metadata_write_temperature;
+ Status s = fs->NewWritableFile(file_name, file_options, &wf, nullptr);
if (!s.ok()) {
return s;
}
"lowest_used_cache_tier=kNonVolatileBlockTier;"
"allow_data_in_errors=false;"
"enforce_single_del_contracts=false;"
- "daily_offpeak_time_utc=08:30-19:00;",
+ "daily_offpeak_time_utc=08:30-19:00;"
+ "follower_refresh_catchup_period_ms=123;"
+ "follower_catchup_retry_count=456;"
+ "follower_catchup_retry_wait_ms=789;"
+ "metadata_write_temperature=kCold;"
+ "wal_write_temperature=kHot;",
new_options));
ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
projects / rocksdb.git / commitdiff