cout << " datalog list list data log\n";
cout << " datalog trim trim data log\n";
cout << " datalog status read data log status\n";
+ cout << " datalog type change datalog type to --log_type={fifo,omap}\n";
cout << " orphans find deprecated -- init and run search for leaked rados objects (use job-id, pool)\n";
cout << " orphans finish deprecated -- clean up search for leaked rados objects\n";
cout << " orphans list-jobs deprecated -- list the current job-ids for orphans search\n";
DATALOG_STATUS,
DATALOG_AUTOTRIM,
DATALOG_TRIM,
+ DATALOG_TYPE,
+ DATALOG_PRUNE,
REALM_CREATE,
REALM_DELETE,
REALM_GET,
{ "datalog status", OPT::DATALOG_STATUS },
{ "datalog autotrim", OPT::DATALOG_AUTOTRIM },
{ "datalog trim", OPT::DATALOG_TRIM },
+ { "datalog type", OPT::DATALOG_TYPE },
+ { "datalog prune", OPT::DATALOG_PRUNE },
{ "realm create", OPT::REALM_CREATE },
{ "realm delete", OPT::REALM_DELETE },
{ "realm get", OPT::REALM_GET },
return BIIndexType::Invalid;
}
+log_type get_log_type(const string& type_str) {
+ if (strcasecmp(type_str.c_str(), "fifo") == 0)
+ return log_type::fifo;
+ if (strcasecmp(type_str.c_str(), "omap") == 0)
+ return log_type::omap;
+
+ return static_cast<log_type>(0xff);
+}
+
void dump_bi_entry(bufferlist& bl, BIIndexType index_type, Formatter *formatter)
{
auto iter = bl.cbegin();
uint64_t min_rewrite_stripe_size = 0;
BIIndexType bi_index_type = BIIndexType::Plain;
+ std::optional<log_type> opt_log_type;
string job_id;
int num_shards = 0;
cerr << "ERROR: invalid bucket index entry type" << std::endl;
return EINVAL;
}
+ } else if (ceph_argparse_witharg(args, i, &val, "--log-type", (char*)NULL)) {
+ string log_type_str = val;
+ auto l = get_log_type(log_type_str);
+ if (l == static_cast<log_type>(0xff)) {
+ cerr << "ERROR: invalid log type" << std::endl;
+ return EINVAL;
+ }
+ opt_log_type = l;
} else if (ceph_argparse_binary_flag(args, i, &is_master_int, NULL, "--master", (char*)NULL)) {
is_master = (bool)is_master_int;
is_master_set = true;
}
}
+ if (opt_cmd == OPT::DATALOG_TYPE) {
+ if (!opt_log_type) {
+ std::cerr << "log-type not specified." << std::endl;
+ return -EINVAL;
+ }
+ auto datalog = static_cast<rgw::sal::RGWRadosStore*>(store)->svc()->datalog_rados;
+ ret = datalog->change_format(*opt_log_type, null_yield);
+ if (ret < 0) {
+ cerr << "ERROR: change_format(): " << cpp_strerror(-ret) << std::endl;
+ return -ret;
+ }
+ }
+
+ if (opt_cmd == OPT::DATALOG_PRUNE) {
+ auto datalog = static_cast<rgw::sal::RGWRadosStore*>(store)->svc()->datalog_rados;
+ std::optional<uint64_t> through;
+ ret = datalog->trim_generations(through);
+
+ if (ret < 0) {
+ cerr << "ERROR: trim_generations(): " << cpp_strerror(-ret) << std::endl;
+ return -ret;
+ }
+
+ if (through) {
+ std::cout << "Pruned " << *through << " empty generations." << std::endl;
+ } else {
+ std::cout << "No empty generations." << std::endl;
+ }
+ }
+
bool quota_op = (opt_cmd == OPT::QUOTA_SET || opt_cmd == OPT::QUOTA_ENABLE || opt_cmd == OPT::QUOTA_DISABLE);
if (quota_op) {
std::string_view max_marker() const override {
return "99999999"sv;
}
+ int is_empty() override {
+ for (auto shard = 0u; shard < oids.size(); ++shard) {
+ std::list<cls_log_entry> log_entries;
+ lr::ObjectReadOperation op;
+ std::string out_marker;
+ bool truncated;
+ cls_log_list(op, {}, {}, {}, 1, log_entries, &out_marker, &truncated);
+ auto r = rgw_rados_operate(ioctx, oids[shard], &op, nullptr, null_yield);
+ if (r == -ENOENT) {
+ continue;
+ }
+ if (r < 0) {
+ lderr(cct) << __PRETTY_FUNCTION__
+ << ": failed to list " << oids[shard]
+ << cpp_strerror(-r) << dendl;
+ return r;
+ }
+ if (!log_entries.empty()) {
+ return 0;
+ }
+ }
+ return 1;
+ }
};
class RGWDataChangesFIFO final : public RGWDataChangesBE {
rgw::cls::fifo::marker::max().to_string();
return std::string_view(mm);
}
+ int is_empty() override {
+ std::vector<rgw::cls::fifo::list_entry> log_entries;
+ bool more = false;
+ for (auto shard = 0u; shard < fifos.size(); ++shard) {
+ auto r = fifos[shard]->list(1, {}, &log_entries, &more,
+ null_yield);
+ if (r < 0) {
+ lderr(cct) << __PRETTY_FUNCTION__
+ << ": unable to list FIFO: " << get_oid(shard)
+ << ": " << cpp_strerror(-r) << dendl;
+ return r;
+ }
+ if (!log_entries.empty()) {
+ return 0;
+ }
+ }
+ return 1;
+ }
};
RGWDataChangesLog::RGWDataChangesLog(CephContext* cct)
GenTrim(DataLogBackends* bes, int shard_id, uint64_t target_gen, std::string cursor,
uint64_t head_gen, uint64_t tail_gen,
- boost::intrusive_ptr<RGWDataChangesBE>&& be,
+ boost::intrusive_ptr<RGWDataChangesBE> be,
lr::AioCompletion* super)
: Completion(super), bes(bes), shard_id(shard_id), target_gen(target_gen),
cursor(std::move(cursor)), head_gen(head_gen), tail_gen(tail_gen),
be.reset();
if (r == -ENOENT) r = -ENODATA;
if (r == -ENODATA && gen_id < target_gen) r = 0;
+ r = 0;
if (r < 0) {
complete(std::move(p), r);
return;
be = i->second;
}
auto c = be->gen_id == target_gen ? cursor : be->max_marker();
- r = be->trim(shard_id, c, call(std::move(p)));
+ be->trim(shard_id, c, call(std::move(p)));
}
};
const auto tail_gen = begin()->first;
if (target_gen < tail_gen) {
l.unlock();
- rgw_complete_aio_completion(c, 0);
+ rgw_complete_aio_completion(c, -ENODATA);
return;
}
- auto be = lower_bound(0)->second;
+ auto be = begin()->second;
l.unlock();
- auto p = be.get();
auto gt = std::make_unique<GenTrim>(this, shard_id, target_gen,
std::string(cursor), head_gen, tail_gen,
- std::move(be), c);
+ be, c);
+
+ auto cc = be->gen_id == target_gen ? cursor : be->max_marker();
+ be->trim(shard_id, cc, GenTrim::call(std::move(gt)));
+}
+
+int DataLogBackends::trim_generations(std::optional<uint64_t>& through) {
+ if (size() == 1) {
+ return 0;
+ }
- p->trim(shard_id, cursor, GenTrim::call(std::move(gt)));
+ std::vector<mapped_type> candidates;
+ {
+ std::scoped_lock l(m);
+ auto e = cend() - 1;
+ for (auto i = cbegin(); i < e; ++i) {
+ candidates.push_back(i->second);
+ }
+ }
+
+ std::optional<uint64_t> highest;
+ for (auto& be : candidates) {
+ auto r = be->is_empty();
+ if (r < 0) {
+ return r;
+ } else if (r == 1) {
+ highest = be->gen_id;
+ } else {
+ break;
+ }
+ }
+
+ through = highest;
+ if (!highest) {
+ return 0;
+ }
+ auto ec = empty_to(*highest, null_yield);
+ if (ec) {
+ return ceph::from_error_code(ec);
+ }
+
+ return ceph::from_error_code(remove_empty(null_yield));
}
+
int RGWDataChangesLog::trim_entries(int shard_id, std::string_view marker,
librados::AioCompletion* c)
{
return gencursor(std::numeric_limits<uint64_t>::max(),
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
}
+
+int RGWDataChangesLog::change_format(log_type type, optional_yield y) {
+ return ceph::from_error_code(bes->new_backing(type, y));
+}
+
+int RGWDataChangesLog::trim_generations(std::optional<uint64_t>& through) {
+ return bes->trim_generations(through);
+}
bs::error_code handle_init(entries_t e) noexcept override;
bs::error_code handle_new_gens(entries_t e) noexcept override;
bs::error_code handle_empty_to(uint64_t new_tail) noexcept override;
+
+ int trim_generations(std::optional<uint64_t>& through);
};
class RGWDataChangesLog {
// a marker that compares greater than any other
std::string max_marker() const;
std::string get_oid(uint64_t gen_id, int shard_id) const;
+
+
+ int change_format(log_type type, optional_yield y);
+ int trim_generations(std::optional<uint64_t>& through);
};
class RGWDataChangesBE : public boost::intrusive_ref_counter<RGWDataChangesBE> {
virtual int trim(int index, std::string_view marker,
librados::AioCompletion* c) = 0;
virtual std::string_view max_marker() const = 0;
+ // 1 on empty, 0 on non-empty, negative on error.
+ virtual int is_empty() = 0;
};
projects / ceph.git / commitdiff