if (id.length() > MAX_ID_LEN) {
return false;
}
- else if (expiration.get_days() <= 0) {
+ else if(expiration.empty() && noncur_expiration.empty()) {
+ return false;
+ }
+ else if (!expiration.empty() && expiration.get_days() <= 0) {
+ return false;
+ }
+ else if (!noncur_expiration.empty() && noncur_expiration.get_days() <=0) {
return false;
}
return true;
string id;
rule->get_id(id); // not that this will return false for groups, but that's ok, we won't search groups
rule_map.insert(pair<string, LCRule>(id, *rule));
- _add_rule(rule);
}
-void RGWLifecycleConfiguration::_add_rule(LCRule *rule)
+bool RGWLifecycleConfiguration::_add_rule(LCRule *rule)
{
- prefix_map[rule->get_prefix()] = rule->get_expiration().get_days();
+ lc_op op;
+ if (rule->get_status().compare("Enabled") == 0) {
+ op.status = true;
+ }
+ if (!rule->get_expiration().empty()) {
+ op.expiration = rule->get_expiration().get_days();
+ }
+ if (!rule->get_noncur_expiration().empty()) {
+ op.noncur_expiration = rule->get_noncur_expiration().get_days();
+ }
+ auto ret = prefix_map.insert(pair<string, lc_op>(rule->get_prefix(), op));
+ return ret.second;
}
int RGWLifecycleConfiguration::check_and_add_rule(LCRule *rule)
}
rule_map.insert(pair<string, LCRule>(id, *rule));
- auto ret = prefix_map.insert(pair<string, int>(rule->get_prefix(), rule->get_expiration().get_days()));
- //Now prefix shouldn't be the same. When we add noncurrent expiration or other action, prefix may be same.
- if (!ret.second) {
+ if (!_add_rule(rule)) {
return -ERR_INVALID_REQUEST;
}
return 0;
if (prefix_map.size() < 2) {
return true;
}
- auto next_iter = prefix_map.begin();
- auto cur_iter = next_iter++;
- while (next_iter != prefix_map.end()) {
- string c_pre = cur_iter->first;
- string n_pre = next_iter->first;
- if (n_pre.compare(0, c_pre.length(), c_pre) == 0) {
- return false;
- }
+ auto cur_iter = prefix_map.begin();
+ while (cur_iter != prefix_map.end()) {
+ auto next_iter = cur_iter;
++next_iter;
+ while (next_iter != prefix_map.end()) {
+ string c_pre = cur_iter->first;
+ string n_pre = next_iter->first;
+ if (n_pre.compare(0, c_pre.length(), c_pre) == 0) {
+ if ((cur_iter->second.expiration > 0 && next_iter->second.expiration > 0) ||
+ (cur_iter->second.noncur_expiration > 0 && next_iter->second.noncur_expiration > 0)) {
+ return false;
+ } else {
+ ++next_iter;
+ }
+ } else {
+ break;
+ }
+ }
++cur_iter;
}
return true;
return (timediff >= cmp);
}
+int RGWLC::remove_expired_obj(RGWBucketInfo& bucket_info, rgw_obj_key obj_key, bool remove_indeed)
+{
+ if (remove_indeed) {
+ return rgw_remove_object(store, bucket_info, bucket_info.bucket, obj_key);
+ } else {
+ obj_key.instance.clear();
+ RGWObjectCtx rctx(store);
+ rgw_obj obj(bucket_info.bucket, obj_key);
+ return store->delete_obj(rctx, bucket_info, obj, bucket_info.versioning_status());
+ }
+}
+
int RGWLC::bucket_lc_process(string& shard_id)
{
RGWLifecycleConfiguration config(cct);
map<string, bufferlist> bucket_attrs;
string next_marker, no_ns, list_versions;
bool is_truncated;
- bool default_config = false;
- int default_days = 0;
vector<RGWObjEnt> objs;
RGWObjectCtx obj_ctx(store);
vector<std::string> result;
return -1;
}
- map<string, int>& prefix_map = config.get_prefix_map();
- for(map<string, int>::iterator prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
- if (prefix_iter->first.empty()) {
- default_config = true;
- default_days = prefix_iter->second;
- break;
- }
- }
-
- if (default_config) {
- do {
-
- objs.clear();
- list_op.params.marker = list_op.get_next_marker();
- ret = list_op.list_objects(1000, &objs, NULL, &is_truncated);
- if (ret < 0) {
- if (ret == -ENOENT)
- return 0;
- ldout(cct, 0) << "ERROR: store->list_objects():" <<dendl;
- return ret;
- }
-
- vector<RGWObjEnt>::iterator obj_iter;
- int pos = 0;
- utime_t now = ceph_clock_now();
- for (obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
- bool prefix_match = false;
- int match_days = 0;
- map<string, int>& prefix_map = config.get_prefix_map();
-
- for(map<string, int>::iterator prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
- if (prefix_iter->first.empty()) {
- continue;
- }
- pos = (*obj_iter).key.name.find(prefix_iter->first, 0);
- if (pos != 0) {
- continue;
- }
- prefix_match = true;
- match_days = prefix_iter->second;
- break;
- }
- int days = 0;
- if (prefix_match) {
- days = match_days;
- } else if (default_config) {
- days = default_days;
- } else {
- continue;
- }
- if (obj_has_expired(now - ceph::real_clock::to_time_t((*obj_iter).mtime), days)) {
- RGWObjectCtx rctx(store);
- rgw_obj obj(bucket_info.bucket, (*obj_iter).key.name);
- RGWObjState *state;
- int ret = store->get_obj_state(&rctx, obj, &state, false);
- if (ret < 0) {
- return ret;
- }
- if (state->mtime != (*obj_iter).mtime) //Check mtime again to avoid delete a recently update object as much as possible
- continue;
- ret = rgw_remove_object(store, bucket_info, bucket_info.bucket, (*obj_iter).key);
- if (ret < 0) {
- ldout(cct, 0) << "ERROR: rgw_remove_object " << dendl;
- } else {
- ldout(cct, 10) << "DELETED:" << bucket_name << ":" << (*obj_iter).key.name <<dendl;
- }
- }
- }
- } while (is_truncated);
- } else {
- for(map<string, int>::iterator prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
- if (prefix_iter->first.empty()) {
+ map<string, lc_op>& prefix_map = config.get_prefix_map();
+ list_op.params.list_versions = bucket_info.versioned();
+ if (!bucket_info.versioned()) {
+ for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
+ if (!prefix_iter->second.status || prefix_iter->second.expiration <=0) {
continue;
}
list_op.params.prefix = prefix_iter->first;
-
do {
-
objs.clear();
list_op.params.marker = list_op.get_next_marker();
ret = list_op.list_objects(1000, &objs, NULL, &is_truncated);
return ret;
}
- vector<RGWObjEnt>::iterator obj_iter;
- int days = prefix_iter->second;
utime_t now = ceph_clock_now();
- for (obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
- if (obj_has_expired(now - ceph::real_clock::to_time_t((*obj_iter).mtime), days)) {
+ for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
+ if (obj_has_expired(now - ceph::real_clock::to_time_t(obj_iter->mtime), prefix_iter->second.expiration)) {
RGWObjectCtx rctx(store);
- rgw_obj obj(bucket_info.bucket, (*obj_iter).key.name);
+ rgw_obj obj(bucket_info.bucket, obj_iter->key.name);
RGWObjState *state;
int ret = store->get_obj_state(&rctx, obj, &state, false);
if (ret < 0) {
return ret;
}
- if (state->mtime != (*obj_iter).mtime)//Check mtime again to avoid delete a recently update object as much as possible
+ if (state->mtime != obj_iter->mtime)//Check mtime again to avoid delete a recently update object as much as possible
continue;
- ret = rgw_remove_object(store, bucket_info, bucket_info.bucket, (*obj_iter).key);
+ ret = remove_expired_obj(bucket_info, obj_iter->key, true);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: remove_expired_obj " << dendl;
+ } else {
+ ldout(cct, 10) << "DELETED:" << bucket_name << ":" << obj_iter->key.name << dendl;
+ }
+ }
+ }
+ } while (is_truncated);
+ }
+ } else {
+ //bucket versioning is enabled or suspended
+ rgw_obj_key pre_marker;
+ for(auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
+ if (!prefix_iter->second.status) {
+ continue;
+ }
+ if (prefix_iter != prefix_map.begin() &&
+ (prefix_iter->first.compare(0, prev(prefix_iter)->first.length(), prev(prefix_iter)->first) == 0)) {
+ list_op.next_marker = pre_marker;
+ } else {
+ pre_marker = list_op.get_next_marker();
+ }
+ list_op.params.prefix = prefix_iter->first;
+ RGWObjEnt pre_obj;
+ do {
+ if (!objs.empty()) {
+ pre_obj = objs.back();
+ }
+ objs.clear();
+ list_op.params.marker = list_op.get_next_marker();
+ ret = list_op.list_objects(1000, &objs, NULL, &is_truncated);
+
+ if (ret < 0) {
+ if (ret == (-ENOENT))
+ return 0;
+ ldout(cct, 0) << "ERROR: store->list_objects():" <<dendl;
+ return ret;
+ }
+
+ utime_t now = ceph_clock_now();
+ ceph::real_time mtime;
+ bool remove_indeed = true;
+ int expiration;
+ for (auto obj_iter = objs.begin(); obj_iter != objs.end(); ++obj_iter) {
+ if (obj_iter->is_current()) {
+ if (prefix_iter->second.expiration <= 0) {
+ continue;
+ }
+ if (obj_iter->is_delete_marker()) {
+ if ((obj_iter + 1)==objs.end()) {
+ if (is_truncated) {
+ //deal with it in next round because we can't judge whether this marker is the only version
+ list_op.next_marker = obj_iter->key;
+ break;
+ }
+ } else if (obj_iter->key.name.compare((obj_iter + 1)->key.name) == 0) { //*obj_iter is delete marker and isn't the only version, do nothing.
+ continue;
+ }
+ remove_indeed = true; //we should remove the delete marker if it's the only version
+ } else {
+ remove_indeed = false;
+ }
+ mtime = obj_iter->mtime;
+ expiration = prefix_iter->second.expiration;
+ } else {
+ if (prefix_iter->second.noncur_expiration <=0) {
+ continue;
+ }
+ remove_indeed = true;
+ mtime = (obj_iter == objs.begin())?pre_obj.mtime:(obj_iter - 1)->mtime;
+ expiration = prefix_iter->second.noncur_expiration;
+ }
+ if (obj_has_expired(now - ceph::real_clock::to_time_t(mtime), expiration)) {
+ if (obj_iter->is_visible()) {
+ RGWObjectCtx rctx(store);
+ rgw_obj obj(bucket_info.bucket, obj_iter->key.name);
+ obj.set_instance(obj_iter->key.instance);
+ RGWObjState *state;
+ int ret = store->get_obj_state(&rctx, obj, &state, false);
+ if (ret < 0) {
+ return ret;
+ }
+ if (state->mtime != obj_iter->mtime)//Check mtime again to avoid delete a recently update object as much as possible
+ continue;
+ }
+ ret = remove_expired_obj(bucket_info, obj_iter->key, remove_indeed);
if (ret < 0) {
- ldout(cct, 0) << "ERROR: rgw_remove_object " << dendl;
+ ldout(cct, 0) << "ERROR: remove_expired_obj " << dendl;
} else {
- ldout(cct, 10) << "DELETED:" << bucket_name << ":" << (*obj_iter).key.name << dendl;
+ ldout(cct, 10) << "DELETED:" << bucket_name << ":" << obj_iter->key.name << dendl;
}
}
}
// static void generate_test_instances(list<ACLOwner*>& o);
void set_days(const string& _days) { days = _days; }
int get_days() {return atoi(days.c_str()); }
+ bool empty() const{
+ return days.empty();
+ }
};
WRITE_CLASS_ENCODER(LCExpiration)
string prefix;
string status;
LCExpiration expiration;
+ LCExpiration noncur_expiration;
public:
return expiration;
}
+ LCExpiration& get_noncur_expiration() {
+ return noncur_expiration;
+ }
+
void set_id(string*_id) {
id = *_id;
}
expiration = *_expiration;
}
+ void set_noncur_expiration(LCExpiration*_noncur_expiration) {
+ noncur_expiration = *_noncur_expiration;
+ }
+
bool validate();
void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
+ ENCODE_START(2, 1, bl);
::encode(id, bl);
::encode(prefix, bl);
::encode(status, bl);
::encode(expiration, bl);
+ ::encode(noncur_expiration, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
- DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(2, 1, 1, bl);
::decode(id, bl);
::decode(prefix, bl);
::decode(status, bl);
::decode(expiration, bl);
+ if (struct_v >=2) {
+ ::decode(noncur_expiration, bl);
+ }
DECODE_FINISH(bl);
}
};
WRITE_CLASS_ENCODER(LCRule)
+struct lc_op
+{
+ bool status;
+ int expiration;
+ int noncur_expiration;
+
+ lc_op() : status(false), expiration(0), noncur_expiration(0) {}
+
+};
+
class RGWLifecycleConfiguration
{
protected:
CephContext *cct;
- map<string, int> prefix_map;
+ map<string, lc_op> prefix_map;
multimap<string, LCRule> rule_map;
- void _add_rule(LCRule *rule);
+ bool _add_rule(LCRule *rule);
public:
RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
RGWLifecycleConfiguration() : cct(NULL) {}
bool validate();
multimap<string, LCRule>& get_rule_map() { return rule_map; }
- map<string, int>& get_prefix_map() { return prefix_map; }
+ map<string, lc_op>& get_prefix_map() { return prefix_map; }
/*
void create_default(string id, string name) {
ACLGrant grant;
void stop_processor();
private:
+ int remove_expired_obj(RGWBucketInfo& bucket_info, rgw_obj_key obj_key, bool remove_indeed = true);
bool obj_has_expired(double timediff, int days);
};
string& to_str() { return data; }
};
+class LCNoncurDays_S3 : public XMLObj
+{
+public:
+ LCNoncurDays_S3() {}
+ ~LCNoncurDays_S3() {}
+ string& to_str() {
+ return data;
+ }
+};
+
class LCExpiration_S3 : public LCExpiration, public XMLObj
{
public:
}
};
+class LCNoncurExpiration_S3 : public LCExpiration, public XMLObj
+{
+public:
+ LCNoncurExpiration_S3() {}
+ ~LCNoncurExpiration_S3() {}
+
+ bool xml_end(const char *el);
+ void to_xml(ostream& out) {
+ out << "<NoncurrentVersionExpiration>" << "<NoncurrentDays>" << days << "</NoncurrentDays>"<< "</NoncurrentVersionExpiration>";
+ }
+ void dump_xml(Formatter *f) const {
+ f->open_object_section("NoncurrentVersionExpiration");
+ encode_xml("NoncurrentDays", days, f);
+ f->close_section();
+ }
+};
+
class LCRule_S3 : public LCRule, public XMLObj
{
public:
bool xml_end(const char *el);
bool xml_start(const char *el, const char **attr);
void dump_xml(Formatter *f) const {
- const LCExpiration_S3& expir = static_cast<const LCExpiration_S3&>(expiration);
-
- f->open_object_section("Rule");
- encode_xml("ID", id, f);
- encode_xml("Prefix", prefix, f);
- encode_xml("Status", status, f);
- expir.dump_xml(f);
- f->close_section(); // Rule
+ f->open_object_section("Rule");
+ encode_xml("ID", id, f);
+ encode_xml("Prefix", prefix, f);
+ encode_xml("Status", status, f);
+ if (!expiration.empty()) {
+ const LCExpiration_S3& expir = static_cast<const LCExpiration_S3&>(expiration);
+ expir.dump_xml(f);
+ }
+ if (!noncur_expiration.empty()) {
+ const LCNoncurExpiration_S3& noncur_expir = static_cast<const LCNoncurExpiration_S3&>(noncur_expiration);
+ noncur_expir.dump_xml(f);
+ }
+ f->close_section(); // Rule
}
};
projects / ceph-ci.git / commitdiff