From b0cc9e4553096ad2fc4e87c412b3d2839fcc98c2 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 9 Oct 2014 17:00:36 -0700 Subject: [PATCH] rgw: purge intent log We don't use the intent log anymore, just remove it. Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_dencoder.cc | 10 --- src/rgw/rgw_json_enc.cc | 10 --- src/rgw/rgw_log.cc | 43 --------- src/rgw/rgw_log.h | 26 ------ src/rgw/rgw_main.cc | 8 -- src/rgw/rgw_rados.cc | 187 ---------------------------------------- src/rgw/rgw_rados.h | 25 +----- 7 files changed, 2 insertions(+), 307 deletions(-) diff --git a/src/rgw/rgw_dencoder.cc b/src/rgw/rgw_dencoder.cc index f0f4fce87133a..10e9961e69e21 100644 --- a/src/rgw/rgw_dencoder.cc +++ b/src/rgw/rgw_dencoder.cc @@ -214,16 +214,6 @@ void rgw_log_entry::generate_test_instances(list& o) o.push_back(new rgw_log_entry); } -void rgw_intent_log_entry::generate_test_instances(list& o) -{ - rgw_intent_log_entry *e = new rgw_intent_log_entry; - rgw_bucket b("bucket", "pool", ".index_pool", "marker", "10", "region"); - e->obj = rgw_obj(b, "object"); - e->intent = DEL_OBJ; - o.push_back(e); - o.push_back(new rgw_intent_log_entry); -} - void ACLPermission::generate_test_instances(list& o) { ACLPermission *p = new ACLPermission; diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index 9a21f26cb2c2a..e55665d317458 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -101,16 +101,6 @@ void rgw_log_entry::dump(Formatter *f) const f->dump_string("bucket_id", bucket_id); } -void rgw_intent_log_entry::dump(Formatter *f) const -{ - f->open_object_section("obj"); - obj.dump(f); - f->close_section(); - f->dump_stream("op_time") << op_time; - f->dump_unsigned("intent", intent); -} - - void ACLPermission::dump(Formatter *f) const { f->dump_int("flags", flags); diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index 5c3d924d99228..795d78787327e 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -371,46 +371,3 @@ done: return ret; } -int rgw_log_intent(RGWRados *store, rgw_obj& obj, RGWIntentEvent intent, const utime_t& timestamp, bool utc) -{ - rgw_bucket intent_log_bucket(store->zone.intent_log_pool); - - rgw_intent_log_entry entry; - entry.obj = obj; - entry.intent = (uint32_t)intent; - entry.op_time = timestamp; - - struct tm bdt; - time_t t = timestamp.sec(); - if (utc) - gmtime_r(&t, &bdt); - else - localtime_r(&t, &bdt); - - struct rgw_bucket& bucket = obj.bucket; - - char buf[bucket.name.size() + bucket.bucket_id.size() + 16]; - sprintf(buf, "%.4d-%.2d-%.2d-%s-%s", (bdt.tm_year+1900), (bdt.tm_mon+1), bdt.tm_mday, - bucket.bucket_id.c_str(), obj.bucket.name.c_str()); - string oid(buf); - rgw_obj log_obj(intent_log_bucket, oid); - - bufferlist bl; - ::encode(entry, bl); - - int ret = store->append_async(log_obj, bl.length(), bl); - if (ret == -ENOENT) { - ret = store->create_pool(intent_log_bucket); - if (ret < 0) - goto done; - ret = store->append_async(log_obj, bl.length(), bl); - } - -done: - return ret; -} - -int rgw_log_intent(RGWRados *store, struct req_state *s, rgw_obj& obj, RGWIntentEvent intent) -{ - return rgw_log_intent(store, obj, intent, s->time, s->cct->_conf->rgw_intent_log_object_name_utc); -} diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h index 9af2bb6ceed71..3622737dae43e 100644 --- a/src/rgw/rgw_log.h +++ b/src/rgw/rgw_log.h @@ -101,30 +101,6 @@ struct rgw_log_entry { }; WRITE_CLASS_ENCODER(rgw_log_entry) -struct rgw_intent_log_entry { - rgw_obj obj; - utime_t op_time; - uint32_t intent; - - void encode(bufferlist &bl) const { - ENCODE_START(2, 2, bl); - ::encode(obj, bl); - ::encode(op_time, bl); - ::encode(intent, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator &p) { - DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, p); - ::decode(obj, p); - ::decode(op_time, p); - ::decode(intent, p); - DECODE_FINISH(p); - } - void dump(Formatter *f) const; - static void generate_test_instances(list& o); -}; -WRITE_CLASS_ENCODER(rgw_intent_log_entry) - class OpsLogSocket : public OutputDataSocket { Formatter *formatter; Mutex lock; @@ -142,8 +118,6 @@ public: }; int rgw_log_op(RGWRados *store, struct req_state *s, const string& op_name, OpsLogSocket *olog); -int rgw_log_intent(RGWRados *store, rgw_obj& obj, RGWIntentEvent intent, const utime_t& timestamp, bool utc); -int rgw_log_intent(RGWRados *store, struct req_state *s, rgw_obj& obj, RGWIntentEvent intent); void rgw_log_usage_init(CephContext *cct, RGWRados *store); void rgw_log_usage_finalize(); void rgw_format_ops_log_entry(struct rgw_log_entry& entry, Formatter *formatter); diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 9ed0e636ec2db..11520e11567fa 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -500,13 +500,6 @@ static void godown_alarm(int signum) _exit(0); } -static int call_log_intent(RGWRados *store, void *ctx, rgw_obj& obj, RGWIntentEvent intent) -{ - struct req_state *s = (struct req_state *)ctx; - return rgw_log_intent(store, s, obj, intent); -} - - static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWClientIO *client_io, OpsLogSocket *olog) { int ret = 0; @@ -526,7 +519,6 @@ static int process_request(RGWRados *store, RGWREST *rest, RGWRequest *req, RGWC RGWObjectCtx rados_ctx(store, s); s->obj_ctx = &rados_ctx; - store->set_intent_cb(s->obj_ctx, call_log_intent); s->req_id = store->unique_id(req->id); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index dec7733451958..663c0abe42800 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -7140,193 +7140,6 @@ int RGWRados::check_quota(const string& bucket_owner, rgw_bucket& bucket, return quota_handler->check_quota(bucket_owner, bucket, user_quota, bucket_quota, 1, obj_size); } -class IntentLogNameFilter : public RGWAccessListFilter -{ - string prefix; - bool filter_exact_date; -public: - IntentLogNameFilter(const char *date, struct tm *tm) : prefix(date) { - filter_exact_date = !(tm->tm_hour || tm->tm_min || tm->tm_sec); /* if time was specified and is not 00:00:00 - we should look at objects from that date */ - } - bool filter(string& name, string& key) { - if (filter_exact_date) - return name.compare(prefix) < 0; - else - return name.compare(0, prefix.size(), prefix) <= 0; - } -}; - -enum IntentFlags { // bitmask - I_DEL_OBJ = 1, - I_DEL_DIR = 2, -}; - - -int RGWRados::remove_temp_objects(string date, string time) -{ - struct tm tm; - - string format = "%Y-%m-%d"; - string datetime = date; - if (datetime.size() != 10) { - cerr << "bad date format" << std::endl; - return -EINVAL; - } - - if (!time.empty()) { - if (time.size() != 5 && time.size() != 8) { - cerr << "bad time format" << std::endl; - return -EINVAL; - } - format.append(" %H:%M:%S"); - datetime.append(time.c_str()); - } - memset(&tm, 0, sizeof(tm)); - const char *s = strptime(datetime.c_str(), format.c_str(), &tm); - if (s && *s) { - cerr << "failed to parse date/time" << std::endl; - return -EINVAL; - } - time_t epoch = mktime(&tm); - - vector objs; - - int max = 1000; - bool is_truncated; - IntentLogNameFilter filter(date.c_str(), &tm); - RGWPoolIterCtx iter_ctx; - int r = pool_iterate_begin(zone.intent_log_pool, iter_ctx); - if (r < 0) { - cerr << "failed to list objects" << std::endl; - return r; - } - do { - objs.clear(); - r = pool_iterate(iter_ctx, max, objs, &is_truncated, &filter); - if (r == -ENOENT) - break; - if (r < 0) { - cerr << "failed to list objects" << std::endl; - } - vector::iterator iter; - for (iter = objs.begin(); iter != objs.end(); ++iter) { - process_intent_log(zone.intent_log_pool, (*iter).key.name, epoch, I_DEL_OBJ | I_DEL_DIR, true); - } - } while (is_truncated); - - return 0; -} - -int RGWRados::process_intent_log(rgw_bucket& bucket, string& oid, - time_t epoch, int flags, bool purge) -{ - cout << "processing intent log " << oid << std::endl; - rgw_obj obj(bucket, oid); - - unsigned chunk = 1024 * 1024; - off_t pos = 0; - bool eof = false; - bool complete = true; - int ret = 0; - int r; - - bufferlist bl; - bufferlist::iterator iter; - off_t off; - - string no_owner; - - while (!eof || !iter.end()) { - off = iter.get_off(); - if (!eof && (bl.length() - off) < chunk / 2) { - bufferlist more; - r = read(NULL, obj, pos, chunk, more); - if (r < 0) { - cerr << "error while reading from " << bucket << ":" << oid - << " " << cpp_strerror(-r) << std::endl; - return -r; - } - eof = (more.length() < (off_t)chunk); - pos += more.length(); - bufferlist old; - old.substr_of(bl, off, bl.length() - off); - bl.clear(); - bl.claim(old); - bl.claim_append(more); - iter = bl.begin(); - } - - struct rgw_intent_log_entry entry; - RGWObjectCtx obj_ctx(this); - try { - ::decode(entry, iter); - } catch (buffer::error& err) { - cerr << "failed to decode intent log entry in " << bucket << ":" << oid << std::endl; - cerr << "skipping log" << std::endl; // no use to continue - ret = -EIO; - complete = false; - break; - } - if (entry.op_time.sec() > epoch) { - cerr << "skipping entry for obj=" << obj << " entry.op_time=" << entry.op_time.sec() << " requested epoch=" << epoch << std::endl; - cerr << "skipping log" << std::endl; // no use to continue - complete = false; - break; - } - switch (entry.intent) { - case DEL_OBJ: - if (!(flags & I_DEL_OBJ)) { - complete = false; - break; - } - r = delete_obj(obj_ctx, no_owner, entry.obj, false); - if (r < 0 && r != -ENOENT) { - cerr << "failed to remove obj: " << entry.obj << std::endl; - complete = false; - } - break; - case DEL_DIR: - if (!(flags & I_DEL_DIR)) { - complete = false; - break; - } else { - librados::IoCtx index_ctx; - string oid; - int r = open_bucket_index(entry.obj.bucket, index_ctx, oid); - if (r < 0) - return r; - ObjectWriteOperation op; - op.remove(); - oid.append(entry.obj.bucket.marker); - librados::AioCompletion *completion = rados->aio_create_completion(NULL, NULL, NULL); - r = index_ctx.aio_operate(oid, completion, &op); - completion->release(); - if (r < 0 && r != -ENOENT) { - cerr << "failed to remove bucket: " << entry.obj.bucket << std::endl; - complete = false; - } - } - break; - default: - complete = false; - } - } - - if (complete) { - rgw_obj obj(bucket, oid); - cout << "completed intent log: " << obj << (purge ? ", purging it" : "") << std::endl; - if (purge) { - r = delete_system_obj(obj); - if (r < 0) - cerr << "failed to remove obj: " << obj << std::endl; - } - } - - return ret; -} - - void RGWStateLog::oid_str(int shard, string& oid) { oid = RGW_STATELOG_OBJ_PREFIX + module_name + "."; char buf[16]; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index b90b167ffeaf8..09addd268525f 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1111,27 +1111,14 @@ public: struct RGWObjectCtx { RGWRados *store; map objs_state; - int (*intent_cb)(RGWRados *store, void *user_ctx, rgw_obj& obj, RGWIntentEvent intent); void *user_ctx; - RGWObjectCtx(RGWRados *_store) : store(_store), intent_cb(NULL), user_ctx(NULL) { } - RGWObjectCtx(RGWRados *_store, void *_user_ctx) : store(_store), intent_cb(NULL), user_ctx(_user_ctx) { } + RGWObjectCtx(RGWRados *_store) : store(_store), user_ctx(NULL) { } + RGWObjectCtx(RGWRados *_store, void *_user_ctx) : store(_store), user_ctx(_user_ctx) { } RGWObjState *get_state(rgw_obj& obj); void set_atomic(rgw_obj& obj); void set_prefetch_data(rgw_obj& obj); - - void set_intent_cb(int (*cb)(RGWRados *store, void *user_ctx, rgw_obj& obj, RGWIntentEvent intent)) { - intent_cb = cb; - } - - int notify_intent(RGWRados *store, rgw_obj& obj, RGWIntentEvent intent) { - if (intent_cb) { - return intent_cb(store, user_ctx, obj, intent); - } - return 0; - } - void invalidate(rgw_obj& obj); }; @@ -1743,12 +1730,6 @@ public: RGWObjectCtx *rctx = static_cast(ctx); rctx->set_prefetch_data(obj); } - // to notify upper layer that we need to do some operation on an object, and it's up to - // the upper layer to schedule this operation.. e.g., log intent in intent log - void set_intent_cb(void *ctx, int (*cb)(RGWRados *store, void *user_ctx, rgw_obj& obj, RGWIntentEvent intent)) { - RGWObjectCtx *rctx = static_cast(ctx); - rctx->set_intent_cb(cb); - } int decode_policy(bufferlist& bl, ACLOwner *owner); int get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map& stats, @@ -1868,8 +1849,6 @@ public: } private: - int process_intent_log(rgw_bucket& bucket, string& oid, - time_t epoch, int flags, bool purge); /** * Check the actual on-disk state of the object specified * by list_state, and fill in the time and size of object. -- 2.39.5