From bf5d51e2887f465fbb67843303e030faa4efdfbe Mon Sep 17 00:00:00 2001 From: Zhiqiang Wang Date: Wed, 27 May 2015 14:02:33 +0800 Subject: [PATCH] osd: explicitly set the reqid when proxying the write op This is needed as in the following scenario: - Client sends 3 writes and a read on the same object to base tier - Set up cache tiering - Client retries ops and sends the 3 writes and 1 read to the cache tier - The 3 writes finished on the base tier, say with versions v1, v2 and v3 - Cache tier proxies the 1st write, and start to promote the object for the 2nd write, the 2nd and 3rd writes and the read are blocked - The proxied 1st write finishes on the base tier with version v4, and returns to cache tier. But somehow the cache tier fails to send the reply due to socket failure injecting - Client retries the writes and the read again, the writes are identified as dup ops - The promotion finishes, it copies the pg_log entries from the base tier and put it in the cache tier's pg_log. This includes the 3 writes on the base tier and the proxied write - The writes dispatches after the promotion, they are identified as completed dup ops. Cache tier replies these write ops with the version from the base tier (v1, v2 and v3) - In the last, the read dispatches, it reads the version of the proxied write (v4) and replies to client - Client complains that 'racing read got wrong version' Signed-off-by: Zhiqiang Wang --- src/osd/ReplicatedPG.cc | 5 +++-- src/osd/ReplicatedPG.h | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 85dcf9d43d05e..c9710bec31c1a 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -2244,7 +2244,7 @@ void ReplicatedPG::do_proxy_write(OpRequestRef op, const hobject_t& missing_oid) unsigned flags = CEPH_OSD_FLAG_IGNORE_CACHE | CEPH_OSD_FLAG_IGNORE_OVERLAY; dout(10) << __func__ << " Start proxy write for " << *m << dendl; - ProxyWriteOpRef pwop(new ProxyWriteOp(op, soid, m->ops)); + ProxyWriteOpRef pwop(new ProxyWriteOp(op, soid, m->ops, m->get_reqid())); pwop->ctx = new OpContext(op, m->get_reqid(), pwop->ops, this); pwop->mtime = m->get_mtime(); @@ -2257,7 +2257,8 @@ void ReplicatedPG::do_proxy_write(OpRequestRef op, const hobject_t& missing_oid) snapc, pwop->mtime, flags, NULL, new C_OnFinisher(fin, &osd->objecter_finisher), - &pwop->user_version); + &pwop->user_version, + pwop->reqid); fin->tid = tid; pwop->objecter_tid = tid; proxywrite_ops[tid] = pwop; diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index ec64b21a4a6b1..cc3ce84a4b1a8 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -267,12 +267,14 @@ public: bool sent_ack; utime_t mtime; bool canceled; + osd_reqid_t &reqid; - ProxyWriteOp(OpRequestRef _op, hobject_t oid, vector& _ops) + ProxyWriteOp(OpRequestRef _op, hobject_t oid, vector& _ops, osd_reqid_t _reqid) : ctx(NULL), op(_op), soid(oid), objecter_tid(0), ops(_ops), user_version(0), sent_disk(false), - sent_ack(false), canceled(false) { } + sent_ack(false), canceled(false), + reqid(_reqid) { } }; typedef boost::shared_ptr ProxyWriteOpRef; -- 2.39.5