From: Sage Weil <sage@newdream.net>
Date: Thu, 17 Sep 2009 23:15:21 +0000 (-0700)
Subject: kclient: fix osd request kicking
X-Git-Tag: v0.15~42
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=24c84ab49408cfb8b66547c4dd1bd98273c43e2f;p=ceph.git

kclient: fix osd request kicking

We often need to kick the request to the same osd.  Keep track of
an osd incarnation, incremented each time we reset_osd or see the
connection drop, and kick requests when the incarnation doesn't
match.
---

diff --git a/src/kernel/osd_client.c b/src/kernel/osd_client.c
index 88b2d58c5702..013646d80e10 100644
--- a/src/kernel/osd_client.c
+++ b/src/kernel/osd_client.c
@@ -295,6 +295,7 @@ static void osd_reset(struct ceph_connection *con)
 		return;
 	dout("osd_reset osd%d\n", osd->o_osd);
 	osdc = osd->o_osdc;
+	osd->o_incarnation++;
 	down_read(&osdc->map_sem);
 	kick_requests(osdc, osd);
 	up_read(&osdc->map_sem);
@@ -314,6 +315,7 @@ static struct ceph_osd *create_osd(struct ceph_osd_client *osdc)
 	atomic_set(&osd->o_ref, 1);
 	osd->o_osdc = osdc;
 	INIT_LIST_HEAD(&osd->o_requests);
+	osd->o_incarnation = 1;
 
 	ceph_con_init(osdc->client->msgr, &osd->o_con);
 	osd->o_con.private = osd;
@@ -369,6 +371,7 @@ static int reset_osd(struct ceph_osd_client *osdc, struct ceph_osd *osd)
 	} else {
 		ceph_con_close(&osd->o_con);
 		ceph_con_open(&osd->o_con, &osdc->osdmap->osd_addr[osd->o_osd]);
+		osd->o_incarnation++;
 	}
 	return ret;
 }
@@ -485,7 +488,7 @@ static void __cancel_request(struct ceph_osd_request *req)
 {
 	if (req->r_sent) {
 		ceph_con_revoke(&req->r_osd->o_con, req->r_request);
-		req->r_sent = false;
+		req->r_sent = 0;
 	}
 }
 
@@ -515,7 +518,8 @@ static int __map_osds(struct ceph_osd_client *osdc,
 	pgid.pg64 = le64_to_cpu(reqhead->layout.ol_pgid);
 	o = ceph_calc_pg_primary(osdc->osdmap, pgid);
 
-	if ((req->r_osd && req->r_osd->o_osd == o) ||
+	if ((req->r_osd && req->r_osd->o_osd == o &&
+	     req->r_sent >= req->r_osd->o_incarnation) ||
 	    (req->r_osd == NULL && o == -1))
 		return 0;  /* no change */
 
@@ -594,7 +598,7 @@ static int __send_request(struct ceph_osd_client *osdc,
 
 	ceph_msg_get(req->r_request); /* send consumes a ref */
 	ceph_con_send(&req->r_osd->o_con, req->r_request);
-	req->r_sent = true;
+	req->r_sent = req->r_osd->o_incarnation;
 	return 0;
 }
 
diff --git a/src/kernel/osd_client.h b/src/kernel/osd_client.h
index 36a5e4be6cf6..89b5cf151a94 100644
--- a/src/kernel/osd_client.h
+++ b/src/kernel/osd_client.h
@@ -25,6 +25,7 @@ struct ceph_osd {
 	atomic_t o_ref;
 	struct ceph_osd_client *o_osdc;
 	int o_osd;
+	int o_incarnation;
 	struct rb_node o_node;
 	struct ceph_connection o_con;
 	struct list_head o_requests;
@@ -40,7 +41,7 @@ struct ceph_osd_request {
 	struct ceph_msg  *r_request, *r_reply;
 	int               r_result;
 	int               r_flags;     /* any additional flags for the osd */
-	bool              r_sent;      /* true if r_request is sending/sent */
+	u32               r_sent;      /* >0 if r_request is sending/sent */
 	int r_prepared_pages, r_got_reply;
 
 	struct ceph_osd_client *r_osdc;