kclient: fix osd request kicking

We often need to kick the request to the same osd.  Keep track of
an osd incarnation, incremented each time we reset_osd or see the
connection drop, and kick requests when the incarnation doesn't
match.

diff --git a/src/kernel/osd_client.c b/src/kernel/osd_client.c
index 88b2d58c5702aa6f7e0dbe5ba1977f701643da34..013646d80e10730ab13051b3c5dc9368466b2b04 100644 (file)
--- a/src/kernel/osd_client.c
+++ b/src/kernel/osd_client.c
@@ -295,6 +295,7 @@ static void osd_reset(struct ceph_connection *con)
                return;
        dout("osd_reset osd%d\n", osd->o_osd);
        osdc = osd->o_osdc;
+       osd->o_incarnation++;
        down_read(&osdc->map_sem);
        kick_requests(osdc, osd);
        up_read(&osdc->map_sem);
@@ -314,6 +315,7 @@ static struct ceph_osd *create_osd(struct ceph_osd_client *osdc)
        atomic_set(&osd->o_ref, 1);
        osd->o_osdc = osdc;
        INIT_LIST_HEAD(&osd->o_requests);
+       osd->o_incarnation = 1;
 
        ceph_con_init(osdc->client->msgr, &osd->o_con);
        osd->o_con.private = osd;
@@ -369,6 +371,7 @@ static int reset_osd(struct ceph_osd_client *osdc, struct ceph_osd *osd)
        } else {
                ceph_con_close(&osd->o_con);
                ceph_con_open(&osd->o_con, &osdc->osdmap->osd_addr[osd->o_osd]);
+               osd->o_incarnation++;
        }
        return ret;
 }
@@ -485,7 +488,7 @@ static void __cancel_request(struct ceph_osd_request *req)
 {
        if (req->r_sent) {
                ceph_con_revoke(&req->r_osd->o_con, req->r_request);
-               req->r_sent = false;
+               req->r_sent = 0;
        }
 }
 
@@ -515,7 +518,8 @@ static int __map_osds(struct ceph_osd_client *osdc,
        pgid.pg64 = le64_to_cpu(reqhead->layout.ol_pgid);
        o = ceph_calc_pg_primary(osdc->osdmap, pgid);
 
-       if ((req->r_osd && req->r_osd->o_osd == o) ||
+       if ((req->r_osd && req->r_osd->o_osd == o &&
+            req->r_sent >= req->r_osd->o_incarnation) ||
            (req->r_osd == NULL && o == -1))
                return 0;  /* no change */
 
@@ -594,7 +598,7 @@ static int __send_request(struct ceph_osd_client *osdc,
 
        ceph_msg_get(req->r_request); /* send consumes a ref */
        ceph_con_send(&req->r_osd->o_con, req->r_request);
-       req->r_sent = true;
+       req->r_sent = req->r_osd->o_incarnation;
        return 0;
 }
 

diff --git a/src/kernel/osd_client.h b/src/kernel/osd_client.h
index 36a5e4be6cf633c2e44c1d4eb0dbc415e0ab2a7e..89b5cf151a94499fa7767e5ebdecc84f885dd6ab 100644 (file)
--- a/src/kernel/osd_client.h
+++ b/src/kernel/osd_client.h
@@ -25,6 +25,7 @@ struct ceph_osd {
        atomic_t o_ref;
        struct ceph_osd_client *o_osdc;
        int o_osd;
+       int o_incarnation;
        struct rb_node o_node;
        struct ceph_connection o_con;
        struct list_head o_requests;
@@ -40,7 +41,7 @@ struct ceph_osd_request {
        struct ceph_msg  *r_request, *r_reply;
        int               r_result;
        int               r_flags;     /* any additional flags for the osd */
-       bool              r_sent;      /* true if r_request is sending/sent */
+       u32               r_sent;      /* >0 if r_request is sending/sent */
        int r_prepared_pages, r_got_reply;
 
        struct ceph_osd_client *r_osdc;