From bea73a7c040ad2048e14458370da8b69f35e9642 Mon Sep 17 00:00:00 2001 From: Patience Warnick Date: Mon, 31 Dec 2007 14:13:42 -0800 Subject: [PATCH] complete close from peer, increase delay for retries --- src/kernel/ktcp.c | 9 +++++---- src/kernel/mds_client.c | 2 +- src/kernel/messenger.c | 21 ++++++++++++++++----- src/kernel/messenger.h | 7 ++++--- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/kernel/ktcp.c b/src/kernel/ktcp.c index de3d5301ce759..cace2061376e9 100644 --- a/src/kernel/ktcp.c +++ b/src/kernel/ktcp.c @@ -4,7 +4,6 @@ #include #include "messenger.h" #include "ktcp.h" -#include "ktcp.h" int ceph_tcp_debug = 50; #define DOUT_VAR ceph_tcp_debug @@ -49,7 +48,8 @@ static void ceph_write_space(struct sock *sk) dout(30, "ceph_write_space %p state = %lu\n", con, con->state); /* only queue to workqueue if a WRITE is pending */ - if (con && test_bit(WRITE_PENDING, &con->state)) { + if (con && (test_bit(WRITE_PENDING, &con->state) || + test_bit(CLOSING, &con->state))) { dout(30, "ceph_write_space %p queuing write work\n", con); queue_work(send_wq, &con->swork.work); } @@ -65,10 +65,11 @@ static void ceph_state_change(struct sock *sk) dout(30, "ceph_state_change %p state = %lu sk_state = %u\n", con, con->state, sk->sk_state); switch (sk->sk_state) { - case TCP_CLOSE_WAIT: case TCP_CLOSE: - set_bit(CLOSED,&con->state); + set_bit(CLOSED, &con->state); break; + case TCP_CLOSE_WAIT: + set_bit(CLOSING, &con->state); case TCP_ESTABLISHED: ceph_write_space(sk); break; diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c index a22bf13879486..342488adf229e 100644 --- a/src/kernel/mds_client.c +++ b/src/kernel/mds_client.c @@ -17,7 +17,7 @@ static void send_msg_mds(struct ceph_mds_client *mdsc, struct ceph_msg *msg, int msg->hdr.dst.addr = *ceph_mdsmap_get_addr(mdsc->mdsmap, mds); msg->hdr.dst.name.type = CEPH_ENTITY_TYPE_MDS; msg->hdr.dst.name.num = mds; - ceph_msg_send(mdsc->client->msgr, msg, BASE_RETRY_INTERVAL); + ceph_msg_send(mdsc->client->msgr, msg, BASE_DELAY_INTERVAL); } diff --git a/src/kernel/messenger.c b/src/kernel/messenger.c index 4d6778460fae7..5e4530e0d822d 100644 --- a/src/kernel/messenger.c +++ b/src/kernel/messenger.c @@ -54,6 +54,7 @@ static void ceph_send_fault(struct ceph_connection *con, int error) case -ENETUNREACH: derr(1, "ENETUNREACH set\n"); spin_lock(&con->out_queue_lock); + /* TBD: reset buffer correctly */ list_splice_init(&con->out_sent, &con->out_queue); spin_unlock(&con->out_queue_lock); /* retry with delay */ @@ -68,6 +69,7 @@ static void ceph_send_fault(struct ceph_connection *con, int error) /* TBD: setup timeout here */ if (!test_and_clear_bit(CONNECTING, &con->state)){ derr(1, "CONNECTING bit not set\n"); + /* TBD: reset buffer correctly */ /* reset buffer */ spin_lock(&con->out_queue_lock); list_splice_init(&con->out_sent, @@ -87,6 +89,10 @@ static void ceph_send_fault(struct ceph_connection *con, int error) /* if we ever hit here ... */ derr(1, "unrecognized error %d\n", error); } + if (con->delay < MAX_DELAY_INTERVAL) + con->delay *= 2; + else + con->delay = MAX_DELAY_INTERVAL; } @@ -107,7 +113,7 @@ static struct ceph_connection *new_connection(struct ceph_messenger *msgr) return NULL; con->msgr = msgr; - con->delay = BASE_RETRY_INTERVAL; + con->delay = BASE_DELAY_INTERVAL; set_bit(NEW, &con->state); atomic_set(&con->nref, 1); @@ -446,6 +452,15 @@ static void try_write(struct work_struct *work) con = container_of(work, struct ceph_connection, swork.work); msgr = con->msgr; dout(30, "try_write start %p state %lu\n", con, con->state); + + /* Peer initiated close, other end is waiting for us to close */ + if (test_and_clear_bit(CLOSING, &con->state)) { + dout(5, "try_write peer initiated close\n"); + remove_connection(msgr, con); + set_bit(CLOSED, &con->state); + put_connection(con); + goto done; + } more: dout(30, "try_write out_kvec_bytes %d\n", con->out_kvec_bytes); @@ -819,10 +834,6 @@ retry: clear_bit(READABLE, &con->state); more: - /* - * TBD: maybe store error in ceph_connection - */ - if (test_bit(CLOSED, &con->state)) { dout(20, "try_read closed\n"); goto done; diff --git a/src/kernel/messenger.h b/src/kernel/messenger.h index 7f75b369fcd4f..e5e84e113b50e 100644 --- a/src/kernel/messenger.h +++ b/src/kernel/messenger.h @@ -52,8 +52,8 @@ struct ceph_msg_pos { }; /* ceph connection fault delay defaults */ -#define BASE_RETRY_INTERVAL (3U * HZ) -#define MAX_RETRY_INTERVAL (5U * 60 * HZ) +#define BASE_DELAY_INTERVAL (3U * HZ) +#define MAX_DELAY_INTERVAL (5U * 60 * HZ) /* ceph_connection state bit flags */ #define NEW 0 @@ -64,7 +64,8 @@ struct ceph_msg_pos { #define READABLE 5 /* set when socket gets new data */ #define READING 6 /* provides mutual exclusion, protecting in_* */ #define REJECTING 7 -#define CLOSED 8 +#define CLOSING 8 +#define CLOSED 9 struct ceph_connection { struct ceph_messenger *msgr; -- 2.39.5