kclient: stop using msgpools for incoming msgs

Even though we may only dispatch N messages at a time (if we
have N cores), we may have many more connections, and be
reading the same message types in parallel from multiple
peers.  Msgpool_alloc blocks when teh pool is empty, which
will make the messenger thread hang, and stall forward.

Badness.

There is no easy solution here.

diff --git a/src/kernel/msgpool.c b/src/kernel/msgpool.c
index e656e6be1f28003ffde93450ef36fd716b8104a4..d8276af8e3484f94d8420c2e0d27a964eff9fa87 100644 (file)
--- a/src/kernel/msgpool.c
+++ b/src/kernel/msgpool.c
@@ -65,7 +65,7 @@ int ceph_msgpool_init(struct ceph_msg_pool *pool,
 {
        int ret;
 
-       dout("msgpool_init %p front_len %d min %d\n", pool, front_len, min);
+       pr_err("msgpool_init %p front_len %d min %d\n", pool, front_len, min);
        spin_lock_init(&pool->lock);
        pool->front_len = front_len;
        INIT_LIST_HEAD(&pool->msgs);

diff --git a/src/kernel/super.c b/src/kernel/super.c
index 359123125416bf584577fea9e6ca14a452799f23..1fc12722ae3327fc6b16d7ccd44297eedbef29d5 100644 (file)
--- a/src/kernel/super.c
+++ b/src/kernel/super.c
@@ -710,34 +710,6 @@ static struct ceph_client *ceph_create_client(void)
                goto fail;
 
        /* msg pools */
-       /* simple pools: */
-       err = ceph_msgpool_init(&client->msgpool_mount_ack, 16384, 1);
-       if (err < 0)
-               goto fail;
-       err = ceph_msgpool_init(&client->msgpool_unmount, 4096, 1);
-       if (err < 0)
-               goto fail;
-       err = ceph_msgpool_init(&client->msgpool_mds_map, 16384, 1);
-       if (err < 0)
-               goto fail;
-       err = ceph_msgpool_init(&client->msgpool_client_session,
-                               sizeof(struct ceph_mds_session_head), 1);
-       if (err < 0)
-               goto fail;
-       err = ceph_msgpool_init(&client->msgpool_client_request_forward, 32, 1);
-       if (err < 0)
-               goto fail;
-       err = ceph_msgpool_init(&client->msgpool_client_lease,
-                               sizeof(struct ceph_mds_lease) + PATH_MAX, 1);
-       if (err < 0)
-               goto fail;
-       err = ceph_msgpool_init(&client->msgpool_client_caps, 4096, 1);
-       if (err < 0)
-               goto fail;
-       err = ceph_msgpool_init(&client->msgpool_osd_opreply, 4096, 1);
-       if (err < 0)
-               goto fail;
-
        /* preallocated at request time: */
        err = ceph_msgpool_init(&client->msgpool_statfs_reply, 4096, 0);
        if (err < 0)
@@ -781,18 +753,7 @@ static void ceph_destroy_client(struct ceph_client *client)
                mempool_destroy(client->wb_pagevec_pool);
 
        /* msg pools */
-       ceph_msgpool_destroy(&client->msgpool_mount_ack);
-       ceph_msgpool_destroy(&client->msgpool_unmount);
        ceph_msgpool_destroy(&client->msgpool_statfs_reply);
-       ceph_msgpool_destroy(&client->msgpool_mds_map);
-       ceph_msgpool_destroy(&client->msgpool_client_session);
-       //ceph_msgpool_destroy(&client->msgpool_client_reply);
-       ceph_msgpool_destroy(&client->msgpool_client_request_forward);
-       //ceph_msgpool_destroy(&client->msgpool_client_caps);
-       //ceph_msgpool_destroy(&client->msgpool_client_snap);
-       ceph_msgpool_destroy(&client->msgpool_client_lease);
-       //ceph_msgpool_destroy(&client->msgpool_osd_map);
-       //ceph_msgpool_destroy(&client->msgpool_osd_opreply);
 
        release_mount_args(&client->mount_args);
 
@@ -968,25 +929,16 @@ out:
 static struct ceph_msg_pool *get_pool(struct ceph_client *client, int type)
 {
        switch (type) {
-       case CEPH_MSG_CLIENT_MOUNT_ACK:
-               return &client->msgpool_mount_ack;
-       case CEPH_MSG_CLIENT_UNMOUNT:
-               return &client->msgpool_unmount;
        case CEPH_MSG_STATFS_REPLY:
                return &client->msgpool_statfs_reply;
+       case CEPH_MSG_CLIENT_MOUNT_ACK:
+       case CEPH_MSG_CLIENT_UNMOUNT:
        case CEPH_MSG_MDS_MAP:
-               return &client->msgpool_mds_map;
        case CEPH_MSG_CLIENT_SESSION:
-               return &client->msgpool_client_session;
        case CEPH_MSG_CLIENT_REQUEST_FORWARD:
-               return &client->msgpool_client_request_forward;
        case CEPH_MSG_CLIENT_LEASE:
-               return &client->msgpool_client_lease;
        case CEPH_MSG_CLIENT_CAPS:
-               return &client->msgpool_client_caps;
        case CEPH_MSG_OSD_OPREPLY:
-               return &client->msgpool_osd_opreply;
-
        case CEPH_MSG_OSD_MAP:
        case CEPH_MSG_CLIENT_REPLY:
        case CEPH_MSG_CLIENT_SNAP:

diff --git a/src/kernel/super.h b/src/kernel/super.h
index 706ffaaff09bacf011ad7a36550475d62ce83442..561adf7e83b52dfef59e5143cb7fc11a15446134 100644 (file)
--- a/src/kernel/super.h
+++ b/src/kernel/super.h
@@ -136,18 +136,7 @@ struct ceph_client {
        struct ceph_osd_client osdc;
 
        /* msg pools */
-       struct ceph_msg_pool msgpool_mount_ack;
-       struct ceph_msg_pool msgpool_unmount;
        struct ceph_msg_pool msgpool_statfs_reply;
-       struct ceph_msg_pool msgpool_mds_map;
-       struct ceph_msg_pool msgpool_client_session;
-       struct ceph_msg_pool msgpool_client_reply;
-       struct ceph_msg_pool msgpool_client_request_forward;
-       struct ceph_msg_pool msgpool_client_caps;
-       struct ceph_msg_pool msgpool_client_snap;
-       struct ceph_msg_pool msgpool_client_lease;
-       struct ceph_msg_pool msgpool_osd_map;
-       struct ceph_msg_pool msgpool_osd_opreply;
 
        /* writeback */
        mempool_t *wb_pagevec_pool;