kclient: use mempool for osd req in writeback paths

Only in writeback path.  For reads and sync writes, avoid the
mempool.

diff --git a/src/TODO b/src/TODO
index 99587724bbd6ccbf045c7726142dc485b19b6a50..9db6635841aa16f7dee8fac4c7c64d8d53d95483 100644 (file)
--- a/src/TODO
+++ b/src/TODO
@@ -46,6 +46,13 @@ v0.11
 /- respond to control-c on slow/hung mount
 
 v0.12
+- mapping_set_error on failed writepage
+- document correct debugfs mount point
+- clean up layout ioctls
+- fix bad kmalloc
+- use mempool for write path allocations where appropriate
+
+
 - osdmap: allow explicit pg 'override' mappings
 - http gw
 

diff --git a/src/kernel/addr.c b/src/kernel/addr.c
index 862b4fbec2067bab2dc65c5ce9165e08764497d7..c8d4577c1f099d3d8a0dd3276a5f5783a9fefe25 100644 (file)
--- a/src/kernel/addr.c
+++ b/src/kernel/addr.c
@@ -767,7 +767,7 @@ get_more_pages:
                                            snapc, do_sync,
                                            ci->i_truncate_seq,
                                            ci->i_truncate_size,
-                                           &inode->i_mtime);
+                                           &inode->i_mtime, true);
                                max_pages = req->r_num_pages;
 
                                rc = alloc_page_vec(client, req);

diff --git a/src/kernel/file.c b/src/kernel/file.c
index 84c39d503e55b95f5de3094747a4febc105b0c56..436e5c026e1f00dac6a111e6c052f3275c146a74 100644 (file)
--- a/src/kernel/file.c
+++ b/src/kernel/file.c
@@ -578,7 +578,7 @@ more:
                                    ci->i_snap_realm->cached_context,
                                    do_sync,
                                    ci->i_truncate_seq, ci->i_truncate_size,
-                                   &mtime);
+                                   &mtime, false);
        if (IS_ERR(req))
                return PTR_ERR(req);
 

diff --git a/src/kernel/osd_client.c b/src/kernel/osd_client.c
index 8514552fbce309840b78e9e03ee5081013d6f7e9..1d81917aee879ab5e0b280608527709e0e068ac7 100644 (file)
--- a/src/kernel/osd_client.c
+++ b/src/kernel/osd_client.c
@@ -86,7 +86,10 @@ void ceph_osdc_put_request(struct ceph_osd_request *req)
                        ceph_release_page_vector(req->r_pages,
                                                 req->r_num_pages);
                ceph_put_snap_context(req->r_snapc);
-               kfree(req);
+               if (req->r_mempool)
+                       mempool_free(req, req->r_osdc->req_mempool);
+               else
+                       kfree(req);
        }
 }
 
@@ -110,7 +113,8 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc,
                                               int do_sync,
                                               u32 truncate_seq,
                                               u64 truncate_size,
-                                              struct timespec *mtime)
+                                              struct timespec *mtime,
+                                              bool use_mempool)
 {
        struct ceph_osd_request *req;
        struct ceph_msg *msg;
@@ -123,10 +127,17 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc,
        int i;
        u64 prevofs;
 
-       req = kzalloc(sizeof(*req), GFP_NOFS);
+       if (use_mempool) {
+               req = mempool_alloc(osdc->req_mempool, GFP_NOFS);
+               memset(req, 0, sizeof(*req));
+       } else {
+               req = kzalloc(sizeof(*req), GFP_NOFS);
+       }
        if (req == NULL)
                return ERR_PTR(-ENOMEM);
 
+       req->r_osdc = osdc;
+       req->r_mempool = use_mempool;
        atomic_set(&req->r_ref, 1);
        init_completion(&req->r_completion);
        init_completion(&req->r_safe_completion);
@@ -912,7 +923,7 @@ void ceph_osdc_sync(struct ceph_osd_client *osdc)
 /*
  * init, shutdown
  */
-void ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
+int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
 {
        dout("init\n");
        osdc->client = client;
@@ -926,6 +937,12 @@ void ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
        osdc->requests = RB_ROOT;
        osdc->num_requests = 0;
        INIT_DELAYED_WORK(&osdc->timeout_work, handle_timeout);
+
+       osdc->req_mempool = mempool_create_kmalloc_pool(10,
+                                       sizeof(struct ceph_osd_request));
+       if (!osdc->req_mempool)
+               return -ENOMEM;
+       return 0;
 }
 
 void ceph_osdc_stop(struct ceph_osd_client *osdc)
@@ -935,6 +952,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc)
                ceph_osdmap_destroy(osdc->osdmap);
                osdc->osdmap = NULL;
        }
+       mempool_destroy(osdc->req_mempool);
 }
 
 /*
@@ -956,7 +974,8 @@ int ceph_osdc_readpages(struct ceph_osd_client *osdc,
             vino.snap, off, len);
        req = ceph_osdc_new_request(osdc, layout, vino, off, &len,
                                    CEPH_OSD_OP_READ, CEPH_OSD_FLAG_READ,
-                                   NULL, 0, truncate_seq, truncate_size, NULL);
+                                   NULL, 0, truncate_seq, truncate_size, NULL,
+                                   false);
        if (IS_ERR(req))
                return PTR_ERR(req);
 
@@ -1033,7 +1052,8 @@ int ceph_osdc_writepages(struct ceph_osd_client *osdc, struct ceph_vino vino,
                                    flags | CEPH_OSD_FLAG_ONDISK |
                                            CEPH_OSD_FLAG_WRITE,
                                    snapc, do_sync,
-                                   truncate_seq, truncate_size, mtime);
+                                   truncate_seq, truncate_size, mtime,
+                                   true);
        if (IS_ERR(req))
                return PTR_ERR(req);
 

diff --git a/src/kernel/osd_client.h b/src/kernel/osd_client.h
index 5aa2b6abc407ce01154cd13b4e277c969d7a2380..a0a5117f8e16b672250e2674b15c5b32495eca3d 100644 (file)
--- a/src/kernel/osd_client.h
+++ b/src/kernel/osd_client.h
@@ -3,6 +3,7 @@
 
 #include <linux/radix-tree.h>
 #include <linux/completion.h>
+#include <linux/mempool.h>
 
 #include "types.h"
 #include "osdmap.h"
@@ -10,6 +11,7 @@
 struct ceph_msg;
 struct ceph_snap_context;
 struct ceph_osd_request;
+struct ceph_osd_client;
 
 /*
  * completion callback for async writepages
@@ -27,7 +29,9 @@ struct ceph_osd_request {
        int               r_flags;     /* any additional flags for the osd */
        int               r_aborted;   /* set if we cancel this request */
 
+       struct ceph_osd_client *r_osdc;
        atomic_t          r_ref;
+       bool              r_mempool;
        struct completion r_completion, r_safe_completion;
        ceph_osdc_callback_t r_callback, r_safe_callback;
        struct ceph_eversion r_reassert_version;
@@ -65,10 +69,12 @@ struct ceph_osd_client {
        int                    num_requests;
        struct delayed_work    timeout_work;
        struct dentry          *debugfs_file;
+
+       mempool_t              *req_mempool;
 };
 
-extern void ceph_osdc_init(struct ceph_osd_client *osdc,
-                          struct ceph_client *client);
+extern int ceph_osdc_init(struct ceph_osd_client *osdc,
+                         struct ceph_client *client);
 extern void ceph_osdc_stop(struct ceph_osd_client *osdc);
 
 extern void ceph_osdc_handle_reset(struct ceph_osd_client *osdc,
@@ -90,7 +96,8 @@ extern struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *,
                                      struct ceph_snap_context *snapc,
                                      int do_sync, u32 truncate_seq,
                                      u64 truncate_size,
-                                     struct timespec *mtime);
+                                     struct timespec *mtime,
+                                     bool use_mempool);
 
 static inline void ceph_osdc_get_request(struct ceph_osd_request *req)
 {

diff --git a/src/kernel/super.c b/src/kernel/super.c
index 8439888c6eb098992da93219b158cfd46e94fcfe..30c5e767b8cd9d6893be9cb58fd100f94cdeebe9 100644 (file)
--- a/src/kernel/super.c
+++ b/src/kernel/super.c
@@ -681,6 +681,7 @@ static struct ceph_client *ceph_create_client(void)
        client->signed_ticket = NULL;
        client->signed_ticket_len = 0;
 
+       err = -ENOMEM;
        client->wb_wq = create_workqueue("ceph-writeback");
        if (client->wb_wq == NULL)
                goto fail;
@@ -694,14 +695,15 @@ static struct ceph_client *ceph_create_client(void)
        /* subsystems */
        err = ceph_monc_init(&client->monc, client);
        if (err < 0)
-               return ERR_PTR(err);
+               goto fail;
+       err = ceph_osdc_init(&client->osdc, client);
+       if (err < 0)
+               goto fail;
        ceph_mdsc_init(&client->mdsc, client);
-       ceph_osdc_init(&client->osdc, client);
-
        return client;
 
 fail:
-       return ERR_PTR(-ENOMEM);
+       return ERR_PTR(err);
 }
 
 static void ceph_destroy_client(struct ceph_client *client)