From: Sage Weil Date: Tue, 28 Jul 2009 20:01:49 +0000 (-0700) Subject: kclient: use mempool for osd req in writeback paths X-Git-Tag: v0.12~65 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=dcdd7d0d418dc45f3b4d181116b0af8efe64e731;p=ceph.git kclient: use mempool for osd req in writeback paths Only in writeback path. For reads and sync writes, avoid the mempool. --- diff --git a/src/TODO b/src/TODO index 99587724bbd6..9db6635841aa 100644 --- a/src/TODO +++ b/src/TODO @@ -46,6 +46,13 @@ v0.11 /- respond to control-c on slow/hung mount v0.12 +- mapping_set_error on failed writepage +- document correct debugfs mount point +- clean up layout ioctls +- fix bad kmalloc +- use mempool for write path allocations where appropriate + + - osdmap: allow explicit pg 'override' mappings - http gw diff --git a/src/kernel/addr.c b/src/kernel/addr.c index 862b4fbec206..c8d4577c1f09 100644 --- a/src/kernel/addr.c +++ b/src/kernel/addr.c @@ -767,7 +767,7 @@ get_more_pages: snapc, do_sync, ci->i_truncate_seq, ci->i_truncate_size, - &inode->i_mtime); + &inode->i_mtime, true); max_pages = req->r_num_pages; rc = alloc_page_vec(client, req); diff --git a/src/kernel/file.c b/src/kernel/file.c index 84c39d503e55..436e5c026e1f 100644 --- a/src/kernel/file.c +++ b/src/kernel/file.c @@ -578,7 +578,7 @@ more: ci->i_snap_realm->cached_context, do_sync, ci->i_truncate_seq, ci->i_truncate_size, - &mtime); + &mtime, false); if (IS_ERR(req)) return PTR_ERR(req); diff --git a/src/kernel/osd_client.c b/src/kernel/osd_client.c index 8514552fbce3..1d81917aee87 100644 --- a/src/kernel/osd_client.c +++ b/src/kernel/osd_client.c @@ -86,7 +86,10 @@ void ceph_osdc_put_request(struct ceph_osd_request *req) ceph_release_page_vector(req->r_pages, req->r_num_pages); ceph_put_snap_context(req->r_snapc); - kfree(req); + if (req->r_mempool) + mempool_free(req, req->r_osdc->req_mempool); + else + kfree(req); } } @@ -110,7 +113,8 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc, int do_sync, u32 truncate_seq, u64 truncate_size, - struct timespec *mtime) + struct timespec *mtime, + bool use_mempool) { struct ceph_osd_request *req; struct ceph_msg *msg; @@ -123,10 +127,17 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc, int i; u64 prevofs; - req = kzalloc(sizeof(*req), GFP_NOFS); + if (use_mempool) { + req = mempool_alloc(osdc->req_mempool, GFP_NOFS); + memset(req, 0, sizeof(*req)); + } else { + req = kzalloc(sizeof(*req), GFP_NOFS); + } if (req == NULL) return ERR_PTR(-ENOMEM); + req->r_osdc = osdc; + req->r_mempool = use_mempool; atomic_set(&req->r_ref, 1); init_completion(&req->r_completion); init_completion(&req->r_safe_completion); @@ -912,7 +923,7 @@ void ceph_osdc_sync(struct ceph_osd_client *osdc) /* * init, shutdown */ -void ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) +int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) { dout("init\n"); osdc->client = client; @@ -926,6 +937,12 @@ void ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) osdc->requests = RB_ROOT; osdc->num_requests = 0; INIT_DELAYED_WORK(&osdc->timeout_work, handle_timeout); + + osdc->req_mempool = mempool_create_kmalloc_pool(10, + sizeof(struct ceph_osd_request)); + if (!osdc->req_mempool) + return -ENOMEM; + return 0; } void ceph_osdc_stop(struct ceph_osd_client *osdc) @@ -935,6 +952,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc) ceph_osdmap_destroy(osdc->osdmap); osdc->osdmap = NULL; } + mempool_destroy(osdc->req_mempool); } /* @@ -956,7 +974,8 @@ int ceph_osdc_readpages(struct ceph_osd_client *osdc, vino.snap, off, len); req = ceph_osdc_new_request(osdc, layout, vino, off, &len, CEPH_OSD_OP_READ, CEPH_OSD_FLAG_READ, - NULL, 0, truncate_seq, truncate_size, NULL); + NULL, 0, truncate_seq, truncate_size, NULL, + false); if (IS_ERR(req)) return PTR_ERR(req); @@ -1033,7 +1052,8 @@ int ceph_osdc_writepages(struct ceph_osd_client *osdc, struct ceph_vino vino, flags | CEPH_OSD_FLAG_ONDISK | CEPH_OSD_FLAG_WRITE, snapc, do_sync, - truncate_seq, truncate_size, mtime); + truncate_seq, truncate_size, mtime, + true); if (IS_ERR(req)) return PTR_ERR(req); diff --git a/src/kernel/osd_client.h b/src/kernel/osd_client.h index 5aa2b6abc407..a0a5117f8e16 100644 --- a/src/kernel/osd_client.h +++ b/src/kernel/osd_client.h @@ -3,6 +3,7 @@ #include #include +#include #include "types.h" #include "osdmap.h" @@ -10,6 +11,7 @@ struct ceph_msg; struct ceph_snap_context; struct ceph_osd_request; +struct ceph_osd_client; /* * completion callback for async writepages @@ -27,7 +29,9 @@ struct ceph_osd_request { int r_flags; /* any additional flags for the osd */ int r_aborted; /* set if we cancel this request */ + struct ceph_osd_client *r_osdc; atomic_t r_ref; + bool r_mempool; struct completion r_completion, r_safe_completion; ceph_osdc_callback_t r_callback, r_safe_callback; struct ceph_eversion r_reassert_version; @@ -65,10 +69,12 @@ struct ceph_osd_client { int num_requests; struct delayed_work timeout_work; struct dentry *debugfs_file; + + mempool_t *req_mempool; }; -extern void ceph_osdc_init(struct ceph_osd_client *osdc, - struct ceph_client *client); +extern int ceph_osdc_init(struct ceph_osd_client *osdc, + struct ceph_client *client); extern void ceph_osdc_stop(struct ceph_osd_client *osdc); extern void ceph_osdc_handle_reset(struct ceph_osd_client *osdc, @@ -90,7 +96,8 @@ extern struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *, struct ceph_snap_context *snapc, int do_sync, u32 truncate_seq, u64 truncate_size, - struct timespec *mtime); + struct timespec *mtime, + bool use_mempool); static inline void ceph_osdc_get_request(struct ceph_osd_request *req) { diff --git a/src/kernel/super.c b/src/kernel/super.c index 8439888c6eb0..30c5e767b8cd 100644 --- a/src/kernel/super.c +++ b/src/kernel/super.c @@ -681,6 +681,7 @@ static struct ceph_client *ceph_create_client(void) client->signed_ticket = NULL; client->signed_ticket_len = 0; + err = -ENOMEM; client->wb_wq = create_workqueue("ceph-writeback"); if (client->wb_wq == NULL) goto fail; @@ -694,14 +695,15 @@ static struct ceph_client *ceph_create_client(void) /* subsystems */ err = ceph_monc_init(&client->monc, client); if (err < 0) - return ERR_PTR(err); + goto fail; + err = ceph_osdc_init(&client->osdc, client); + if (err < 0) + goto fail; ceph_mdsc_init(&client->mdsc, client); - ceph_osdc_init(&client->osdc, client); - return client; fail: - return ERR_PTR(-ENOMEM); + return ERR_PTR(err); } static void ceph_destroy_client(struct ceph_client *client)