From 7dc730b2c0f0d282d35099cb20c60478aad200f5 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 20 Dec 2007 15:03:49 -0800 Subject: [PATCH] refactored osd_reqid_t --- src/include/ceph_fs.h | 45 ++++++++++++++++------------------- src/kernel/client.c | 2 +- src/kernel/osd_client.c | 15 +++++++++++- src/kernel/osd_client.h | 3 ++- src/messages/MOSDOp.h | 27 ++++++++++----------- src/messages/MOSDOpReply.h | 7 +++--- src/messages/MOSDSubOp.h | 6 ++--- src/messages/MOSDSubOpReply.h | 2 +- src/osd/PG.h | 10 ++++---- src/osd/ReplicatedPG.cc | 8 +++---- src/osd/ReplicatedPG.h | 4 ++-- src/osd/osd_types.h | 25 ++++++++++++------- src/osdc/Objecter.cc | 2 -- 13 files changed, 84 insertions(+), 72 deletions(-) diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index ac20982e003e2..769cb5cf96c71 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -188,11 +188,11 @@ struct ceph_entity_name { #define CEPH_ENTITY_TYPE_CLIENT 4 #define CEPH_ENTITY_TYPE_ADMIN 5 -#define CEPH_MSGR_TAG_READY 1 // server -> client + cseq: ready for messages -#define CEPH_MSGR_TAG_REJECT 2 // server -> client + cseq: decline socket -#define CEPH_MSGR_TAG_MSG 3 // message -#define CEPH_MSGR_TAG_ACK 4 // message ack -#define CEPH_MSGR_TAG_CLOSE 5 // closing pipe +#define CEPH_MSGR_TAG_READY 1 /* server -> client + cseq: ready for messages */ +#define CEPH_MSGR_TAG_REJECT 2 /* server -> client + cseq: decline socket */ +#define CEPH_MSGR_TAG_MSG 3 /* message */ +#define CEPH_MSGR_TAG_ACK 4 /* message ack */ +#define CEPH_MSGR_TAG_CLOSE 5 /* closing pipe */ /* @@ -254,19 +254,19 @@ struct ceph_msg_header { #define CEPH_MSG_MDS_GETMAP 20 #define CEPH_MSG_MDS_MAP 21 -#define CEPH_MSG_CLIENT_SESSION 22 // start or stop +#define CEPH_MSG_CLIENT_SESSION 22 #define CEPH_MSG_CLIENT_RECONNECT 23 #define CEPH_MSG_CLIENT_REQUEST 24 #define CEPH_MSG_CLIENT_REQUEST_FORWARD 25 #define CEPH_MSG_CLIENT_REPLY 26 -#define CEPH_MSG_CLIENT_FILECAPS 0x310 // +#define CEPH_MSG_CLIENT_FILECAPS 0x310 /* osd */ #define CEPH_MSG_OSD_GETMAP 40 #define CEPH_MSG_OSD_MAP 41 -#define CEPH_MSG_OSD_OP 42 // delete, etc. -#define CEPH_MSG_OSD_OPREPLY 43 // delete, etc. +#define CEPH_MSG_OSD_OP 42 +#define CEPH_MSG_OSD_OPREPLY 43 /* client_session message op values */ @@ -394,13 +394,6 @@ struct ceph_mds_reply_dirfrag { /* * osd ops */ -struct ceph_osd_reqid { - struct ceph_entity_name name; /* who */ - __u32 inc; /* incarnation */ - ceph_tid_t tid; -} __attribute__ ((packed)); -typedef struct ceph_osd_reqid ceph_osd_reqid_t; - enum { CEPH_OSD_OP_READ = 1, CEPH_OSD_OP_STAT = 2, @@ -418,7 +411,7 @@ enum { CEPH_OSD_OP_RDUNLOCK = 23, CEPH_OSD_OP_UPLOCK = 24, CEPH_OSD_OP_DNLOCK = 25, - CEPH_OSD_OP_MININCLOCK = 26, // minimum incarnation lock + CEPH_OSD_OP_MININCLOCK = 26, /* minimum incarnation lock */ CEPH_OSD_OP_PULL = 30, CEPH_OSD_OP_PUSH = 31, @@ -427,14 +420,17 @@ enum { CEPH_OSD_OP_UNBALANCEREADS = 102 }; +/* + * osd op flags + */ enum { - CEPH_OSD_OP_ACK = 1, - CEPH_OSD_OP_SAFE = 2, - CEPH_OSD_OP_RETRY = 4 + CEPH_OSD_OP_ACK = 1, /* want (or is) "ack" ack */ + CEPH_OSD_OP_SAFE = 2, /* want (or is) "safe" ack */ + CEPH_OSD_OP_RETRY = 4 /* resend attempt */ }; struct ceph_osd_peer_stat { - ceph_timeval stamp; + struct ceph_timeval stamp; float oprate; float qlen; float recent_qlen; @@ -446,8 +442,9 @@ struct ceph_osd_peer_stat { typedef struct ceph_osd_peer_stat ceph_osd_peer_stat_t; struct ceph_osd_request_head { - struct ceph_entity_addr client_addr; - ceph_osd_reqid_t reqid; /* fixme: this dups client.name */ + struct ceph_entity_inst client_inst; + ceph_tid_t tid; + __u32 client_inc; __u32 op; __u64 offset, length; ceph_object_t oid; @@ -464,7 +461,7 @@ struct ceph_osd_request_head { } __attribute__ ((packed)); struct ceph_osd_reply_head { - ceph_osd_reqid_t reqid; + ceph_tid_t tid; __u32 op; __u32 flags; ceph_object_t oid; diff --git a/src/kernel/client.c b/src/kernel/client.c index 8156e3e5d9901..d6a99b67c2fb9 100644 --- a/src/kernel/client.c +++ b/src/kernel/client.c @@ -81,7 +81,7 @@ static struct ceph_client *create_client(struct ceph_mount_args *args) cl->whoami = -1; ceph_monc_init(&cl->monc); ceph_mdsc_init(&cl->mdsc, cl); - ceph_osdc_init(&cl->osdc); + ceph_osdc_init(&cl->osdc, cl); return cl; diff --git a/src/kernel/osd_client.c b/src/kernel/osd_client.c index e11470ff76410..f9d433c196700 100644 --- a/src/kernel/osd_client.c +++ b/src/kernel/osd_client.c @@ -9,10 +9,11 @@ int ceph_osdc_debug = 50; #define DOUT_PREFIX "osdc: " #include "super.h" -void ceph_osdc_init(struct ceph_osd_client *osdc) +void ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) { dout(5, "init\n"); spin_lock_init(&osdc->lock); + osdc->client = client; osdc->osdmap = NULL; osdc->last_requested_map = 0; osdc->last_tid = 0; @@ -118,5 +119,17 @@ ceph_osdc_create_request(struct ceph_osd_client *osdc, int op) { struct ceph_msg *req; struct ceph_osd_request_head *head; + + req = ceph_msg_new(CEPH_MSG_OSD_OP, sizeof(struct ceph_osd_request_head), 0, 0); + if (IS_ERR(req)) + return req; + memset(req->front.iov_base, 0, req->front.iov_len); + head = req->front.iov_base; + /* encode head */ + head->op = cpu_to_le32(op); + ceph_encode_inst(&head->client_inst, &osdc->client->msgr->inst); + head->client_inc = 1; /* always, for now. */ + + return req; } diff --git a/src/kernel/osd_client.h b/src/kernel/osd_client.h index f63044ef9ec7e..602869b0e4141 100644 --- a/src/kernel/osd_client.h +++ b/src/kernel/osd_client.h @@ -27,6 +27,7 @@ struct ceph_osd_request { struct ceph_osd_client { spinlock_t lock; + struct ceph_client *client; struct ceph_osdmap *osdmap; /* current map */ __u64 last_requested_map; __u64 last_tid; /* tid of last request */ @@ -34,7 +35,7 @@ struct ceph_osd_client { struct completion map_waiters; }; -extern void ceph_osdc_init(struct ceph_osd_client *osdc); +extern void ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client); extern void ceph_osdc_handle_reply(struct ceph_osd_client *osdc, struct ceph_msg *msg); extern void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg); diff --git a/src/messages/MOSDOp.h b/src/messages/MOSDOp.h index 5f0b3a35604bb..3ce6071fcf240 100644 --- a/src/messages/MOSDOp.h +++ b/src/messages/MOSDOp.h @@ -66,15 +66,13 @@ private: friend class MOSDOpReply; public: - const ceph_osd_reqid_t& get_reqid() { return head.reqid; } - const tid_t get_client_tid() { return head.reqid.tid; } - int get_client_inc() { return head.reqid.inc; } - - entity_name_t get_client() { return head.reqid.name; } - entity_inst_t get_client_inst() { - return entity_inst_t(head.reqid.name, head.client_addr); - } - void set_client_addr(const entity_addr_t& a) { head.client_addr = a.v; } + osd_reqid_t get_reqid() { return osd_reqid_t(head.client_inst.name, head.client_inc, head.tid); } + int get_client_inc() { return head.client_inc; } + tid_t get_client_tid() { return head.tid; } + + entity_name_t get_client() { return head.client_inst.name; } + entity_inst_t get_client_inst() { return head.client_inst; } + void set_client_addr(const entity_addr_t& a) { head.client_inst.addr = a.v; } object_t get_oid() { return object_t(head.oid); } pg_t get_pg() { return head.layout.pgid; } @@ -107,11 +105,10 @@ public: object_t oid, ceph_object_layout_t ol, epoch_t mapepoch, int op) : Message(CEPH_MSG_OSD_OP) { memset(&head, 0, sizeof(head)); - head.client_addr = asker.addr.v; - head.reqid.name = asker.name.v; - head.reqid.inc = inc; - head.reqid.tid = tid; - + head.client_inst.name = asker.name.v; + head.client_inst.addr = asker.addr.v; + head.tid = tid; + head.client_inc = inc; head.oid = oid; head.layout = ol; head.osdmap_epoch = mapepoch; @@ -150,7 +147,7 @@ public: virtual char *get_type_name() { return "osd_op"; } void print(ostream& out) { - out << "osd_op(" << head.reqid + out << "osd_op(" << get_reqid() << " " << get_opname(head.op) << " " << head.oid; if (head.length) out << " " << head.offset << "~" << head.length; diff --git a/src/messages/MOSDOpReply.h b/src/messages/MOSDOpReply.h index 291e8bd73dc06..60eeeddc11db1 100644 --- a/src/messages/MOSDOpReply.h +++ b/src/messages/MOSDOpReply.h @@ -34,8 +34,7 @@ class MOSDOpReply : public Message { map attrset; public: - const ceph_osd_reqid_t& get_reqid() { return head.reqid; } - long get_tid() { return head.reqid.tid; } + long get_tid() { return head.tid; } object_t get_oid() { return head.oid; } pg_t get_pg() { return head.layout.pgid; } int get_op() { return head.op; } @@ -63,7 +62,7 @@ public: MOSDOpReply(MOSDOp *req, int result, epoch_t e, bool commit) : Message(CEPH_MSG_OSD_OPREPLY) { memset(&head, 0, sizeof(head)); - head.reqid = req->head.reqid; + head.tid = req->head.tid; head.op = req->head.op; head.flags = commit ? CEPH_OSD_OP_SAFE:0; head.oid = req->head.oid; @@ -92,7 +91,7 @@ public: virtual char *get_type_name() { return "osd_op_reply"; } void print(ostream& out) { - out << "osd_op_reply(" << head.reqid + out << "osd_op_reply(" << get_tid() << " " << MOSDOp::get_opname(head.op) << " " << head.oid; if (head.length) out << " " << head.offset << "~" << head.length; diff --git a/src/messages/MOSDSubOp.h b/src/messages/MOSDSubOp.h index 4b307c9b15ef1..5feb07a24fabd 100644 --- a/src/messages/MOSDSubOp.h +++ b/src/messages/MOSDSubOp.h @@ -30,7 +30,7 @@ private: epoch_t map_epoch; // metadata from original request - ceph_osd_reqid_t reqid; + osd_reqid_t reqid; // subop pg_t pgid; @@ -53,7 +53,7 @@ private: public: const epoch_t get_map_epoch() { return st.map_epoch; } - const ceph_osd_reqid_t& get_reqid() { return st.reqid; } + const osd_reqid_t& get_reqid() { return st.reqid; } bool wants_reply() { if (st.op < 100) return true; @@ -78,7 +78,7 @@ public: void set_peer_stat(const osd_peer_stat_t& stat) { st.peer_stat = stat; } const osd_peer_stat_t& get_peer_stat() { return st.peer_stat; } - MOSDSubOp(ceph_osd_reqid_t r, pg_t p, pobject_t po, int o, off_t of, off_t le, + MOSDSubOp(osd_reqid_t r, pg_t p, pobject_t po, int o, off_t of, off_t le, epoch_t mape, tid_t rtid, eversion_t v) : Message(MSG_OSD_SUBOP) { memset(&st, 0, sizeof(st)); diff --git a/src/messages/MOSDSubOpReply.h b/src/messages/MOSDSubOpReply.h index 9e91b2f91eb9e..716729cea0a6f 100644 --- a/src/messages/MOSDSubOpReply.h +++ b/src/messages/MOSDSubOpReply.h @@ -34,7 +34,7 @@ class MOSDSubOpReply : public Message { epoch_t map_epoch; // subop metadata - ceph_osd_reqid_t reqid; + osd_reqid_t reqid; pg_t pgid; tid_t rep_tid; int32_t op; diff --git a/src/osd/PG.h b/src/osd/PG.h index f9adda4425e16..b6dac4d65d2e4 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -218,11 +218,11 @@ public: object_t oid; eversion_t version; - ceph_osd_reqid_t reqid; // caller+tid to uniquely identify request + osd_reqid_t reqid; // caller+tid to uniquely identify request Entry() : op(0) {} Entry(int _op, object_t _oid, const eversion_t& v, - const ceph_osd_reqid_t& rid) : + const osd_reqid_t& rid) : op(_op), oid(_oid), version(v), reqid(rid) {} bool is_delete() const { return op == DELETE; } @@ -275,7 +275,7 @@ public: class IndexedLog : public Log { public: hash_map objects; // ptrs into log. be careful! - hash_set caller_ops; + hash_set caller_ops; // recovery pointers list::iterator requested_to; // not inclusive of referenced item @@ -293,7 +293,7 @@ public: bool logged_object(object_t oid) { return objects.count(oid); } - bool logged_req(const ceph_osd_reqid_t &r) { + bool logged_req(const osd_reqid_t &r) { return caller_ops.count(r); } @@ -633,7 +633,7 @@ public: void trim_ondisklog_to(ObjectStore::Transaction& t, eversion_t v); - bool is_dup(ceph_osd_reqid_t rid) { + bool is_dup(osd_reqid_t rid) { return log.logged_req(rid); } diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index b07b425e90ad4..7b48093739b70 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -573,7 +573,7 @@ void ReplicatedPG::op_read(MOSDOp *op) // MODIFY void ReplicatedPG::prepare_log_transaction(ObjectStore::Transaction& t, - ceph_osd_reqid_t reqid, pobject_t poid, int op, eversion_t version, + osd_reqid_t reqid, pobject_t poid, int op, eversion_t version, objectrev_t crev, objectrev_t rev, eversion_t trim_to) { @@ -612,7 +612,7 @@ void ReplicatedPG::prepare_log_transaction(ObjectStore::Transaction& t, /** prepare_op_transaction * apply an op to the store wrapped in a transaction. */ -void ReplicatedPG::prepare_op_transaction(ObjectStore::Transaction& t, const ceph_osd_reqid_t& reqid, +void ReplicatedPG::prepare_op_transaction(ObjectStore::Transaction& t, const osd_reqid_t& reqid, pg_t pgid, int op, pobject_t poid, off_t offset, off_t length, bufferlist& bl, eversion_t& version, objectrev_t crev, objectrev_t rev) @@ -1340,7 +1340,7 @@ void ReplicatedPG::pull(pobject_t poid) << dendl; // send op - ceph_osd_reqid_t rid; + osd_reqid_t rid; tid_t tid = osd->get_tid(); MOSDSubOp *subop = new MOSDSubOp(rid, info.pgid, poid, CEPH_OSD_OP_PULL, 0, 0, @@ -1382,7 +1382,7 @@ void ReplicatedPG::push(pobject_t poid, int peer) osd->logger->inc("r_pushb", bl.length()); // send - ceph_osd_reqid_t rid; // useless? + osd_reqid_t rid; // useless? MOSDSubOp *subop = new MOSDSubOp(rid, info.pgid, poid, CEPH_OSD_OP_PUSH, 0, bl.length(), osd->osdmap->get_epoch(), osd->get_tid(), v); subop->set_data(bl); // note: claims bl, set length above here! diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index af51687b5dc75..45d5da7f3b6be 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -100,10 +100,10 @@ protected: void sub_op_modify_commit(MOSDSubOp *op, int ackerosd, eversion_t last_complete); void prepare_log_transaction(ObjectStore::Transaction& t, - ceph_osd_reqid_t reqid, pobject_t poid, int op, eversion_t version, + osd_reqid_t reqid, pobject_t poid, int op, eversion_t version, objectrev_t crev, objectrev_t rev, eversion_t trim_to); - void prepare_op_transaction(ObjectStore::Transaction& t, const ceph_osd_reqid_t& reqid, + void prepare_op_transaction(ObjectStore::Transaction& t, const osd_reqid_t& reqid, pg_t pgid, int op, pobject_t poid, off_t offset, off_t length, bufferlist& bl, eversion_t& version, objectrev_t crev, objectrev_t rev); diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 688da22561cc1..08de55b59e871 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -22,31 +22,38 @@ /* osdreqid_t - caller name + incarnation# + tid to unique identify this request * use for metadata and osd ops. */ +struct osd_reqid_t { + entity_name_t name; // who + int32_t inc; // incarnation + tid_t tid; + osd_reqid_t() : inc(0), tid(0) {} + osd_reqid_t(const entity_name_t& a, int i, tid_t t) : name(a), inc(i), tid(t) {} +}; -inline ostream& operator<<(ostream& out, const ceph_osd_reqid_t& r) { +inline ostream& operator<<(ostream& out, const osd_reqid_t& r) { return out << r.name << "." << r.inc << ":" << r.tid; } -inline bool operator==(const ceph_osd_reqid_t& l, const ceph_osd_reqid_t& r) { +inline bool operator==(const osd_reqid_t& l, const osd_reqid_t& r) { return (l.name == r.name) && (l.inc == r.inc) && (l.tid == r.tid); } -inline bool operator!=(const ceph_osd_reqid_t& l, const ceph_osd_reqid_t& r) { +inline bool operator!=(const osd_reqid_t& l, const osd_reqid_t& r) { return (l.name != r.name) || (l.inc != r.inc) || (l.tid != r.tid); } -inline bool operator<(const ceph_osd_reqid_t& l, const ceph_osd_reqid_t& r) { +inline bool operator<(const osd_reqid_t& l, const osd_reqid_t& r) { return (l.name < r.name) || (l.inc < r.inc) || (l.name == r.name && l.inc == r.inc && l.tid < r.tid); } -inline bool operator<=(const ceph_osd_reqid_t& l, const ceph_osd_reqid_t& r) { +inline bool operator<=(const osd_reqid_t& l, const osd_reqid_t& r) { return (l.name < r.name) || (l.inc < r.inc) || (l.name == r.name && l.inc == r.inc && l.tid <= r.tid); } -inline bool operator>(const ceph_osd_reqid_t& l, const ceph_osd_reqid_t& r) { return !(l <= r); } -inline bool operator>=(const ceph_osd_reqid_t& l, const ceph_osd_reqid_t& r) { return !(l < r); } +inline bool operator>(const osd_reqid_t& l, const osd_reqid_t& r) { return !(l <= r); } +inline bool operator>=(const osd_reqid_t& l, const osd_reqid_t& r) { return !(l < r); } namespace __gnu_cxx { - template<> struct hash { - size_t operator()(const ceph_osd_reqid_t &r) const { + template<> struct hash { + size_t operator()(const osd_reqid_t &r) const { static blobhash H; return H((const char*)&r, sizeof(r)); } diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc index ea3972916b43f..99a16d32c8da7 100644 --- a/src/osdc/Objecter.cc +++ b/src/osdc/Objecter.cc @@ -821,7 +821,6 @@ void Objecter::handle_osd_modify_reply(MOSDOpReply *m) // ack or safe? if (m->is_safe()) { - //dout(15) << " handle_osd_write_reply commit on " << tid << dendl; assert(wr->tid_version.count(tid) == 0 || m->get_version() == wr->tid_version[tid]); @@ -845,7 +844,6 @@ void Objecter::handle_osd_modify_reply(MOSDOpReply *m) } } else { // ack. - //dout(15) << " handle_osd_write_reply ack on " << tid << dendl; assert(wr->waitfor_ack.count(tid)); wr->waitfor_ack.erase(tid); -- 2.39.5