msg: fixup for atomic_t spinlocks in XioConnection

Signed-off-by: Jesse Williamson <jwilliamson@suse.de>

diff --git a/src/msg/xio/XioConnection.cc b/src/msg/xio/XioConnection.cc
index 36b946de08fb07e4b6905d320083f8a35dbf8905..9fd699752e2b22c5f0c014f02395ea0ffcabe392 100644 (file)
--- a/src/msg/xio/XioConnection.cc
+++ b/src/msg/xio/XioConnection.cc
@@ -100,7 +100,6 @@ XioConnection::XioConnection(XioMessenger *m, XioConnection::type _type,
   in_seq(),
   cstate(this)
 {
-  pthread_spin_init(&sp, PTHREAD_PROCESS_PRIVATE);
   set_peer_type(peer.name.type());
   set_peer_addr(peer.addr);
 
@@ -160,7 +159,7 @@ void XioConnection::send_keepalive_or_ack(bool ack, const utime_t *tp)
 {
   /* If con is not in READY state, we need to queue the request */
   if (cstate.session_state.read() != XioConnection::UP) {
-    pthread_spin_lock(&sp);
+    std::lock_guad<ceph::util::spinlock> lg(sp);
     if (cstate.session_state.read() != XioConnection::UP) {
       if (ack) {
        outgoing.ack = true;
@@ -169,10 +168,8 @@ void XioConnection::send_keepalive_or_ack(bool ack, const utime_t *tp)
       else {
        outgoing.keepalive = true;
       }
-      pthread_spin_unlock(&sp);
       return;
     }
-    pthread_spin_unlock(&sp);
   }
 
   send_keepalive_or_ack_internal(ack, tp);
@@ -612,7 +609,7 @@ void XioConnection::msg_release_fail(struct xio_msg *msg, int code)
 int XioConnection::flush_out_queues(uint32_t flags) {
   XioMessenger* msgr = static_cast<XioMessenger*>(get_messenger());
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_lock(&sp);
+    sp.lock();
 
   if (outgoing.keepalive) {
     outgoing.keepalive = false;
@@ -637,7 +634,7 @@ int XioConnection::flush_out_queues(uint32_t flags) {
     msgr->_send_message_impl(m, this);
   }
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_unlock(&sp);
+    sp.unlock();
   return 0;
 }
 
@@ -647,7 +644,7 @@ int XioConnection::discard_out_queues(uint32_t flags)
   XioSubmit::Queue deferred_q;
 
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_lock(&sp);
+    sp.lock();
 
   /* the two send queues contain different objects:
    * - anything on the mqueue is a Message
@@ -662,7 +659,7 @@ int XioConnection::discard_out_queues(uint32_t flags)
   outgoing.keepalive = outgoing.ack = false;
 
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_unlock(&sp);
+    sp.unlock();
 
   // mqueue
   while (!disc_q.empty()) {
@@ -700,11 +697,11 @@ int XioConnection::discard_out_queues(uint32_t flags)
 int XioConnection::adjust_clru(uint32_t flags)
 {
   if (flags & CState::OP_FLAG_LOCKED)
-    pthread_spin_unlock(&sp);
+    sp.unlock();
 
   XioMessenger* msgr = static_cast<XioMessenger*>(get_messenger());
   msgr->conns_sp.lock();
-  pthread_spin_lock(&sp);
+  sp.lock();
 
   if (cstate.flags & CState::FLAG_MAPPED) {
     XioConnection::ConnList::iterator citer =
@@ -716,7 +713,7 @@ int XioConnection::adjust_clru(uint32_t flags)
   msgr->conns_sp.unlock();
 
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_unlock(&sp);
+    sp.unlock();
 
   return 0;
 }
@@ -742,7 +739,7 @@ void XioConnection::mark_down()
 int XioConnection::_mark_down(uint32_t flags)
 {
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_lock(&sp);
+    sp.lock();
 
   // per interface comment, we only stage a remote reset if the
   // current policy required it
@@ -756,7 +753,7 @@ int XioConnection::_mark_down(uint32_t flags)
   discard_out_queues(flags|CState::OP_FLAG_LOCKED);
 
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_unlock(&sp);
+    sp.unlock();
 
   return 0;
 }
@@ -769,12 +766,12 @@ void XioConnection::mark_disposable()
 int XioConnection::_mark_disposable(uint32_t flags)
 {
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_lock(&sp);
+    sp.lock();
 
   cstate.policy.lossy = true;
 
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_unlock(&sp);
+    sp.unlock();
 
   return 0;
 }
@@ -782,7 +779,7 @@ int XioConnection::_mark_disposable(uint32_t flags)
 int XioConnection::CState::state_up_ready(uint32_t flags)
 {
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_lock(&xcon->sp);
+    xcon->sp.lock();
 
   xcon->flush_out_queues(flags|CState::OP_FLAG_LOCKED);
 
@@ -790,7 +787,7 @@ int XioConnection::CState::state_up_ready(uint32_t flags)
   startup_state = session_startup_states::READY;
 
   if (! (flags & CState::OP_FLAG_LOCKED))
-    pthread_spin_unlock(&xcon->sp);
+    xcon->sp.unlock();
 
   return (0);
 }
@@ -806,12 +803,12 @@ int XioConnection::CState::state_discon()
 int XioConnection::CState::state_flow_controlled(uint32_t flags)
 {
   if (! (flags & OP_FLAG_LOCKED))
-    pthread_spin_lock(&xcon->sp);
+    xcon->sp.lock();
 
   session_state = session_states::FLOW_CONTROLLED;
 
   if (! (flags & OP_FLAG_LOCKED))
-    pthread_spin_unlock(&xcon->sp);
+    xcon->sp.unlock();
 
   return (0);
 }
@@ -819,7 +816,7 @@ int XioConnection::CState::state_flow_controlled(uint32_t flags)
 int XioConnection::CState::state_fail(Message* m, uint32_t flags)
 {
   if (! (flags & OP_FLAG_LOCKED))
-    pthread_spin_lock(&xcon->sp);
+    xcon->sp.lock();
 
   // advance to state FAIL, drop queued, msgs, adjust LRU
   session_state = session_states::DISCONNECTED);
@@ -831,7 +828,7 @@ int XioConnection::CState::state_fail(Message* m, uint32_t flags)
   xcon->disconnect();
 
   if (! (flags & OP_FLAG_LOCKED))
-    pthread_spin_unlock(&xcon->sp);
+    xcon->sp.unlock();
 
   // notify ULP
   XioMessenger* msgr = static_cast<XioMessenger*>(xcon->get_messenger());

diff --git a/src/msg/xio/XioConnection.h b/src/msg/xio/XioConnection.h
index c92b6176e3a7be2955af1365952be68a4715c118..a6dc950c2d70ca336ff683b68addf844426ffeec 100644 (file)
--- a/src/msg/xio/XioConnection.h
+++ b/src/msg/xio/XioConnection.h
@@ -72,7 +72,7 @@ private:
   entity_inst_t peer;
   struct xio_session *session;
   struct xio_connection        *conn;
-  std::atomic_flag sp = ATOMIC_FLAG_INIT;
+  ceph::util::spinlock sp;
   std::atomic<int64_t> send = { 0 };
   std::atomic<int64_t> recv = { 0 };
   uint32_t n_reqs; // Accelio-initiated reqs in progress (!counting partials)