Ibv-conduit: Verbs Work Request API

acconfig.h

@@ -513,6 +513,7 @@
 #undef HAVE_IBV_SRQ
 #undef HAVE_IBV_TRANSPORT_TYPE
 #undef HAVE_IBV_CREATE_QP_EX
+#undef GASNETC_HAVE_IBV_WR_API
 #undef GASNETC_IBV_MAX_MEDIUM
 #undef GASNETC_IBV_ODP
 #undef GASNETC_IBV_ODP_MLNX

configure.in

@@ -3912,6 +3912,31 @@ if test "$enabled_ibv" = yes; then
     ], [
       (void) ibv_create_qp_ex(NULL, NULL);
     ], [ AC_DEFINE(HAVE_IBV_CREATE_QP_EX) ] )
+
+    GASNET_IF_DISABLED(ibv-wr-api,
+      [Disable use of verbs work request API (default is to probe)],
+      [:],
+      [GASNET_TRY_CACHE_LINK(for verbs work request API support, ibv_wr_api, [
+         INCLUDE_VERBS_H
+         #if !HAVE_IBV_CREATE_QP_EX
+         #error WR API requires ibv_create_qp_ex
+         #endif
+       ], [
+         struct ibv_qp_init_attr_ex init_attr_ex;
+         init_attr_ex.send_ops_flags = IBV_QP_EX_WITH_SEND_WITH_IMM |
+                                       IBV_QP_EX_WITH_RDMA_WRITE |
+                                       IBV_QP_EX_WITH_RDMA_READ;
+         init_attr_ex.comp_mask = IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+         struct ibv_qp *qp = NULL;
+         struct ibv_qp_ex *qpx = ibv_qp_to_qp_ex(qp);
+         ibv_wr_start(qpx);
+         ibv_wr_send_imm(qpx, 0);
+         ibv_wr_rdma_write(qpx, 0, 0);
+         ibv_wr_rdma_read(qpx, 0, 0);
+         ibv_wr_atomic_fetch_add(qpx, 0, 0, 0);
+         ibv_wr_complete(qpx);
+       ], [ AC_DEFINE(GASNETC_HAVE_IBV_WR_API) ] )
+     ])
   fi
 
   popdef([INCLUDE_VERBS_H])

ibv-conduit/README

@@ -204,6 +204,11 @@ Paul H. Hargrove <PHHargrove@lbl.gov>
   behavior.  For more information, see the `GASNET_RCV_THREAD_POLL_MODE`
   environment variable documentation, below.
 
+  By default, ibv-conduit will use the Verbs Work Request API if support is
+  detected at configure time.  One can configure using `--disable-ibv-wr-api`
+  to force use of the older `ibv_post_send()` interface.  There is no mechanism
+  to choose between the two implementations at run time.
+
 @ Section: Job Spawning @
 
   If using UPC++, Chapel, etc. the language-specific commands should be used

ibv-conduit/gasnet_core.c

@@ -66,6 +66,10 @@ GASNETI_IDENT(gasnetc_IdentString_MaxHCAs, "$GASNetIbvMaxHCAs: " _STRINGIFY(GASN
   GASNETI_IDENT(gasnetc_IdentString_SerializeCqPoll, "$GASNetIbvSerializeCqPoll: 1 $");
 #endif
 
+#if GASNETC_HAVE_IBV_WR_API
+  GASNETI_IDENT(gasnetc_IdentString_WR, "$GASNetIbvWR: 1 $");
+#endif
+
 int gex_System_QueryHiddenAMConcurrencyLevel(void) {
 #if !GASNETC_USE_RCV_THREAD
   gasneti_assert(! gasnetc_use_rcv_thread);
@@ -4305,9 +4309,6 @@ int gasnetc_am_commit(    gasnetc_buffer_t *buf, gasnetc_buffer_t *buf_alloc,
       GASNETC_DECL_SR_DESC(sr_desc, 2);
       int numargs_field = have_flow ? GASNETC_MAX_ARGS : numargs;
 
-      sr_desc->imm_data   = GASNETC_MSG_GENFLAGS(!is_reply, category, numargs_field, handler,
-						 gasneti_mynode);
-      sr_desc->opcode     = IBV_WR_SEND_WITH_IMM;
       sr_desc->num_sge    = 1;
       sr_desc->sg_list[0].addr   = (uintptr_t)buf;
       sr_desc->sg_list[0].length = head_len + (in_place ? nbytes : copy_len);
@@ -4338,8 +4339,10 @@ int gasnetc_am_commit(    gasnetc_buffer_t *buf, gasnetc_buffer_t *buf_alloc,
       }
       #endif
 
+      uint32_t imm_data = GASNETC_MSG_GENFLAGS(!is_reply, category, numargs_field,
+                                               handler, gasneti_mynode);
       int reserved = (immediate != 0); // CQ slot was pre-reserved if and only if immediate
-      gasnetc_snd_post_common(sreq, sr_desc, reserved, !buf_alloc GASNETI_THREAD_PASS);
+      gasnetc_post_send_imm(sreq, sr_desc, imm_data, reserved, !buf_alloc GASNETI_THREAD_PASS);
     }
 
     return 0;

ibv-conduit/gasnet_core_connect.c

@@ -177,12 +177,35 @@ gasnetc_parse_filename(const char *filename)
 
 #if HAVE_IBV_CREATE_QP_EX
   typedef struct ibv_qp_init_attr_ex gasnetc_qp_init_attr_t;
-  #define gasnetc_create_qp(hca, attr) ibv_create_qp_ex((hca)->handle, attr)
 #else
   typedef struct ibv_qp_init_attr gasnetc_qp_init_attr_t;
-  #define gasnetc_create_qp(hca, attr) ibv_create_qp((hca)->pd, attr)
 #endif
 
+GASNETI_INLINE(gasnetc_create_qp)
+struct ibv_qp *gasnetc_create_qp(gasnetc_hca_t *hca, gasnetc_qp_init_attr_t *init_attr_p)
+{
+#if GASNETC_HAVE_IBV_WR_API
+  uint64_t ops = IBV_QP_EX_WITH_SEND_WITH_IMM |
+                 IBV_QP_EX_WITH_RDMA_WRITE |
+                 IBV_QP_EX_WITH_RDMA_READ;
+  #if GASNETC_BUILD_IBVRATOMIC
+    ops       |= IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD |
+                 IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP;
+  #elif GASNETC_HAVE_FENCED_PUTS
+    if (gasnetc_use_fenced_puts) {
+      ops     |= IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD;
+    }
+  #endif
+  init_attr_p->send_ops_flags  = ops;
+  init_attr_p->comp_mask      |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+#endif
+#if HAVE_IBV_CREATE_QP_EX
+  return ibv_create_qp_ex(hca->handle, init_attr_p);
+#else
+  return ibv_create_qp(hca->pd, init_attr_p);
+#endif
+}
+
 /* ------------------------------------------------------------------------------------ */
 #if GASNETC_IBV_XRC
 typedef struct gasnetc_xrc_snd_qp_s {
@@ -749,6 +772,9 @@ gasnetc_qp_create(gasnetc_conn_info_t *conn_info)
     #endif
 
       cep->qp_handle = hndl;
+    #if GASNETC_HAVE_IBV_WR_API
+      cep->qp_ex_handle = ibv_qp_to_qp_ex(hndl);
+    #endif
       conn_info->local_qpn[qpi] = hndl->qp_num;
     }
 

ibv-conduit/gasnet_core_fwd.h

@@ -268,8 +268,13 @@
 	CNT(C, GET_RATOMICBUF, cnt)               \
 	TIME(C, GET_RATOMICBUF_STALL, stalled time) \
 	VAL(C, ALLOC_SREQ, sreqs)                 \
-	VAL(C, POST_SR, segments)                 \
-	CNT(C, POST_INLINE_SR, cnt)               \
+	CNT(C, POST_INLINE_SEND, cnt)             \
+	CNT(C, POST_INLINE_WRITE, cnt)            \
+	VAL(C, POST_SEND_IMM, segments)           \
+	VAL(C, POST_WRITE, segments)              \
+	VAL(C, POST_READ, segments)               \
+	CNT(C, POST_FADD, cnt)                    \
+	CNT(C, POST_FCAS, cnt)                    \
 	TIME(C, POST_SR_STALL_CQ, stalled time)   \
 	TIME(C, POST_SR_STALL_SQ, stalled time)   \
 	TIME(C, POST_SR_STALL_SQ2, stalled time)  \

ibv-conduit/gasnet_core_internal.h

@@ -40,6 +40,11 @@
     #endif
   #endif
 
+// Sanity check
+#if GASNETC_HAVE_IBV_WR_API && !HAVE_IBV_CREATE_QP_EX
+  #error GASNETC_HAVE_IBV_WR_API requires HAVE_IBV_CREATE_QP_EX
+#endif
+
 /*  whether or not to use spin-locking for HSL's */
 #define GASNETC_HSL_SPINLOCK 1
 
@@ -658,7 +663,12 @@ struct gasnetc_cep_t_ {
   gasnetc_lifo_head_t   *rbuf_freelist; /* Source of rcv buffers for AMs.
                                            Copy of &hca->rbuf_freelist */
   struct ibv_qp         *qp_handle;
+#if GASNETC_HAVE_IBV_WR_API
+  struct ibv_qp_ex      *qp_ex_handle;
+  #define _GASNETC_CEP_PTR_0 6*sizeof(void*)
+#else
   #define _GASNETC_CEP_PTR_0 5*sizeof(void*)
+#endif
 #if GASNETC_IBV_SRQ
   struct ibv_srq        *srq;           // Copy of hca->repl_srq OR hca->rqst_srq
   #define _GASNETC_CEP_PTR_1 1*sizeof(void*)
@@ -1075,9 +1085,32 @@ extern gasnetc_epid_t gasnetc_epid_select_qpi(gasnetc_cep_t *ceps, gasnetc_epid_
 #endif
 extern int gasnetc_snd_cq_reserve(gasnetc_cep_t * const cep);
 
-extern void gasnetc_snd_post_common(
-                  gasnetc_sreq_t *sreq, struct ibv_send_wr *sr_desc,
-                  int reserved, int is_inline GASNETI_THREAD_FARG);
+extern void gasnetc_post_send_imm(
+                  gasnetc_sreq_t *sreq,
+                  struct ibv_send_wr *sr_desc,
+                  uint32_t imm_data,
+                  int reserved, int is_inline
+                  GASNETI_THREAD_FARG);
+// Currently write and read are static and/or inline within gasnet_core_sndrcv.c
+//extern void gasnetc_post_write(
+//                  gasnetc_sreq_t *sreq,
+//                  struct ibv_send_wr *sr_desc,
+//                  int is_inline
+//                  GASNETI_THREAD_FARG);
+//extern void gasnetc_post_read(
+//                  gasnetc_sreq_t *sreq,
+//                  struct ibv_send_wr *sr_desc
+//                  GASNETI_THREAD_FARG);
+extern void gasnetc_post_fetch_add(
+                  gasnetc_sreq_t *sreq,
+                  struct ibv_send_wr *sr_desc,
+                  uint64_t op1
+                  GASNETI_THREAD_FARG);
+extern void gasnetc_post_cmp_swp(
+                  gasnetc_sreq_t *sreq,
+                  struct ibv_send_wr *sr_desc,
+                  uint64_t op1, uint64_t op2
+                  GASNETI_THREAD_FARG);
 
 extern void gasnetc_poll_rcv_hca(gasnetc_EP_t ep, gasnetc_hca_t *hca, int limit GASNETI_THREAD_FARG);
 extern void gasnetc_poll_rcv_all(gasnetc_EP_t ep, int limit GASNETI_THREAD_FARG);