pyverbs/efa: Support EFA DV CQ

Expose the create CQ and read SGID direct verbs.

Signed-off-by: Daniel Kranzdorf <[email protected]>

Author: dkkranz

pyverbs/providers/efa/efa_enums.pyx

@@ -11,3 +11,6 @@ cdef extern from 'infiniband/efadv.h':
 
     cpdef enum:
         EFADV_QP_DRIVER_TYPE_SRD
+
+    cpdef enum:
+        EFADV_WC_EX_WITH_SGID

pyverbs/providers/efa/efadv.pxd

@@ -1,12 +1,13 @@
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
-# Copyright 2020 Amazon.com, Inc. or its affiliates. All rights reserved.
+# Copyright 2020-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
 
 #cython: language_level=3
 
 cimport pyverbs.providers.efa.libefa as dv
 
 from pyverbs.addr cimport AH
 from pyverbs.base cimport PyverbsObject
+from pyverbs.cq cimport CQEX
 from pyverbs.device cimport Context
 from pyverbs.qp cimport QP, QPEx
 
@@ -37,3 +38,11 @@ cdef class SRDQPEx(QPEx):
 
 cdef class EfaQPInitAttr(PyverbsObject):
     cdef dv.efadv_qp_init_attr qp_init_attr
+
+
+cdef class EfaCQ(CQEX):
+    cdef dv.efadv_cq *dv_cq
+
+
+cdef class EfaDVCQInitAttr(PyverbsObject):
+    cdef dv.efadv_cq_init_attr cq_init_attr

pyverbs/providers/efa/efadv.pyx

@@ -4,8 +4,11 @@
 cimport pyverbs.providers.efa.efadv_enums as dve
 cimport pyverbs.providers.efa.libefa as dv
 
+from pyverbs.addr cimport GID
 from pyverbs.base import PyverbsRDMAErrno, PyverbsRDMAError
+from pyverbs.cq cimport CQEX, CqInitAttrEx
 import pyverbs.enums as e
+cimport pyverbs.libibverbs as v
 from pyverbs.pd cimport PD
 from pyverbs.qp cimport QP, QPEx, QPInitAttr, QPInitAttrEx
 
@@ -194,3 +197,55 @@ cdef class SRDQPEx(QPEx):
                     'RTR': 0,
                     'RTS': e.IBV_QP_SQ_PSN}
         return srd_mask [dst] | e.IBV_QP_STATE
+
+
+cdef class EfaDVCQInitAttr(PyverbsObject):
+    """
+    Represents efadv_cq_init_attr struct.
+    """
+    def __init__(self, wc_flags=0):
+        super().__init__()
+        self.cq_init_attr.wc_flags = wc_flags
+
+    @property
+    def comp_mask(self):
+        return self.cq_init_attr.comp_mask
+
+    @property
+    def wc_flags(self):
+        return self.cq_init_attr.wc_flags
+
+    @wc_flags.setter
+    def wc_flags(self, val):
+        self.cq_init_attr.wc_flags = val
+
+
+cdef class EfaCQ(CQEX):
+    """
+    Initializes an Efa CQ according to the user-provided data.
+    :param ctx: Context object
+    :param attr_ex: CQInitAttrEx object
+    :param efa_init_attr: EfaDVCQInitAttr object
+    :return: An initialized EfaCQ
+    """
+    def __init__(self, Context ctx not None, CqInitAttrEx attr_ex not None, EfaDVCQInitAttr efa_init_attr):
+        if efa_init_attr is None:
+            efa_init_attr = EfaDVCQInitAttr()
+        self.cq = dv.efadv_create_cq(ctx.context, &attr_ex.attr, &efa_init_attr.cq_init_attr, sizeof(efa_init_attr.cq_init_attr))
+        if self.cq == NULL:
+            raise PyverbsRDMAErrno('Failed to create EFA CQ')
+        self.ibv_cq = v.ibv_cq_ex_to_cq(self.cq)
+        self.dv_cq = dv.efadv_cq_from_ibv_cq_ex(self.cq)
+        self.context = ctx
+        ctx.add_ref(self)
+        super().__init__(ctx, attr_ex)
+
+    def read_sgid(self):
+        """
+        Read SGID from last work completion, if AH is unknown.
+        """
+        sgid = GID()
+        err = dv.efadv_wc_read_sgid(self.dv_cq, &sgid.gid)
+        if err:
+            return None
+        return sgid

pyverbs/providers/efa/efadv_enums.pxd

@@ -12,3 +12,6 @@ cdef extern from 'infiniband/efadv.h':
 
     cpdef enum:
         EFADV_QP_DRIVER_TYPE_SRD
+
+    cpdef enum:
+        EFADV_WC_EX_WITH_SGID

pyverbs/providers/efa/libefa.pxd

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
-# Copyright 2020 Amazon.com, Inc. or its affiliates. All rights reserved.
+# Copyright 2020-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
 
 from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t
 cimport pyverbs.libibverbs as v
@@ -28,6 +28,13 @@ cdef extern from 'infiniband/efadv.h':
         uint32_t driver_qp_type;
         uint8_t reserved[4];
 
+    cdef struct efadv_cq_init_attr:
+        uint64_t comp_mask;
+        uint64_t wc_flags;
+
+    cdef struct efadv_cq:
+        uint64_t comp_mask;
+
     int efadv_query_device(v.ibv_context *ibvctx, efadv_device_attr *attrs,
                            uint32_t inlen)
     int efadv_query_ah(v.ibv_ah *ibvah, efadv_ah_attr *attr,
@@ -38,3 +45,9 @@ cdef extern from 'infiniband/efadv.h':
                                  v.ibv_qp_init_attr_ex *attr_ex,
                                  efadv_qp_init_attr *efa_attr,
                                  uint32_t inlen)
+    v.ibv_cq_ex *efadv_create_cq(v.ibv_context *ibvctx,
+                                 v.ibv_cq_init_attr_ex *attr_ex,
+                                 efadv_cq_init_attr *efa_attr,
+                                 uint32_t inlen)
+    efadv_cq *efadv_cq_from_ibv_cq_ex(v.ibv_cq_ex *ibvcqx)
+    int efadv_wc_read_sgid(efadv_cq *efadv_cq, v.ibv_gid *sgid)