efa: Read source GID from EFA DV CQ entry

Add direct verb to read source address from EFA CQ completion entry, for
completion from unknown AH. Adjust the CQ entry size requested from the
device accordingly.

Signed-off-by: Daniel Kranzdorf <[email protected]>

Author: dkkranz

providers/efa/efa.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2019-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2019-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include <stdio.h>
@@ -75,6 +75,7 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev,
 	ctx->sub_cqs_per_cq = resp.sub_cqs_per_cq;
 	ctx->cmds_supp_udata_mask = resp.cmds_supp_udata_mask;
 	ctx->cqe_size = sizeof(struct efa_io_rx_cdesc);
+	ctx->ex_cqe_size = sizeof(struct efa_io_rx_cdesc_ex);
 	ctx->inline_buf_size = resp.inline_buf_size;
 	ctx->max_llq_size = resp.max_llq_size;
 	ctx->max_tx_batch = resp.max_tx_batch;

providers/efa/efa.h

@@ -42,6 +42,7 @@ struct efa_context {
 	uint16_t max_tx_batch;
 	uint16_t min_sq_wr;
 	size_t cqe_size;
+	size_t ex_cqe_size;
 	struct efa_qp **qp_table;
 	unsigned int qp_table_sz_m1;
 	pthread_spinlock_t qp_table_lock;
@@ -176,6 +177,11 @@ static inline struct efa_cq *to_efa_cq_ex(struct ibv_cq_ex *ibvcqx)
 	return container_of(ibvcqx, struct efa_cq, verbs_cq.cq_ex);
 }
 
+static inline struct efa_cq *efadv_cq_to_efa_cq(struct efadv_cq *efadv_cq)
+{
+	return container_of(efadv_cq, struct efa_cq, dv_cq);
+}
+
 static inline struct efa_qp *to_efa_qp(struct ibv_qp *ibvqp)
 {
 	return container_of(ibvqp, struct efa_qp, verbs_qp.qp);

providers/efa/efa_io_defs.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_IO_H_
@@ -219,9 +219,7 @@ struct efa_io_cdesc_common {
 	 * 2:1 : q_type - enum efa_io_queue_type: send/recv
 	 * 3 : has_imm - indicates that immediate data is
 	 *    present - for RX completions only
-	 * 4 : wide_completion - indicates that wide
-	 *    completion format is used
-	 * 7:5 : reserved29
+	 * 7:4 : reserved28 - MBZ
 	 */
 	uint8_t flags;
 
@@ -253,33 +251,15 @@ struct efa_io_rx_cdesc {
 };
 
 /* Extended Rx Completion Descriptor */
-struct efa_io_rx_cdesc_wide {
+struct efa_io_rx_cdesc_ex {
 	/* Base RX completion info */
 	struct efa_io_rx_cdesc rx_cdesc_base;
 
 	/*
-	 * Word 0 of remote (source) address, needed only for in-band
-	 * ad-hoc AH support
+	 * Valid only in case of unknown AH (0xFFFF) and CQ set_src_addr is
+	 * enabled.
 	 */
-	uint32_t src_addr_0;
-
-	/*
-	 * Word 1 of remote (source) address, needed only for in-band
-	 * ad-hoc AH support
-	 */
-	uint32_t src_addr_1;
-
-	/*
-	 * Word 2 of remote (source) address, needed only for in-band
-	 * ad-hoc AH support
-	 */
-	uint32_t src_addr_2;
-
-	/*
-	 * Word 3 of remote (source) address, needed only for in-band
-	 * ad-hoc AH support
-	 */
-	uint32_t src_addr_3;
+	uint8_t src_addr[16];
 };
 
 /* tx_meta_desc */
@@ -305,6 +285,5 @@ struct efa_io_rx_cdesc_wide {
 #define EFA_IO_CDESC_COMMON_PHASE_MASK                      BIT(0)
 #define EFA_IO_CDESC_COMMON_Q_TYPE_MASK                     GENMASK(2, 1)
 #define EFA_IO_CDESC_COMMON_HAS_IMM_MASK                    BIT(3)
-#define EFA_IO_CDESC_COMMON_WIDE_COMPLETION_MASK            BIT(4)
 
 #endif /* _EFA_IO_H_ */

providers/efa/efadv.h

@@ -38,6 +38,7 @@ struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
 enum {
 	EFADV_DEVICE_ATTR_CAPS_RDMA_READ = 1 << 0,
 	EFADV_DEVICE_ATTR_CAPS_RNR_RETRY = 1 << 1,
+	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID = 1 << 2,
 };
 
 struct efadv_device_attr {
@@ -67,6 +68,11 @@ int efadv_query_ah(struct ibv_ah *ibvah, struct efadv_ah_attr *attr,
 
 struct efadv_cq {
 	uint64_t comp_mask;
+	int (*wc_read_sgid)(struct efadv_cq *efadv_cq, union ibv_gid *sgid);
+};
+
+enum {
+	EFADV_WC_EX_WITH_SGID = 1 << 0,
 };
 
 struct efadv_cq_init_attr {
@@ -81,6 +87,12 @@ struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
 
 struct efadv_cq *efadv_cq_from_ibv_cq_ex(struct ibv_cq_ex *ibvcqx);
 
+static inline int efadv_wc_read_sgid(struct efadv_cq *efadv_cq,
+				     union ibv_gid *sgid)
+{
+	return efadv_cq->wc_read_sgid(efadv_cq, sgid);
+}
+
 #ifdef __cplusplus
 }
 #endif

providers/efa/libefa.map

@@ -17,4 +17,5 @@ EFA_1.2 {
 	global:
 		efadv_cq_from_ibv_cq_ex;
 		efadv_create_cq;
+		efadv_wc_read_sgid;
 } EFA_1.1;

providers/efa/man/efadv_create_cq.3.md

@@ -21,6 +21,9 @@ struct ibv_cq_ex *efadv_create_cq(struct ibv_context *context,
 				  struct ibv_cq_init_attr_ex *attr_ex,
 				  struct efadv_cq_init_attr *efa_attr,
 				  uint32_t inlen);
+
+static inline int efadv_wc_read_sgid(struct efadv_cq *efadv_cq,
+				     union ibv_gid *sgid);
 ```
 
 # DESCRIPTION
@@ -54,7 +57,17 @@ struct efadv_cq_init_attr {
 :	Compatibility mask.
 
 *wc_flags*
-:	Required WC fields.
+:       A bitwise OR of the various values described below.
+
+	EFADV_WC_EX_WITH_SGID:
+		if source AH is unknown, require sgid in WC.
+
+
+# Completion iterator functions
+
+*efadv_wc_read_sgid*
+:	Get the source GID field from the current completion.
+	If the AH is known, a negative error value is returned.
 
 
 # RETURN VALUE

providers/efa/man/efadv_query_device.3.md

@@ -72,6 +72,10 @@ struct efadv_device_attr {
 	EFADV_DEVICE_ATTR_CAPS_RNR_RETRY:
 		RNR retry is supported for SRD QPs.
 
+	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID:
+		Reading source address (SGID) from receive completion descriptors is supported.
+		Valid only for unknown AH.
+
 *max_rdma_size*
 :	Maximum RDMA transfer size in bytes.
 

providers/efa/verbs.c

@@ -168,6 +168,9 @@ int efadv_query_device(struct ibv_context *ibvctx,
 	if (vext_field_avail(typeof(*attr), device_caps, inlen)) {
 		if (EFA_DEV_CAP(ctx, RNR_RETRY))
 			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_RNR_RETRY;
+
+		if (EFA_DEV_CAP(ctx, CQ_WITH_SGID))
+			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID;
 	}
 
 	if (vext_field_avail(typeof(*attr), max_rdma_size, inlen)) {
@@ -691,9 +694,28 @@ static uint8_t efa_wc_read_dlid_path_bits(struct ibv_cq_ex *ibvcqx)
 	return 0;
 }
 
-static void efa_cq_fill_pfns(struct ibv_cq_ex *ibvcqx,
-			     struct ibv_cq_init_attr_ex *attr)
+static int efa_wc_read_sgid(struct efadv_cq *efadv_cq, union ibv_gid *sgid)
+{
+	struct efa_cq *cq = efadv_cq_to_efa_cq(efadv_cq);
+	struct efa_io_rx_cdesc_ex *rcqex;
+
+	rcqex = container_of(cq->cur_cqe, struct efa_io_rx_cdesc_ex,
+			     rx_cdesc_base.common);
+	if (rcqex->rx_cdesc_base.ah != 0xFFFF) {
+		/* SGID is only available if AH is unknown. */
+		return -ENOENT;
+	}
+	memcpy(sgid->raw, rcqex->src_addr, sizeof(sgid->raw));
+
+	return 0;
+}
+
+static void efa_cq_fill_pfns(struct efa_cq *cq,
+			     struct ibv_cq_init_attr_ex *attr,
+			     struct efadv_cq_init_attr *efa_attr)
 {
+	struct ibv_cq_ex *ibvcqx = &cq->verbs_cq.cq_ex;
+
 	ibvcqx->start_poll = efa_start_poll;
 	ibvcqx->end_poll = efa_end_poll;
 	ibvcqx->next_poll = efa_next_poll;
@@ -716,6 +738,9 @@ static void efa_cq_fill_pfns(struct ibv_cq_ex *ibvcqx,
 		ibvcqx->read_sl = efa_wc_read_sl;
 	if (attr->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS)
 		ibvcqx->read_dlid_path_bits = efa_wc_read_dlid_path_bits;
+
+	if (efa_attr && (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID))
+		cq->dv_cq.wc_read_sgid = efa_wc_read_sgid;
 }
 
 static void efa_sub_cq_initialize(struct efa_sub_cq *sub_cq, uint8_t *buf,
@@ -730,10 +755,12 @@ static void efa_sub_cq_initialize(struct efa_sub_cq *sub_cq, uint8_t *buf,
 }
 
 static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
-				   struct ibv_cq_init_attr_ex *attr)
+				   struct ibv_cq_init_attr_ex *attr,
+				   struct efadv_cq_init_attr *efa_attr)
 {
 	struct efa_context *ctx = to_efa_context(ibvctx);
 	struct efa_create_cq_resp resp = {};
+	uint16_t cqe_size = ctx->cqe_size;
 	struct efa_create_cq cmd = {};
 	uint16_t num_sub_cqs;
 	struct efa_cq *cq;
@@ -762,9 +789,14 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 	if (!cq)
 		return NULL;
 
+	if (efa_attr && (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID)) {
+		cmd.flags |= EFA_CREATE_CQ_WITH_SGID;
+		cqe_size = ctx->ex_cqe_size;
+	}
+
 	num_sub_cqs = ctx->sub_cqs_per_cq;
 	cmd.num_sub_cqs = num_sub_cqs;
-	cmd.cq_entry_size = ctx->cqe_size;
+	cmd.cq_entry_size = cqe_size;
 	if (attr->channel)
 		cmd.flags |= EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL;
 
@@ -781,7 +813,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 	cq->cqn = resp.cq_idx;
 	cq->buf_size = resp.q_mmap_size;
 	cq->num_sub_cqs = num_sub_cqs;
-	cq->cqe_size = ctx->cqe_size;
+	cq->cqe_size = cqe_size;
 
 	cq->buf = mmap(NULL, cq->buf_size, PROT_READ, MAP_SHARED,
 		       ibvctx->cmd_fd, resp.q_mmap_key);
@@ -806,7 +838,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 		cq->db = (uint32_t *)((uint8_t *)cq->db + resp.db_off);
 	}
 
-	efa_cq_fill_pfns(&cq->verbs_cq.cq_ex, attr);
+	efa_cq_fill_pfns(cq, attr, efa_attr);
 	pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE);
 
 	return &cq->verbs_cq.cq_ex;
@@ -831,22 +863,25 @@ struct ibv_cq *efa_create_cq(struct ibv_context *ibvctx, int ncqe,
 	};
 	struct ibv_cq_ex *ibvcqx;
 
-	ibvcqx = create_cq(ibvctx, &attr_ex);
+	ibvcqx = create_cq(ibvctx, &attr_ex, NULL);
 
 	return ibvcqx ? ibv_cq_ex_to_cq(ibvcqx) : NULL;
 }
 
 struct ibv_cq_ex *efa_create_cq_ex(struct ibv_context *ibvctx,
 				   struct ibv_cq_init_attr_ex *attr_ex)
 {
-	return create_cq(ibvctx, attr_ex);
+	return create_cq(ibvctx, attr_ex, NULL);
 }
 
 struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
 				  struct ibv_cq_init_attr_ex *attr_ex,
 				  struct efadv_cq_init_attr *efa_attr,
 				  uint32_t inlen)
 {
+	struct efa_context *ctx;
+	uint64_t supp_wc_flags;
+
 	if (!is_efa_dev(ibvctx->device)) {
 		verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n");
 		errno = EOPNOTSUPP;
@@ -861,14 +896,16 @@ struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
 		return NULL;
 	}
 
-	if (efa_attr->wc_flags) {
+	ctx = to_efa_context(ibvctx);
+	supp_wc_flags = EFA_DEV_CAP(ctx, CQ_WITH_SGID) ? EFADV_WC_EX_WITH_SGID : 0;
+	if (!check_comp_mask(efa_attr->wc_flags, supp_wc_flags)) {
 		verbs_err(verbs_get_ctx(ibvctx),
 			  "Invalid EFA wc_flags[%#x]\n", efa_attr->wc_flags);
 		errno = EOPNOTSUPP;
 		return NULL;
 	}
 
-	return create_cq(ibvctx, attr_ex);
+	return create_cq(ibvctx, attr_ex, efa_attr);
 }
 
 struct efadv_cq *efadv_cq_from_ibv_cq_ex(struct ibv_cq_ex *ibvcqx)