Merge pull request #1203 from mrgolin/efa-dv-cq

efa: Add DV CQ with source GID capability

Author: rleon

debian/ibverbs-providers.symbols

@@ -165,7 +165,10 @@ libefa.so.1 ibverbs-providers #MINVER#
 * Build-Depends-Package: libibverbs-dev
  EFA_1.0@EFA_1.0 24
  EFA_1.1@EFA_1.1 26
+ EFA_1.2@EFA_1.2 43
  efadv_create_driver_qp@EFA_1.0 24
  efadv_create_qp_ex@EFA_1.1 26
  efadv_query_device@EFA_1.1 26
  efadv_query_ah@EFA_1.1 26
+ efadv_cq_from_ibv_cq_ex@EFA_1.2 43
+ efadv_create_cq@EFA_1.2 43

kernel-headers/rdma/efa-abi.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
 /*
- * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef EFA_ABI_USER_H
@@ -54,6 +54,7 @@ struct efa_ibv_alloc_pd_resp {
 
 enum {
 	EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL = 1 << 0,
+	EFA_CREATE_CQ_WITH_SGID               = 1 << 1,
 };
 
 struct efa_ibv_create_cq {
@@ -118,6 +119,7 @@ enum {
 	EFA_QUERY_DEVICE_CAPS_RDMA_READ = 1 << 0,
 	EFA_QUERY_DEVICE_CAPS_RNR_RETRY = 1 << 1,
 	EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS = 1 << 2,
+	EFA_QUERY_DEVICE_CAPS_CQ_WITH_SGID     = 1 << 3,
 };
 
 struct efa_ibv_ex_query_device_resp {

providers/efa/CMakeLists.txt

@@ -1,5 +1,5 @@
 rdma_shared_provider(efa libefa.map
-	1 1.1.${PACKAGE_VERSION}
+	1 1.2.${PACKAGE_VERSION}
 	efa.c
 	verbs.c
 )

providers/efa/efa.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2019-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2019-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include <stdio.h>
@@ -75,6 +75,7 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev,
 	ctx->sub_cqs_per_cq = resp.sub_cqs_per_cq;
 	ctx->cmds_supp_udata_mask = resp.cmds_supp_udata_mask;
 	ctx->cqe_size = sizeof(struct efa_io_rx_cdesc);
+	ctx->ex_cqe_size = sizeof(struct efa_io_rx_cdesc_ex);
 	ctx->inline_buf_size = resp.inline_buf_size;
 	ctx->max_llq_size = resp.max_llq_size;
 	ctx->max_tx_batch = resp.max_tx_batch;

providers/efa/efa.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2019-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef __EFA_H__
@@ -15,6 +15,7 @@
 
 #include "efa-abi.h"
 #include "efa_io_defs.h"
+#include "efadv.h"
 
 #define EFA_GET(ptr, mask) FIELD_GET(mask##_MASK, *(ptr))
 
@@ -41,6 +42,7 @@ struct efa_context {
 	uint16_t max_tx_batch;
 	uint16_t min_sq_wr;
 	size_t cqe_size;
+	size_t ex_cqe_size;
 	struct efa_qp **qp_table;
 	unsigned int qp_table_sz_m1;
 	pthread_spinlock_t qp_table_lock;
@@ -62,6 +64,7 @@ struct efa_sub_cq {
 
 struct efa_cq {
 	struct verbs_cq verbs_cq;
+	struct efadv_cq dv_cq;
 	uint32_t cqn;
 	size_t cqe_size;
 	uint8_t *buf;
@@ -174,6 +177,11 @@ static inline struct efa_cq *to_efa_cq_ex(struct ibv_cq_ex *ibvcqx)
 	return container_of(ibvcqx, struct efa_cq, verbs_cq.cq_ex);
 }
 
+static inline struct efa_cq *efadv_cq_to_efa_cq(struct efadv_cq *efadv_cq)
+{
+	return container_of(efadv_cq, struct efa_cq, dv_cq);
+}
+
 static inline struct efa_qp *to_efa_qp(struct ibv_qp *ibvqp)
 {
 	return container_of(ibvqp, struct efa_qp, verbs_qp.qp);

providers/efa/efa_io_defs.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_IO_H_
@@ -219,9 +219,7 @@ struct efa_io_cdesc_common {
 	 * 2:1 : q_type - enum efa_io_queue_type: send/recv
 	 * 3 : has_imm - indicates that immediate data is
 	 *    present - for RX completions only
-	 * 4 : wide_completion - indicates that wide
-	 *    completion format is used
-	 * 7:5 : reserved29
+	 * 7:4 : reserved28 - MBZ
 	 */
 	uint8_t flags;
 
@@ -253,33 +251,15 @@ struct efa_io_rx_cdesc {
 };
 
 /* Extended Rx Completion Descriptor */
-struct efa_io_rx_cdesc_wide {
+struct efa_io_rx_cdesc_ex {
 	/* Base RX completion info */
 	struct efa_io_rx_cdesc rx_cdesc_base;
 
 	/*
-	 * Word 0 of remote (source) address, needed only for in-band
-	 * ad-hoc AH support
+	 * Valid only in case of unknown AH (0xFFFF) and CQ set_src_addr is
+	 * enabled.
 	 */
-	uint32_t src_addr_0;
-
-	/*
-	 * Word 1 of remote (source) address, needed only for in-band
-	 * ad-hoc AH support
-	 */
-	uint32_t src_addr_1;
-
-	/*
-	 * Word 2 of remote (source) address, needed only for in-band
-	 * ad-hoc AH support
-	 */
-	uint32_t src_addr_2;
-
-	/*
-	 * Word 3 of remote (source) address, needed only for in-band
-	 * ad-hoc AH support
-	 */
-	uint32_t src_addr_3;
+	uint8_t src_addr[16];
 };
 
 /* tx_meta_desc */
@@ -305,6 +285,5 @@ struct efa_io_rx_cdesc_wide {
 #define EFA_IO_CDESC_COMMON_PHASE_MASK                      BIT(0)
 #define EFA_IO_CDESC_COMMON_Q_TYPE_MASK                     GENMASK(2, 1)
 #define EFA_IO_CDESC_COMMON_HAS_IMM_MASK                    BIT(3)
-#define EFA_IO_CDESC_COMMON_WIDE_COMPLETION_MASK            BIT(4)
 
 #endif /* _EFA_IO_H_ */

providers/efa/efadv.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2019-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef __EFADV_H__
@@ -38,6 +38,7 @@ struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
 enum {
 	EFADV_DEVICE_ATTR_CAPS_RDMA_READ = 1 << 0,
 	EFADV_DEVICE_ATTR_CAPS_RNR_RETRY = 1 << 1,
+	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID = 1 << 2,
 };
 
 struct efadv_device_attr {
@@ -65,6 +66,33 @@ struct efadv_ah_attr {
 int efadv_query_ah(struct ibv_ah *ibvah, struct efadv_ah_attr *attr,
 		   uint32_t inlen);
 
+struct efadv_cq {
+	uint64_t comp_mask;
+	int (*wc_read_sgid)(struct efadv_cq *efadv_cq, union ibv_gid *sgid);
+};
+
+enum {
+	EFADV_WC_EX_WITH_SGID = 1 << 0,
+};
+
+struct efadv_cq_init_attr {
+	uint64_t comp_mask;
+	uint64_t wc_flags;
+};
+
+struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
+				  struct ibv_cq_init_attr_ex *attr_ex,
+				  struct efadv_cq_init_attr *efa_attr,
+				  uint32_t inlen);
+
+struct efadv_cq *efadv_cq_from_ibv_cq_ex(struct ibv_cq_ex *ibvcqx);
+
+static inline int efadv_wc_read_sgid(struct efadv_cq *efadv_cq,
+				     union ibv_gid *sgid)
+{
+	return efadv_cq->wc_read_sgid(efadv_cq, sgid);
+}
+
 #ifdef __cplusplus
 }
 #endif

providers/efa/libefa.map

@@ -12,3 +12,10 @@ EFA_1.1 {
 		efadv_query_ah;
 		efadv_query_device;
 } EFA_1.0;
+
+EFA_1.2 {
+	global:
+		efadv_cq_from_ibv_cq_ex;
+		efadv_create_cq;
+		efadv_wc_read_sgid;
+} EFA_1.1;

providers/efa/man/efadv_create_cq.3.md

@@ -0,0 +1,84 @@
+---
+layout: page
+title: EFADV_CREATE_CQ
+section: 3
+tagline: Verbs
+date: 2021-01-04
+header: "EFA Direct Verbs Manual"
+footer: efa
+---
+
+# NAME
+
+efadv_create_cq - Create EFA specific Completion Queue (CQ)
+
+# SYNOPSIS
+
+```c
+#include <infiniband/efadv.h>
+
+struct ibv_cq_ex *efadv_create_cq(struct ibv_context *context,
+				  struct ibv_cq_init_attr_ex *attr_ex,
+				  struct efadv_cq_init_attr *efa_attr,
+				  uint32_t inlen);
+
+static inline int efadv_wc_read_sgid(struct efadv_cq *efadv_cq,
+				     union ibv_gid *sgid);
+```
+
+# DESCRIPTION
+
+**efadv_create_cq()** creates a Completion Queue (CQ) with specific driver
+properties.
+
+The argument attr_ex is an ibv_cq_init_attr_ex struct,
+as defined in <infiniband/verbs.h>.
+
+The EFADV work completions APIs (efadv_wc_\*) is an extension for IBV work
+completions API (ibv_wc_\*) with efa specific features for polling fields in
+the completion. This may be used together with or without ibv_wc_* calls.
+
+Use efadv_cq_from_ibv_cq_ex() to get the efadv_cq for accessing the work
+completion interface.
+
+Compatibility is handled using the comp_mask and inlen fields.
+
+```c
+struct efadv_cq_init_attr {
+	uint64_t comp_mask;
+	uint64_t wc_flags;
+};
+```
+
+*inlen*
+:	In: Size of struct efadv_cq_init_attr.
+
+*comp_mask*
+:	Compatibility mask.
+
+*wc_flags*
+:       A bitwise OR of the various values described below.
+
+	EFADV_WC_EX_WITH_SGID:
+		if source AH is unknown, require sgid in WC.
+
+
+# Completion iterator functions
+
+*efadv_wc_read_sgid*
+:	Get the source GID field from the current completion.
+	If the AH is known, a negative error value is returned.
+
+
+# RETURN VALUE
+
+efadv_create_cq() returns a pointer to the created extended CQ, or NULL if the
+request fails.
+
+# SEE ALSO
+
+**efadv**(7), **ibv_create_cq_ex**(3)
+
+# AUTHORS
+
+Daniel Kranzdorf <[email protected]>

providers/efa/man/efadv_query_device.3.md

@@ -72,6 +72,10 @@ struct efadv_device_attr {
 	EFADV_DEVICE_ATTR_CAPS_RNR_RETRY:
 		RNR retry is supported for SRD QPs.
 
+	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID:
+		Reading source address (SGID) from receive completion descriptors is supported.
+		Valid only for unknown AH.
+
 *max_rdma_size*
 :	Maximum RDMA transfer size in bytes.