prov/efa: Fix the unsolicited write recv check

Currently, efa provider uses the device & env to
check the unsolicited write recv. This check is wrong
and can cause false positive when the efadv_cq creation
failed. This patch fixes this issue by tracking the unsolicited
write recv status in both efa_ibv_cq and efa_qp.

Signed-off-by: Shi Jin <[email protected]>

Author: shijin-aws

prov/efa/src/efa_base_ep.c

@@ -195,9 +195,12 @@ static int efa_base_ep_modify_qp_rst2rts(struct efa_base_ep *base_ep,
  *
  * @param qp double pointer of struct efa_qp
  * @param init_attr_ex ibv_qp_init_attr_ex
+ * @param tclass traffic class (QoS) of the qp
+ * @param use_unsolicited_write_recv whether to use unsolicited write recv in the qp
  * @return int 0 on success, negative integer on failure
  */
-int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex, uint32_t tclass)
+int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex,
+		   uint32_t tclass, bool use_unsolicited_write_recv)
 {
 	struct efadv_qp_init_attr efa_attr = { 0 };
 
@@ -219,8 +222,9 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex,
 			init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
 			init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM;
 		}
+		(*qp)->unsolicited_write_recv_enabled = use_unsolicited_write_recv;
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-		if (efa_use_unsolicited_write_recv())
+		if (use_unsolicited_write_recv)
 			efa_attr.flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV;
 #endif
 		efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;
@@ -313,7 +317,10 @@ static int efa_base_ep_create_qp(struct efa_base_ep *base_ep,
 
 	efa_base_ep_construct_ibv_qp_init_attr_ex(base_ep, &attr_ex, tx_cq->ibv_cq_ex, rx_cq->ibv_cq_ex);
 
-	ret = efa_qp_create(&base_ep->qp, &attr_ex, base_ep->info->tx_attr->tclass);
+	assert(tx_cq->unsolicited_write_recv_enabled == rx_cq->unsolicited_write_recv_enabled);
+
+	ret = efa_qp_create(&base_ep->qp, &attr_ex, base_ep->info->tx_attr->tclass,
+			    tx_cq->unsolicited_write_recv_enabled);
 	if (ret)
 		return ret;
 
@@ -332,7 +339,7 @@ static int efa_base_ep_create_qp(struct efa_base_ep *base_ep,
 #endif
 
 	if (create_user_recv_qp) {
-		ret = efa_qp_create(&base_ep->user_recv_qp, &attr_ex, base_ep->info->tx_attr->tclass);
+		ret = efa_qp_create(&base_ep->user_recv_qp, &attr_ex, base_ep->info->tx_attr->tclass, tx_cq->unsolicited_write_recv_enabled);
 		if (ret) {
 			efa_base_ep_destruct_qp(base_ep);
 			return ret;
@@ -669,7 +676,7 @@ int efa_base_ep_check_qp_in_order_aligned_128_bytes(struct efa_base_ep *ep,
 	/* Create a dummy qp for query only */
 	efa_base_ep_construct_ibv_qp_init_attr_ex(ep, &attr_ex, ibv_cq.ibv_cq_ex, ibv_cq.ibv_cq_ex);
 
-	ret = efa_qp_create(&qp, &attr_ex, FI_TC_UNSPEC);
+	ret = efa_qp_create(&qp, &attr_ex, FI_TC_UNSPEC, ibv_cq.unsolicited_write_recv_enabled);
 	if (ret)
 		goto out;
 

prov/efa/src/efa_base_ep.h

@@ -51,6 +51,7 @@ struct efa_qp {
 #if HAVE_EFA_DATA_PATH_DIRECT
 	struct efa_data_path_direct_qp data_path_direct_qp;
 #endif
+	bool unsolicited_write_recv_enabled;
 };
 
 struct efa_av;
@@ -111,7 +112,8 @@ int efa_base_ep_getname(fid_t fid, void *addr, size_t *addrlen);
 int efa_ep_open(struct fid_domain *domain_fid, struct fi_info *user_info,
 		struct fid_ep **ep_fid, void *context);
 
-int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex, uint32_t tclass);
+int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex,
+		   uint32_t tclass, bool enable_unsolicited_write_recv);
 
 void efa_qp_destruct(struct efa_qp *qp);
 

prov/efa/src/efa_cq.h

@@ -22,6 +22,7 @@ struct efa_ibv_cq {
 	struct efa_data_path_direct_cq data_path_direct;
 #endif
 	struct ibv_comp_channel	*channel;
+	bool unsolicited_write_recv_enabled;
 };
 
 struct efa_ibv_cq_poll_list_entry {
@@ -216,6 +217,7 @@ int efa_cq_open_ibv_cq(struct fi_cq_attr *attr,
 		.wc_flags = EFADV_WC_EX_WITH_SGID,
 	};
 
+	ibv_cq->unsolicited_write_recv_enabled = false;
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
 	if (efa_use_unsolicited_write_recv())
 		efadv_cq_init_attr.wc_flags |= EFADV_WC_EX_WITH_IS_UNSOLICITED;
@@ -251,6 +253,11 @@ int efa_cq_open_ibv_cq(struct fi_cq_attr *attr,
 			&init_attr_ex, ibv_ctx, &ibv_cq->ibv_cq_ex, &ibv_cq->ibv_cq_ex_type);
 	}
 
+#if HAVE_CAPS_UNSOLICITED_WRITE_RECV
+	if (efadv_cq_init_attr.wc_flags & EFADV_WC_EX_WITH_IS_UNSOLICITED)
+		ibv_cq->unsolicited_write_recv_enabled = true;
+#endif
+
 	ibv_cq->ibv_cq_ex_type = EFADV_CQ;
 	return 0;
 }
@@ -283,6 +290,7 @@ int efa_cq_open_ibv_cq(struct fi_cq_attr *attr,
 	};
 
 	ibv_cq->data_path_direct_enabled = false;
+	ibv_cq->unsolicited_write_recv_enabled = false;
 	return efa_cq_open_ibv_cq_with_ibv_create_cq_ex(
 		&init_attr_ex, ibv_ctx, &ibv_cq->ibv_cq_ex, &ibv_cq->ibv_cq_ex_type);
 }

prov/efa/src/efa_data_path_ops.h

@@ -320,7 +320,7 @@ static inline int efa_ibv_cq_wc_read_sgid(struct efa_ibv_cq *ibv_cq, union ibv_g
  */
 static inline bool efa_cq_wc_is_unsolicited(struct efa_ibv_cq *ibv_cq)
 {
-	return efa_use_unsolicited_write_recv() && efa_ibv_cq_wc_is_unsolicited(ibv_cq);
+	return ibv_cq->unsolicited_write_recv_enabled && efa_ibv_cq_wc_is_unsolicited(ibv_cq);
 }
 
 static inline bool efa_cq_wc_available(struct efa_ibv_cq *cq)

prov/efa/src/rdm/efa_rdm_ep_fiops.c

@@ -1074,7 +1074,7 @@ void efa_rdm_ep_set_extra_info(struct efa_rdm_ep *ep)
 
 	ep->extra_info[0] |= EFA_RDM_EXTRA_FEATURE_DELIVERY_COMPLETE;
 
-	if (efa_use_unsolicited_write_recv())
+	if (ep->base_ep.qp->unsolicited_write_recv_enabled)
 		ep->extra_info[0] |= EFA_RDM_EXTRA_FEATURE_UNSOLICITED_WRITE_RECV;
 
 	if (ep->use_zcpy_rx) {

prov/efa/src/rdm/efa_rdm_rma.c

@@ -395,7 +395,7 @@ ssize_t efa_rdm_rma_post_write(struct efa_rdm_ep *ep, struct efa_rdm_ope *txe)
 				"This is usually caused by inconsistent efa driver, libfabric, or rdma-core versions.\n"
 				"Please use consistent software versions on both hosts, or disable the unsolicited write "
 				"recv feature by setting environment variable FI_EFA_USE_UNSOLICITED_WRITE_RECV=0\n",
-				efa_use_unsolicited_write_recv(), efa_rdm_peer_support_unsolicited_write_recv(txe->peer),
+				ep->base_ep.qp->unsolicited_write_recv_enabled, efa_rdm_peer_support_unsolicited_write_recv(txe->peer),
 				err_msg);
 			return -FI_EOPNOTSUPP;
 		}

prov/efa/test/efa_unit_test_cq.c

@@ -388,6 +388,24 @@ void test_rdm_cq_handshake_bad_send_status_unsupported_op(struct efa_resource **
 	test_rdm_cq_handshake_bad_send_status_impl(state, EFA_IO_COMP_STATUS_LOCAL_ERROR_UNSUPPORTED_OP, true);
 }
 
+
+/**
+ * @brief Verify that unsolicited write recv status is tracked in efa_ibv_cq correctly
+ *
+ * @param[in]	state		struct efa_resource that is managed by the framework
+ */
+void test_ibv_cq_unsolicited_write_recv_status(struct efa_resource **state)
+{
+	struct efa_resource *resource = *state;
+	struct efa_cq *efa_cq;
+
+
+	efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
+	efa_cq = container_of(resource->cq, struct efa_cq, util_cq.cq_fid);
+
+	assert_true(efa_use_unsolicited_write_recv() == efa_cq->ibv_cq.unsolicited_write_recv_enabled);
+}
+
 /**
  * @brief verify that fi_cq_read/fi_cq_readerr works properly when rdma-core return bad status for recv.
  *
@@ -460,7 +478,7 @@ void test_ibv_cq_ex_read_bad_recv_status(struct efa_resource **state)
 	ibv_cq->ibv_cq_ex->status = IBV_WC_GENERAL_ERR;
 
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-	if (efa_use_unsolicited_write_recv()) {
+	if (ibv_cq->unsolicited_write_recv_enabled) {
 		g_efa_unit_test_mocks.efa_ibv_cq_wc_is_unsolicited = &efa_mock_efa_ibv_cq_wc_is_unsolicited_return_mock;
 		will_return(efa_mock_efa_ibv_cq_wc_is_unsolicited_return_mock, false);
 	}
@@ -536,19 +554,18 @@ void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(struct efa_resource
 	will_return_always(efa_mock_efa_ibv_cq_wc_read_qp_num_return_mock, efa_rdm_ep->base_ep.qp->qp_num);
 	will_return(efa_mock_efa_ibv_cq_wc_read_vendor_err_return_mock, EFA_IO_COMP_STATUS_FLUSHED);
 
-	g_efa_unit_test_mocks.efa_device_support_unsolicited_write_recv = &efa_mock_efa_device_support_unsolicited_write_recv;
 
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
 	if (use_unsolicited_recv) {
 		g_efa_unit_test_mocks.efa_ibv_cq_wc_is_unsolicited = &efa_mock_efa_ibv_cq_wc_is_unsolicited_return_mock;
-		will_return(efa_mock_efa_device_support_unsolicited_write_recv, true);
+		ibv_cq->unsolicited_write_recv_enabled = true;
 		will_return(efa_mock_efa_ibv_cq_wc_is_unsolicited_return_mock, true);
 		ibv_cq->ibv_cq_ex->wr_id = 0;
 	} else {
 		/*
 		 * For solicited write recv, it will consume an internal rx pkt
 		 */
-		will_return(efa_mock_efa_device_support_unsolicited_write_recv, false);
+		ibv_cq->unsolicited_write_recv_enabled = false;
 		struct efa_rdm_pke *pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
 		assert_non_null(pkt_entry);
 		efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep);
@@ -558,7 +575,7 @@ void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(struct efa_resource
 	/*
 	 * Always test with solicited recv
 	 */
-	will_return(efa_mock_efa_device_support_unsolicited_write_recv, false);
+	ibv_cq->unsolicited_write_recv_enabled = false;
 	struct efa_rdm_pke *pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL);
 	assert_non_null(pkt_entry);
 	efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep);
@@ -1065,7 +1082,7 @@ static void test_efa_cq_read_prep(struct efa_resource *resource,
 
 
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-    if (efa_use_unsolicited_write_recv()) {
+    if (ibv_cq->unsolicited_write_recv_enabled) {
         g_efa_unit_test_mocks.efa_ibv_cq_wc_is_unsolicited = &efa_mock_efa_ibv_cq_wc_is_unsolicited_return_mock;
         will_return_maybe(efa_mock_efa_ibv_cq_wc_is_unsolicited_return_mock, is_unsolicited_write_recv);
     }

prov/efa/test/efa_unit_test_ep.c

@@ -1461,7 +1461,7 @@ void test_efa_rdm_ep_rx_refill_threshold_larger_than_rx_size(struct efa_resource
 }
 
 /**
- * @brief when unsolicited write recv is supported (by device + env),
+ * @brief when unsolicited write recv is supported (checked by cq),
  * efa_rdm_ep_support_unsolicited_write_recv
  * should return true, otherwise it should return false
  *
@@ -1471,13 +1471,15 @@ void test_efa_rdm_ep_rx_refill_threshold_larger_than_rx_size(struct efa_resource
 void test_efa_rdm_ep_support_unsolicited_write_recv(struct efa_resource **state)
 {
 	struct efa_rdm_ep *efa_rdm_ep;
+	struct efa_cq *efa_cq;
 	struct efa_resource *resource = *state;
 
 	efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
 
 	efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
+	efa_cq = container_of(resource->cq, struct efa_cq, util_cq.cq_fid);
 
-	assert_int_equal(efa_use_unsolicited_write_recv(),
+	assert_int_equal(efa_cq->ibv_cq.unsolicited_write_recv_enabled,
 			 efa_rdm_ep_support_unsolicited_write_recv(efa_rdm_ep));
 }
 

prov/efa/test/efa_unit_tests.c

@@ -200,6 +200,7 @@ int main(void)
 		cmocka_unit_test_setup_teardown(test_rdm_cq_handshake_bad_send_status_unreach_remote, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_rdm_cq_handshake_bad_send_status_remote_abort, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_rdm_cq_handshake_bad_send_status_unsupported_op, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_ibv_cq_unsolicited_write_recv_status, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_ibv_cq_ex_read_bad_recv_status, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_use_unsolicited_recv, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_use_solicited_recv, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),

prov/efa/test/efa_unit_tests.h

@@ -173,6 +173,7 @@ void test_rdm_cq_handshake_bad_send_status_unresp_remote();
 void test_rdm_cq_handshake_bad_send_status_unreach_remote();
 void test_rdm_cq_handshake_bad_send_status_remote_abort();
 void test_rdm_cq_handshake_bad_send_status_unsupported_op();
+void test_ibv_cq_unsolicited_write_recv_status();
 void test_ibv_cq_ex_read_bad_recv_status();
 void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_use_unsolicited_recv();
 void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_use_solicited_recv();