prov/efa: Fail early in rma when iov count exceeds device support

efa-direct offloads the rma request to rdma wqes in efa device
directly. But today efa device doesn't support > 1 iov (queried as
ibv_device_attr.max_sge_rd). This
patch makes efa_post_read and efa_post_write fail early with
clear error message for this scenario to avoid unexpected crashing.

Signed-off-by: Shi Jin <[email protected]>

Author: shijin-aws

prov/efa/src/efa_rma.c

@@ -56,8 +56,10 @@ static inline ssize_t efa_rma_post_read(struct efa_base_ep *base_ep,
 		ofi_total_iov_len(msg->msg_iov, msg->iov_count),
 		msg->addr, (size_t) msg->context, flags);
 
-	assert(msg->iov_count > 0 &&
-	       msg->iov_count <= base_ep->domain->info->tx_attr->iov_limit);
+	if (OFI_UNLIKELY(msg->iov_count > base_ep->domain->device->ibv_attr.max_sge_rd)) {
+		EFA_WARN(FI_LOG_EP_DATA, "EFA device currently doesn't support > %d iov for rdma work request\n", base_ep->domain->device->ibv_attr.max_sge_rd);
+		return -FI_EINVAL;
+	}
 	assert(msg->rma_iov_count > 0 &&
 	       msg->rma_iov_count <= base_ep->domain->info->tx_attr->rma_iov_limit);
 	assert(ofi_total_iov_len(msg->msg_iov, msg->iov_count) <=
@@ -207,6 +209,11 @@ static inline ssize_t efa_rma_post_write(struct efa_base_ep *base_ep,
 		return -FI_ENOSYS;
 	}
 
+	if (OFI_UNLIKELY(msg->iov_count > base_ep->domain->device->ibv_attr.max_sge_rd)) {
+		EFA_WARN(FI_LOG_EP_DATA, "EFA device currently doesn't support > %d iov for rdma work request\n", base_ep->domain->device->ibv_attr.max_sge_rd);
+		return -FI_EINVAL;
+	}
+
 	efa_tracepoint(write_begin_msg_context, (size_t) msg->context, (size_t) msg->addr);
 
 	EFA_DBG(FI_LOG_EP_DATA,

prov/efa/test/efa_unit_test_rma.c

@@ -299,3 +299,63 @@ void test_efa_rma_writemsg_with_inject(struct efa_resource **state)
 
 	efa_unit_test_buff_destruct(&local_buff);
 }
+
+void test_efa_rma_readv_multiple_iov_einval(struct efa_resource **state)
+{
+	struct efa_resource *resource = *state;
+	struct efa_unit_test_buff local_buff1, local_buff2;
+	struct iovec iov[2];
+	fi_addr_t src_addr;
+	void *desc[2];
+	int ret;
+	uint64_t remote_addr = 0x87654321;
+	uint64_t remote_key = 123456;
+
+	test_efa_rma_prep(resource, &src_addr);
+	efa_unit_test_buff_construct(&local_buff1, resource, 2048);
+	efa_unit_test_buff_construct(&local_buff2, resource, 2048);
+
+	iov[0].iov_base = local_buff1.buff;
+	iov[0].iov_len = local_buff1.size;
+	iov[1].iov_base = local_buff2.buff;
+	iov[1].iov_len = local_buff2.size;
+	desc[0] = fi_mr_desc(local_buff1.mr);
+	desc[1] = fi_mr_desc(local_buff2.mr);
+
+	ret = fi_readv(resource->ep, iov, desc, 2, src_addr, remote_addr,
+		       remote_key, NULL);
+	assert_int_equal(ret, -FI_EINVAL);
+
+	efa_unit_test_buff_destruct(&local_buff1);
+	efa_unit_test_buff_destruct(&local_buff2);
+}
+
+void test_efa_rma_writev_multiple_iov_einval(struct efa_resource **state)
+{
+	struct efa_resource *resource = *state;
+	struct efa_unit_test_buff local_buff1, local_buff2;
+	struct iovec iov[2];
+	fi_addr_t dest_addr;
+	void *desc[2];
+	int ret;
+	uint64_t remote_addr = 0x87654321;
+	uint64_t remote_key = 123456;
+
+	test_efa_rma_prep(resource, &dest_addr);
+	efa_unit_test_buff_construct(&local_buff1, resource, 2048);
+	efa_unit_test_buff_construct(&local_buff2, resource, 2048);
+
+	iov[0].iov_base = local_buff1.buff;
+	iov[0].iov_len = local_buff1.size;
+	iov[1].iov_base = local_buff2.buff;
+	iov[1].iov_len = local_buff2.size;
+	desc[0] = fi_mr_desc(local_buff1.mr);
+	desc[1] = fi_mr_desc(local_buff2.mr);
+
+	ret = fi_writev(resource->ep, iov, desc, 2, dest_addr, remote_addr,
+			remote_key, NULL);
+	assert_int_equal(ret, -FI_EINVAL);
+
+	efa_unit_test_buff_destruct(&local_buff1);
+	efa_unit_test_buff_destruct(&local_buff2);
+}

prov/efa/test/efa_unit_tests.c

@@ -370,6 +370,8 @@ int main(void)
 		cmocka_unit_test_setup_teardown(test_efa_rma_inject_write, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rma_inject_writedata, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rma_writemsg_with_inject, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_rma_readv_multiple_iov_einval, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_rma_writev_multiple_iov_einval, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_cq_read_no_completion, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_cq_read_send_success, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_cq_read_senddata_success, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),

prov/efa/test/efa_unit_tests.h

@@ -332,6 +332,8 @@ void test_efa_rma_writedata();
 void test_efa_rma_inject_write();
 void test_efa_rma_inject_writedata();
 void test_efa_rma_writemsg_with_inject();
+void test_efa_rma_readv_multiple_iov_einval();
+void test_efa_rma_writev_multiple_iov_einval();
 void test_efa_cq_read_no_completion();
 void test_efa_cq_read_send_success();
 void test_efa_cq_read_senddata_success();