mlx5: Allow an application to use the indirect mkey tag

An application should be able to update mkey tag bits (Mkey[7:0]) post
the mkey creation, but currently, this option is ignored when building
the UMR WQE.

An application interested in updating the mkey tag should call
mlx5dv_create_mkey with MLX5DV_MKEY_INIT_ATTR_FLAGS_UPDATE_TAG. If this
option is not supported the creation will fail, otherwise, it can go and
update the mkey tag bits and it may be applicable.

Signed-off-by: Aharon Landau <[email protected]>
Reviewed-by: Michael Guralnik <[email protected]>
Signed-off-by: Yishai Hadas <[email protected]>

Author: aharonl-nvidia

providers/mlx5/man/mlx5dv_create_mkey.3.md

@@ -9,7 +9,7 @@ tagline: Verbs
 
 mlx5dv_create_mkey -  Creates an indirect mkey
 
-mlx5dv_create_mkey -  Destroys an indirect mkey
+mlx5dv_destroy_mkey -  Destroys an indirect mkey
 
 # SYNOPSIS
 
@@ -59,6 +59,10 @@ Create an indirect mkey to enable application uses its specific device functiona
 		was not configured for crypto properties using
 		**mlx5dv_wr_set_mkey_crypto()**, then running traffic with the
 		MKey will fail, generating a CQE with error.
+	MLX5DV_MKEY_INIT_ATTR_FLAGS_UPDATE_TAG:
+		Enable update tag support for mkey.
+		Setting this flag allows an application to set the mkey tag post of creating the mkey.
+		If the kernel does not support updating the mkey tag, mkey creation will fail.
 
 *max_entries*
 :	Requested max number of pointed entries by this indirect mkey.

providers/mlx5/mlx5.c

@@ -2407,6 +2407,10 @@ static int mlx5_set_context(struct mlx5_context *context,
 	if (resp->comp_mask & MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS)
 		context->flags |= MLX5_CTX_FLAGS_REAL_TIME_TS_SUPPORTED;
 
+	if (resp->comp_mask &
+	    MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG)
+		context->flags |= MLX5_CTX_FLAGS_MKEY_UPDATE_TAG_SUPPORTED;
+
 	if (resp->comp_mask & MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY) {
 		context->dump_fill_mkey = resp->dump_fill_mkey;
 		/* Have the BE value ready to be used in data path */

providers/mlx5/mlx5.h

@@ -263,6 +263,7 @@ enum mlx5_ctx_flags {
 	MLX5_CTX_FLAGS_ECE_SUPPORTED = 1 << 2,
 	MLX5_CTX_FLAGS_SQD2RTS_SUPPORTED = 1 << 3,
 	MLX5_CTX_FLAGS_REAL_TIME_TS_SUPPORTED = 1 << 4,
+	MLX5_CTX_FLAGS_MKEY_UPDATE_TAG_SUPPORTED = 1 << 5,
 };
 
 struct mlx5_entropy_caps {

providers/mlx5/mlx5dv.h

@@ -283,6 +283,7 @@ enum mlx5dv_mkey_init_attr_flags {
 	MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT = 1 << 0,
 	MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE = 1 << 1,
 	MLX5DV_MKEY_INIT_ATTR_FLAGS_CRYPTO = 1 << 2,
+	MLX5DV_MKEY_INIT_ATTR_FLAGS_UPDATE_TAG = 1 << 3,
 };
 
 struct mlx5dv_mkey_init_attr {

providers/mlx5/qp.c

@@ -2796,6 +2796,7 @@ static void mlx5_send_wr_mkey_configure(struct mlx5dv_qp_ex *dv_qp,
 					struct mlx5dv_mkey_conf_attr *attr)
 {
 	struct mlx5_qp *mqp = mqp_from_mlx5dv_qp_ex(dv_qp);
+	struct mlx5_context *mctx = to_mctx(mqp->ibv_qp->context);
 	struct ibv_qp_ex *ibqp = &mqp->verbs_qp.qp_ex;
 	struct mlx5_wqe_umr_ctrl_seg *umr_ctrl;
 	struct mlx5_wqe_mkey_context_seg *mk;
@@ -2838,7 +2839,14 @@ static void mlx5_send_wr_mkey_configure(struct mlx5dv_qp_ex *dv_qp,
 
 	mk = seg;
 	memset(mk, 0, sizeof(*mk));
+	if (unlikely(dv_mkey->lkey & 0xff &&
+		     !(mctx->flags &
+		       MLX5_CTX_FLAGS_MKEY_UPDATE_TAG_SUPPORTED))) {
+		mqp->err = EOPNOTSUPP;
+		return;
+	}
 	mk->qpn_mkey = htobe32(0xffffff00 | (dv_mkey->lkey & 0xff));
+	mkey_mask |= MLX5_WQE_UMR_CTRL_MKEY_MASK_MKEY;
 
 	seg += sizeof(*mk);
 	mqp->cur_size += (sizeof(*mk) / 16);

providers/mlx5/verbs.c

@@ -6727,17 +6727,22 @@ _mlx5dv_create_mkey(struct mlx5dv_mkey_init_attr *mkey_init_attr)
 	uint32_t out[DEVX_ST_SZ_DW(create_mkey_out)] = {};
 	uint32_t in[DEVX_ST_SZ_DW(create_mkey_in)] = {};
 	struct mlx5_mkey *mkey;
+	bool update_tag;
 	bool sig_mkey;
 	bool crypto_mkey;
 	struct ibv_pd *pd = mkey_init_attr->pd;
 	size_t bsf_size = 0;
 	void *mkc;
 
+	update_tag = to_mctx(pd->context)->flags &
+		     MLX5_CTX_FLAGS_MKEY_UPDATE_TAG_SUPPORTED;
 	if (!mkey_init_attr->create_flags ||
 	    !check_comp_mask(mkey_init_attr->create_flags,
 			     MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT |
 			     MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE |
-			     MLX5DV_MKEY_INIT_ATTR_FLAGS_CRYPTO)) {
+			     MLX5DV_MKEY_INIT_ATTR_FLAGS_CRYPTO |
+			     (update_tag ?
+			      MLX5DV_MKEY_INIT_ATTR_FLAGS_UPDATE_TAG : 0))) {
 		errno = EOPNOTSUPP;
 		return NULL;
 	}