Skip to content

prov/efa: make efa object allocations cache line aligned #11458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+39 −28
Open

prov/efa: make efa object allocations cache line aligned #11458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions prov/efa/src/efa_av.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ struct efa_ah *efa_ah_alloc(struct efa_domain *domain, const uint8_t *gid)
return efa_ah;
}

efa_ah = malloc(sizeof(struct efa_ah));
ofi_memalign((void **)&efa_ah, EFA_MEM_ALIGNMENT, sizeof(struct efa_ah));
if (!efa_ah) {
errno = FI_ENOMEM;
EFA_WARN(FI_LOG_AV, "cannot allocate memory for efa_ah\n");
Expand Down Expand Up @@ -284,7 +284,7 @@ struct efa_ah *efa_ah_alloc(struct efa_domain *domain, const uint8_t *gid)
err_destroy_ibv_ah:
ibv_destroy_ah(efa_ah->ibv_ah);
err_free_efa_ah:
free(efa_ah);
ofi_freealign(efa_ah);
ofi_genlock_unlock(&domain->util_domain.lock);
return NULL;
}
Expand Down Expand Up @@ -313,7 +313,7 @@ void efa_ah_release(struct efa_domain *domain, struct efa_ah *ah)
err = ibv_destroy_ah(ah->ibv_ah);
if (err)
EFA_WARN(FI_LOG_AV, "ibv_destroy_ah failed! err=%d\n", err);
free(ah);
ofi_freealign(ah);
}
ofi_genlock_unlock(&domain->util_domain.lock);
}
Expand Down Expand Up @@ -469,7 +469,7 @@ int efa_av_reverse_av_add(struct efa_av *av,

HASH_FIND(hh, *cur_reverse_av, &cur_key, sizeof(cur_key), cur_entry);
if (!cur_entry) {
cur_entry = malloc(sizeof(*cur_entry));
ofi_memalign((void **)&cur_entry, EFA_MEM_ALIGNMENT, sizeof(*cur_entry));
if (!cur_entry) {
EFA_WARN(FI_LOG_AV, "Cannot allocate memory for cur_reverse_av entry\n");
return -FI_ENOMEM;
Expand All @@ -487,7 +487,7 @@ int efa_av_reverse_av_add(struct efa_av *av,
* and only RDM endpoint can reach here. hence the following assertion
*/
assert(av->domain->info_type == EFA_INFO_RDM);
prv_entry = malloc(sizeof(*prv_entry));
ofi_memalign((void **)&prv_entry, EFA_MEM_ALIGNMENT, sizeof(*prv_entry));
if (!prv_entry) {
EFA_WARN(FI_LOG_AV, "Cannot allocate memory for prv_reverse_av entry\n");
return -FI_ENOMEM;
Expand Down Expand Up @@ -533,7 +533,7 @@ static void efa_av_reverse_av_remove(struct efa_cur_reverse_av **cur_reverse_av,
cur_reverse_av_entry);
if (cur_reverse_av_entry) {
HASH_DEL(*cur_reverse_av, cur_reverse_av_entry);
free(cur_reverse_av_entry);
ofi_freealign(cur_reverse_av_entry);
} else {
memset(&prv_key, 0, sizeof(prv_key));
prv_key.ahn = conn->ah->ahn;
Expand All @@ -543,7 +543,7 @@ static void efa_av_reverse_av_remove(struct efa_cur_reverse_av **cur_reverse_av,
prv_reverse_av_entry);
assert(prv_reverse_av_entry);
HASH_DEL(*prv_reverse_av, prv_reverse_av_entry);
free(prv_reverse_av_entry);
ofi_freealign(prv_reverse_av_entry);
}
}

Expand Down Expand Up @@ -1251,7 +1251,7 @@ static int efa_av_close(struct fid *fid)
free(ep_addr_hashable);
}

free(av);
ofi_freealign(av);
return err;
}

Expand Down Expand Up @@ -1315,9 +1315,10 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr,
else
attr->count = MAX(attr->count, EFA_MIN_AV_SIZE);

av = calloc(1, sizeof(*av));
ofi_memalign((void **)&av, EFA_MEM_ALIGNMENT, sizeof(*av));
if (!av)
return -FI_ENOMEM;
memset(av, 0x0, sizeof(*av));

if (attr->type == FI_AV_MAP) {
EFA_INFO(FI_LOG_AV, "FI_AV_MAP is deprecated in Libfabric 2.x. Please use FI_AV_TABLE. "
Expand Down Expand Up @@ -1396,6 +1397,6 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr,
"Unable to close util_av_implicit: %s\n", fi_strerror(-retv));

err:
free(av);
ofi_freealign(av);
return ret;
}
12 changes: 8 additions & 4 deletions prov/efa/src/efa_base_ep.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,10 @@ int efa_base_ep_destruct(struct efa_base_ep *base_ep)
err = efa_base_ep_destruct_qp(base_ep);

if (base_ep->efa_recv_wr_vec)
free(base_ep->efa_recv_wr_vec);
ofi_freealign(base_ep->efa_recv_wr_vec);

if (base_ep->user_recv_wr_vec)
free(base_ep->user_recv_wr_vec);
ofi_freealign(base_ep->user_recv_wr_vec);

return err;
}
Expand Down Expand Up @@ -483,16 +483,20 @@ int efa_base_ep_construct(struct efa_base_ep *base_ep,
/* This is SRD qp's default behavior */
base_ep->rnr_retry = EFA_RNR_INFINITE_RETRY;

base_ep->efa_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
ofi_memalign((void **)&base_ep->efa_recv_wr_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
if (!base_ep->efa_recv_wr_vec) {
EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for base_ep->efa_recv_wr_vec!\n");
return -FI_ENOMEM;
}
base_ep->user_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
memset(base_ep->efa_recv_wr_vec, 0x0, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);

ofi_memalign((void **)&base_ep->user_recv_wr_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
if (!base_ep->user_recv_wr_vec) {
EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for base_ep->user_recv_wr_vec!\n");
return -FI_ENOMEM;
}
memset(base_ep->user_recv_wr_vec, 0x0, sizeof(struct efa_recv_wr) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);

base_ep->recv_wr_index = 0;
base_ep->efa_qp_enabled = false;
base_ep->qp = NULL;
Expand Down
7 changes: 4 additions & 3 deletions prov/efa/src/efa_cq.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -863,7 +863,7 @@ int efa_cq_close(fid_t fid)
if (cq->err_buf)
free(cq->err_buf);

free(cq);
ofi_freealign(cq);

return 0;
}
Expand Down Expand Up @@ -936,11 +936,12 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr,
struct fi_cq_attr tmp_attr;
int err, retv;

cq = calloc(1, sizeof(*cq));
ofi_memalign((void **)&cq, EFA_MEM_ALIGNMENT, sizeof(*cq));
if (!cq) {
EFA_WARN(FI_LOG_CQ, "Failed to allocate memory for CQ\n");
return -FI_ENOMEM;
}
memset(cq, 0x0, sizeof(*cq));

cq->poll_ibv_cq = efa_cq_poll_ibv_cq;

Expand Down Expand Up @@ -1068,6 +1069,6 @@ int efa_cq_open(struct fid_domain *domain_fid, struct fi_cq_attr *attr,
EFA_WARN(FI_LOG_CQ, "Unable to close util cq: %s\n",
fi_strerror(-retv));
err_free_cq:
free(cq);
ofi_freealign(cq);
return err;
}
7 changes: 4 additions & 3 deletions prov/efa/src/efa_domain.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,9 +191,10 @@ int efa_domain_open(struct fid_fabric *fabric_fid, struct fi_info *info,
int ret = 0, err;
bool use_lock;

efa_domain = calloc(1, sizeof(struct efa_domain));
ofi_memalign((void **)&efa_domain, EFA_MEM_ALIGNMENT, sizeof(struct efa_domain));
if (!efa_domain)
return -FI_ENOMEM;
memset(efa_domain, 0x0, sizeof(struct efa_domain));

dlist_init(&efa_domain->list_entry);
efa_domain->fabric = container_of(fabric_fid, struct efa_fabric,
Expand Down Expand Up @@ -368,7 +369,7 @@ static int efa_domain_close(fid_t fid)
if (ret)
EFA_WARN(FI_LOG_DOMAIN, "ibv_destroy_ah failed during cleanup! err=%d\n", ret);
HASH_DEL(efa_domain->ah_map, ah_entry);
free(ah_entry);
ofi_freealign(ah_entry);
}
}
ofi_genlock_unlock(&efa_domain->util_domain.lock);
Expand Down Expand Up @@ -398,7 +399,7 @@ static int efa_domain_close(fid_t fid)

ofi_genlock_destroy(&efa_domain->srx_lock);
free(efa_domain->qp_table);
free(efa_domain);
ofi_freealign(efa_domain);
return 0;
}

Expand Down
7 changes: 4 additions & 3 deletions prov/efa/src/rdm/efa_rdm_cq.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ int efa_rdm_cq_close(struct fid *fid)
ret = ofi_cq_cleanup(&cq->efa_cq.util_cq);
if (ret)
return ret;
free(cq);
ofi_freealign(cq);
return retv;
}

Expand Down Expand Up @@ -929,9 +929,10 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
if (attr->wait_obj != FI_WAIT_NONE)
return -FI_ENOSYS;

cq = calloc(1, sizeof(*cq));
ofi_memalign((void **)&cq, EFA_MEM_ALIGNMENT, sizeof(*cq));
if (!cq)
return -FI_ENOMEM;
memset(cq, 0x0, sizeof(*cq));

efa_domain = container_of(domain, struct efa_domain,
util_domain.domain_fid);
Expand Down Expand Up @@ -987,6 +988,6 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
EFA_WARN(FI_LOG_CQ, "Unable to close util cq: %s\n",
fi_strerror(-retv));
free:
free(cq);
ofi_freealign(cq);
return ret;
}
13 changes: 8 additions & 5 deletions prov/efa/src/rdm/efa_rdm_ep_fiops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -466,10 +466,12 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
int ret, retv, i;
enum fi_hmem_iface iface;

efa_rdm_ep = calloc(1, sizeof(*efa_rdm_ep));
ofi_memalign((void **)&efa_rdm_ep, EFA_MEM_ALIGNMENT, sizeof(*efa_rdm_ep));
if (!efa_rdm_ep)
return -FI_ENOMEM;

memset(efa_rdm_ep, 0x0, sizeof(*efa_rdm_ep));

efa_domain = container_of(domain, struct efa_domain,
util_domain.domain_fid);

Expand Down Expand Up @@ -582,12 +584,13 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
efa_rdm_ep->write_in_order_aligned_128_bytes = false;
efa_rdm_ep->homogeneous_peers = false;

efa_rdm_ep->pke_vec = calloc(sizeof(struct efa_rdm_pke *), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
ofi_memalign((void **)&efa_rdm_ep->pke_vec, EFA_MEM_ALIGNMENT, sizeof(struct efa_rdm_pke *) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
if (!efa_rdm_ep->pke_vec) {
EFA_WARN(FI_LOG_EP_CTRL, "cannot alloc memory for efa_rdm_ep->pke_vec!\n");
ret = -FI_ENOMEM;
goto err_close_shm_ep;
}
memset(efa_rdm_ep->pke_vec, 0x0, sizeof(struct efa_rdm_pke *) * EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);

*ep = &efa_rdm_ep->base_ep.util_ep.ep_fid;
(*ep)->msg = &efa_rdm_msg_ops;
Expand All @@ -610,7 +613,7 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
efa_base_ep_destruct(&efa_rdm_ep->base_ep);
err_free_ep:
if (efa_rdm_ep)
free(efa_rdm_ep);
ofi_freealign(efa_rdm_ep);
return ret;
}

Expand Down Expand Up @@ -1040,11 +1043,11 @@ static int efa_rdm_ep_close(struct fid *fid)
efa_rdm_ep_destroy_buffer_pools(efa_rdm_ep);

if (efa_rdm_ep->pke_vec)
free(efa_rdm_ep->pke_vec);
ofi_freealign(efa_rdm_ep->pke_vec);

ofi_genlock_unlock(&domain->srx_lock);

free(efa_rdm_ep);
ofi_freealign(efa_rdm_ep);
return retv;
}

Expand Down