Skip to content

Commit

Permalink
No commit message
Browse files Browse the repository at this point in the history
  • Loading branch information
@aingerson
aingerson committed Feb 9, 2023
1 parent 48ca1a1 commit 3d4168d
Show file tree
Hide file tree
Showing 6 changed files with 207 additions and 257 deletions.
46 changes: 12 additions & 34 deletions prov/shm/src/smr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,7 @@ struct smr_rx_entry {
uint64_t ignore;
int multi_recv_ref;
uint64_t err;
enum fi_hmem_iface iface;
uint64_t device;
struct ofi_mr *mr[SMR_IOV_LIMIT];
};

struct smr_tx_entry {
Expand All @@ -132,8 +131,7 @@ struct smr_tx_entry {
int next;
void *map_ptr;
struct smr_ep_name *map_name;
enum fi_hmem_iface iface;
uint64_t device;
struct ofi_mr *mr[SMR_IOV_LIMIT];
int fd;
};

Expand All @@ -145,8 +143,7 @@ struct smr_sar_entry {
int next;
struct iovec iov[SMR_IOV_LIMIT];
size_t iov_count;
enum fi_hmem_iface iface;
uint64_t device;
struct ofi_mr *mr[SMR_IOV_LIMIT];
};

struct smr_match_attr {
Expand All @@ -166,24 +163,6 @@ static inline int smr_match_tag(uint64_t tag, uint64_t ignore, uint64_t match_ta
return ((tag | ignore) == (match_tag | ignore));
}

static inline enum fi_hmem_iface smr_get_mr_hmem_iface(struct util_domain *domain,
void **desc, uint64_t *device)
{
if (!(domain->mr_mode & FI_MR_HMEM) || !desc || !*desc) {
*device = 0;
return FI_HMEM_SYSTEM;
}

*device = ((struct ofi_mr *) *desc)->device;
return ((struct ofi_mr *) *desc)->iface;
}

static inline uint64_t smr_get_mr_flags(void **desc)
{
assert(desc && *desc);
return ((struct ofi_mr *) *desc)->flags;
}

struct smr_cmd_ctx {
struct dlist_entry entry;
struct smr_ep *ep;
Expand Down Expand Up @@ -336,27 +315,26 @@ int smr_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
int64_t smr_verify_peer(struct smr_ep *ep, fi_addr_t fi_addr);

void smr_format_pend_resp(struct smr_tx_entry *pend, struct smr_cmd *cmd,
void *context, enum fi_hmem_iface iface, uint64_t device,
void *context, struct ofi_mr **mr,
const struct iovec *iov, uint32_t iov_count,
uint64_t op_flags, int64_t id, struct smr_resp *resp);
void smr_generic_format(struct smr_cmd *cmd, int64_t peer_id, uint32_t op,
uint64_t tag, uint64_t data, uint64_t op_flags);
size_t smr_copy_to_sar(struct smr_freestack *sar_pool, struct smr_resp *resp,
struct smr_cmd *cmd, enum fi_hmem_iface, uint64_t device,
struct smr_cmd *cmd, struct ofi_mr **mr,
const struct iovec *iov, size_t count,
size_t *bytes_done, int *next);
size_t smr_copy_from_sar(struct smr_freestack *sar_pool, struct smr_resp *resp,
struct smr_cmd *cmd, enum fi_hmem_iface iface,
uint64_t device, const struct iovec *iov, size_t count,
struct smr_cmd *cmd, struct ofi_mr **mr,
const struct iovec *iov, size_t count,
size_t *bytes_done, int *next);

int smr_select_proto(bool use_ipc, bool cma_avail, enum fi_hmem_iface iface,
uint32_t op, uint64_t total_len, uint64_t op_flags);
int smr_select_proto(bool use_ipc, bool cma_avail, uint32_t op,
uint64_t total_len, uint64_t op_flags);
typedef ssize_t (*smr_proto_func)(struct smr_ep *ep, struct smr_region *peer_smr,
int64_t id, int64_t peer_id, uint32_t op, uint64_t tag,
uint64_t data, uint64_t op_flags, enum fi_hmem_iface iface,
uint64_t device, const struct iovec *iov, size_t iov_count,
size_t total_len, void *context);
uint64_t data, uint64_t op_flags, struct ofi_mr **desc,
const struct iovec *iov, size_t iov_count, size_t total_len,
void *context);
extern smr_proto_func smr_proto_ops[smr_src_max];

int smr_write_err_comp(struct util_cq *cq, void *context,
Expand Down
56 changes: 21 additions & 35 deletions prov/shm/src/smr_atomic.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
#include <sys/uio.h>

#include "ofi_iov.h"
#include "ofi_hmem.h"
#include "smr.h"


Expand All @@ -54,20 +53,17 @@ static void smr_generic_atomic_format(struct smr_cmd *cmd, uint8_t datatype,
}

static void smr_format_inline_atomic(struct smr_cmd *cmd,
enum fi_hmem_iface iface, uint64_t device,
const struct iovec *iov, size_t count)
{
cmd->msg.hdr.op_src = smr_src_inline;

cmd->msg.hdr.size = ofi_copy_from_hmem_iov(cmd->msg.data.msg,
SMR_MSG_DATA_LEN, iface, device,
iov, count, 0);
cmd->msg.hdr.size = ofi_copy_from_iov(cmd->msg.data.msg,
SMR_MSG_DATA_LEN, iov, count, 0);
}

static void smr_do_atomic_inline(struct smr_ep *ep, struct smr_region *peer_smr,
int64_t id, int64_t peer_id, uint32_t op,
uint64_t op_flags, enum fi_hmem_iface iface,
uint64_t device, uint8_t datatype, uint8_t atomic_op,
uint64_t op_flags, uint8_t datatype, uint8_t atomic_op,
const struct iovec *iov, size_t iov_count,
size_t total_len)
{
Expand All @@ -76,14 +72,13 @@ static void smr_do_atomic_inline(struct smr_ep *ep, struct smr_region *peer_smr,
cmd = ofi_cirque_next(smr_cmd_queue(peer_smr));
smr_generic_format(cmd, peer_id, op, 0, 0, op_flags);
smr_generic_atomic_format(cmd, datatype, atomic_op);
smr_format_inline_atomic(cmd, iface, device, iov, iov_count);
smr_format_inline_atomic(cmd, iov, iov_count);

ofi_cirque_commit(smr_cmd_queue(peer_smr));
peer_smr->cmd_cnt--;
}

static void smr_format_inject_atomic(struct smr_cmd *cmd,
enum fi_hmem_iface iface, uint64_t device,
const struct iovec *iov, size_t count,
const struct iovec *resultv, size_t result_count,
const struct iovec *compv, size_t comp_count,
Expand All @@ -96,24 +91,22 @@ static void smr_format_inject_atomic(struct smr_cmd *cmd,

switch (cmd->msg.hdr.op) {
case ofi_op_atomic:
cmd->msg.hdr.size = ofi_copy_from_hmem_iov(tx_buf->data,
SMR_INJECT_SIZE, iface, device,
iov, count, 0);
cmd->msg.hdr.size = ofi_copy_from_iov(tx_buf->data,
SMR_INJECT_SIZE, iov, count, 0);
break;
case ofi_op_atomic_fetch:
if (cmd->msg.hdr.atomic_op == FI_ATOMIC_READ)
cmd->msg.hdr.size = ofi_total_iov_len(resultv, result_count);
else
cmd->msg.hdr.size = ofi_copy_from_hmem_iov(tx_buf->data,
SMR_INJECT_SIZE, iface, device,
iov, count, 0);
cmd->msg.hdr.size = ofi_copy_from_iov(tx_buf->data,
SMR_INJECT_SIZE, iov, count, 0);
break;
case ofi_op_atomic_compare:
cmd->msg.hdr.size = ofi_copy_from_hmem_iov(tx_buf->buf,
SMR_COMP_INJECT_SIZE, iface, device,
iov, count, 0);
comp_size = ofi_copy_from_hmem_iov(tx_buf->comp, SMR_COMP_INJECT_SIZE,
iface, device, compv, comp_count, 0);
cmd->msg.hdr.size = ofi_copy_from_iov(tx_buf->buf,
SMR_COMP_INJECT_SIZE,
iov, count, 0);
comp_size = ofi_copy_from_iov(tx_buf->comp, SMR_COMP_INJECT_SIZE,
compv, comp_count, 0);
if (comp_size != cmd->msg.hdr.size)
FI_WARN(&smr_prov, FI_LOG_EP_CTRL,
"atomic and compare buffer size mismatch\n");
Expand All @@ -125,8 +118,7 @@ static void smr_format_inject_atomic(struct smr_cmd *cmd,

static ssize_t smr_do_atomic_inject(struct smr_ep *ep, struct smr_region *peer_smr,
int64_t id, int64_t peer_id, uint32_t op,
uint64_t op_flags, enum fi_hmem_iface iface,
uint64_t device, uint8_t datatype, uint8_t atomic_op,
uint64_t op_flags, uint8_t datatype, uint8_t atomic_op,
const struct iovec *iov, size_t iov_count,
const struct iovec *resultv, size_t result_count,
const struct iovec *compv, size_t comp_count,
Expand All @@ -142,7 +134,7 @@ static ssize_t smr_do_atomic_inject(struct smr_ep *ep, struct smr_region *peer_s

smr_generic_format(cmd, peer_id, op, 0, 0, op_flags);
smr_generic_atomic_format(cmd, datatype, atomic_op);
smr_format_inject_atomic(cmd, iface, device, iov, iov_count,
smr_format_inject_atomic(cmd, iov, iov_count,
resultv, result_count, compv, comp_count,
peer_smr, tx_buf);

Expand All @@ -153,7 +145,7 @@ static ssize_t smr_do_atomic_inject(struct smr_ep *ep, struct smr_region *peer_s
}
resp = ofi_cirque_next(smr_resp_queue(ep->region));
pend = ofi_freestack_pop(ep->pend_fs);
smr_format_pend_resp(pend, cmd, context, iface, device, resultv,
smr_format_pend_resp(pend, cmd, context, NULL, resultv,
result_count, op_flags, id, resp);
cmd->msg.hdr.data = smr_get_offset(ep->region, resp);
ofi_cirque_commit(smr_resp_queue(ep->region));
Expand Down Expand Up @@ -191,8 +183,6 @@ static ssize_t smr_generic_atomic(struct smr_ep *ep,
struct iovec iov[SMR_IOV_LIMIT];
struct iovec compare_iov[SMR_IOV_LIMIT];
struct iovec result_iov[SMR_IOV_LIMIT];
enum fi_hmem_iface iface;
uint64_t device;
uint16_t smr_flags = 0;
int64_t id, peer_id;
int proto;
Expand Down Expand Up @@ -245,17 +235,15 @@ static ssize_t smr_generic_atomic(struct smr_ep *ep,
break;
}

iface = smr_get_mr_hmem_iface(ep->util_ep.domain, desc, &device);

proto = smr_select_atomic_proto(op, total_len, op_flags);

if (proto == smr_src_inline) {
smr_do_atomic_inline(ep, peer_smr, id, peer_id, ofi_op_atomic,
op_flags, iface, device, datatype, atomic_op,
op_flags, datatype, atomic_op,
iov, count, total_len);
} else {
ret = smr_do_atomic_inject(ep, peer_smr, id, peer_id, op,
op_flags, iface, device, datatype, atomic_op,
op_flags, datatype, atomic_op,
iov, count, result_iov, result_count,
compare_iov, compare_count, total_len, context,
smr_flags);
Expand Down Expand Up @@ -379,13 +367,11 @@ static ssize_t smr_atomic_inject(struct fid_ep *ep_fid, const void *buf,

if (total_len <= SMR_MSG_DATA_LEN) {
smr_do_atomic_inline(ep, peer_smr, id, peer_id, ofi_op_atomic,
0, FI_HMEM_SYSTEM, 0, datatype, op,
&iov, 1, total_len);
0, datatype, op, &iov, 1, total_len);
} else if (total_len <= SMR_INJECT_SIZE) {
ret = smr_do_atomic_inject(ep, peer_smr, id, peer_id,
ofi_op_atomic, 0, FI_HMEM_SYSTEM, 0, datatype,
op, &iov, 1, NULL, 0, NULL, 0, total_len,
NULL, 0);
ofi_op_atomic, 0, datatype, op, &iov, 1, NULL,
0, NULL, 0, total_len, NULL, 0);
if (ret)
goto unlock_region;
}
Expand Down
Loading

0 comments on commit 3d4168d

Please sign in to comment.