Skip to content

Commit f92daa6

Browse files
jladd-mlnxjladd-mlnx
authored
Merge pull request #5392 from hoopoepg/topic/bitwise-atomics-c
OSHMEM/ATOMICS: added C implementation of and/or/xor ops
2 parents 0d4cd42 + 0212410 commit f92daa6

File tree

19 files changed

+732
-7
lines changed

19 files changed

+732
-7
lines changed

‎config/ompi_check_ucx.m4‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,14 @@ AC_DEFUN([OMPI_CHECK_UCX],[
112112
ucp_request_check_status, ucp_put_nb, ucp_get_nb],
113113
[], [],
114114
[#include <ucp/api/ucp.h>])
115+
AC_CHECK_DECLS([UCP_ATOMIC_POST_OP_AND,
116+
UCP_ATOMIC_POST_OP_OR,
117+
UCP_ATOMIC_POST_OP_XOR,
118+
UCP_ATOMIC_FETCH_OP_FAND,
119+
UCP_ATOMIC_FETCH_OP_FOR,
120+
UCP_ATOMIC_FETCH_OP_FXOR],
121+
[], [],
122+
[#include <ucp/api/ucp.h>])
115123
CPPFLAGS=$old_CPPFLAGS
116124

117125
OPAL_SUMMARY_ADD([[Transports]],[[Open UCX]],[$1],[$ompi_check_ucx_happy])])])

‎oshmem/include/pshmem.h‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,42 @@ OSHMEM_DECLSPEC long long pshmem_longlong_fadd(long long *target, long long valu
378378
long long*: pshmem_longlong_fadd)(dst, val, pe)
379379
#endif
380380

381+
/* Atomic Fetch&And */
382+
OSHMEM_DECLSPEC int pshmem_int_atomic_fand(int *target, int value, int pe);
383+
OSHMEM_DECLSPEC long pshmem_long_atomic_fand(long *target, long value, int pe);
384+
OSHMEM_DECLSPEC long long pshmem_longlong_atomic_fand(long long *target, long long value, int pe);
385+
#if OSHMEMP_HAVE_C11
386+
#define pshmem_atomic_fand(dst, val, pe) \
387+
_Generic(&*(dst), \
388+
int*: pshmem_int_atomic_fand, \
389+
long*: pshmem_long_atomic_fand, \
390+
long long*: pshmem_longlong_atomic_fand)(dst, val, pe)
391+
#endif
392+
393+
/* Atomic Fetch&Or */
394+
OSHMEM_DECLSPEC int pshmem_int_atomic_for(int *target, int value, int pe);
395+
OSHMEM_DECLSPEC long pshmem_long_atomic_for(long *target, long value, int pe);
396+
OSHMEM_DECLSPEC long long pshmem_longlong_atomic_for(long long *target, long long value, int pe);
397+
#if OSHMEMP_HAVE_C11
398+
#define pshmem_atomic_for(dst, val, pe) \
399+
_Generic(&*(dst), \
400+
int*: pshmem_int_atomic_for, \
401+
long*: pshmem_long_atomic_for, \
402+
long long*: pshmem_longlong_atomic_for)(dst, val, pe)
403+
#endif
404+
405+
/* Atomic Fetch&Xor */
406+
OSHMEM_DECLSPEC int pshmem_int_atomic_fxor(int *target, int value, int pe);
407+
OSHMEM_DECLSPEC long pshmem_long_atomic_fxor(long *target, long value, int pe);
408+
OSHMEM_DECLSPEC long long pshmem_longlong_atomic_fxor(long long *target, long long value, int pe);
409+
#if OSHMEMP_HAVE_C11
410+
#define pshmem_atomic_fxor(dst, val, pe) \
411+
_Generic(&*(dst), \
412+
int*: pshmem_int_atomic_fxor, \
413+
long*: pshmem_long_atomic_fxor, \
414+
long long*: pshmem_longlong_atomic_fxor)(dst, val, pe)
415+
#endif
416+
381417
/* Atomic Fetch */
382418
OSHMEM_DECLSPEC int pshmem_int_fetch(const int *target, int pe);
383419
OSHMEM_DECLSPEC long pshmem_long_fetch(const long *target, int pe);
@@ -406,7 +442,7 @@ OSHMEM_DECLSPEC long long pshmem_longlong_finc(long long *target, int pe);
406442
long long*: pshmem_longlong_finc)(dst, val, pe)
407443
#endif
408444

409-
/* Atomic Add*/
445+
/* Atomic Add */
410446
OSHMEM_DECLSPEC void pshmem_int_add(int *target, int value, int pe);
411447
OSHMEM_DECLSPEC void pshmem_long_add(long *target, long value, int pe);
412448
OSHMEM_DECLSPEC void pshmem_longlong_add(long long *target, long long value, int pe);
@@ -418,6 +454,42 @@ OSHMEM_DECLSPEC void pshmem_longlong_add(long long *target, long long value, int
418454
long long*: pshmem_longlong_add)(dst, val, pe)
419455
#endif
420456

457+
/* Atomic And */
458+
OSHMEM_DECLSPEC void pshmem_int_atomic_and(int *target, int value, int pe);
459+
OSHMEM_DECLSPEC void pshmem_long_atomic_and(long *target, long value, int pe);
460+
OSHMEM_DECLSPEC void pshmem_longlong_atomic_and(long long *target, long long value, int pe);
461+
#if OSHMEMP_HAVE_C11
462+
#define pshmem_atomic_and(dst, val, pe) \
463+
_Generic(&*(dst), \
464+
int*: pshmem_int_atomic_and, \
465+
long*: pshmem_long_atomic_and, \
466+
long long*: pshmem_longlong_atomic_and)(dst, val, pe)
467+
#endif
468+
469+
/* Atomic Or */
470+
OSHMEM_DECLSPEC void pshmem_int_atomic_or(int *target, int value, int pe);
471+
OSHMEM_DECLSPEC void pshmem_long_atomic_or(long *target, long value, int pe);
472+
OSHMEM_DECLSPEC void pshmem_longlong_atomic_or(long long *target, long long value, int pe);
473+
#if OSHMEMP_HAVE_C11
474+
#define pshmem_atomic_or(dst, val, pe) \
475+
_Generic(&*(dst), \
476+
int*: pshmem_int_atomic_or, \
477+
long*: pshmem_long_atomic_or, \
478+
long long*: pshmem_longlong_atomic_or)(dst, val, pe)
479+
#endif
480+
481+
/* Atomic Xor */
482+
OSHMEM_DECLSPEC void pshmem_int_atomic_xor(int *target, int value, int pe);
483+
OSHMEM_DECLSPEC void pshmem_long_atomic_xor(long *target, long value, int pe);
484+
OSHMEM_DECLSPEC void pshmem_longlong_atomic_xor(long long *target, long long value, int pe);
485+
#if OSHMEMP_HAVE_C11
486+
#define pshmem_atomic_xor(dst, val, pe) \
487+
_Generic(&*(dst), \
488+
int*: pshmem_int_atomic_xor, \
489+
long*: pshmem_long_atomic_xor, \
490+
long long*: pshmem_longlong_atomic_xor)(dst, val, pe)
491+
#endif
492+
421493
/* Atomic Inc */
422494
OSHMEM_DECLSPEC void pshmem_int_inc(int *target, int pe);
423495
OSHMEM_DECLSPEC void pshmem_long_inc(long *target, int pe);

‎oshmem/include/pshmemx.h‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,18 @@ OSHMEM_DECLSPEC int64_t pshmemx_int64_cswap(int64_t *target, int64_t cond, int64
8989
OSHMEM_DECLSPEC int32_t pshmemx_int32_fadd(int32_t *target, int32_t value, int pe);
9090
OSHMEM_DECLSPEC int64_t pshmemx_int64_fadd(int64_t *target, int64_t value, int pe);
9191

92+
/* Atomic Fetch&And */
93+
OSHMEM_DECLSPEC int32_t pshmemx_int32_atomic_fand(int32_t *target, int32_t value, int pe);
94+
OSHMEM_DECLSPEC int64_t pshmemx_int64_atomic_fand(int64_t *target, int64_t value, int pe);
95+
96+
/* Atomic Fetch&Or */
97+
OSHMEM_DECLSPEC int32_t pshmemx_int32_atomic_for(int32_t *target, int32_t value, int pe);
98+
OSHMEM_DECLSPEC int64_t pshmemx_int64_atomic_for(int64_t *target, int64_t value, int pe);
99+
100+
/* Atomic Fetch&Xor */
101+
OSHMEM_DECLSPEC int32_t pshmemx_int32_atomic_fxor(int32_t *target, int32_t value, int pe);
102+
OSHMEM_DECLSPEC int64_t pshmemx_int64_atomic_fxor(int64_t *target, int64_t value, int pe);
103+
92104
/* Atomic Fetch */
93105
OSHMEM_DECLSPEC int32_t pshmemx_int32_fetch(const int32_t *target, int pe);
94106
OSHMEM_DECLSPEC int64_t pshmemx_int64_fetch(const int64_t *target, int pe);
@@ -97,10 +109,22 @@ OSHMEM_DECLSPEC int64_t pshmemx_int64_fetch(const int64_t *target, int pe);
97109
OSHMEM_DECLSPEC int32_t pshmemx_int32_finc(int32_t *target, int pe);
98110
OSHMEM_DECLSPEC int64_t pshmemx_int64_finc(int64_t *target, int pe);
99111

100-
/* Atomic Add*/
112+
/* Atomic Add */
101113
OSHMEM_DECLSPEC void pshmemx_int32_add(int32_t *target, int32_t value, int pe);
102114
OSHMEM_DECLSPEC void pshmemx_int64_add(int64_t *target, int64_t value, int pe);
103115

116+
/* Atomic And */
117+
OSHMEM_DECLSPEC void pshmemx_int32_atomic_and(int32_t *target, int32_t value, int pe);
118+
OSHMEM_DECLSPEC void pshmemx_int64_atomic_and(int64_t *target, int64_t value, int pe);
119+
120+
/* Atomic Or */
121+
OSHMEM_DECLSPEC void pshmemx_int32_atomic_or(int32_t *target, int32_t value, int pe);
122+
OSHMEM_DECLSPEC void pshmemx_int64_atomic_or(int64_t *target, int64_t value, int pe);
123+
124+
/* Atomic Xor */
125+
OSHMEM_DECLSPEC void pshmemx_int32_atomic_xor(int32_t *target, int32_t value, int pe);
126+
OSHMEM_DECLSPEC void pshmemx_int64_atomic_xor(int64_t *target, int64_t value, int pe);
127+
104128
/* Atomic Inc */
105129
OSHMEM_DECLSPEC void pshmemx_int32_inc(int32_t *target, int pe);
106130
OSHMEM_DECLSPEC void pshmemx_int64_inc(int64_t *target, int pe);

‎oshmem/include/shmem.h.in‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,42 @@ OSHMEM_DECLSPEC long long shmem_longlong_fadd(long long *target, long long value
463463
long long*: shmem_longlong_fadd)(dst, val, pe)
464464
#endif
465465

466+
/* Atomic Fetch&And */
467+
OSHMEM_DECLSPEC int shmem_int_atomic_fand(int *target, int value, int pe);
468+
OSHMEM_DECLSPEC long shmem_long_atomic_fand(long *target, long value, int pe);
469+
OSHMEM_DECLSPEC long long shmem_longlong_atomic_fand(long long *target, long long value, int pe);
470+
#if OSHMEM_HAVE_C11
471+
#define shmem_atomic_fand(dst, val, pe) \
472+
_Generic(&*(dst), \
473+
int*: shmem_int_atomic_fand, \
474+
long*: shmem_long_atomic_fand, \
475+
long long*: shmem_longlong_atomic_fand)(dst, val, pe)
476+
#endif
477+
478+
/* Atomic Fetch&Or */
479+
OSHMEM_DECLSPEC int shmem_int_atomic_for(int *target, int value, int pe);
480+
OSHMEM_DECLSPEC long shmem_long_atomic_for(long *target, long value, int pe);
481+
OSHMEM_DECLSPEC long long shmem_longlong_atomic_for(long long *target, long long value, int pe);
482+
#if OSHMEM_HAVE_C11
483+
#define shmem_atomic_for(dst, val, pe) \
484+
_Generic(&*(dst), \
485+
int*: shmem_int_atomic_for, \
486+
long*: shmem_long_atomic_for, \
487+
long long*: shmem_longlong_atomic_for)(dst, val, pe)
488+
#endif
489+
490+
/* Atomic Fetch&Xor */
491+
OSHMEM_DECLSPEC int shmem_int_atomic_fxor(int *target, int value, int pe);
492+
OSHMEM_DECLSPEC long shmem_long_atomic_fxor(long *target, long value, int pe);
493+
OSHMEM_DECLSPEC long long shmem_longlong_atomic_fxor(long long *target, long long value, int pe);
494+
#if OSHMEM_HAVE_C11
495+
#define shmem_atomic_fxor(dst, val, pe) \
496+
_Generic(&*(dst), \
497+
int*: shmem_int_atomic_fxor, \
498+
long*: shmem_long_atomic_fxor, \
499+
long long*: shmem_longlong_atomic_fxor)(dst, val, pe)
500+
#endif
501+
466502
/* Atomic Fetch */
467503
OSHMEM_DECLSPEC int shmem_int_fetch(const int *target, int pe);
468504
OSHMEM_DECLSPEC long shmem_long_fetch(const long *target, int pe);
@@ -491,7 +527,7 @@ OSHMEM_DECLSPEC long long shmem_longlong_finc(long long *target, int pe);
491527
long long*: shmem_longlong_finc)(dst, pe)
492528
#endif
493529

494-
/* Atomic Add*/
530+
/* Atomic Add */
495531
OSHMEM_DECLSPEC void shmem_int_add(int *target, int value, int pe);
496532
OSHMEM_DECLSPEC void shmem_long_add(long *target, long value, int pe);
497533
OSHMEM_DECLSPEC void shmem_longlong_add(long long *target, long long value, int pe);
@@ -503,6 +539,42 @@ OSHMEM_DECLSPEC void shmem_longlong_add(long long *target, long long value, int
503539
long long*: shmem_longlong_add)(dst, val, pe)
504540
#endif
505541

542+
/* Atomic And */
543+
OSHMEM_DECLSPEC void shmem_int_atomic_and(int *target, int value, int pe);
544+
OSHMEM_DECLSPEC void shmem_long_atomic_and(long *target, long value, int pe);
545+
OSHMEM_DECLSPEC void shmem_longlong_atomic_and(long long *target, long long value, int pe);
546+
#if OSHMEM_HAVE_C11
547+
#define shmem_atomic_and(dst, val, pe) \
548+
_Generic(&*(dst), \
549+
int*: shmem_int_atomic_and, \
550+
long*: shmem_long_atomic_and, \
551+
long long*: shmem_longlong_atomic_and)(dst, val, pe)
552+
#endif
553+
554+
/* Atomic Or */
555+
OSHMEM_DECLSPEC void shmem_int_atomic_or(int *target, int value, int pe);
556+
OSHMEM_DECLSPEC void shmem_long_atomic_or(long *target, long value, int pe);
557+
OSHMEM_DECLSPEC void shmem_longlong_atomic_or(long long *target, long long value, int pe);
558+
#if OSHMEM_HAVE_C11
559+
#define shmem_atomic_or(dst, val, pe) \
560+
_Generic(&*(dst), \
561+
int*: shmem_int_atomic_or, \
562+
long*: shmem_long_atomic_or, \
563+
long long*: shmem_longlong_atomic_or)(dst, val, pe)
564+
#endif
565+
566+
/* Atomic Xor */
567+
OSHMEM_DECLSPEC void shmem_int_atomic_xor(int *target, int value, int pe);
568+
OSHMEM_DECLSPEC void shmem_long_atomic_xor(long *target, long value, int pe);
569+
OSHMEM_DECLSPEC void shmem_longlong_atomic_xor(long long *target, long long value, int pe);
570+
#if OSHMEM_HAVE_C11
571+
#define shmem_atomic_xor(dst, val, pe) \
572+
_Generic(&*(dst), \
573+
int*: shmem_int_atomic_xor, \
574+
long*: shmem_long_atomic_xor, \
575+
long long*: shmem_longlong_atomic_xor)(dst, val, pe)
576+
#endif
577+
506578
/* Atomic Inc */
507579
OSHMEM_DECLSPEC void shmem_int_inc(int *target, int pe);
508580
OSHMEM_DECLSPEC void shmem_long_inc(long *target, int pe);

‎oshmem/include/shmemx.h‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,18 @@ OSHMEM_DECLSPEC int64_t shmemx_int64_cswap(int64_t *target, int64_t cond, int64_
7676
OSHMEM_DECLSPEC int32_t shmemx_int32_fadd(int32_t *target, int32_t value, int pe);
7777
OSHMEM_DECLSPEC int64_t shmemx_int64_fadd(int64_t *target, int64_t value, int pe);
7878

79+
/* Atomic Fetch&And */
80+
OSHMEM_DECLSPEC int32_t shmemx_int32_atomic_fand(int32_t *target, int32_t value, int pe);
81+
OSHMEM_DECLSPEC int64_t shmemx_int64_atomic_fand(int64_t *target, int64_t value, int pe);
82+
83+
/* Atomic Fetch&Or */
84+
OSHMEM_DECLSPEC int32_t shmemx_int32_atomic_for(int32_t *target, int32_t value, int pe);
85+
OSHMEM_DECLSPEC int64_t shmemx_int64_atomic_for(int64_t *target, int64_t value, int pe);
86+
87+
/* Atomic Fetch&Xor */
88+
OSHMEM_DECLSPEC int32_t shmemx_int32_atomic_fxor(int32_t *target, int32_t value, int pe);
89+
OSHMEM_DECLSPEC int64_t shmemx_int64_atomic_fxor(int64_t *target, int64_t value, int pe);
90+
7991
/* Atomic Fetch */
8092
OSHMEM_DECLSPEC int32_t shmemx_int32_fetch(const int32_t *target, int pe);
8193
OSHMEM_DECLSPEC int64_t shmemx_int64_fetch(const int64_t *target, int pe);
@@ -84,10 +96,22 @@ OSHMEM_DECLSPEC int64_t shmemx_int64_fetch(const int64_t *target, int pe);
8496
OSHMEM_DECLSPEC int32_t shmemx_int32_finc(int32_t *target, int pe);
8597
OSHMEM_DECLSPEC int64_t shmemx_int64_finc(int64_t *target, int pe);
8698

87-
/* Atomic Add*/
99+
/* Atomic Add */
88100
OSHMEM_DECLSPEC void shmemx_int32_add(int32_t *target, int32_t value, int pe);
89101
OSHMEM_DECLSPEC void shmemx_int64_add(int64_t *target, int64_t value, int pe);
90102

103+
/* Atomic And */
104+
OSHMEM_DECLSPEC void shmemx_int32_atomic_and(int32_t *target, int32_t value, int pe);
105+
OSHMEM_DECLSPEC void shmemx_int64_atomic_and(int64_t *target, int64_t value, int pe);
106+
107+
/* Atomic Or */
108+
OSHMEM_DECLSPEC void shmemx_int32_atomic_or(int32_t *target, int32_t value, int pe);
109+
OSHMEM_DECLSPEC void shmemx_int64_atomic_or(int64_t *target, int64_t value, int pe);
110+
111+
/* Atomic Xor */
112+
OSHMEM_DECLSPEC void shmemx_int32_atomic_xor(int32_t *target, int32_t value, int pe);
113+
OSHMEM_DECLSPEC void shmemx_int64_atomic_xor(int64_t *target, int64_t value, int pe);
114+
91115
/* Atomic Inc */
92116
OSHMEM_DECLSPEC void shmemx_int32_inc(int32_t *target, int pe);
93117
OSHMEM_DECLSPEC void shmemx_int64_inc(int64_t *target, int pe);

‎oshmem/mca/atomic/atomic.h‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,49 @@ BEGIN_C_DECLS
3535

3636
#define OSHMEM_ATOMIC_PTR_2_INT(ptr, size) ((size) == 8 ? *(uint64_t*)(ptr) : *(uint32_t*)(ptr))
3737

38+
#define OSHMEM_TYPE_OP(type_name, type, prefix, op) \
39+
void prefix##type_name##_atomic_##op(type *target, type value, int pe) \
40+
{ \
41+
int rc = OSHMEM_SUCCESS; \
42+
size_t size = 0; \
43+
\
44+
RUNTIME_CHECK_INIT(); \
45+
RUNTIME_CHECK_PE(pe); \
46+
RUNTIME_CHECK_ADDR(target); \
47+
\
48+
size = sizeof(value); \
49+
rc = MCA_ATOMIC_CALL(op( \
50+
(void*)target, \
51+
value, \
52+
size, \
53+
pe)); \
54+
RUNTIME_CHECK_RC(rc); \
55+
\
56+
return; \
57+
}
58+
59+
#define OSHMEM_TYPE_FOP(type_name, type, prefix, op) \
60+
type prefix##type_name##_atomic_##op(type *target, type value, int pe) \
61+
{ \
62+
int rc = OSHMEM_SUCCESS; \
63+
size_t size = 0; \
64+
type out_value; \
65+
\
66+
RUNTIME_CHECK_INIT(); \
67+
RUNTIME_CHECK_PE(pe); \
68+
RUNTIME_CHECK_ADDR(target); \
69+
\
70+
size = sizeof(out_value); \
71+
rc = MCA_ATOMIC_CALL(op( \
72+
(void*)target, \
73+
(void*)&out_value, \
74+
value, \
75+
size, \
76+
pe)); \
77+
RUNTIME_CHECK_RC(rc); \
78+
\
79+
return out_value; \
80+
}
3881
/* ******************************************************************** */
3982

4083
struct oshmem_op_t;
@@ -92,11 +135,38 @@ struct mca_atomic_base_module_1_0_0_t {
92135
uint64_t value,
93136
size_t size,
94137
int pe);
138+
int (*atomic_and)(void *target,
139+
uint64_t value,
140+
size_t size,
141+
int pe);
142+
int (*atomic_or)(void *target,
143+
uint64_t value,
144+
size_t size,
145+
int pe);
146+
int (*atomic_xor)(void *target,
147+
uint64_t value,
148+
size_t size,
149+
int pe);
95150
int (*atomic_fadd)(void *target,
96151
void *prev,
97152
uint64_t value,
98153
size_t size,
99154
int pe);
155+
int (*atomic_fand)(void *target,
156+
void *prev,
157+
uint64_t value,
158+
size_t size,
159+
int pe);
160+
int (*atomic_for)(void *target,
161+
void *prev,
162+
uint64_t value,
163+
size_t size,
164+
int pe);
165+
int (*atomic_fxor)(void *target,
166+
void *prev,
167+
uint64_t value,
168+
size_t size,
169+
int pe);
100170
int (*atomic_swap)(void *target,
101171
void *prev,
102172
uint64_t value,

0 commit comments

Comments
 (0)