@@ -523,7 +523,9 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
523523 setTruncStoreAction (VT, MVT::f16 , Expand);
524524 }
525525 setOperationAction (ISD::LOAD, MVT::f16 , Custom);
526+ setOperationAction (ISD::ATOMIC_LOAD, MVT::f16 , Custom);
526527 setOperationAction (ISD::STORE, MVT::f16 , Custom);
528+ setOperationAction (ISD::ATOMIC_STORE, MVT::f16 , Custom);
527529 setOperationAction (ISD::FP_ROUND, MVT::f16 , Custom);
528530 setOperationAction (ISD::FP_EXTEND, MVT::f32 , Custom);
529531 setOperationAction (ISD::FP_EXTEND, MVT::f64 , Custom);
@@ -4596,6 +4598,22 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
45964598 return DAG.getNode (ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand (0 ));
45974599}
45984600
4601+ SDValue SystemZTargetLowering::lowerATOMIC_LOAD (SDValue Op,
4602+ SelectionDAG &DAG) const {
4603+ MVT RegVT = Op.getSimpleValueType ();
4604+ if (RegVT.getSizeInBits () == 128 )
4605+ return lowerATOMIC_LDST_I128 (Op, DAG);
4606+ return lowerLoadF16 (Op, DAG);
4607+ }
4608+
4609+ SDValue SystemZTargetLowering::lowerATOMIC_STORE (SDValue Op,
4610+ SelectionDAG &DAG) const {
4611+ auto *Node = cast<AtomicSDNode>(Op.
getNode ());
4612+ if (Node->getMemoryVT ().getSizeInBits () == 128 )
4613+ return lowerATOMIC_LDST_I128 (Op, DAG);
4614+ return lowerStoreF16 (Op, DAG);
4615+ }
4616+
45994617SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128 (SDValue Op,
46004618 SelectionDAG &DAG) const {
46014619 auto *Node = cast<AtomicSDNode>(Op.
getNode ());
@@ -6217,15 +6235,25 @@ SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
62176235 MVT RegVT = Op.getSimpleValueType ();
62186236 if (RegVT != MVT::f16 )
62196237 return SDValue ();
6220- LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode ());
6221- SDLoc DL (Ld);
6222- assert (EVT (RegVT) == Ld->getMemoryVT () && " Expected non-extending f16 load"
6238+
6239+ SDLoc DL (Op);
6240+ SDValue NewLd;
6241+ if (
auto *AtomicLd = dyn_cast<AtomicSDNode>(Op.
getNode ())) {
6242+ assert (EVT (RegVT) == AtomicLd->getMemoryVT () && " Unhandled f16 load"
6243+ NewLd = DAG.getAtomic (ISD::ATOMIC_LOAD, DL, MVT::i16 , MVT::i32 ,
6244+ AtomicLd->getChain (), AtomicLd->getBasePtr (),
6245+ AtomicLd->getMemOperand ());
6246+ cast<AtomicSDNode>(NewLd)->setExtensionType (ISD::EXTLOAD);
6247+ } else {
6248+ LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode ());
6249+ assert (EVT (RegVT) == Ld->getMemoryVT () && " Unhandled f16 load"
6250+ NewLd = DAG.getExtLoad (ISD::EXTLOAD, DL, MVT::i32 , Ld->getChain (),
6251+ Ld->getBasePtr (), Ld->getPointerInfo (),
6252+ MVT::i16 , Ld->getOriginalAlign (),
6253+ Ld->getMemOperand ()->getFlags ());
6254+ }
62236255 // Load as integer, shift and insert into upper 2 bytes of the FP register.
62246256 // TODO: Use VLEH if available.
6225- SDValue NewLd = DAG.getExtLoad (ISD::EXTLOAD, DL, MVT::i32 , Ld->getChain (),
6226- Ld->getBasePtr (), Ld->getPointerInfo (),
6227- MVT::i16 , Ld->getOriginalAlign (),
6228- Ld->getMemOperand ()->getFlags ());
62296257 SDValue Shft = DAG.getNode (ISD::SHL, DL, MVT::i32 , NewLd,
62306258 DAG.getConstant (16 , DL, MVT::i32 ));
62316259 SDValue BCast = DAG.getNode (ISD::BITCAST, DL, MVT::f32 , Shft);
@@ -6236,20 +6264,25 @@ SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
62366264
62376265SDValue SystemZTargetLowering::lowerStoreF16 (SDValue Op,
62386266 SelectionDAG &DAG) const {
6239- StoreSDNode *St = cast<StoreSDNode>(Op.getNode ());
6240- SDLoc DL (St);
6241- SDValue StoredVal = St->getValue ();
6267+ SDValue StoredVal = Op->getOperand (1 );
62426268 MVT StoreVT = StoredVal.getSimpleValueType ();
62436269 if (StoreVT != MVT::f16 )
62446270 return SDValue ();
6245- // Move into a GPR, shift and store the 2 bytes.
6246- // TODO: Use VSTEH if available.
6271+
6272+ // Move into a GPR, shift and store the 2 bytes. TODO: Use VSTEH if available.
6273+ SDLoc DL (Op);
62476274 SDNode *U32 = DAG.getMachineNode (TargetOpcode::IMPLICIT_DEF, DL, MVT::f32 );
62486275 SDValue In32 = DAG.getTargetInsertSubreg (SystemZ::subreg_h16, DL,
62496276 MVT::f32 , SDValue (U32, 0 ), StoredVal);
62506277 SDValue BCast = DAG.getNode (ISD::BITCAST, DL, MVT::i32 , In32);
62516278 SDValue Shft = DAG.getNode (ISD::SRL, DL, MVT::i32 , BCast,
62526279 DAG.getConstant (16 , DL, MVT::i32 ));
6280+
6281+ if (
auto *AtomicSt = dyn_cast<AtomicSDNode>(Op.
getNode ()))
6282+ return DAG.getAtomic (ISD::ATOMIC_STORE, DL, MVT::i16 , AtomicSt->getChain (),
6283+ Shft, AtomicSt->getBasePtr (), AtomicSt->getMemOperand ());
6284+
6285+ StoreSDNode *St = cast<StoreSDNode>(Op.getNode ());
62536286 return DAG.getTruncStore (St->getChain (), DL, Shft, St->getBasePtr (),
62546287 MVT::i16 , St->getMemOperand ());
62556288}
@@ -6373,8 +6406,9 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
63736406 case ISD::ATOMIC_SWAP:
63746407 return lowerATOMIC_LOAD_OP (Op, DAG, SystemZISD::ATOMIC_SWAPW);
63756408 case ISD::ATOMIC_STORE:
6409+ return lowerATOMIC_STORE (Op, DAG);
63766410 case ISD::ATOMIC_LOAD:
6377- return lowerATOMIC_LDST_I128 (Op, DAG);
6411+ return lowerATOMIC_LOAD (Op, DAG);
63786412 case ISD::ATOMIC_LOAD_ADD:
63796413 return lowerATOMIC_LOAD_OP (Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
63806414 case ISD::ATOMIC_LOAD_SUB:
0 commit comments