1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers atomic intrinsics to non-atomic form for use in a known
10 // non-preemptible environment.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/Utils/LowerAtomic.h"
15 #include "llvm/IR/Function.h"
16 #include "llvm/IR/IRBuilder.h"
17 #include "llvm/InitializePasses.h"
18 #include "llvm/Pass.h"
19 using namespace llvm;
20
21 #define DEBUG_TYPE "loweratomic"
22
lowerAtomicCmpXchgInst(AtomicCmpXchgInst * CXI)23 bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
24 IRBuilder<> Builder(CXI);
25 Value *Ptr = CXI->getPointerOperand();
26 Value *Cmp = CXI->getCompareOperand();
27 Value *Val = CXI->getNewValOperand();
28
29 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
30 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
31 Value *Res = Builder.CreateSelect(Equal, Val, Orig);
32 Builder.CreateStore(Res, Ptr);
33
34 Res = Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0);
35 Res = Builder.CreateInsertValue(Res, Equal, 1);
36
37 CXI->replaceAllUsesWith(Res);
38 CXI->eraseFromParent();
39 return true;
40 }
41
buildAtomicRMWValue(AtomicRMWInst::BinOp Op,IRBuilderBase & Builder,Value * Loaded,Value * Val)42 Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
43 IRBuilderBase &Builder, Value *Loaded,
44 Value *Val) {
45 Value *NewVal;
46 switch (Op) {
47 case AtomicRMWInst::Xchg:
48 return Val;
49 case AtomicRMWInst::Add:
50 return Builder.CreateAdd(Loaded, Val, "new");
51 case AtomicRMWInst::Sub:
52 return Builder.CreateSub(Loaded, Val, "new");
53 case AtomicRMWInst::And:
54 return Builder.CreateAnd(Loaded, Val, "new");
55 case AtomicRMWInst::Nand:
56 return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new");
57 case AtomicRMWInst::Or:
58 return Builder.CreateOr(Loaded, Val, "new");
59 case AtomicRMWInst::Xor:
60 return Builder.CreateXor(Loaded, Val, "new");
61 case AtomicRMWInst::Max:
62 NewVal = Builder.CreateICmpSGT(Loaded, Val);
63 return Builder.CreateSelect(NewVal, Loaded, Val, "new");
64 case AtomicRMWInst::Min:
65 NewVal = Builder.CreateICmpSLE(Loaded, Val);
66 return Builder.CreateSelect(NewVal, Loaded, Val, "new");
67 case AtomicRMWInst::UMax:
68 NewVal = Builder.CreateICmpUGT(Loaded, Val);
69 return Builder.CreateSelect(NewVal, Loaded, Val, "new");
70 case AtomicRMWInst::UMin:
71 NewVal = Builder.CreateICmpULE(Loaded, Val);
72 return Builder.CreateSelect(NewVal, Loaded, Val, "new");
73 case AtomicRMWInst::FAdd:
74 return Builder.CreateFAdd(Loaded, Val, "new");
75 case AtomicRMWInst::FSub:
76 return Builder.CreateFSub(Loaded, Val, "new");
77 case AtomicRMWInst::FMax:
78 return Builder.CreateMaxNum(Loaded, Val);
79 case AtomicRMWInst::FMin:
80 return Builder.CreateMinNum(Loaded, Val);
81 case AtomicRMWInst::UIncWrap: {
82 Constant *One = ConstantInt::get(Loaded->getType(), 1);
83 Value *Inc = Builder.CreateAdd(Loaded, One);
84 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
85 Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
86 return Builder.CreateSelect(Cmp, Zero, Inc, "new");
87 }
88 case AtomicRMWInst::UDecWrap: {
89 Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
90 Constant *One = ConstantInt::get(Loaded->getType(), 1);
91
92 Value *Dec = Builder.CreateSub(Loaded, One);
93 Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero);
94 Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val);
95 Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal);
96 return Builder.CreateSelect(Or, Val, Dec, "new");
97 }
98 default:
99 llvm_unreachable("Unknown atomic op");
100 }
101 }
102
lowerAtomicRMWInst(AtomicRMWInst * RMWI)103 bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
104 IRBuilder<> Builder(RMWI);
105 Value *Ptr = RMWI->getPointerOperand();
106 Value *Val = RMWI->getValOperand();
107
108 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
109 Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val);
110 Builder.CreateStore(Res, Ptr);
111 RMWI->replaceAllUsesWith(Orig);
112 RMWI->eraseFromParent();
113 return true;
114 }
115