Importing rustc-1.65.0 Bug: 250026064 Test: ./build.py --lto=thin Change-Id: If144bbb779543363527e335b12d3ae6fbb4be8f0
diff --git a/src/llvm-project/llvm/lib/Analysis/AliasAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/AliasAnalysis.cpp index a8132e5..e249c38 100644 --- a/src/llvm-project/llvm/lib/Analysis/AliasAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -42,7 +42,6 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/InitializePasses.h" @@ -680,7 +679,7 @@ } } - const MemoryLocation &Loc = OptLoc.getValueOr(MemoryLocation()); + const MemoryLocation &Loc = OptLoc.value_or(MemoryLocation()); switch (I->getOpcode()) { case Instruction::VAArg: @@ -988,6 +987,28 @@ return isa<AllocaInst>(V) || isNoAliasCall(V) || isNoAliasOrByValArgument(V); } +bool llvm::isEscapeSource(const Value *V) { + if (auto *CB = dyn_cast<CallBase>(V)) + return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(CB, + true); + + // The load case works because isNonEscapingLocalObject considers all + // stores to be escapes (it passes true for the StoreCaptures argument + // to PointerMayBeCaptured). + if (isa<LoadInst>(V)) + return true; + + // The inttoptr case works because isNonEscapingLocalObject considers all + // means of converting or equating a pointer to an int (ptrtoint, ptr store + // which could be followed by an integer load, ptr<->int compare) as + // escaping, and objects located at well-known addresses via platform-specific + // means cannot be considered non-escaping local objects. + if (isa<IntToPtrInst>(V)) + return true; + + return false; +} + bool llvm::isNotVisibleOnUnwind(const Value *Object, bool &RequiresNoCaptureBeforeUnwind) { RequiresNoCaptureBeforeUnwind = false;
diff --git a/src/llvm-project/llvm/lib/Analysis/AliasAnalysisEvaluator.cpp b/src/llvm-project/llvm/lib/Analysis/AliasAnalysisEvaluator.cpp index 1577f1e..e3446a1 100644 --- a/src/llvm-project/llvm/lib/Analysis/AliasAnalysisEvaluator.cpp +++ b/src/llvm-project/llvm/lib/Analysis/AliasAnalysisEvaluator.cpp
@@ -9,9 +9,7 @@ #include "llvm/Analysis/AliasAnalysisEvaluator.h" #include "llvm/ADT/SetVector.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" @@ -19,7 +17,6 @@ #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -41,30 +38,48 @@ static cl::opt<bool> EvalAAMD("evaluate-aa-metadata", cl::ReallyHidden); -static void PrintResults(AliasResult AR, bool P, const Value *V1, - const Value *V2, const Module *M) { +static void PrintResults(AliasResult AR, bool P, + std::pair<const Value *, Type *> Loc1, + std::pair<const Value *, Type *> Loc2, + const Module *M) { if (PrintAll || P) { + Type *Ty1 = Loc1.second, *Ty2 = Loc2.second; + unsigned AS1 = Loc1.first->getType()->getPointerAddressSpace(); + unsigned AS2 = Loc2.first->getType()->getPointerAddressSpace(); std::string o1, o2; { raw_string_ostream os1(o1), os2(o2); - V1->printAsOperand(os1, true, M); - V2->printAsOperand(os2, true, M); + Loc1.first->printAsOperand(os1, false, M); + Loc2.first->printAsOperand(os2, false, M); } if (o2 < o1) { std::swap(o1, o2); + std::swap(Ty1, Ty2); + std::swap(AS1, AS2); // Change offset sign for the local AR, for printing only. AR.swap(); } - errs() << " " << AR << ":\t" << o1 << ", " << o2 << "\n"; + errs() << " " << AR << ":\t"; + Ty1->print(errs(), false, /* NoDetails */ true); + if (AS1 != 0) + errs() << " addrspace(" << AS1 << ")"; + errs() << "* " << o1 << ", "; + Ty2->print(errs(), false, /* NoDetails */ true); + if (AS2 != 0) + errs() << " addrspace(" << AS2 << ")"; + errs() << "* " << o2 << "\n"; } } -static inline void PrintModRefResults(const char *Msg, bool P, Instruction *I, - Value *Ptr, Module *M) { +static inline void PrintModRefResults( + const char *Msg, bool P, Instruction *I, + std::pair<const Value *, Type *> Loc, Module *M) { if (PrintAll || P) { errs() << " " << Msg << ": Ptr: "; - Ptr->printAsOperand(errs(), true, M); + Loc.second->print(errs(), false, /* NoDetails */ true); + errs() << "* "; + Loc.first->printAsOperand(errs(), false, M); errs() << "\t<->" << *I << '\n'; } } @@ -84,11 +99,6 @@ } } -static inline bool isInterestingPointer(Value *V) { - return V->getType()->isPointerTy() - && !isa<ConstantPointerNull>(V); -} - PreservedAnalyses AAEvaluator::run(Function &F, FunctionAnalysisManager &AM) { runInternal(F, AM.getResult<AAManager>(F)); return PreservedAnalyses::all(); @@ -99,38 +109,21 @@ ++FunctionCount; - SetVector<Value *> Pointers; + SetVector<std::pair<const Value *, Type *>> Pointers; SmallSetVector<CallBase *, 16> Calls; SetVector<Value *> Loads; SetVector<Value *> Stores; - for (auto &I : F.args()) - if (I.getType()->isPointerTy()) // Add all pointer arguments. - Pointers.insert(&I); - for (Instruction &Inst : instructions(F)) { - if (Inst.getType()->isPointerTy()) // Add all pointer instructions. - Pointers.insert(&Inst); - if (EvalAAMD && isa<LoadInst>(&Inst)) - Loads.insert(&Inst); - if (EvalAAMD && isa<StoreInst>(&Inst)) - Stores.insert(&Inst); - if (auto *Call = dyn_cast<CallBase>(&Inst)) { - Value *Callee = Call->getCalledOperand(); - // Skip actual functions for direct function calls. - if (!isa<Function>(Callee) && isInterestingPointer(Callee)) - Pointers.insert(Callee); - // Consider formals. - for (Use &DataOp : Call->data_ops()) - if (isInterestingPointer(DataOp)) - Pointers.insert(DataOp); - Calls.insert(Call); - } else { - // Consider all operands. - for (Use &Op : Inst.operands()) - if (isInterestingPointer(Op)) - Pointers.insert(Op); - } + if (auto *LI = dyn_cast<LoadInst>(&Inst)) { + Pointers.insert({LI->getPointerOperand(), LI->getType()}); + Loads.insert(LI); + } else if (auto *SI = dyn_cast<StoreInst>(&Inst)) { + Pointers.insert({SI->getPointerOperand(), + SI->getValueOperand()->getType()}); + Stores.insert(SI); + } else if (auto *CB = dyn_cast<CallBase>(&Inst)) + Calls.insert(CB); } if (PrintAll || PrintNoAlias || PrintMayAlias || PrintPartialAlias || @@ -139,20 +132,12 @@ << " pointers, " << Calls.size() << " call sites\n"; // iterate over the worklist, and run the full (n^2)/2 disambiguations - for (SetVector<Value *>::iterator I1 = Pointers.begin(), E = Pointers.end(); - I1 != E; ++I1) { - auto I1Size = LocationSize::afterPointer(); - Type *I1ElTy = (*I1)->getType()->getPointerElementType(); - if (I1ElTy->isSized()) - I1Size = LocationSize::precise(DL.getTypeStoreSize(I1ElTy)); - - for (SetVector<Value *>::iterator I2 = Pointers.begin(); I2 != I1; ++I2) { - auto I2Size = LocationSize::afterPointer(); - Type *I2ElTy = (*I2)->getType()->getPointerElementType(); - if (I2ElTy->isSized()) - I2Size = LocationSize::precise(DL.getTypeStoreSize(I2ElTy)); - - AliasResult AR = AA.alias(*I1, I1Size, *I2, I2Size); + for (auto I1 = Pointers.begin(), E = Pointers.end(); I1 != E; ++I1) { + LocationSize Size1 = LocationSize::precise(DL.getTypeStoreSize(I1->second)); + for (auto I2 = Pointers.begin(); I2 != I1; ++I2) { + LocationSize Size2 = + LocationSize::precise(DL.getTypeStoreSize(I2->second)); + AliasResult AR = AA.alias(I1->first, Size1, I2->first, Size2); switch (AR) { case AliasResult::NoAlias: PrintResults(AR, PrintNoAlias, *I1, *I2, F.getParent()); @@ -231,13 +216,10 @@ // Mod/ref alias analysis: compare all pairs of calls and values for (CallBase *Call : Calls) { - for (auto Pointer : Pointers) { - auto Size = LocationSize::afterPointer(); - Type *ElTy = Pointer->getType()->getPointerElementType(); - if (ElTy->isSized()) - Size = LocationSize::precise(DL.getTypeStoreSize(ElTy)); - - switch (AA.getModRefInfo(Call, Pointer, Size)) { + for (const auto &Pointer : Pointers) { + LocationSize Size = + LocationSize::precise(DL.getTypeStoreSize(Pointer.second)); + switch (AA.getModRefInfo(Call, Pointer.first, Size)) { case ModRefInfo::NoModRef: PrintModRefResults("NoModRef", PrintNoModRef, Call, Pointer, F.getParent());
diff --git a/src/llvm-project/llvm/lib/Analysis/AliasSetTracker.cpp b/src/llvm-project/llvm/lib/Analysis/AliasSetTracker.cpp index 288928a..bb25244 100644 --- a/src/llvm-project/llvm/lib/Analysis/AliasSetTracker.cpp +++ b/src/llvm-project/llvm/lib/Analysis/AliasSetTracker.cpp
@@ -13,16 +13,12 @@ #include "llvm/Analysis/AliasSetTracker.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/GuardUtils.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Config/llvm-config.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/PatternMatch.h" #include "llvm/IR/Value.h" @@ -258,31 +254,6 @@ return false; } -Instruction* AliasSet::getUniqueInstruction() { - if (AliasAny) - // May have collapses alias set - return nullptr; - if (begin() != end()) { - if (!UnknownInsts.empty()) - // Another instruction found - return nullptr; - if (std::next(begin()) != end()) - // Another instruction found - return nullptr; - Value *Addr = begin()->getValue(); - assert(!Addr->user_empty() && - "where's the instruction which added this pointer?"); - if (std::next(Addr->user_begin()) != Addr->user_end()) - // Another instruction found -- this is really restrictive - // TODO: generalize! - return nullptr; - return cast<Instruction>(*(Addr->user_begin())); - } - if (1 != UnknownInsts.size()) - return nullptr; - return cast<Instruction>(UnknownInsts[0]); -} - void AliasSetTracker::clear() { // Delete all the PointerRec entries. for (auto &I : PointerMap) @@ -608,7 +579,7 @@ AliasAnyAS->Access = AliasSet::ModRefAccess; AliasAnyAS->AliasAny = true; - for (auto Cur : ASVector) { + for (auto *Cur : ASVector) { // If Cur was already forwarding, just forward to the new AS instead. AliasSet *FwdTo = Cur->Forward; if (FwdTo) {
diff --git a/src/llvm-project/llvm/lib/Analysis/Analysis.cpp b/src/llvm-project/llvm/lib/Analysis/Analysis.cpp index 177f38a..460dddc 100644 --- a/src/llvm-project/llvm/lib/Analysis/Analysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/Analysis.cpp
@@ -40,14 +40,14 @@ initializeDelinearizationPass(Registry); initializeDemandedBitsWrapperPassPass(Registry); initializeDominanceFrontierWrapperPassPass(Registry); - initializeDomViewerPass(Registry); - initializeDomPrinterPass(Registry); - initializeDomOnlyViewerPass(Registry); - initializePostDomViewerPass(Registry); - initializeDomOnlyPrinterPass(Registry); - initializePostDomPrinterPass(Registry); - initializePostDomOnlyViewerPass(Registry); - initializePostDomOnlyPrinterPass(Registry); + initializeDomViewerWrapperPassPass(Registry); + initializeDomPrinterWrapperPassPass(Registry); + initializeDomOnlyViewerWrapperPassPass(Registry); + initializePostDomViewerWrapperPassPass(Registry); + initializeDomOnlyPrinterWrapperPassPass(Registry); + initializePostDomPrinterWrapperPassPass(Registry); + initializePostDomOnlyViewerWrapperPassPass(Registry); + initializePostDomOnlyPrinterWrapperPassPass(Registry); initializeAAResultsWrapperPassPass(Registry); initializeGlobalsAAWrapperPassPass(Registry); initializeIVUsersWrapperPassPass(Registry);
diff --git a/src/llvm-project/llvm/lib/Analysis/AssumeBundleQueries.cpp b/src/llvm-project/llvm/lib/Analysis/AssumeBundleQueries.cpp index 9d4fe12..7440dbd 100644 --- a/src/llvm-project/llvm/lib/Analysis/AssumeBundleQueries.cpp +++ b/src/llvm-project/llvm/lib/Analysis/AssumeBundleQueries.cpp
@@ -10,8 +10,8 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/PatternMatch.h" #include "llvm/Support/DebugCounter.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/AssumptionCache.cpp b/src/llvm-project/llvm/lib/Analysis/AssumptionCache.cpp index 3e0214e..e7e476d 100644 --- a/src/llvm-project/llvm/lib/Analysis/AssumptionCache.cpp +++ b/src/llvm-project/llvm/lib/Analysis/AssumptionCache.cpp
@@ -11,18 +11,17 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Analysis/AssumeBundleQueries.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/AssumeBundleQueries.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/PatternMatch.h" #include "llvm/InitializePasses.h" @@ -31,7 +30,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <utility>
diff --git a/src/llvm-project/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/BasicAliasAnalysis.cpp index 0a0b537..c3b032a 100644 --- a/src/llvm-project/llvm/lib/Analysis/BasicAliasAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -22,7 +22,6 @@ #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/CaptureTracking.h" -#include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/PhiValues.h" @@ -45,7 +44,6 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/IR/Metadata.h" #include "llvm/IR/Operator.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" @@ -105,29 +103,6 @@ // Useful predicates //===----------------------------------------------------------------------===// -/// Returns true if the pointer is one which would have been considered an -/// escape by isNonEscapingLocalObject. -static bool isEscapeSource(const Value *V) { - if (isa<CallBase>(V)) - return true; - - // The load case works because isNonEscapingLocalObject considers all - // stores to be escapes (it passes true for the StoreCaptures argument - // to PointerMayBeCaptured). - if (isa<LoadInst>(V)) - return true; - - // The inttoptr case works because isNonEscapingLocalObject considers all - // means of converting or equating a pointer to an int (ptrtoint, ptr store - // which could be followed by an integer load, ptr<->int compare) as - // escaping, and objects located at well-known addresses via platform-specific - // means cannot be considered non-escaping local objects. - if (isa<IntToPtrInst>(V)) - return true; - - return false; -} - /// Returns the size of the object specified by V or UnknownSize if unknown. static uint64_t getObjectSize(const Value *V, const DataLayout &DL, const TargetLibraryInfo &TLI, @@ -234,7 +209,7 @@ if (Iter.second) { Instruction *EarliestCapture = FindEarliestCapture( Object, *const_cast<Function *>(I->getFunction()), - /*ReturnCaptures=*/false, /*StoreCaptures=*/true, DT); + /*ReturnCaptures=*/false, /*StoreCaptures=*/true, DT, EphValues); if (EarliestCapture) { auto Ins = Inst2Obj.insert({EarliestCapture, {}}); Ins.first->second.push_back(Object); @@ -661,8 +636,8 @@ unsigned TypeSize = DL.getTypeAllocSize(GTI.getIndexedType()).getFixedSize(); LE = LE.mul(APInt(IndexSize, TypeSize), GEPOp->isInBounds()); - Decomposed.Offset += LE.Offset.sextOrSelf(MaxIndexSize); - APInt Scale = LE.Scale.sextOrSelf(MaxIndexSize); + Decomposed.Offset += LE.Offset.sext(MaxIndexSize); + APInt Scale = LE.Scale.sext(MaxIndexSize); // If we already had an occurrence of this index variable, merge this // scale into it. For example, we want to handle: @@ -1299,8 +1274,31 @@ const VariableGEPIndex &Var = DecompGEP1.VarIndices[0]; if (Var.Val.TruncBits == 0 && isKnownNonZero(Var.Val.V, DL, 0, &AC, Var.CxtI, DT)) { - // If V != 0 then abs(VarIndex) >= abs(Scale). - MinAbsVarIndex = Var.Scale.abs(); + // If V != 0, then abs(VarIndex) > 0. + MinAbsVarIndex = APInt(Var.Scale.getBitWidth(), 1); + + // Check if abs(V*Scale) >= abs(Scale) holds in the presence of + // potentially wrapping math. + auto MultiplyByScaleNoWrap = [](const VariableGEPIndex &Var) { + if (Var.IsNSW) + return true; + + int ValOrigBW = Var.Val.V->getType()->getPrimitiveSizeInBits(); + // If Scale is small enough so that abs(V*Scale) >= abs(Scale) holds. + // The max value of abs(V) is 2^ValOrigBW - 1. Multiplying with a + // constant smaller than 2^(bitwidth(Val) - ValOrigBW) won't wrap. + int MaxScaleValueBW = Var.Val.getBitWidth() - ValOrigBW; + if (MaxScaleValueBW <= 0) + return false; + return Var.Scale.ule( + APInt::getMaxValue(MaxScaleValueBW).zext(Var.Scale.getBitWidth())); + }; + // Refine MinAbsVarIndex, if abs(Scale*V) >= abs(Scale) holds in the + // presence of potentially wrapping math. + if (MultiplyByScaleNoWrap(Var)) { + // If V != 0 then abs(VarIndex) >= abs(Scale). + MinAbsVarIndex = Var.Scale.abs(); + } } } else if (DecompGEP1.VarIndices.size() == 2) { // VarIndex = Scale*V0 + (-Scale)*V1. @@ -1370,15 +1368,15 @@ // If both arms of the Select node NoAlias or MustAlias V2, then returns // NoAlias / MustAlias. Otherwise, returns MayAlias. - AliasResult Alias = getBestAAResults().alias( - MemoryLocation(V2, V2Size), - MemoryLocation(SI->getTrueValue(), SISize), AAQI); + AliasResult Alias = + getBestAAResults().alias(MemoryLocation(SI->getTrueValue(), SISize), + MemoryLocation(V2, V2Size), AAQI); if (Alias == AliasResult::MayAlias) return AliasResult::MayAlias; - AliasResult ThisAlias = getBestAAResults().alias( - MemoryLocation(V2, V2Size), - MemoryLocation(SI->getFalseValue(), SISize), AAQI); + AliasResult ThisAlias = + getBestAAResults().alias(MemoryLocation(SI->getFalseValue(), SISize), + MemoryLocation(V2, V2Size), AAQI); return MergeAliasResults(ThisAlias, Alias); } @@ -1500,8 +1498,7 @@ AAQueryInfo *UseAAQI = BlockInserted ? &NewAAQI : &AAQI; AliasResult Alias = getBestAAResults().alias( - MemoryLocation(V2, V2Size), - MemoryLocation(V1Srcs[0], PNSize), *UseAAQI); + MemoryLocation(V1Srcs[0], PNSize), MemoryLocation(V2, V2Size), *UseAAQI); // Early exit if the check of the first PHI source against V2 is MayAlias. // Other results are not possible. @@ -1518,7 +1515,7 @@ Value *V = V1Srcs[i]; AliasResult ThisAlias = getBestAAResults().alias( - MemoryLocation(V2, V2Size), MemoryLocation(V, PNSize), *UseAAQI); + MemoryLocation(V, PNSize), MemoryLocation(V2, V2Size), *UseAAQI); Alias = MergeAliasResults(ThisAlias, Alias); if (Alias == AliasResult::MayAlias) break; @@ -1767,7 +1764,7 @@ // Make sure that the visited phis cannot reach the Value. This ensures that // the Values cannot come from different iterations of a potential cycle the // phi nodes could be involved in. - for (auto *P : VisitedPhiBBs) + for (const auto *P : VisitedPhiBBs) if (isPotentiallyReachable(&P->front(), Inst, nullptr, DT)) return false;
diff --git a/src/llvm-project/llvm/lib/Analysis/BlockFrequencyInfo.cpp b/src/llvm-project/llvm/lib/Analysis/BlockFrequencyInfo.cpp index b464071..436b017 100644 --- a/src/llvm-project/llvm/lib/Analysis/BlockFrequencyInfo.cpp +++ b/src/llvm-project/llvm/lib/Analysis/BlockFrequencyInfo.cpp
@@ -25,7 +25,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/GraphWriter.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <string>
diff --git a/src/llvm-project/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp b/src/llvm-project/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp index 2a5e1f6..ec8d318 100644 --- a/src/llvm-project/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp +++ b/src/llvm-project/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
@@ -13,7 +13,6 @@ #include "llvm/Analysis/BlockFrequencyInfoImpl.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/None.h" #include "llvm/ADT/SCCIterator.h" #include "llvm/Config/llvm-config.h" @@ -22,8 +21,8 @@ #include "llvm/Support/BranchProbability.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/ScaledNumber.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/ScaledNumber.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> @@ -48,7 +47,7 @@ "for debugging missed BFI updates")); cl::opt<bool> UseIterativeBFIInference( - "use-iterative-bfi-inference", cl::init(false), cl::Hidden, cl::ZeroOrMore, + "use-iterative-bfi-inference", cl::Hidden, cl::desc("Apply an iterative post-processing to infer correct BFI counts")); cl::opt<unsigned> IterativeBFIMaxIterationsPerBlock(
diff --git a/src/llvm-project/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/src/llvm-project/llvm/lib/Analysis/BranchProbabilityInfo.cpp index ffb8013..f457287 100644 --- a/src/llvm-project/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/src/llvm-project/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -15,6 +15,7 @@ #include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/TargetLibraryInfo.h" @@ -414,8 +415,7 @@ const LoopBlock DstLoopBB = getLoopBlock(TI->getSuccessor(I - 1)); auto EstimatedWeight = getEstimatedEdgeWeight({SrcLoopBB, DstLoopBB}); if (EstimatedWeight && - EstimatedWeight.getValue() <= - static_cast<uint32_t>(BlockExecWeight::UNREACHABLE)) + *EstimatedWeight <= static_cast<uint32_t>(BlockExecWeight::UNREACHABLE)) UnreachableIdxs.push_back(I - 1); else ReachableIdxs.push_back(I - 1); @@ -630,9 +630,10 @@ if (!CmpLHSConst || !llvm::is_contained(successors(BB), B)) continue; // First collapse InstChain + const DataLayout &DL = BB->getModule()->getDataLayout(); for (Instruction *I : llvm::reverse(InstChain)) { - CmpLHSConst = ConstantExpr::get(I->getOpcode(), CmpLHSConst, - cast<Constant>(I->getOperand(1)), true); + CmpLHSConst = ConstantFoldBinaryOpOperands( + I->getOpcode(), CmpLHSConst, cast<Constant>(I->getOperand(1)), DL); if (!CmpLHSConst) break; } @@ -688,7 +689,7 @@ if (!Weight) return None; - if (!MaxWeight || MaxWeight.getValue() < Weight.getValue()) + if (!MaxWeight || *MaxWeight < *Weight) MaxWeight = Weight; } @@ -827,9 +828,8 @@ if (auto BBWeight = getInitialEstimatedBlockWeight(BB)) // If we were able to find estimated weight for the block set it to this // block and propagate up the IR. - propagateEstimatedBlockWeight(getLoopBlock(BB), DT, PDT, - BBWeight.getValue(), BlockWorkList, - LoopWorkList); + propagateEstimatedBlockWeight(getLoopBlock(BB), DT, PDT, BBWeight.value(), + BlockWorkList, LoopWorkList); // BlockWorklist/LoopWorkList contains blocks/loops with at least one // successor/exit having estimated weight. Try to propagate weight to such @@ -852,8 +852,7 @@ if (LoopWeight <= static_cast<uint32_t>(BlockExecWeight::UNREACHABLE)) LoopWeight = static_cast<uint32_t>(BlockExecWeight::LOWEST_NON_ZERO); - EstimatedLoopWeight.insert( - {LoopBB.getLoopData(), LoopWeight.getValue()}); + EstimatedLoopWeight.insert({LoopBB.getLoopData(), *LoopWeight}); // Add all blocks entering the loop into working list. getLoopEnterBlocks(LoopBB, BlockWorkList); } @@ -875,7 +874,7 @@ auto MaxWeight = getMaxEstimatedEdgeWeight(LoopBB, successors(BB)); if (MaxWeight) - propagateEstimatedBlockWeight(LoopBB, DT, PDT, MaxWeight.getValue(), + propagateEstimatedBlockWeight(LoopBB, DT, PDT, *MaxWeight, BlockWorkList, LoopWorkList); } } while (!BlockWorkList.empty() || !LoopWorkList.empty()); @@ -913,7 +912,7 @@ // Scale down loop exiting weight by trip count. Weight = std::max( static_cast<uint32_t>(BlockExecWeight::LOWEST_NON_ZERO), - Weight.getValueOr(static_cast<uint32_t>(BlockExecWeight::DEFAULT)) / + Weight.value_or(static_cast<uint32_t>(BlockExecWeight::DEFAULT)) / TC); } bool IsUnlikelyEdge = LoopBB.getLoop() && UnlikelyBlocks.contains(SuccBB); @@ -923,15 +922,14 @@ // 'Unlikely' blocks have twice lower weight. Weight = std::max( static_cast<uint32_t>(BlockExecWeight::LOWEST_NON_ZERO), - Weight.getValueOr(static_cast<uint32_t>(BlockExecWeight::DEFAULT)) / - 2); + Weight.value_or(static_cast<uint32_t>(BlockExecWeight::DEFAULT)) / 2); } if (Weight) FoundEstimatedWeight = true; auto WeightVal = - Weight.getValueOr(static_cast<uint32_t>(BlockExecWeight::DEFAULT)); + Weight.value_or(static_cast<uint32_t>(BlockExecWeight::DEFAULT)); TotalWeight += WeightVal; SuccWeights.push_back(WeightVal); } @@ -1252,7 +1250,7 @@ // Walk the basic blocks in post-order so that we can build up state about // the successors of a block iteratively. - for (auto BB : post_order(&F.getEntryBlock())) { + for (const auto *BB : post_order(&F.getEntryBlock())) { LLVM_DEBUG(dbgs() << "Computing probabilities for " << BB->getName() << "\n"); // If there is no at least two successors, no sense to set probability.
diff --git a/src/llvm-project/llvm/lib/Analysis/CFG.cpp b/src/llvm-project/llvm/lib/Analysis/CFG.cpp index ec25ee1..e5dd458 100644 --- a/src/llvm-project/llvm/lib/Analysis/CFG.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CFG.cpp
@@ -127,11 +127,7 @@ // the outermost loop in the loop nest that contains BB. static const Loop *getOutermostLoop(const LoopInfo *LI, const BasicBlock *BB) { const Loop *L = LI->getLoopFor(BB); - if (L) { - while (const Loop *Parent = L->getParentLoop()) - L = Parent; - } - return L; + return L ? L->getOutermostLoop() : nullptr; } bool llvm::isPotentiallyReachableFromMany( @@ -153,7 +149,7 @@ // untrue. SmallPtrSet<const Loop *, 8> LoopsWithHoles; if (LI && ExclusionSet) { - for (auto BB : *ExclusionSet) { + for (auto *BB : *ExclusionSet) { if (const Loop *L = getOutermostLoop(LI, BB)) LoopsWithHoles.insert(L); }
diff --git a/src/llvm-project/llvm/lib/Analysis/CFGPrinter.cpp b/src/llvm-project/llvm/lib/Analysis/CFGPrinter.cpp index 04ccdc5..f8eba1a 100644 --- a/src/llvm-project/llvm/lib/Analysis/CFGPrinter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CFGPrinter.cpp
@@ -23,7 +23,7 @@ #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" -#include <algorithm> +#include "llvm/Support/GraphWriter.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/CFLAndersAliasAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/CFLAndersAliasAnalysis.cpp index 1216d03..d70e1b2 100644 --- a/src/llvm-project/llvm/lib/Analysis/CFLAndersAliasAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CFLAndersAliasAnalysis.cpp
@@ -615,7 +615,7 @@ auto Src = InstantiatedValue{Val, I}; // If there's an assignment edge from X to Y, it means Y is reachable from // X at S3 and X is reachable from Y at S1 - for (auto &Edge : ValueInfo.getNodeInfoAtLevel(I).Edges) { + for (const auto &Edge : ValueInfo.getNodeInfoAtLevel(I).Edges) { propagate(Edge.Other, Src, MatchState::FlowFromReadOnly, ReachSet, WorkList); propagate(Src, Edge.Other, MatchState::FlowToWriteOnly, ReachSet, @@ -831,14 +831,14 @@ scan(Fn); Iter = Cache.find(&Fn); assert(Iter != Cache.end()); - assert(Iter->second.hasValue()); + assert(Iter->second); } return Iter->second; } const AliasSummary *CFLAndersAAResult::getAliasSummary(const Function &Fn) { auto &FunInfo = ensureCached(Fn); - if (FunInfo.hasValue()) + if (FunInfo) return &FunInfo->getAliasSummary(); else return nullptr;
diff --git a/src/llvm-project/llvm/lib/Analysis/CFLGraph.h b/src/llvm-project/llvm/lib/Analysis/CFLGraph.h index 02a13d6..47bb02a 100644 --- a/src/llvm-project/llvm/lib/Analysis/CFLGraph.h +++ b/src/llvm-project/llvm/lib/Analysis/CFLGraph.h
@@ -403,7 +403,7 @@ auto &RetParamRelations = Summary->RetParamRelations; for (auto &Relation : RetParamRelations) { auto IRelation = instantiateExternalRelation(Relation, Call); - if (IRelation.hasValue()) { + if (IRelation) { Graph.addNode(IRelation->From); Graph.addNode(IRelation->To); Graph.addEdge(IRelation->From, IRelation->To); @@ -413,7 +413,7 @@ auto &RetParamAttributes = Summary->RetParamAttributes; for (auto &Attribute : RetParamAttributes) { auto IAttr = instantiateExternalAttribute(Attribute, Call); - if (IAttr.hasValue()) + if (IAttr) Graph.addNode(IAttr->IValue, IAttr->Attr); } } @@ -434,7 +434,8 @@ // introduce any aliases. // TODO: address other common library functions such as realloc(), // strdup(), etc. - if (isMallocOrCallocLikeFn(&Call, &TLI) || isFreeCall(&Call, &TLI)) + if (isMallocOrCallocLikeFn(&Call, &TLI) || + getFreedOperand(&Call, &TLI) != nullptr) return; // TODO: Add support for noalias args/all the other fun function
diff --git a/src/llvm-project/llvm/lib/Analysis/CFLSteensAliasAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/CFLSteensAliasAnalysis.cpp index 090dccc..33ed6f8 100644 --- a/src/llvm-project/llvm/lib/Analysis/CFLSteensAliasAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CFLSteensAliasAnalysis.cpp
@@ -165,7 +165,7 @@ assert(RetVal != nullptr); assert(RetVal->getType()->isPointerTy()); auto RetInfo = Sets.find(InstantiatedValue{RetVal, 0}); - if (RetInfo.hasValue()) + if (RetInfo) AddToRetParamRelations(0, RetInfo->Index); } @@ -174,7 +174,7 @@ for (auto &Param : Fn.args()) { if (Param.getType()->isPointerTy()) { auto ParamInfo = Sets.find(InstantiatedValue{&Param, 0}); - if (ParamInfo.hasValue()) + if (ParamInfo) AddToRetParamRelations(I + 1, ParamInfo->Index); } ++I; @@ -216,7 +216,7 @@ for (unsigned I = 0, E = ValueInfo.getNumLevels(); I < E; ++I) { auto Src = InstantiatedValue{Val, I}; - for (auto &Edge : ValueInfo.getNodeInfoAtLevel(I).Edges) + for (const auto &Edge : ValueInfo.getNodeInfoAtLevel(I).Edges) SetBuilder.addWith(Src, Edge.Other); } } @@ -250,14 +250,14 @@ scan(Fn); Iter = Cache.find(Fn); assert(Iter != Cache.end()); - assert(Iter->second.hasValue()); + assert(Iter->second); } return Iter->second; } const AliasSummary *CFLSteensAAResult::getAliasSummary(Function &Fn) { auto &FunInfo = ensureCached(&Fn); - if (FunInfo.hasValue()) + if (FunInfo) return &FunInfo->getAliasSummary(); else return nullptr; @@ -293,15 +293,15 @@ assert(Fn != nullptr); auto &MaybeInfo = ensureCached(Fn); - assert(MaybeInfo.hasValue()); + assert(MaybeInfo); auto &Sets = MaybeInfo->getStratifiedSets(); auto MaybeA = Sets.find(InstantiatedValue{ValA, 0}); - if (!MaybeA.hasValue()) + if (!MaybeA) return AliasResult::MayAlias; auto MaybeB = Sets.find(InstantiatedValue{ValB, 0}); - if (!MaybeB.hasValue()) + if (!MaybeB) return AliasResult::MayAlias; auto SetA = *MaybeA;
diff --git a/src/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp b/src/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp index c60b70a..b2e7422 100644 --- a/src/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -9,6 +9,7 @@ #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/PriorityWorklist.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" @@ -27,7 +28,6 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> #include <iterator> @@ -164,9 +164,9 @@ InlinedInternalEdges; CGSCCUpdateResult UR = { - RCWorklist, CWorklist, InvalidRefSCCSet, InvalidSCCSet, - nullptr, nullptr, PreservedAnalyses::all(), InlinedInternalEdges, - {}}; + RCWorklist, CWorklist, InvalidRefSCCSet, + InvalidSCCSet, nullptr, PreservedAnalyses::all(), + InlinedInternalEdges, {}}; // Request PassInstrumentation from analysis manager, will use it to run // instrumenting callbacks for the passes later. @@ -174,9 +174,8 @@ PreservedAnalyses PA = PreservedAnalyses::all(); CG.buildRefSCCs(); - for (auto RCI = CG.postorder_ref_scc_begin(), - RCE = CG.postorder_ref_scc_end(); - RCI != RCE;) { + for (LazyCallGraph::RefSCC &RC : + llvm::make_early_inc_range(CG.postorder_ref_sccs())) { assert(RCWorklist.empty() && "Should always start with an empty RefSCC worklist"); // The postorder_ref_sccs range we are walking is lazily constructed, so @@ -190,7 +189,7 @@ // // We also eagerly increment the iterator to the next position because // the CGSCC passes below may delete the current RefSCC. - RCWorklist.insert(&*RCI++); + RCWorklist.insert(&RC); do { LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val(); @@ -230,11 +229,15 @@ LLVM_DEBUG(dbgs() << "Skipping redundant run on SCC: " << *C << "\n"); continue; } - if (&C->getOuterRefSCC() != RC) { - LLVM_DEBUG(dbgs() << "Skipping an SCC that is now part of some other " - "RefSCC...\n"); - continue; - } + // We used to also check if the current SCC is part of the current + // RefSCC and bail if it wasn't, since it should be in RCWorklist. + // However, this can cause compile time explosions in some cases on + // modules with a huge RefSCC. If a non-trivial amount of SCCs in the + // huge RefSCC can become their own child RefSCC, we create one child + // RefSCC, bail on the current RefSCC, visit the child RefSCC, revisit + // the huge RefSCC, and repeat. By visiting all SCCs in the original + // RefSCC we create all the child RefSCCs in one pass of the RefSCC, + // rather one pass of the RefSCC creating one child RefSCC at a time. // Ensure we can proxy analysis updates from the CGSCC analysis manager // into the the Function analysis manager by getting a proxy here. @@ -264,11 +267,8 @@ // Check that we didn't miss any update scenario. assert(!InvalidSCCSet.count(C) && "Processing an invalid SCC!"); assert(C->begin() != C->end() && "Cannot have an empty SCC!"); - assert(&C->getOuterRefSCC() == RC && - "Processing an SCC in a different RefSCC!"); LastUpdatedC = UR.UpdatedC; - UR.UpdatedRC = nullptr; UR.UpdatedC = nullptr; // Check the PassInstrumentation's BeforePass callbacks before @@ -290,7 +290,6 @@ // Update the SCC and RefSCC if necessary. C = UR.UpdatedC ? UR.UpdatedC : C; - RC = UR.UpdatedRC ? UR.UpdatedRC : RC; if (UR.UpdatedC) { // If we're updating the SCC, also update the FAM inside the proxy's @@ -1213,10 +1212,8 @@ assert(!UR.InvalidatedRefSCCs.count(RC) && "Invalidated the current RefSCC!"); assert(&C->getOuterRefSCC() == RC && "Current SCC not in current RefSCC!"); - // Record the current RefSCC and SCC for higher layers of the CGSCC pass - // manager now that all the updates have been applied. - if (RC != &InitialRC) - UR.UpdatedRC = RC; + // Record the current SCC for higher layers of the CGSCC pass manager now that + // all the updates have been applied. if (C != &InitialC) UR.UpdatedC = C;
diff --git a/src/llvm-project/llvm/lib/Analysis/CMakeLists.txt b/src/llvm-project/llvm/lib/Analysis/CMakeLists.txt index aec8412..e59725c 100644 --- a/src/llvm-project/llvm/lib/Analysis/CMakeLists.txt +++ b/src/llvm-project/llvm/lib/Analysis/CMakeLists.txt
@@ -2,10 +2,7 @@ include(TensorFlowCompile) set(LLVM_INLINER_MODEL_PATH_DEFAULT "models/inliner-Oz") - # This url points to the most recent most which is known to be compatible with - # LLVM. When better models are published, this url should be updated to aid - # discoverability. - set(LLVM_INLINER_MODEL_CURRENT_URL "https://github.com/google/ml-compiler-opt/releases/download/inlining-Oz-v1.1/inlining-Oz-99f0063-v1.1.tar.gz") + set(LLVM_INLINER_MODEL_CURRENT_URL "<UNSPECIFIED>") if (DEFINED LLVM_HAVE_TF_AOT) tf_find_and_compile( @@ -101,6 +98,7 @@ MemoryBuiltins.cpp MemoryDependenceAnalysis.cpp MemoryLocation.cpp + MemoryProfileInfo.cpp MemorySSA.cpp MemorySSAUpdater.cpp ModelUnderTrainingRunner.cpp @@ -131,6 +129,7 @@ SyncDependenceAnalysis.cpp SyntheticCountsUtils.cpp TFUtils.cpp + TensorSpec.cpp TargetLibraryInfo.cpp TargetTransformInfo.cpp Trace.cpp
diff --git a/src/llvm-project/llvm/lib/Analysis/CallGraph.cpp b/src/llvm-project/llvm/lib/Analysis/CallGraph.cpp index dfbd29b..f855271 100644 --- a/src/llvm-project/llvm/lib/Analysis/CallGraph.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CallGraph.cpp
@@ -21,7 +21,6 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> #include <cassert> using namespace llvm; @@ -70,8 +69,7 @@ // Check whether the analysis, all analyses on functions, or the function's // CFG have been preserved. auto PAC = PA.getChecker<CallGraphAnalysis>(); - return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>() || - PAC.preservedSet<CFGAnalyses>()); + return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>()); } void CallGraph::addToCallGraph(Function *F) {
diff --git a/src/llvm-project/llvm/lib/Analysis/CallGraphSCCPass.cpp b/src/llvm-project/llvm/lib/Analysis/CallGraphSCCPass.cpp index 930cb13..8438f33 100644 --- a/src/llvm-project/llvm/lib/Analysis/CallGraphSCCPass.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CallGraphSCCPass.cpp
@@ -28,7 +28,6 @@ #include "llvm/IR/OptBisect.h" #include "llvm/IR/PassTimingInfo.h" #include "llvm/IR/PrintPasses.h" -#include "llvm/IR/StructuralHash.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -271,7 +270,7 @@ Calls.count(Call) || // If the call edge is not from a call or invoke, or it is a - // instrinsic call, then the function pass RAUW'd a call with + // intrinsic call, then the function pass RAUW'd a call with // another value. This can happen when constant folding happens // of well known functions etc. (Call->getCalledFunction() && @@ -470,7 +469,7 @@ initializeAnalysisImpl(P); #ifdef EXPENSIVE_CHECKS - uint64_t RefHash = StructuralHash(CG.getModule()); + uint64_t RefHash = P->structuralHash(CG.getModule()); #endif // Actually run this pass on the current SCC. @@ -480,7 +479,7 @@ Changed |= LocalChanged; #ifdef EXPENSIVE_CHECKS - if (!LocalChanged && (RefHash != StructuralHash(CG.getModule()))) { + if (!LocalChanged && (RefHash != P->structuralHash(CG.getModule()))) { llvm::errs() << "Pass modifies its input and doesn't report it: " << P->getPassName() << "\n"; llvm_unreachable("Pass modifies its input and doesn't report it");
diff --git a/src/llvm-project/llvm/lib/Analysis/CallPrinter.cpp b/src/llvm-project/llvm/lib/Analysis/CallPrinter.cpp index 829532a..65e3184 100644 --- a/src/llvm-project/llvm/lib/Analysis/CallPrinter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CallPrinter.cpp
@@ -14,18 +14,23 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/CallPrinter.h" -#include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/BranchProbabilityInfo.h" -#include "llvm/Analysis/CallGraph.h" -#include "llvm/Analysis/DOTGraphTraitsPass.h" -#include "llvm/Analysis/HeatUtils.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/InitializePasses.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/Analysis/HeatUtils.h" +#include "llvm/IR/Instructions.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/DOTGraphTraits.h" +#include "llvm/Support/GraphWriter.h" using namespace llvm; +namespace llvm { +template <class GraphType> struct GraphTraits; +} + // This option shows static (relative) call counts. // FIXME: // Need to show real counts when profile data is available @@ -213,6 +218,71 @@ } // end llvm namespace namespace { +void doCallGraphDOTPrinting( + Module &M, function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) { + std::string Filename; + if (!CallGraphDotFilenamePrefix.empty()) + Filename = (CallGraphDotFilenamePrefix + ".callgraph.dot"); + else + Filename = (std::string(M.getModuleIdentifier()) + ".callgraph.dot"); + errs() << "Writing '" << Filename << "'..."; + + std::error_code EC; + raw_fd_ostream File(Filename, EC, sys::fs::OF_Text); + + CallGraph CG(M); + CallGraphDOTInfo CFGInfo(&M, &CG, LookupBFI); + + if (!EC) + WriteGraph(File, &CFGInfo); + else + errs() << " error opening file for writing!"; + errs() << "\n"; +} + +void viewCallGraph(Module &M, + function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) { + CallGraph CG(M); + CallGraphDOTInfo CFGInfo(&M, &CG, LookupBFI); + + std::string Title = + DOTGraphTraits<CallGraphDOTInfo *>::getGraphName(&CFGInfo); + ViewGraph(&CFGInfo, "callgraph", true, Title); +} +} // namespace + +namespace llvm { +PreservedAnalyses CallGraphDOTPrinterPass::run(Module &M, + ModuleAnalysisManager &AM) { + FunctionAnalysisManager &FAM = + AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + + auto LookupBFI = [&FAM](Function &F) { + return &FAM.getResult<BlockFrequencyAnalysis>(F); + }; + + doCallGraphDOTPrinting(M, LookupBFI); + + return PreservedAnalyses::all(); +} + +PreservedAnalyses CallGraphViewerPass::run(Module &M, + ModuleAnalysisManager &AM) { + + FunctionAnalysisManager &FAM = + AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + + auto LookupBFI = [&FAM](Function &F) { + return &FAM.getResult<BlockFrequencyAnalysis>(F); + }; + + viewCallGraph(M, LookupBFI); + + return PreservedAnalyses::all(); +} +} // namespace llvm + +namespace { // Viewer class CallGraphViewer : public ModulePass { public: @@ -234,12 +304,7 @@ return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI(); }; - CallGraph CG(M); - CallGraphDOTInfo CFGInfo(&M, &CG, LookupBFI); - - std::string Title = - DOTGraphTraits<CallGraphDOTInfo *>::getGraphName(&CFGInfo); - ViewGraph(&CFGInfo, "callgraph", true, Title); + viewCallGraph(M, LookupBFI); return false; } @@ -266,24 +331,7 @@ return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI(); }; - std::string Filename; - if (!CallGraphDotFilenamePrefix.empty()) - Filename = (CallGraphDotFilenamePrefix + ".callgraph.dot"); - else - Filename = (std::string(M.getModuleIdentifier()) + ".callgraph.dot"); - errs() << "Writing '" << Filename << "'..."; - - std::error_code EC; - raw_fd_ostream File(Filename, EC, sys::fs::OF_Text); - - CallGraph CG(M); - CallGraphDOTInfo CFGInfo(&M, &CG, LookupBFI); - - if (!EC) - WriteGraph(File, &CFGInfo); - else - errs() << " error opening file for writing!"; - errs() << "\n"; + doCallGraphDOTPrinting(M, LookupBFI); return false; }
diff --git a/src/llvm-project/llvm/lib/Analysis/CaptureTracking.cpp b/src/llvm-project/llvm/lib/Analysis/CaptureTracking.cpp index ba8462e..f4fd660 100644 --- a/src/llvm-project/llvm/lib/Analysis/CaptureTracking.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CaptureTracking.cpp
@@ -16,6 +16,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/CaptureTracking.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -44,15 +45,15 @@ /// use it where possible. The caching version can use much higher limit or /// don't have this cap at all. static cl::opt<unsigned> -DefaultMaxUsesToExplore("capture-tracking-max-uses-to-explore", cl::Hidden, - cl::desc("Maximal number of uses to explore."), - cl::init(20)); + DefaultMaxUsesToExplore("capture-tracking-max-uses-to-explore", cl::Hidden, + cl::desc("Maximal number of uses to explore."), + cl::init(100)); unsigned llvm::getDefaultMaxUsesToExploreForCaptureTracking() { return DefaultMaxUsesToExplore; } -CaptureTracker::~CaptureTracker() {} +CaptureTracker::~CaptureTracker() = default; bool CaptureTracker::shouldExplore(const Use *U) { return true; } @@ -74,8 +75,10 @@ namespace { struct SimpleCaptureTracker : public CaptureTracker { - explicit SimpleCaptureTracker(bool ReturnCaptures) - : ReturnCaptures(ReturnCaptures) {} + explicit SimpleCaptureTracker( + + const SmallPtrSetImpl<const Value *> &EphValues, bool ReturnCaptures) + : EphValues(EphValues), ReturnCaptures(ReturnCaptures) {} void tooManyUses() override { Captured = true; } @@ -83,10 +86,15 @@ if (isa<ReturnInst>(U->getUser()) && !ReturnCaptures) return false; + if (EphValues.contains(U->getUser())) + return false; + Captured = true; return true; } + const SmallPtrSetImpl<const Value *> &EphValues; + bool ReturnCaptures; bool Captured = false; @@ -154,8 +162,9 @@ // escape are not in a cycle. struct EarliestCaptures : public CaptureTracker { - EarliestCaptures(bool ReturnCaptures, Function &F, const DominatorTree &DT) - : DT(DT), ReturnCaptures(ReturnCaptures), F(F) {} + EarliestCaptures(bool ReturnCaptures, Function &F, const DominatorTree &DT, + const SmallPtrSetImpl<const Value *> &EphValues) + : EphValues(EphValues), DT(DT), ReturnCaptures(ReturnCaptures), F(F) {} void tooManyUses() override { Captured = true; @@ -167,6 +176,9 @@ if (isa<ReturnInst>(I) && !ReturnCaptures) return false; + if (EphValues.contains(I)) + return false; + if (!EarliestCapture) { EarliestCapture = I; } else if (EarliestCapture->getParent() == I->getParent()) { @@ -193,6 +205,8 @@ return false; } + const SmallPtrSetImpl<const Value *> &EphValues; + Instruction *EarliestCapture = nullptr; const DominatorTree &DT; @@ -212,8 +226,18 @@ /// counts as capturing it or not. The boolean StoreCaptures specified whether /// storing the value (or part of it) into memory anywhere automatically /// counts as capturing it or not. -bool llvm::PointerMayBeCaptured(const Value *V, - bool ReturnCaptures, bool StoreCaptures, +bool llvm::PointerMayBeCaptured(const Value *V, bool ReturnCaptures, + bool StoreCaptures, unsigned MaxUsesToExplore) { + SmallPtrSet<const Value *, 1> Empty; + return PointerMayBeCaptured(V, ReturnCaptures, StoreCaptures, Empty, + MaxUsesToExplore); +} + +/// Variant of the above function which accepts a set of Values that are +/// ephemeral and cannot cause pointers to escape. +bool llvm::PointerMayBeCaptured(const Value *V, bool ReturnCaptures, + bool StoreCaptures, + const SmallPtrSetImpl<const Value *> &EphValues, unsigned MaxUsesToExplore) { assert(!isa<GlobalValue>(V) && "It doesn't make sense to ask whether a global is captured."); @@ -224,7 +248,7 @@ // take advantage of this. (void)StoreCaptures; - SimpleCaptureTracker SCT(ReturnCaptures); + SimpleCaptureTracker SCT(EphValues, ReturnCaptures); PointerMayBeCaptured(V, &SCT, MaxUsesToExplore); if (SCT.Captured) ++NumCaptured; @@ -266,14 +290,16 @@ return CB.Captured; } -Instruction *llvm::FindEarliestCapture(const Value *V, Function &F, - bool ReturnCaptures, bool StoreCaptures, - const DominatorTree &DT, - unsigned MaxUsesToExplore) { +Instruction * +llvm::FindEarliestCapture(const Value *V, Function &F, bool ReturnCaptures, + bool StoreCaptures, const DominatorTree &DT, + + const SmallPtrSetImpl<const Value *> &EphValues, + unsigned MaxUsesToExplore) { assert(!isa<GlobalValue>(V) && "It doesn't make sense to ask whether a global is captured."); - EarliestCaptures CB(ReturnCaptures, F, DT); + EarliestCaptures CB(ReturnCaptures, F, DT, EphValues); PointerMayBeCaptured(V, &CB, MaxUsesToExplore); if (CB.Captured) ++NumCapturedBefore; @@ -282,6 +308,132 @@ return CB.EarliestCapture; } +UseCaptureKind llvm::DetermineUseCaptureKind( + const Use &U, + function_ref<bool(Value *, const DataLayout &)> IsDereferenceableOrNull) { + Instruction *I = cast<Instruction>(U.getUser()); + + switch (I->getOpcode()) { + case Instruction::Call: + case Instruction::Invoke: { + auto *Call = cast<CallBase>(I); + // Not captured if the callee is readonly, doesn't return a copy through + // its return value and doesn't unwind (a readonly function can leak bits + // by throwing an exception or not depending on the input value). + if (Call->onlyReadsMemory() && Call->doesNotThrow() && + Call->getType()->isVoidTy()) + return UseCaptureKind::NO_CAPTURE; + + // The pointer is not captured if returned pointer is not captured. + // NOTE: CaptureTracking users should not assume that only functions + // marked with nocapture do not capture. This means that places like + // getUnderlyingObject in ValueTracking or DecomposeGEPExpression + // in BasicAA also need to know about this property. + if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(Call, true)) + return UseCaptureKind::PASSTHROUGH; + + // Volatile operations effectively capture the memory location that they + // load and store to. + if (auto *MI = dyn_cast<MemIntrinsic>(Call)) + if (MI->isVolatile()) + return UseCaptureKind::MAY_CAPTURE; + + // Calling a function pointer does not in itself cause the pointer to + // be captured. This is a subtle point considering that (for example) + // the callee might return its own address. It is analogous to saying + // that loading a value from a pointer does not cause the pointer to be + // captured, even though the loaded value might be the pointer itself + // (think of self-referential objects). + if (Call->isCallee(&U)) + return UseCaptureKind::NO_CAPTURE; + + // Not captured if only passed via 'nocapture' arguments. + if (Call->isDataOperand(&U) && + !Call->doesNotCapture(Call->getDataOperandNo(&U))) { + // The parameter is not marked 'nocapture' - captured. + return UseCaptureKind::MAY_CAPTURE; + } + return UseCaptureKind::NO_CAPTURE; + } + case Instruction::Load: + // Volatile loads make the address observable. + if (cast<LoadInst>(I)->isVolatile()) + return UseCaptureKind::MAY_CAPTURE; + return UseCaptureKind::NO_CAPTURE; + case Instruction::VAArg: + // "va-arg" from a pointer does not cause it to be captured. + return UseCaptureKind::NO_CAPTURE; + case Instruction::Store: + // Stored the pointer - conservatively assume it may be captured. + // Volatile stores make the address observable. + if (U.getOperandNo() == 0 || cast<StoreInst>(I)->isVolatile()) + return UseCaptureKind::MAY_CAPTURE; + return UseCaptureKind::NO_CAPTURE; + case Instruction::AtomicRMW: { + // atomicrmw conceptually includes both a load and store from + // the same location. + // As with a store, the location being accessed is not captured, + // but the value being stored is. + // Volatile stores make the address observable. + auto *ARMWI = cast<AtomicRMWInst>(I); + if (U.getOperandNo() == 1 || ARMWI->isVolatile()) + return UseCaptureKind::MAY_CAPTURE; + return UseCaptureKind::NO_CAPTURE; + } + case Instruction::AtomicCmpXchg: { + // cmpxchg conceptually includes both a load and store from + // the same location. + // As with a store, the location being accessed is not captured, + // but the value being stored is. + // Volatile stores make the address observable. + auto *ACXI = cast<AtomicCmpXchgInst>(I); + if (U.getOperandNo() == 1 || U.getOperandNo() == 2 || ACXI->isVolatile()) + return UseCaptureKind::MAY_CAPTURE; + return UseCaptureKind::NO_CAPTURE; + } + case Instruction::BitCast: + case Instruction::GetElementPtr: + case Instruction::PHI: + case Instruction::Select: + case Instruction::AddrSpaceCast: + // The original value is not captured via this if the new value isn't. + return UseCaptureKind::PASSTHROUGH; + case Instruction::ICmp: { + unsigned Idx = U.getOperandNo(); + unsigned OtherIdx = 1 - Idx; + if (auto *CPN = dyn_cast<ConstantPointerNull>(I->getOperand(OtherIdx))) { + // Don't count comparisons of a no-alias return value against null as + // captures. This allows us to ignore comparisons of malloc results + // with null, for example. + if (CPN->getType()->getAddressSpace() == 0) + if (isNoAliasCall(U.get()->stripPointerCasts())) + return UseCaptureKind::NO_CAPTURE; + if (!I->getFunction()->nullPointerIsDefined()) { + auto *O = I->getOperand(Idx)->stripPointerCastsSameRepresentation(); + // Comparing a dereferenceable_or_null pointer against null cannot + // lead to pointer escapes, because if it is not null it must be a + // valid (in-bounds) pointer. + const DataLayout &DL = I->getModule()->getDataLayout(); + if (IsDereferenceableOrNull && IsDereferenceableOrNull(O, DL)) + return UseCaptureKind::NO_CAPTURE; + } + } + // Comparison against value stored in global variable. Given the pointer + // does not escape, its value cannot be guessed and stored separately in a + // global variable. + auto *LI = dyn_cast<LoadInst>(I->getOperand(OtherIdx)); + if (LI && isa<GlobalVariable>(LI->getPointerOperand())) + return UseCaptureKind::NO_CAPTURE; + // Otherwise, be conservative. There are crazy ways to capture pointers + // using comparisons. + return UseCaptureKind::MAY_CAPTURE; + } + default: + // Something else - be conservative and say it is captured. + return UseCaptureKind::MAY_CAPTURE; + } +} + void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker, unsigned MaxUsesToExplore) { assert(V->getType()->isPointerTy() && "Capture is for pointers only!"); @@ -293,11 +445,10 @@ SmallSet<const Use *, 20> Visited; auto AddUses = [&](const Value *V) { - unsigned Count = 0; for (const Use &U : V->uses()) { // If there are lots of uses, conservatively say that the value // is captured to avoid taking too much compile time. - if (Count++ >= MaxUsesToExplore) { + if (Visited.size() >= MaxUsesToExplore) { Tracker->tooManyUses(); return false; } @@ -312,144 +463,22 @@ if (!AddUses(V)) return; + auto IsDereferenceableOrNull = [Tracker](Value *V, const DataLayout &DL) { + return Tracker->isDereferenceableOrNull(V, DL); + }; while (!Worklist.empty()) { const Use *U = Worklist.pop_back_val(); - Instruction *I = cast<Instruction>(U->getUser()); - - switch (I->getOpcode()) { - case Instruction::Call: - case Instruction::Invoke: { - auto *Call = cast<CallBase>(I); - // Not captured if the callee is readonly, doesn't return a copy through - // its return value and doesn't unwind (a readonly function can leak bits - // by throwing an exception or not depending on the input value). - if (Call->onlyReadsMemory() && Call->doesNotThrow() && - Call->getType()->isVoidTy()) - break; - - // The pointer is not captured if returned pointer is not captured. - // NOTE: CaptureTracking users should not assume that only functions - // marked with nocapture do not capture. This means that places like - // getUnderlyingObject in ValueTracking or DecomposeGEPExpression - // in BasicAA also need to know about this property. - if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(Call, - true)) { - if (!AddUses(Call)) - return; - break; - } - - // Volatile operations effectively capture the memory location that they - // load and store to. - if (auto *MI = dyn_cast<MemIntrinsic>(Call)) - if (MI->isVolatile()) - if (Tracker->captured(U)) - return; - - // Calling a function pointer does not in itself cause the pointer to - // be captured. This is a subtle point considering that (for example) - // the callee might return its own address. It is analogous to saying - // that loading a value from a pointer does not cause the pointer to be - // captured, even though the loaded value might be the pointer itself - // (think of self-referential objects). - if (Call->isCallee(U)) - break; - - // Not captured if only passed via 'nocapture' arguments. - if (Call->isDataOperand(U) && - !Call->doesNotCapture(Call->getDataOperandNo(U))) { - // The parameter is not marked 'nocapture' - captured. - if (Tracker->captured(U)) - return; - } - break; - } - case Instruction::Load: - // Volatile loads make the address observable. - if (cast<LoadInst>(I)->isVolatile()) - if (Tracker->captured(U)) - return; - break; - case Instruction::VAArg: - // "va-arg" from a pointer does not cause it to be captured. - break; - case Instruction::Store: - // Stored the pointer - conservatively assume it may be captured. - // Volatile stores make the address observable. - if (U->getOperandNo() == 0 || cast<StoreInst>(I)->isVolatile()) - if (Tracker->captured(U)) - return; - break; - case Instruction::AtomicRMW: { - // atomicrmw conceptually includes both a load and store from - // the same location. - // As with a store, the location being accessed is not captured, - // but the value being stored is. - // Volatile stores make the address observable. - auto *ARMWI = cast<AtomicRMWInst>(I); - if (U->getOperandNo() == 1 || ARMWI->isVolatile()) - if (Tracker->captured(U)) - return; - break; - } - case Instruction::AtomicCmpXchg: { - // cmpxchg conceptually includes both a load and store from - // the same location. - // As with a store, the location being accessed is not captured, - // but the value being stored is. - // Volatile stores make the address observable. - auto *ACXI = cast<AtomicCmpXchgInst>(I); - if (U->getOperandNo() == 1 || U->getOperandNo() == 2 || - ACXI->isVolatile()) - if (Tracker->captured(U)) - return; - break; - } - case Instruction::BitCast: - case Instruction::GetElementPtr: - case Instruction::PHI: - case Instruction::Select: - case Instruction::AddrSpaceCast: - // The original value is not captured via this if the new value isn't. - if (!AddUses(I)) - return; - break; - case Instruction::ICmp: { - unsigned Idx = U->getOperandNo(); - unsigned OtherIdx = 1 - Idx; - if (auto *CPN = dyn_cast<ConstantPointerNull>(I->getOperand(OtherIdx))) { - // Don't count comparisons of a no-alias return value against null as - // captures. This allows us to ignore comparisons of malloc results - // with null, for example. - if (CPN->getType()->getAddressSpace() == 0) - if (isNoAliasCall(U->get()->stripPointerCasts())) - break; - if (!I->getFunction()->nullPointerIsDefined()) { - auto *O = I->getOperand(Idx)->stripPointerCastsSameRepresentation(); - // Comparing a dereferenceable_or_null pointer against null cannot - // lead to pointer escapes, because if it is not null it must be a - // valid (in-bounds) pointer. - if (Tracker->isDereferenceableOrNull(O, I->getModule()->getDataLayout())) - break; - } - } - // Comparison against value stored in global variable. Given the pointer - // does not escape, its value cannot be guessed and stored separately in a - // global variable. - auto *LI = dyn_cast<LoadInst>(I->getOperand(OtherIdx)); - if (LI && isa<GlobalVariable>(LI->getPointerOperand())) - break; - // Otherwise, be conservative. There are crazy ways to capture pointers - // using comparisons. + switch (DetermineUseCaptureKind(*U, IsDereferenceableOrNull)) { + case UseCaptureKind::NO_CAPTURE: + continue; + case UseCaptureKind::MAY_CAPTURE: if (Tracker->captured(U)) return; - break; - } - default: - // Something else - be conservative and say it is captured. - if (Tracker->captured(U)) + continue; + case UseCaptureKind::PASSTHROUGH: + if (!AddUses(U->getUser())) return; - break; + continue; } }
diff --git a/src/llvm-project/llvm/lib/Analysis/CmpInstAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/CmpInstAnalysis.cpp index 5b95198..20b1df6 100644 --- a/src/llvm-project/llvm/lib/Analysis/CmpInstAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CmpInstAnalysis.cpp
@@ -18,9 +18,7 @@ using namespace llvm; -unsigned llvm::getICmpCode(const ICmpInst *ICI, bool InvertPred) { - ICmpInst::Predicate Pred = InvertPred ? ICI->getInversePredicate() - : ICI->getPredicate(); +unsigned llvm::getICmpCode(CmpInst::Predicate Pred) { switch (Pred) { // False -> 0 case ICmpInst::ICMP_UGT: return 1; // 001 @@ -63,6 +61,18 @@ (CmpInst::isSigned(P2) && ICmpInst::isEquality(P1)); } +Constant *llvm::getPredForFCmpCode(unsigned Code, Type *OpTy, + CmpInst::Predicate &Pred) { + Pred = static_cast<FCmpInst::Predicate>(Code); + assert(FCmpInst::FCMP_FALSE <= Pred && Pred <= FCmpInst::FCMP_TRUE && + "Unexpected FCmp predicate!"); + if (Pred == FCmpInst::FCMP_FALSE) + return ConstantInt::get(CmpInst::makeCmpResultType(OpTy), 0); + if (Pred == FCmpInst::FCMP_TRUE) + return ConstantInt::get(CmpInst::makeCmpResultType(OpTy), 1); + return nullptr; +} + bool llvm::decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate &Pred, Value *&X, APInt &Mask, bool LookThruTrunc) {
diff --git a/src/llvm-project/llvm/lib/Analysis/CodeMetrics.cpp b/src/llvm-project/llvm/lib/Analysis/CodeMetrics.cpp index 27c5250..ded842b 100644 --- a/src/llvm-project/llvm/lib/Analysis/CodeMetrics.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CodeMetrics.cpp
@@ -15,7 +15,6 @@ #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Function.h" #include "llvm/Support/Debug.h" #include "llvm/Support/InstructionCost.h" @@ -118,13 +117,6 @@ const BasicBlock *BB, const TargetTransformInfo &TTI, const SmallPtrSetImpl<const Value *> &EphValues, bool PrepareForLTO) { ++NumBlocks; - // Use a proxy variable for NumInsts of type InstructionCost, so that it can - // use InstructionCost's arithmetic properties such as saturation when this - // feature is added to InstructionCost. - // When storing the value back to NumInsts, we can assume all costs are Valid - // because the IR should not contain any nodes that cannot be costed. If that - // happens the cost-model is broken. - InstructionCost NumInstsProxy = NumInsts; InstructionCost NumInstsBeforeThisBB = NumInsts; for (const Instruction &I : *BB) { // Skip ephemeral values. @@ -141,7 +133,8 @@ // When preparing for LTO, liberally consider calls as inline // candidates. if (!Call->isNoInline() && IsLoweredToCall && - ((F->hasInternalLinkage() && F->hasOneUse()) || PrepareForLTO)) { + ((F->hasInternalLinkage() && F->hasOneLiveUse()) || + PrepareForLTO)) { ++NumInlineCandidates; } @@ -184,8 +177,7 @@ if (InvI->cannotDuplicate()) notDuplicatable = true; - NumInstsProxy += TTI.getUserCost(&I, TargetTransformInfo::TCK_CodeSize); - NumInsts = *NumInstsProxy.getValue(); + NumInsts += TTI.getUserCost(&I, TargetTransformInfo::TCK_CodeSize); } if (isa<ReturnInst>(BB->getTerminator())) @@ -205,6 +197,6 @@ notDuplicatable |= isa<IndirectBrInst>(BB->getTerminator()); // Remember NumInsts for this BB. - InstructionCost NumInstsThisBB = NumInstsProxy - NumInstsBeforeThisBB; - NumBBInsts[BB] = *NumInstsThisBB.getValue(); + InstructionCost NumInstsThisBB = NumInsts - NumInstsBeforeThisBB; + NumBBInsts[BB] = NumInstsThisBB; }
diff --git a/src/llvm-project/llvm/lib/Analysis/ConstantFolding.cpp b/src/llvm-project/llvm/lib/Analysis/ConstantFolding.cpp index f6b9551..ae927da 100644 --- a/src/llvm-project/llvm/lib/Analysis/ConstantFolding.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ConstantFolding.cpp
@@ -30,6 +30,7 @@ #include "llvm/Analysis/VectorUtils.h" #include "llvm/Config/config.h" #include "llvm/IR/Constant.h" +#include "llvm/IR/ConstantFold.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" @@ -57,7 +58,6 @@ #include <cerrno> #include <cfenv> #include <cmath> -#include <cstddef> #include <cstdint> using namespace llvm; @@ -92,7 +92,7 @@ return ConstantExpr::getBitCast(C, DestTy); Result <<= BitShift; - Result |= ElementCI->getValue().zextOrSelf(Result.getBitWidth()); + Result |= ElementCI->getValue().zext(Result.getBitWidth()); } return nullptr; @@ -634,6 +634,39 @@ return ConstantInt::get(IntType->getContext(), ResultVal); } +} // anonymous namespace + +// If GV is a constant with an initializer read its representation starting +// at Offset and return it as a constant array of unsigned char. Otherwise +// return null. +Constant *llvm::ReadByteArrayFromGlobal(const GlobalVariable *GV, + uint64_t Offset) { + if (!GV->isConstant() || !GV->hasDefinitiveInitializer()) + return nullptr; + + const DataLayout &DL = GV->getParent()->getDataLayout(); + Constant *Init = const_cast<Constant *>(GV->getInitializer()); + TypeSize InitSize = DL.getTypeAllocSize(Init->getType()); + if (InitSize < Offset) + return nullptr; + + uint64_t NBytes = InitSize - Offset; + if (NBytes > UINT16_MAX) + // Bail for large initializers in excess of 64K to avoid allocating + // too much memory. + // Offset is assumed to be less than or equal than InitSize (this + // is enforced in ReadDataFromGlobal). + return nullptr; + + SmallVector<unsigned char, 256> RawBytes(static_cast<size_t>(NBytes)); + unsigned char *CurPtr = RawBytes.data(); + + if (!ReadDataFromGlobal(Init, Offset, CurPtr, NBytes, DL)) + return nullptr; + + return ConstantDataArray::get(GV->getContext(), RawBytes); +} + /// If this Offset points exactly to the start of an aggregate element, return /// that element, otherwise return nullptr. Constant *getConstantAtOffset(Constant *Base, APInt Offset, @@ -662,8 +695,6 @@ return C; } -} // end anonymous namespace - Constant *llvm::ConstantFoldLoadFromConst(Constant *C, Type *Ty, const APInt &Offset, const DataLayout &DL) { @@ -867,21 +898,6 @@ Type *IntIdxTy = DL.getIndexType(Ptr->getType()); - // If this is "gep i8* Ptr, (sub 0, V)", fold this as: - // "inttoptr (sub (ptrtoint Ptr), V)" - if (Ops.size() == 2 && ResElemTy->isIntegerTy(8)) { - auto *CE = dyn_cast<ConstantExpr>(Ops[1]); - assert((!CE || CE->getType() == IntIdxTy) && - "CastGEPIndices didn't canonicalize index types!"); - if (CE && CE->getOpcode() == Instruction::Sub && - CE->getOperand(0)->isNullValue()) { - Constant *Res = ConstantExpr::getPtrToInt(Ptr, CE->getType()); - Res = ConstantExpr::getSub(Res, CE->getOperand(1)); - Res = ConstantExpr::getIntToPtr(Res, ResTy); - return ConstantFoldConstant(Res, DL, TLI); - } - } - for (unsigned i = 1, e = Ops.size(); i != e; ++i) if (!isa<ConstantInt>(Ops[i])) return nullptr; @@ -1015,8 +1031,24 @@ if (Instruction::isUnaryOp(Opcode)) return ConstantFoldUnaryOpOperand(Opcode, Ops[0], DL); - if (Instruction::isBinaryOp(Opcode)) + if (Instruction::isBinaryOp(Opcode)) { + switch (Opcode) { + default: + break; + case Instruction::FAdd: + case Instruction::FSub: + case Instruction::FMul: + case Instruction::FDiv: + case Instruction::FRem: + // Handle floating point instructions separately to account for denormals + // TODO: If a constant expression is being folded rather than an + // instruction, denormals will not be flushed/treated as zero + if (const auto *I = dyn_cast<Instruction>(InstOrCE)) { + return ConstantFoldFPInstOperands(Opcode, Ops[0], Ops[1], DL, I); + } + } return ConstantFoldBinaryOpOperands(Opcode, Ops[0], Ops[1], DL); + } if (Instruction::isCast(Opcode)) return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL); @@ -1030,13 +1062,21 @@ GEP->getInRangeIndex()); } - if (auto *CE = dyn_cast<ConstantExpr>(InstOrCE)) + if (auto *CE = dyn_cast<ConstantExpr>(InstOrCE)) { + if (CE->isCompare()) + return ConstantFoldCompareInstOperands(CE->getPredicate(), Ops[0], Ops[1], + DL, TLI); return CE->getWithOperands(Ops); + } switch (Opcode) { default: return nullptr; case Instruction::ICmp: - case Instruction::FCmp: llvm_unreachable("Invalid for compares"); + case Instruction::FCmp: { + auto *C = cast<CmpInst>(InstOrCE); + return ConstantFoldCompareInstOperands(C->getPredicate(), Ops[0], Ops[1], + DL, TLI, C); + } case Instruction::Freeze: return isGuaranteedNotToBeUndefOrPoison(Ops[0]) ? Ops[0] : nullptr; case Instruction::Call: @@ -1051,13 +1091,22 @@ case Instruction::ExtractElement: return ConstantExpr::getExtractElement(Ops[0], Ops[1]); case Instruction::ExtractValue: - return ConstantExpr::getExtractValue( + return ConstantFoldExtractValueInstruction( Ops[0], cast<ExtractValueInst>(InstOrCE)->getIndices()); case Instruction::InsertElement: return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]); + case Instruction::InsertValue: + return ConstantFoldInsertValueInstruction( + Ops[0], Ops[1], cast<InsertValueInst>(InstOrCE)->getIndices()); case Instruction::ShuffleVector: return ConstantExpr::getShuffleVector( Ops[0], Ops[1], cast<ShuffleVectorInst>(InstOrCE)->getShuffleMask()); + case Instruction::Load: { + const auto *LI = dyn_cast<LoadInst>(InstOrCE); + if (LI->isVolatile()) + return nullptr; + return ConstantFoldLoadFromConstPtr(Ops[0], LI->getType(), DL); + } } } @@ -1095,11 +1144,10 @@ } if (auto *CE = dyn_cast<ConstantExpr>(C)) { - if (CE->isCompare()) - return ConstantFoldCompareInstOperands(CE->getPredicate(), Ops[0], Ops[1], - DL, TLI); - - return ConstantFoldInstOperandsImpl(CE, CE->getOpcode(), Ops, DL, TLI); + if (Constant *Res = + ConstantFoldInstOperandsImpl(CE, CE->getOpcode(), Ops, DL, TLI)) + return Res; + return const_cast<Constant *>(C); } assert(isa<ConstantVector>(C)); @@ -1153,22 +1201,6 @@ Ops.push_back(Op); } - if (const auto *CI = dyn_cast<CmpInst>(I)) - return ConstantFoldCompareInstOperands(CI->getPredicate(), Ops[0], Ops[1], - DL, TLI); - - if (const auto *LI = dyn_cast<LoadInst>(I)) { - if (LI->isVolatile()) - return nullptr; - return ConstantFoldLoadFromConstPtr(Ops[0], LI->getType(), DL); - } - - if (auto *IVI = dyn_cast<InsertValueInst>(I)) - return ConstantExpr::getInsertValue(Ops[0], Ops[1], IVI->getIndices()); - - if (auto *EVI = dyn_cast<ExtractValueInst>(I)) - return ConstantExpr::getExtractValue(Ops[0], EVI->getIndices()); - return ConstantFoldInstOperands(I, Ops, DL, TLI); } @@ -1185,10 +1217,9 @@ return ConstantFoldInstOperandsImpl(I, I->getOpcode(), Ops, DL, TLI); } -Constant *llvm::ConstantFoldCompareInstOperands(unsigned IntPredicate, - Constant *Ops0, Constant *Ops1, - const DataLayout &DL, - const TargetLibraryInfo *TLI) { +Constant *llvm::ConstantFoldCompareInstOperands( + unsigned IntPredicate, Constant *Ops0, Constant *Ops1, const DataLayout &DL, + const TargetLibraryInfo *TLI, const Instruction *I) { CmpInst::Predicate Predicate = (CmpInst::Predicate)IntPredicate; // fold: icmp (inttoptr x), null -> icmp x, 0 // fold: icmp null, (inttoptr x) -> icmp 0, x @@ -1290,6 +1321,11 @@ return ConstantFoldCompareInstOperands(Predicate, Ops1, Ops0, DL, TLI); } + // Flush any denormal constant float input according to denormal handling + // mode. + Ops0 = FlushFPConstant(Ops0, I, /* IsOutput */ false); + Ops1 = FlushFPConstant(Ops1, I, /* IsOutput */ false); + return ConstantExpr::getCompare(Predicate, Ops0, Ops1); } @@ -1308,7 +1344,68 @@ if (Constant *C = SymbolicallyEvaluateBinop(Opcode, LHS, RHS, DL)) return C; - return ConstantExpr::get(Opcode, LHS, RHS); + if (ConstantExpr::isDesirableBinOp(Opcode)) + return ConstantExpr::get(Opcode, LHS, RHS); + return ConstantFoldBinaryInstruction(Opcode, LHS, RHS); +} + +Constant *llvm::FlushFPConstant(Constant *Operand, const Instruction *I, + bool IsOutput) { + if (!I || !I->getParent() || !I->getFunction()) + return Operand; + + ConstantFP *CFP = dyn_cast<ConstantFP>(Operand); + if (!CFP) + return Operand; + + const APFloat &APF = CFP->getValueAPF(); + Type *Ty = CFP->getType(); + DenormalMode DenormMode = + I->getFunction()->getDenormalMode(Ty->getFltSemantics()); + DenormalMode::DenormalModeKind Mode = + IsOutput ? DenormMode.Output : DenormMode.Input; + switch (Mode) { + default: + llvm_unreachable("unknown denormal mode"); + return Operand; + case DenormalMode::IEEE: + return Operand; + case DenormalMode::PreserveSign: + if (APF.isDenormal()) { + return ConstantFP::get( + Ty->getContext(), + APFloat::getZero(Ty->getFltSemantics(), APF.isNegative())); + } + return Operand; + case DenormalMode::PositiveZero: + if (APF.isDenormal()) { + return ConstantFP::get(Ty->getContext(), + APFloat::getZero(Ty->getFltSemantics(), false)); + } + return Operand; + } + return Operand; +} + +Constant *llvm::ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS, + Constant *RHS, const DataLayout &DL, + const Instruction *I) { + if (Instruction::isBinaryOp(Opcode)) { + // Flush denormal inputs if needed. + Constant *Op0 = FlushFPConstant(LHS, I, /* IsOutput */ false); + Constant *Op1 = FlushFPConstant(RHS, I, /* IsOutput */ false); + + // Calculate constant result. + Constant *C = ConstantFoldBinaryOpOperands(Opcode, Op0, Op1, DL); + if (!C) + return nullptr; + + // Flush denormal output if needed. + return FlushFPConstant(C, I, /* IsOutput */ true); + } + // If instruction lacks a parent/function and the denormal mode cannot be + // determined, use the default (IEEE). + return ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL); } Constant *llvm::ConstantFoldCastOperand(unsigned Opcode, Constant *C, @@ -1337,6 +1434,19 @@ DL, BaseOffset, /*AllowNonInbounds=*/true)); if (Base->isNullValue()) { FoldedValue = ConstantInt::get(CE->getContext(), BaseOffset); + } else { + // ptrtoint (gep i8, Ptr, (sub 0, V)) -> sub (ptrtoint Ptr), V + if (GEP->getNumIndices() == 1 && + GEP->getSourceElementType()->isIntegerTy(8)) { + auto *Ptr = cast<Constant>(GEP->getPointerOperand()); + auto *Sub = dyn_cast<ConstantExpr>(GEP->getOperand(1)); + Type *IntIdxTy = DL.getIndexType(Ptr->getType()); + if (Sub && Sub->getType() == IntIdxTy && + Sub->getOpcode() == Instruction::Sub && + Sub->getOperand(0)->isNullValue()) + FoldedValue = ConstantExpr::getSub( + ConstantExpr::getPtrToInt(Ptr, IntIdxTy), Sub->getOperand(1)); + } } } if (FoldedValue) { @@ -1389,6 +1499,8 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) { if (Call->isNoBuiltin()) return false; + if (Call->getFunctionType() != F->getFunctionType()) + return false; switch (F->getIntrinsicID()) { // Operations that do not operate floating-point numbers and do not depend on // FP environment can be folded even in strictfp functions. @@ -1530,6 +1642,8 @@ case Intrinsic::experimental_constrained_trunc: case Intrinsic::experimental_constrained_nearbyint: case Intrinsic::experimental_constrained_rint: + case Intrinsic::experimental_constrained_fcmp: + case Intrinsic::experimental_constrained_fcmps: return true; default: return false; @@ -1801,12 +1915,12 @@ // If evaluation raised FP exception, the result can depend on rounding // mode. If the latter is unknown, folding is not possible. - if (!ORM || *ORM == RoundingMode::Dynamic) + if (ORM && *ORM == RoundingMode::Dynamic) return false; // If FP exceptions are ignored, fold the call, even if such exception is // raised. - if (!EB || *EB != fp::ExceptionBehavior::ebStrict) + if (EB && *EB != fp::ExceptionBehavior::ebStrict) return true; // Leave the calculation for runtime so that exception flags be correctly set @@ -1982,7 +2096,7 @@ case Intrinsic::experimental_constrained_rint: { auto CI = cast<ConstrainedFPIntrinsic>(Call); RM = CI->getRoundingMode(); - if (!RM || RM.getValue() == RoundingMode::Dynamic) + if (!RM || *RM == RoundingMode::Dynamic) return nullptr; break; } @@ -2304,6 +2418,24 @@ return nullptr; } +static Constant *evaluateCompare(const APFloat &Op1, const APFloat &Op2, + const ConstrainedFPIntrinsic *Call) { + APFloat::opStatus St = APFloat::opOK; + auto *FCmp = cast<ConstrainedFPCmpIntrinsic>(Call); + FCmpInst::Predicate Cond = FCmp->getPredicate(); + if (FCmp->isSignaling()) { + if (Op1.isNaN() || Op2.isNaN()) + St = APFloat::opInvalidOp; + } else { + if (Op1.isSignaling() || Op2.isSignaling()) + St = APFloat::opInvalidOp; + } + bool Result = FCmpInst::compare(Op1, Op2, Cond); + if (mayFoldConstrained(const_cast<ConstrainedFPCmpIntrinsic *>(FCmp), St)) + return ConstantInt::get(Call->getType()->getScalarType(), Result); + return nullptr; +} + static Constant *ConstantFoldScalarCall2(StringRef Name, Intrinsic::ID IntrinsicID, Type *Ty, @@ -2332,8 +2464,6 @@ } if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) { - if (!Ty->isFloatingPointTy()) - return nullptr; const APFloat &Op1V = Op1->getValueAPF(); if (const auto *Op2 = dyn_cast<ConstantFP>(Operands[1])) { @@ -2363,6 +2493,9 @@ case Intrinsic::experimental_constrained_frem: St = Res.mod(Op2V); break; + case Intrinsic::experimental_constrained_fcmp: + case Intrinsic::experimental_constrained_fcmps: + return evaluateCompare(Op1V, Op2V, ConstrIntr); } if (mayFoldConstrained(const_cast<ConstrainedFPIntrinsic *>(ConstrIntr), St)) @@ -2487,6 +2620,11 @@ case Intrinsic::smin: case Intrinsic::umax: case Intrinsic::umin: + // This is the same as for binary ops - poison propagates. + // TODO: Poison handling should be consolidated. + if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1])) + return PoisonValue::get(Ty); + if (!C0 && !C1) return UndefValue::get(Ty); if (!C0 || !C1) @@ -2553,6 +2691,11 @@ } case Intrinsic::uadd_sat: case Intrinsic::sadd_sat: + // This is the same as for binary ops - poison propagates. + // TODO: Poison handling should be consolidated. + if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1])) + return PoisonValue::get(Ty); + if (!C0 && !C1) return UndefValue::get(Ty); if (!C0 || !C1) @@ -2563,6 +2706,11 @@ return ConstantInt::get(Ty, C0->sadd_sat(*C1)); case Intrinsic::usub_sat: case Intrinsic::ssub_sat: + // This is the same as for binary ops - poison propagates. + // TODO: Poison handling should be consolidated. + if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1])) + return PoisonValue::get(Ty); + if (!C0 && !C1) return UndefValue::get(Ty); if (!C0 || !C1) @@ -2843,11 +2991,11 @@ unsigned Width = C0->getBitWidth(); assert(Scale < Width && "Illegal scale."); unsigned ExtendedWidth = Width * 2; - APInt Product = (C0->sextOrSelf(ExtendedWidth) * - C1->sextOrSelf(ExtendedWidth)).ashr(Scale); + APInt Product = + (C0->sext(ExtendedWidth) * C1->sext(ExtendedWidth)).ashr(Scale); if (IntrinsicID == Intrinsic::smul_fix_sat) { - APInt Max = APInt::getSignedMaxValue(Width).sextOrSelf(ExtendedWidth); - APInt Min = APInt::getSignedMinValue(Width).sextOrSelf(ExtendedWidth); + APInt Max = APInt::getSignedMaxValue(Width).sext(ExtendedWidth); + APInt Min = APInt::getSignedMinValue(Width).sext(ExtendedWidth); Product = APIntOps::smin(Product, Max); Product = APIntOps::smax(Product, Min); } @@ -2972,7 +3120,7 @@ } return ConstantVector::get(NCs); } - break; + return nullptr; } case Intrinsic::get_active_lane_mask: { auto *Op0 = dyn_cast<ConstantInt>(Operands[0]); @@ -2991,7 +3139,7 @@ } return ConstantVector::get(NCs); } - break; + return nullptr; } default: break; @@ -3001,7 +3149,7 @@ // Gather a column of constants. for (unsigned J = 0, JE = Operands.size(); J != JE; ++J) { // Some intrinsics use a scalar type for certain arguments. - if (hasVectorInstrinsicScalarOpd(IntrinsicID, J)) { + if (isVectorIntrinsicWithScalarOpAtArg(IntrinsicID, J)) { Lane[J] = Operands[J]; continue; }
diff --git a/src/llvm-project/llvm/lib/Analysis/ConstraintSystem.cpp b/src/llvm-project/llvm/lib/Analysis/ConstraintSystem.cpp index 773f71a..2efa474 100644 --- a/src/llvm-project/llvm/lib/Analysis/ConstraintSystem.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ConstraintSystem.cpp
@@ -12,7 +12,6 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/Support/Debug.h" -#include <algorithm> #include <string> using namespace llvm; @@ -111,7 +110,7 @@ if (Constraints.empty()) return; - for (auto &Row : Constraints) { + for (const auto &Row : Constraints) { SmallVector<std::string, 16> Parts; for (unsigned I = 1, S = Row.size(); I < S; ++I) { if (Row[I] == 0)
diff --git a/src/llvm-project/llvm/lib/Analysis/CostModel.cpp b/src/llvm-project/llvm/lib/Analysis/CostModel.cpp index 326baca..3c162f6 100644 --- a/src/llvm-project/llvm/lib/Analysis/CostModel.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CostModel.cpp
@@ -17,7 +17,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/CostModel.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/Passes.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/Function.h" @@ -25,8 +24,8 @@ #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/IR/IntrinsicInst.h" using namespace llvm; static cl::opt<TargetTransformInfo::TargetCostKind> CostKind( @@ -41,6 +40,9 @@ clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency"))); +static cl::opt<bool> TypeBasedIntrinsicCost("type-based-intrinsic-cost", + cl::desc("Calculate intrinsics cost based only on argument types"), + cl::init(false)); #define CM_NAME "cost-model" #define DEBUG_TYPE CM_NAME @@ -105,7 +107,16 @@ for (BasicBlock &B : *F) { for (Instruction &Inst : B) { - InstructionCost Cost = TTI->getInstructionCost(&Inst, CostKind); + InstructionCost Cost; + if (TypeBasedIntrinsicCost && isa<IntrinsicInst>(&Inst)) { + auto *II = dyn_cast<IntrinsicInst>(&Inst); + IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II, + InstructionCost::getInvalid(), true); + Cost = TTI->getIntrinsicInstrCost(ICA, CostKind); + } + else { + Cost = TTI->getInstructionCost(&Inst, CostKind); + } if (auto CostVal = Cost.getValue()) OS << "Cost Model: Found an estimated cost of " << *CostVal; else @@ -119,12 +130,21 @@ PreservedAnalyses CostModelPrinterPass::run(Function &F, FunctionAnalysisManager &AM) { auto &TTI = AM.getResult<TargetIRAnalysis>(F); - OS << "Cost Model for function '" << F.getName() << "'\n"; + OS << "Printing analysis 'Cost Model Analysis' for function '" << F.getName() << "':\n"; for (BasicBlock &B : F) { for (Instruction &Inst : B) { // TODO: Use a pass parameter instead of cl::opt CostKind to determine // which cost kind to print. - InstructionCost Cost = TTI.getInstructionCost(&Inst, CostKind); + InstructionCost Cost; + if (TypeBasedIntrinsicCost && isa<IntrinsicInst>(&Inst)) { + auto *II = dyn_cast<IntrinsicInst>(&Inst); + IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II, + InstructionCost::getInvalid(), true); + Cost = TTI.getIntrinsicInstrCost(ICA, CostKind); + } + else { + Cost = TTI.getInstructionCost(&Inst, CostKind); + } if (auto CostVal = Cost.getValue()) OS << "Cost Model: Found an estimated cost of " << *CostVal; else
diff --git a/src/llvm-project/llvm/lib/Analysis/CycleAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/CycleAnalysis.cpp index 09c7ee6..1799812 100644 --- a/src/llvm-project/llvm/lib/Analysis/CycleAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/CycleAnalysis.cpp
@@ -8,11 +8,15 @@ #include "llvm/Analysis/CycleAnalysis.h" #include "llvm/ADT/GenericCycleImpl.h" -#include "llvm/IR/CFG.h" +#include "llvm/IR/CFG.h" // for successors found by ADL in GenericCycleImpl.h #include "llvm/InitializePasses.h" using namespace llvm; +namespace llvm { +class Module; +} + template class llvm::GenericCycleInfo<SSAContext>; template class llvm::GenericCycle<SSAContext>;
diff --git a/src/llvm-project/llvm/lib/Analysis/DDG.cpp b/src/llvm-project/llvm/lib/Analysis/DDG.cpp index 7e13579..da64ef1 100644 --- a/src/llvm-project/llvm/lib/Analysis/DDG.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DDG.cpp
@@ -17,13 +17,12 @@ using namespace llvm; static cl::opt<bool> SimplifyDDG( - "ddg-simplify", cl::init(true), cl::Hidden, cl::ZeroOrMore, + "ddg-simplify", cl::init(true), cl::Hidden, cl::desc( "Simplify DDG by merging nodes that have less interesting edges.")); -static cl::opt<bool> - CreatePiBlocks("ddg-pi-blocks", cl::init(true), cl::Hidden, cl::ZeroOrMore, - cl::desc("Create pi-block nodes.")); +static cl::opt<bool> CreatePiBlocks("ddg-pi-blocks", cl::init(true), cl::Hidden, + cl::desc("Create pi-block nodes.")); #define DEBUG_TYPE "ddg" @@ -34,7 +33,7 @@ //===--------------------------------------------------------------------===// // DDGNode implementation //===--------------------------------------------------------------------===// -DDGNode::~DDGNode() {} +DDGNode::~DDGNode() = default; bool DDGNode::collectInstructions( llvm::function_ref<bool(Instruction *)> const &Pred, @@ -96,7 +95,7 @@ llvm_unreachable("unimplemented type of node"); OS << (N.getEdges().empty() ? " Edges:none!\n" : " Edges:\n"); - for (auto &E : N.getEdges()) + for (const auto &E : N.getEdges()) OS.indent(2) << *E; return OS; } @@ -189,7 +188,7 @@ // Put the basic blocks in program order for correct dependence // directions. BasicBlockListType BBList; - for (auto &SCC : make_range(scc_begin(&F), scc_end(&F))) + for (const auto &SCC : make_range(scc_begin(&F), scc_end(&F))) append_range(BBList, SCC); std::reverse(BBList.begin(), BBList.end()); DDGBuilder(*this, D, BBList).populate();
diff --git a/src/llvm-project/llvm/lib/Analysis/DDGPrinter.cpp b/src/llvm-project/llvm/lib/Analysis/DDGPrinter.cpp index 0d5a936..6b5acd2 100644 --- a/src/llvm-project/llvm/lib/Analysis/DDGPrinter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DDGPrinter.cpp
@@ -18,8 +18,8 @@ using namespace llvm; -static cl::opt<bool> DotOnly("dot-ddg-only", cl::init(false), cl::Hidden, - cl::ZeroOrMore, cl::desc("simple ddg dot graph")); +static cl::opt<bool> DotOnly("dot-ddg-only", cl::Hidden, + cl::desc("simple ddg dot graph")); static cl::opt<std::string> DDGDotFilenamePrefix( "dot-ddg-filename-prefix", cl::init("ddg"), cl::Hidden, cl::desc("The prefix used for the DDG dot file names."));
diff --git a/src/llvm-project/llvm/lib/Analysis/Delinearization.cpp b/src/llvm-project/llvm/lib/Analysis/Delinearization.cpp index 670532c..7ab91b9 100644 --- a/src/llvm-project/llvm/lib/Analysis/Delinearization.cpp +++ b/src/llvm-project/llvm/lib/Analysis/Delinearization.cpp
@@ -24,9 +24,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/IR/PassManager.h" -#include "llvm/IR/Type.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" @@ -135,7 +133,7 @@ if (auto *Mul = dyn_cast<SCEVMulExpr>(S)) { bool HasAddRec = false; SmallVector<const SCEV *, 0> Operands; - for (auto Op : Mul->operands()) { + for (const auto *Op : Mul->operands()) { const SCEVUnknown *Unknown = dyn_cast<SCEVUnknown>(Op); if (Unknown && !isa<CallInst>(Unknown->getValue())) { Operands.push_back(Op); @@ -523,6 +521,44 @@ return !Subscripts.empty(); } +bool llvm::tryDelinearizeFixedSizeImpl( + ScalarEvolution *SE, Instruction *Inst, const SCEV *AccessFn, + SmallVectorImpl<const SCEV *> &Subscripts, SmallVectorImpl<int> &Sizes) { + Value *SrcPtr = getLoadStorePointerOperand(Inst); + + // Check the simple case where the array dimensions are fixed size. + auto *SrcGEP = dyn_cast<GetElementPtrInst>(SrcPtr); + if (!SrcGEP) + return false; + + getIndexExpressionsFromGEP(*SE, SrcGEP, Subscripts, Sizes); + + // Check that the two size arrays are non-empty and equal in length and + // value. + // TODO: it would be better to let the caller to clear Subscripts, similar + // to how we handle Sizes. + if (Sizes.empty() || Subscripts.size() <= 1) { + Subscripts.clear(); + return false; + } + + // Check that for identical base pointers we do not miss index offsets + // that have been added before this GEP is applied. + Value *SrcBasePtr = SrcGEP->getOperand(0)->stripPointerCasts(); + const SCEVUnknown *SrcBase = + dyn_cast<SCEVUnknown>(SE->getPointerBase(AccessFn)); + if (!SrcBase || SrcBasePtr != SrcBase->getValue()) { + Subscripts.clear(); + return false; + } + + assert(Subscripts.size() == Sizes.size() + 1 && + "Expected equal number of entries in the list of size and " + "subscript."); + + return true; +} + namespace { class Delinearization : public FunctionPass {
diff --git a/src/llvm-project/llvm/lib/Analysis/DemandedBits.cpp b/src/llvm-project/llvm/lib/Analysis/DemandedBits.cpp index 117b12f..e01ed48 100644 --- a/src/llvm-project/llvm/lib/Analysis/DemandedBits.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DemandedBits.cpp
@@ -21,19 +21,13 @@ #include "llvm/Analysis/DemandedBits.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/InstIterator.h" -#include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/DependenceAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/DependenceAnalysis.cpp index f827f74..3d2d84e 100644 --- a/src/llvm-project/llvm/lib/Analysis/DependenceAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -50,7 +50,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/DependenceAnalysis.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/Delinearization.h" @@ -58,10 +57,8 @@ #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/Config/llvm-config.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Module.h" -#include "llvm/IR/Operator.h" #include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -109,11 +106,10 @@ STATISTIC(BanerjeeSuccesses, "Banerjee successes"); static cl::opt<bool> - Delinearize("da-delinearize", cl::init(true), cl::Hidden, cl::ZeroOrMore, + Delinearize("da-delinearize", cl::init(true), cl::Hidden, cl::desc("Try to delinearize array references.")); static cl::opt<bool> DisableDelinearizationChecks( - "da-disable-delinearization-checks", cl::init(false), cl::Hidden, - cl::ZeroOrMore, + "da-disable-delinearization-checks", cl::Hidden, cl::desc( "Disable checks that try to statically verify validity of " "delinearized subscripts. Enabling this option may result in incorrect " @@ -121,7 +117,7 @@ "dimension to underflow or overflow into another dimension.")); static cl::opt<unsigned> MIVMaxLevelThreshold( - "da-miv-max-level-threshold", cl::init(7), cl::Hidden, cl::ZeroOrMore, + "da-miv-max-level-threshold", cl::init(7), cl::Hidden, cl::desc("Maximum depth allowed for the recursive algorithm used to " "explore MIV direction vectors.")); @@ -787,6 +783,8 @@ unsigned DependenceInfo::mapDstLoop(const Loop *DstLoop) const { unsigned D = DstLoop->getLoopDepth(); if (D > CommonLevels) + // This tries to make sure that we assign unique numbers to src and dst when + // the memory accesses reside in different loops that have the same depth. return D - CommonLevels + SrcLevels; else return D; @@ -796,10 +794,16 @@ // Returns true if Expression is loop invariant in LoopNest. bool DependenceInfo::isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const { + // Unlike ScalarEvolution::isLoopInvariant() we consider an access outside of + // any loop as invariant, because we only consier expression evaluation at a + // specific position (where the array access takes place), and not across the + // entire function. if (!LoopNest) return true; - return SE->isLoopInvariant(Expression, LoopNest) && - isLoopInvariant(Expression, LoopNest->getParentLoop()); + + // If the expression is invariant in the outermost loop of the loop nest, it + // is invariant anywhere in the loop nest. + return SE->isLoopInvariant(Expression, LoopNest->getOutermostLoop()); } @@ -890,13 +894,25 @@ } } -// Examine the scev and return true iff it's linear. +// Examine the scev and return true iff it's affine. // Collect any loops mentioned in the set of "Loops". bool DependenceInfo::checkSubscript(const SCEV *Expr, const Loop *LoopNest, SmallBitVector &Loops, bool IsSrc) { const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Expr); if (!AddRec) return isLoopInvariant(Expr, LoopNest); + + // The AddRec must depend on one of the containing loops. Otherwise, + // mapSrcLoop and mapDstLoop return indices outside the intended range. This + // can happen when a subscript in one loop references an IV from a sibling + // loop that could not be replaced with a concrete exit value by + // getSCEVAtScope. + const Loop *L = LoopNest; + while (L && AddRec->getLoop() != L) + L = L->getParentLoop(); + if (!L) + return false; + const SCEV *Start = AddRec->getStart(); const SCEV *Step = AddRec->getStepRecurrence(*SE); const SCEV *UB = SE->getBackedgeTakenCount(AddRec->getLoop()); @@ -3318,59 +3334,45 @@ return true; } +/// Try to delinearize \p SrcAccessFn and \p DstAccessFn if the underlying +/// arrays accessed are fixed-size arrays. Return true if delinearization was +/// successful. bool DependenceInfo::tryDelinearizeFixedSize( Instruction *Src, Instruction *Dst, const SCEV *SrcAccessFn, const SCEV *DstAccessFn, SmallVectorImpl<const SCEV *> &SrcSubscripts, SmallVectorImpl<const SCEV *> &DstSubscripts) { + LLVM_DEBUG({ + const SCEVUnknown *SrcBase = + dyn_cast<SCEVUnknown>(SE->getPointerBase(SrcAccessFn)); + const SCEVUnknown *DstBase = + dyn_cast<SCEVUnknown>(SE->getPointerBase(DstAccessFn)); + assert(SrcBase && DstBase && SrcBase == DstBase && + "expected src and dst scev unknowns to be equal"); + }); - Value *SrcPtr = getLoadStorePointerOperand(Src); - Value *DstPtr = getLoadStorePointerOperand(Dst); - const SCEVUnknown *SrcBase = - dyn_cast<SCEVUnknown>(SE->getPointerBase(SrcAccessFn)); - const SCEVUnknown *DstBase = - dyn_cast<SCEVUnknown>(SE->getPointerBase(DstAccessFn)); - assert(SrcBase && DstBase && SrcBase == DstBase && - "expected src and dst scev unknowns to be equal"); - - // Check the simple case where the array dimensions are fixed size. - auto *SrcGEP = dyn_cast<GetElementPtrInst>(SrcPtr); - auto *DstGEP = dyn_cast<GetElementPtrInst>(DstPtr); - if (!SrcGEP || !DstGEP) + SmallVector<int, 4> SrcSizes; + SmallVector<int, 4> DstSizes; + if (!tryDelinearizeFixedSizeImpl(SE, Src, SrcAccessFn, SrcSubscripts, + SrcSizes) || + !tryDelinearizeFixedSizeImpl(SE, Dst, DstAccessFn, DstSubscripts, + DstSizes)) return false; - SmallVector<int, 4> SrcSizes, DstSizes; - getIndexExpressionsFromGEP(*SE, SrcGEP, SrcSubscripts, SrcSizes); - getIndexExpressionsFromGEP(*SE, DstGEP, DstSubscripts, DstSizes); - // Check that the two size arrays are non-empty and equal in length and // value. - if (SrcSizes.empty() || SrcSubscripts.size() <= 1 || - SrcSizes.size() != DstSizes.size() || + if (SrcSizes.size() != DstSizes.size() || !std::equal(SrcSizes.begin(), SrcSizes.end(), DstSizes.begin())) { SrcSubscripts.clear(); DstSubscripts.clear(); return false; } - Value *SrcBasePtr = SrcGEP->getOperand(0); - Value *DstBasePtr = DstGEP->getOperand(0); - while (auto *PCast = dyn_cast<BitCastInst>(SrcBasePtr)) - SrcBasePtr = PCast->getOperand(0); - while (auto *PCast = dyn_cast<BitCastInst>(DstBasePtr)) - DstBasePtr = PCast->getOperand(0); - - // Check that for identical base pointers we do not miss index offsets - // that have been added before this GEP is applied. - if (SrcBasePtr != SrcBase->getValue() || DstBasePtr != DstBase->getValue()) { - SrcSubscripts.clear(); - DstSubscripts.clear(); - return false; - } - assert(SrcSubscripts.size() == DstSubscripts.size() && - SrcSubscripts.size() == SrcSizes.size() + 1 && - "Expected equal number of entries in the list of sizes and " - "subscripts."); + "Expected equal number of entries in the list of SrcSubscripts and " + "DstSubscripts."); + + Value *SrcPtr = getLoadStorePointerOperand(Src); + Value *DstPtr = getLoadStorePointerOperand(Dst); // In general we cannot safely assume that the subscripts recovered from GEPs // are in the range of values defined for their corresponding array @@ -3406,8 +3408,8 @@ } LLVM_DEBUG({ dbgs() << "Delinearized subscripts of fixed-size array\n" - << "SrcGEP:" << *SrcGEP << "\n" - << "DstGEP:" << *DstGEP << "\n"; + << "SrcGEP:" << *SrcPtr << "\n" + << "DstGEP:" << *DstPtr << "\n"; }); return true; }
diff --git a/src/llvm-project/llvm/lib/Analysis/DependenceGraphBuilder.cpp b/src/llvm-project/llvm/lib/Analysis/DependenceGraphBuilder.cpp index 6b90db4..7ee2adf 100644 --- a/src/llvm-project/llvm/lib/Analysis/DependenceGraphBuilder.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DependenceGraphBuilder.cpp
@@ -12,6 +12,7 @@ #include "llvm/Analysis/DependenceGraphBuilder.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/EnumeratedArray.h" +#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/DDG.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/src/llvm-project/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp index 4a792fc..79ea160 100644 --- a/src/llvm-project/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
@@ -11,7 +11,6 @@ // //===----------------------------------------------------------------------===// #include "llvm/Config/config.h" -#include "llvm/Support/Casting.h" #if defined(LLVM_HAVE_TF_API) #include "llvm/ADT/BitVector.h" @@ -273,8 +272,8 @@ static const std::vector<TensorSpec> getInputFeatures() { std::vector<TensorSpec> InputSpecs; for (size_t I = 0; I < NumberOfFeatures; ++I) - InputSpecs.push_back( - TensorSpec::createSpec<int64_t>(TFFeedPrefix + FeatureNameMap[I], {1})); + InputSpecs.push_back(TensorSpec::createSpec<int64_t>( + TFFeedPrefix + FeatureMap[I].name(), FeatureMap[I].shape())); append_range(InputSpecs, TrainingOnlyFeatures); return InputSpecs; } @@ -290,8 +289,7 @@ std::vector<LoggedFeatureSpec> FT; for (size_t I = 0; I < NumberOfFeatures; ++I) - FT.push_back( - {TensorSpec::createSpec<int64_t>(FeatureNameMap.at(I), {1}), None}); + FT.push_back({FeatureMap.at(I), None}); if (MUTR && MUTR->outputLoggedFeatureSpecs().size() > 1) append_range(FT, drop_begin(MUTR->outputLoggedFeatureSpecs()));
diff --git a/src/llvm-project/llvm/lib/Analysis/DivergenceAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/DivergenceAnalysis.cpp index 39e80c2..02c40d2 100644 --- a/src/llvm-project/llvm/lib/Analysis/DivergenceAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DivergenceAnalysis.cpp
@@ -73,15 +73,14 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/DivergenceAnalysis.h" +#include "llvm/ADT/PostOrderIterator.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/Passes.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Value.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -221,19 +220,19 @@ // phi nodes at the fringes of the dominance region if (!DT.dominates(&LoopHeader, UserBlock)) { // all PHI nodes of UserBlock become divergent - for (auto &Phi : UserBlock->phis()) { + for (const auto &Phi : UserBlock->phis()) { analyzeTemporalDivergence(Phi, OuterDivLoop); } continue; } // Taint outside users of values carried by OuterDivLoop. - for (auto &I : *UserBlock) { + for (const auto &I : *UserBlock) { analyzeTemporalDivergence(I, OuterDivLoop); } // visit all blocks in the dominance region - for (auto *SuccBlock : successors(UserBlock)) { + for (const auto *SuccBlock : successors(UserBlock)) { if (!Visited.insert(SuccBlock).second) { continue; } @@ -400,7 +399,7 @@ } for (const BasicBlock &BB : F) { OS << "\n " << BB.getName() << ":\n"; - for (auto &I : BB.instructionsWithoutDebug()) { + for (const auto &I : BB.instructionsWithoutDebug()) { OS << (DI.isDivergent(I) ? "DIVERGENT: " : " "); OS << I << "\n"; }
diff --git a/src/llvm-project/llvm/lib/Analysis/DomPrinter.cpp b/src/llvm-project/llvm/lib/Analysis/DomPrinter.cpp index 6088de5..e9f5103 100644 --- a/src/llvm-project/llvm/lib/Analysis/DomPrinter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DomPrinter.cpp
@@ -24,74 +24,6 @@ using namespace llvm; -namespace llvm { -template<> -struct DOTGraphTraits<DomTreeNode*> : public DefaultDOTGraphTraits { - - DOTGraphTraits (bool isSimple=false) - : DefaultDOTGraphTraits(isSimple) {} - - std::string getNodeLabel(DomTreeNode *Node, DomTreeNode *Graph) { - - BasicBlock *BB = Node->getBlock(); - - if (!BB) - return "Post dominance root node"; - - - if (isSimple()) - return DOTGraphTraits<DOTFuncInfo *> - ::getSimpleNodeLabel(BB, nullptr); - else - return DOTGraphTraits<DOTFuncInfo *> - ::getCompleteNodeLabel(BB, nullptr); - } -}; - -template<> -struct DOTGraphTraits<DominatorTree*> : public DOTGraphTraits<DomTreeNode*> { - - DOTGraphTraits (bool isSimple=false) - : DOTGraphTraits<DomTreeNode*>(isSimple) {} - - static std::string getGraphName(DominatorTree *DT) { - return "Dominator tree"; - } - - std::string getNodeLabel(DomTreeNode *Node, DominatorTree *G) { - return DOTGraphTraits<DomTreeNode*>::getNodeLabel(Node, G->getRootNode()); - } -}; - -template<> -struct DOTGraphTraits<PostDominatorTree*> - : public DOTGraphTraits<DomTreeNode*> { - - DOTGraphTraits (bool isSimple=false) - : DOTGraphTraits<DomTreeNode*>(isSimple) {} - - static std::string getGraphName(PostDominatorTree *DT) { - return "Post dominator tree"; - } - - std::string getNodeLabel(DomTreeNode *Node, PostDominatorTree *G ) { - return DOTGraphTraits<DomTreeNode*>::getNodeLabel(Node, G->getRootNode()); - } -}; -} - -PreservedAnalyses DomTreePrinterPass::run(Function &F, - FunctionAnalysisManager &AM) { - WriteDOTGraphToFile(F, &AM.getResult<DominatorTreeAnalysis>(F), "dom", false); - return PreservedAnalyses::all(); -} - -PreservedAnalyses DomTreeOnlyPrinterPass::run(Function &F, - FunctionAnalysisManager &AM) { - WriteDOTGraphToFile(F, &AM.getResult<DominatorTreeAnalysis>(F), "domonly", - true); - return PreservedAnalyses::all(); -} void DominatorTree::viewGraph(const Twine &Name, const Twine &Title) { #ifndef NDEBUG @@ -110,166 +42,167 @@ } namespace { -struct DominatorTreeWrapperPassAnalysisGraphTraits { +struct LegacyDominatorTreeWrapperPassAnalysisGraphTraits { static DominatorTree *getGraph(DominatorTreeWrapperPass *DTWP) { return &DTWP->getDomTree(); } }; -struct DomViewer : public DOTGraphTraitsViewer< - DominatorTreeWrapperPass, false, DominatorTree *, - DominatorTreeWrapperPassAnalysisGraphTraits> { +struct DomViewerWrapperPass + : public DOTGraphTraitsViewerWrapperPass< + DominatorTreeWrapperPass, false, DominatorTree *, + LegacyDominatorTreeWrapperPassAnalysisGraphTraits> { static char ID; - DomViewer() - : DOTGraphTraitsViewer<DominatorTreeWrapperPass, false, DominatorTree *, - DominatorTreeWrapperPassAnalysisGraphTraits>( - "dom", ID) { - initializeDomViewerPass(*PassRegistry::getPassRegistry()); + DomViewerWrapperPass() + : DOTGraphTraitsViewerWrapperPass< + DominatorTreeWrapperPass, false, DominatorTree *, + LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("dom", ID) { + initializeDomViewerWrapperPassPass(*PassRegistry::getPassRegistry()); } }; -struct DomOnlyViewer : public DOTGraphTraitsViewer< - DominatorTreeWrapperPass, true, DominatorTree *, - DominatorTreeWrapperPassAnalysisGraphTraits> { +struct DomOnlyViewerWrapperPass + : public DOTGraphTraitsViewerWrapperPass< + DominatorTreeWrapperPass, true, DominatorTree *, + LegacyDominatorTreeWrapperPassAnalysisGraphTraits> { static char ID; - DomOnlyViewer() - : DOTGraphTraitsViewer<DominatorTreeWrapperPass, true, DominatorTree *, - DominatorTreeWrapperPassAnalysisGraphTraits>( - "domonly", ID) { - initializeDomOnlyViewerPass(*PassRegistry::getPassRegistry()); + DomOnlyViewerWrapperPass() + : DOTGraphTraitsViewerWrapperPass< + DominatorTreeWrapperPass, true, DominatorTree *, + LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("domonly", ID) { + initializeDomOnlyViewerWrapperPassPass(*PassRegistry::getPassRegistry()); } }; -struct PostDominatorTreeWrapperPassAnalysisGraphTraits { +struct LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits { static PostDominatorTree *getGraph(PostDominatorTreeWrapperPass *PDTWP) { return &PDTWP->getPostDomTree(); } }; -struct PostDomViewer : public DOTGraphTraitsViewer< - PostDominatorTreeWrapperPass, false, - PostDominatorTree *, - PostDominatorTreeWrapperPassAnalysisGraphTraits> { +struct PostDomViewerWrapperPass + : public DOTGraphTraitsViewerWrapperPass< + PostDominatorTreeWrapperPass, false, PostDominatorTree *, + LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits> { static char ID; - PostDomViewer() : - DOTGraphTraitsViewer<PostDominatorTreeWrapperPass, false, - PostDominatorTree *, - PostDominatorTreeWrapperPassAnalysisGraphTraits>( - "postdom", ID){ - initializePostDomViewerPass(*PassRegistry::getPassRegistry()); - } + PostDomViewerWrapperPass() + : DOTGraphTraitsViewerWrapperPass< + PostDominatorTreeWrapperPass, false, PostDominatorTree *, + LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits>("postdom", + ID) { + initializePostDomViewerWrapperPassPass(*PassRegistry::getPassRegistry()); + } }; -struct PostDomOnlyViewer : public DOTGraphTraitsViewer< - PostDominatorTreeWrapperPass, true, - PostDominatorTree *, - PostDominatorTreeWrapperPassAnalysisGraphTraits> { +struct PostDomOnlyViewerWrapperPass + : public DOTGraphTraitsViewerWrapperPass< + PostDominatorTreeWrapperPass, true, PostDominatorTree *, + LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits> { static char ID; - PostDomOnlyViewer() : - DOTGraphTraitsViewer<PostDominatorTreeWrapperPass, true, - PostDominatorTree *, - PostDominatorTreeWrapperPassAnalysisGraphTraits>( - "postdomonly", ID){ - initializePostDomOnlyViewerPass(*PassRegistry::getPassRegistry()); - } + PostDomOnlyViewerWrapperPass() + : DOTGraphTraitsViewerWrapperPass< + PostDominatorTreeWrapperPass, true, PostDominatorTree *, + LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits>( + "postdomonly", ID) { + initializePostDomOnlyViewerWrapperPassPass( + *PassRegistry::getPassRegistry()); + } }; } // end anonymous namespace -char DomViewer::ID = 0; -INITIALIZE_PASS(DomViewer, "view-dom", +char DomViewerWrapperPass::ID = 0; +INITIALIZE_PASS(DomViewerWrapperPass, "view-dom", "View dominance tree of function", false, false) -char DomOnlyViewer::ID = 0; -INITIALIZE_PASS(DomOnlyViewer, "view-dom-only", +char DomOnlyViewerWrapperPass::ID = 0; +INITIALIZE_PASS(DomOnlyViewerWrapperPass, "view-dom-only", "View dominance tree of function (with no function bodies)", false, false) -char PostDomViewer::ID = 0; -INITIALIZE_PASS(PostDomViewer, "view-postdom", +char PostDomViewerWrapperPass::ID = 0; +INITIALIZE_PASS(PostDomViewerWrapperPass, "view-postdom", "View postdominance tree of function", false, false) -char PostDomOnlyViewer::ID = 0; -INITIALIZE_PASS(PostDomOnlyViewer, "view-postdom-only", +char PostDomOnlyViewerWrapperPass::ID = 0; +INITIALIZE_PASS(PostDomOnlyViewerWrapperPass, "view-postdom-only", "View postdominance tree of function " "(with no function bodies)", false, false) namespace { -struct DomPrinter : public DOTGraphTraitsPrinter< - DominatorTreeWrapperPass, false, DominatorTree *, - DominatorTreeWrapperPassAnalysisGraphTraits> { +struct DomPrinterWrapperPass + : public DOTGraphTraitsPrinterWrapperPass< + DominatorTreeWrapperPass, false, DominatorTree *, + LegacyDominatorTreeWrapperPassAnalysisGraphTraits> { static char ID; - DomPrinter() - : DOTGraphTraitsPrinter<DominatorTreeWrapperPass, false, DominatorTree *, - DominatorTreeWrapperPassAnalysisGraphTraits>( - "dom", ID) { - initializeDomPrinterPass(*PassRegistry::getPassRegistry()); + DomPrinterWrapperPass() + : DOTGraphTraitsPrinterWrapperPass< + DominatorTreeWrapperPass, false, DominatorTree *, + LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("dom", ID) { + initializeDomPrinterWrapperPassPass(*PassRegistry::getPassRegistry()); } }; -struct DomOnlyPrinter : public DOTGraphTraitsPrinter< - DominatorTreeWrapperPass, true, DominatorTree *, - DominatorTreeWrapperPassAnalysisGraphTraits> { +struct DomOnlyPrinterWrapperPass + : public DOTGraphTraitsPrinterWrapperPass< + DominatorTreeWrapperPass, true, DominatorTree *, + LegacyDominatorTreeWrapperPassAnalysisGraphTraits> { static char ID; - DomOnlyPrinter() - : DOTGraphTraitsPrinter<DominatorTreeWrapperPass, true, DominatorTree *, - DominatorTreeWrapperPassAnalysisGraphTraits>( - "domonly", ID) { - initializeDomOnlyPrinterPass(*PassRegistry::getPassRegistry()); + DomOnlyPrinterWrapperPass() + : DOTGraphTraitsPrinterWrapperPass< + DominatorTreeWrapperPass, true, DominatorTree *, + LegacyDominatorTreeWrapperPassAnalysisGraphTraits>("domonly", ID) { + initializeDomOnlyPrinterWrapperPassPass(*PassRegistry::getPassRegistry()); } }; -struct PostDomPrinter - : public DOTGraphTraitsPrinter< - PostDominatorTreeWrapperPass, false, - PostDominatorTree *, - PostDominatorTreeWrapperPassAnalysisGraphTraits> { +struct PostDomPrinterWrapperPass + : public DOTGraphTraitsPrinterWrapperPass< + PostDominatorTreeWrapperPass, false, PostDominatorTree *, + LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits> { static char ID; - PostDomPrinter() : - DOTGraphTraitsPrinter<PostDominatorTreeWrapperPass, false, - PostDominatorTree *, - PostDominatorTreeWrapperPassAnalysisGraphTraits>( - "postdom", ID) { - initializePostDomPrinterPass(*PassRegistry::getPassRegistry()); - } + PostDomPrinterWrapperPass() + : DOTGraphTraitsPrinterWrapperPass< + PostDominatorTreeWrapperPass, false, PostDominatorTree *, + LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits>("postdom", + ID) { + initializePostDomPrinterWrapperPassPass(*PassRegistry::getPassRegistry()); + } }; -struct PostDomOnlyPrinter - : public DOTGraphTraitsPrinter< - PostDominatorTreeWrapperPass, true, - PostDominatorTree *, - PostDominatorTreeWrapperPassAnalysisGraphTraits> { +struct PostDomOnlyPrinterWrapperPass + : public DOTGraphTraitsPrinterWrapperPass< + PostDominatorTreeWrapperPass, true, PostDominatorTree *, + LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits> { static char ID; - PostDomOnlyPrinter() : - DOTGraphTraitsPrinter<PostDominatorTreeWrapperPass, true, - PostDominatorTree *, - PostDominatorTreeWrapperPassAnalysisGraphTraits>( - "postdomonly", ID) { - initializePostDomOnlyPrinterPass(*PassRegistry::getPassRegistry()); - } + PostDomOnlyPrinterWrapperPass() + : DOTGraphTraitsPrinterWrapperPass< + PostDominatorTreeWrapperPass, true, PostDominatorTree *, + LegacyPostDominatorTreeWrapperPassAnalysisGraphTraits>( + "postdomonly", ID) { + initializePostDomOnlyPrinterWrapperPassPass( + *PassRegistry::getPassRegistry()); + } }; } // end anonymous namespace +char DomPrinterWrapperPass::ID = 0; +INITIALIZE_PASS(DomPrinterWrapperPass, "dot-dom", + "Print dominance tree of function to 'dot' file", false, false) - -char DomPrinter::ID = 0; -INITIALIZE_PASS(DomPrinter, "dot-dom", - "Print dominance tree of function to 'dot' file", - false, false) - -char DomOnlyPrinter::ID = 0; -INITIALIZE_PASS(DomOnlyPrinter, "dot-dom-only", +char DomOnlyPrinterWrapperPass::ID = 0; +INITIALIZE_PASS(DomOnlyPrinterWrapperPass, "dot-dom-only", "Print dominance tree of function to 'dot' file " "(with no function bodies)", false, false) -char PostDomPrinter::ID = 0; -INITIALIZE_PASS(PostDomPrinter, "dot-postdom", - "Print postdominance tree of function to 'dot' file", - false, false) +char PostDomPrinterWrapperPass::ID = 0; +INITIALIZE_PASS(PostDomPrinterWrapperPass, "dot-postdom", + "Print postdominance tree of function to 'dot' file", false, + false) -char PostDomOnlyPrinter::ID = 0; -INITIALIZE_PASS(PostDomOnlyPrinter, "dot-postdom-only", +char PostDomOnlyPrinterWrapperPass::ID = 0; +INITIALIZE_PASS(PostDomOnlyPrinterWrapperPass, "dot-postdom-only", "Print postdominance tree of function to 'dot' file " "(with no function bodies)", false, false) @@ -278,34 +211,34 @@ // "include/llvm/LinkAllPasses.h". Otherwise the pass would be deleted by // the link time optimization. -FunctionPass *llvm::createDomPrinterPass() { - return new DomPrinter(); +FunctionPass *llvm::createDomPrinterWrapperPassPass() { + return new DomPrinterWrapperPass(); } -FunctionPass *llvm::createDomOnlyPrinterPass() { - return new DomOnlyPrinter(); +FunctionPass *llvm::createDomOnlyPrinterWrapperPassPass() { + return new DomOnlyPrinterWrapperPass(); } -FunctionPass *llvm::createDomViewerPass() { - return new DomViewer(); +FunctionPass *llvm::createDomViewerWrapperPassPass() { + return new DomViewerWrapperPass(); } -FunctionPass *llvm::createDomOnlyViewerPass() { - return new DomOnlyViewer(); +FunctionPass *llvm::createDomOnlyViewerWrapperPassPass() { + return new DomOnlyViewerWrapperPass(); } -FunctionPass *llvm::createPostDomPrinterPass() { - return new PostDomPrinter(); +FunctionPass *llvm::createPostDomPrinterWrapperPassPass() { + return new PostDomPrinterWrapperPass(); } -FunctionPass *llvm::createPostDomOnlyPrinterPass() { - return new PostDomOnlyPrinter(); +FunctionPass *llvm::createPostDomOnlyPrinterWrapperPassPass() { + return new PostDomOnlyPrinterWrapperPass(); } -FunctionPass *llvm::createPostDomViewerPass() { - return new PostDomViewer(); +FunctionPass *llvm::createPostDomViewerWrapperPassPass() { + return new PostDomViewerWrapperPass(); } -FunctionPass *llvm::createPostDomOnlyViewerPass() { - return new PostDomOnlyViewer(); +FunctionPass *llvm::createPostDomOnlyViewerWrapperPassPass() { + return new PostDomOnlyViewerWrapperPass(); }
diff --git a/src/llvm-project/llvm/lib/Analysis/DomTreeUpdater.cpp b/src/llvm-project/llvm/lib/Analysis/DomTreeUpdater.cpp index 6e29926..888c167 100644 --- a/src/llvm-project/llvm/lib/Analysis/DomTreeUpdater.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DomTreeUpdater.cpp
@@ -14,6 +14,7 @@ #include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/PostDominators.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/GenericDomTree.h" #include <algorithm> @@ -314,98 +315,6 @@ return *PDT; } -void DomTreeUpdater::insertEdge(BasicBlock *From, BasicBlock *To) { - -#ifndef NDEBUG - assert(isUpdateValid({DominatorTree::Insert, From, To}) && - "Inserted edge does not appear in the CFG"); -#endif - - if (!DT && !PDT) - return; - - // Won't affect DomTree and PostDomTree; discard update. - if (From == To) - return; - - if (Strategy == UpdateStrategy::Eager) { - if (DT) - DT->insertEdge(From, To); - if (PDT) - PDT->insertEdge(From, To); - return; - } - - PendUpdates.push_back({DominatorTree::Insert, From, To}); -} - -void DomTreeUpdater::insertEdgeRelaxed(BasicBlock *From, BasicBlock *To) { - if (From == To) - return; - - if (!DT && !PDT) - return; - - if (!isUpdateValid({DominatorTree::Insert, From, To})) - return; - - if (Strategy == UpdateStrategy::Eager) { - if (DT) - DT->insertEdge(From, To); - if (PDT) - PDT->insertEdge(From, To); - return; - } - - PendUpdates.push_back({DominatorTree::Insert, From, To}); -} - -void DomTreeUpdater::deleteEdge(BasicBlock *From, BasicBlock *To) { - -#ifndef NDEBUG - assert(isUpdateValid({DominatorTree::Delete, From, To}) && - "Deleted edge still exists in the CFG!"); -#endif - - if (!DT && !PDT) - return; - - // Won't affect DomTree and PostDomTree; discard update. - if (From == To) - return; - - if (Strategy == UpdateStrategy::Eager) { - if (DT) - DT->deleteEdge(From, To); - if (PDT) - PDT->deleteEdge(From, To); - return; - } - - PendUpdates.push_back({DominatorTree::Delete, From, To}); -} - -void DomTreeUpdater::deleteEdgeRelaxed(BasicBlock *From, BasicBlock *To) { - if (From == To) - return; - - if (!DT && !PDT) - return; - - if (!isUpdateValid({DominatorTree::Delete, From, To})) - return; - - if (Strategy == UpdateStrategy::Eager) { - if (DT) - DT->deleteEdge(From, To); - if (PDT) - PDT->deleteEdge(From, To); - return; - } - - PendUpdates.push_back({DominatorTree::Delete, From, To}); -} - void DomTreeUpdater::dropOutOfDateUpdates() { if (Strategy == DomTreeUpdater::UpdateStrategy::Eager) return;
diff --git a/src/llvm-project/llvm/lib/Analysis/DominanceFrontier.cpp b/src/llvm-project/llvm/lib/Analysis/DominanceFrontier.cpp index a8806fe..ccba913 100644 --- a/src/llvm-project/llvm/lib/Analysis/DominanceFrontier.cpp +++ b/src/llvm-project/llvm/lib/Analysis/DominanceFrontier.cpp
@@ -15,7 +15,6 @@ #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/EHPersonalities.cpp b/src/llvm-project/llvm/lib/Analysis/EHPersonalities.cpp index df8b7e1..277ff6b 100644 --- a/src/llvm-project/llvm/lib/Analysis/EHPersonalities.cpp +++ b/src/llvm-project/llvm/lib/Analysis/EHPersonalities.cpp
@@ -8,6 +8,7 @@ #include "llvm/Analysis/EHPersonalities.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Triple.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" @@ -67,7 +68,10 @@ } EHPersonality llvm::getDefaultEHPersonality(const Triple &T) { - return EHPersonality::GNU_C; + if (T.isPS5()) + return EHPersonality::GNU_CXX; + else + return EHPersonality::GNU_C; } bool llvm::canSimplifyInvokeNoUnwind(const Function *F) {
diff --git a/src/llvm-project/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp index 3351903..782c119 100644 --- a/src/llvm-project/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
@@ -12,48 +12,87 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/FunctionPropertiesAnalysis.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" +#include <deque> using namespace llvm; -FunctionPropertiesInfo -FunctionPropertiesInfo::getFunctionPropertiesInfo(const Function &F, +namespace { +int64_t getNrBlocksFromCond(const BasicBlock &BB) { + int64_t Ret = 0; + if (const auto *BI = dyn_cast<BranchInst>(BB.getTerminator())) { + if (BI->isConditional()) + Ret += BI->getNumSuccessors(); + } else if (const auto *SI = dyn_cast<SwitchInst>(BB.getTerminator())) { + Ret += (SI->getNumCases() + (nullptr != SI->getDefaultDest())); + } + return Ret; +} + +int64_t getUses(const Function &F) { + return ((!F.hasLocalLinkage()) ? 1 : 0) + F.getNumUses(); +} +} // namespace + +void FunctionPropertiesInfo::reIncludeBB(const BasicBlock &BB) { + updateForBB(BB, +1); +} + +void FunctionPropertiesInfo::updateForBB(const BasicBlock &BB, + int64_t Direction) { + assert(Direction == 1 || Direction == -1); + BasicBlockCount += Direction; + BlocksReachedFromConditionalInstruction += + (Direction * getNrBlocksFromCond(BB)); + for (const auto &I : BB) { + if (auto *CS = dyn_cast<CallBase>(&I)) { + const auto *Callee = CS->getCalledFunction(); + if (Callee && !Callee->isIntrinsic() && !Callee->isDeclaration()) + DirectCallsToDefinedFunctions += Direction; + } + if (I.getOpcode() == Instruction::Load) { + LoadInstCount += Direction; + } else if (I.getOpcode() == Instruction::Store) { + StoreInstCount += Direction; + } + } + TotalInstructionCount += Direction * BB.sizeWithoutDebug(); +} + +void FunctionPropertiesInfo::updateAggregateStats(const Function &F, const LoopInfo &LI) { - FunctionPropertiesInfo FPI; - - FPI.Uses = ((!F.hasLocalLinkage()) ? 1 : 0) + F.getNumUses(); - - for (const auto &BB : F) { - ++FPI.BasicBlockCount; - - if (const auto *BI = dyn_cast<BranchInst>(BB.getTerminator())) { - if (BI->isConditional()) - FPI.BlocksReachedFromConditionalInstruction += BI->getNumSuccessors(); - } else if (const auto *SI = dyn_cast<SwitchInst>(BB.getTerminator())) { - FPI.BlocksReachedFromConditionalInstruction += - (SI->getNumCases() + (nullptr != SI->getDefaultDest())); - } - - for (const auto &I : BB) { - if (auto *CS = dyn_cast<CallBase>(&I)) { - const auto *Callee = CS->getCalledFunction(); - if (Callee && !Callee->isIntrinsic() && !Callee->isDeclaration()) - ++FPI.DirectCallsToDefinedFunctions; - } - if (I.getOpcode() == Instruction::Load) { - ++FPI.LoadInstCount; - } else if (I.getOpcode() == Instruction::Store) { - ++FPI.StoreInstCount; - } - } - // Loop Depth of the Basic Block - int64_t LoopDepth; - LoopDepth = LI.getLoopDepth(&BB); - if (FPI.MaxLoopDepth < LoopDepth) - FPI.MaxLoopDepth = LoopDepth; + Uses = getUses(F); + TopLevelLoopCount = llvm::size(LI); + MaxLoopDepth = 0; + std::deque<const Loop *> Worklist; + llvm::append_range(Worklist, LI); + while (!Worklist.empty()) { + const auto *L = Worklist.front(); + MaxLoopDepth = + std::max(MaxLoopDepth, static_cast<int64_t>(L->getLoopDepth())); + Worklist.pop_front(); + llvm::append_range(Worklist, L->getSubLoops()); } - FPI.TopLevelLoopCount += llvm::size(LI); +} + +FunctionPropertiesInfo FunctionPropertiesInfo::getFunctionPropertiesInfo( + const Function &F, FunctionAnalysisManager &FAM) { + + FunctionPropertiesInfo FPI; + // The const casts are due to the getResult API - there's no mutation of F. + const auto &LI = FAM.getResult<LoopAnalysis>(const_cast<Function &>(F)); + const auto &DT = + FAM.getResult<DominatorTreeAnalysis>(const_cast<Function &>(F)); + for (const auto &BB : F) + if (DT.isReachableFromEntry(&BB)) + FPI.reIncludeBB(BB); + FPI.updateAggregateStats(F, LI); return FPI; } @@ -67,15 +106,15 @@ << "LoadInstCount: " << LoadInstCount << "\n" << "StoreInstCount: " << StoreInstCount << "\n" << "MaxLoopDepth: " << MaxLoopDepth << "\n" - << "TopLevelLoopCount: " << TopLevelLoopCount << "\n\n"; + << "TopLevelLoopCount: " << TopLevelLoopCount << "\n" + << "TotalInstructionCount: " << TotalInstructionCount << "\n\n"; } AnalysisKey FunctionPropertiesAnalysis::Key; FunctionPropertiesInfo FunctionPropertiesAnalysis::run(Function &F, FunctionAnalysisManager &FAM) { - return FunctionPropertiesInfo::getFunctionPropertiesInfo( - F, FAM.getResult<LoopAnalysis>(F)); + return FunctionPropertiesInfo::getFunctionPropertiesInfo(F, FAM); } PreservedAnalyses @@ -86,3 +125,127 @@ AM.getResult<FunctionPropertiesAnalysis>(F).print(OS); return PreservedAnalyses::all(); } + +FunctionPropertiesUpdater::FunctionPropertiesUpdater( + FunctionPropertiesInfo &FPI, const CallBase &CB) + : FPI(FPI), CallSiteBB(*CB.getParent()), Caller(*CallSiteBB.getParent()) { + assert(isa<CallInst>(CB) || isa<InvokeInst>(CB)); + // For BBs that are likely to change, we subtract from feature totals their + // contribution. Some features, like max loop counts or depths, are left + // invalid, as they will be updated post-inlining. + SmallPtrSet<const BasicBlock *, 4> LikelyToChangeBBs; + // The CB BB will change - it'll either be split or the callee's body (single + // BB) will be pasted in. + LikelyToChangeBBs.insert(&CallSiteBB); + + // The caller's entry BB may change due to new alloca instructions. + LikelyToChangeBBs.insert(&*Caller.begin()); + + // The successors may become unreachable in the case of `invoke` inlining. + // We track successors separately, too, because they form a boundary, together + // with the CB BB ('Entry') between which the inlined callee will be pasted. + Successors.insert(succ_begin(&CallSiteBB), succ_end(&CallSiteBB)); + + // Inlining only handles invoke and calls. If this is an invoke, and inlining + // it pulls another invoke, the original landing pad may get split, so as to + // share its content with other potential users. So the edge up to which we + // need to invalidate and then re-account BB data is the successors of the + // current landing pad. We can leave the current lp, too - if it doesn't get + // split, then it will be the place traversal stops. Either way, the + // discounted BBs will be checked if reachable and re-added. + if (const auto *II = dyn_cast<InvokeInst>(&CB)) { + const auto *UnwindDest = II->getUnwindDest(); + Successors.insert(succ_begin(UnwindDest), succ_end(UnwindDest)); + } + + // Exclude the CallSiteBB, if it happens to be its own successor (1-BB loop). + // We are only interested in BBs the graph moves past the callsite BB to + // define the frontier past which we don't want to re-process BBs. Including + // the callsite BB in this case would prematurely stop the traversal in + // finish(). + Successors.erase(&CallSiteBB); + + for (const auto *BB : Successors) + LikelyToChangeBBs.insert(BB); + + // Commit the change. While some of the BBs accounted for above may play dual + // role - e.g. caller's entry BB may be the same as the callsite BB - set + // insertion semantics make sure we account them once. This needs to be + // followed in `finish`, too. + for (const auto *BB : LikelyToChangeBBs) + FPI.updateForBB(*BB, -1); +} + +void FunctionPropertiesUpdater::finish(FunctionAnalysisManager &FAM) const { + // Update feature values from the BBs that were copied from the callee, or + // might have been modified because of inlining. The latter have been + // subtracted in the FunctionPropertiesUpdater ctor. + // There could be successors that were reached before but now are only + // reachable from elsewhere in the CFG. + // One example is the following diamond CFG (lines are arrows pointing down): + // A + // / \ + // B C + // | | + // | D + // | | + // | E + // \ / + // F + // There's a call site in C that is inlined. Upon doing that, it turns out + // it expands to + // call void @llvm.trap() + // unreachable + // F isn't reachable from C anymore, but we did discount it when we set up + // FunctionPropertiesUpdater, so we need to re-include it here. + // At the same time, D and E were reachable before, but now are not anymore, + // so we need to leave D out (we discounted it at setup), and explicitly + // remove E. + SetVector<const BasicBlock *> Reinclude; + SetVector<const BasicBlock *> Unreachable; + const auto &DT = + FAM.getResult<DominatorTreeAnalysis>(const_cast<Function &>(Caller)); + + if (&CallSiteBB != &*Caller.begin()) + Reinclude.insert(&*Caller.begin()); + + // Distribute the successors to the 2 buckets. + for (const auto *Succ : Successors) + if (DT.isReachableFromEntry(Succ)) + Reinclude.insert(Succ); + else + Unreachable.insert(Succ); + + // For reinclusion, we want to stop at the reachable successors, who are at + // the beginning of the worklist; but, starting from the callsite bb and + // ending at those successors, we also want to perform a traversal. + // IncludeSuccessorsMark is the index after which we include successors. + const auto IncludeSuccessorsMark = Reinclude.size(); + bool CSInsertion = Reinclude.insert(&CallSiteBB); + (void)CSInsertion; + assert(CSInsertion); + for (size_t I = 0; I < Reinclude.size(); ++I) { + const auto *BB = Reinclude[I]; + FPI.reIncludeBB(*BB); + if (I >= IncludeSuccessorsMark) + Reinclude.insert(succ_begin(BB), succ_end(BB)); + } + + // For exclusion, we don't need to exclude the set of BBs that were successors + // before and are now unreachable, because we already did that at setup. For + // the rest, as long as a successor is unreachable, we want to explicitly + // exclude it. + const auto AlreadyExcludedMark = Unreachable.size(); + for (size_t I = 0; I < Unreachable.size(); ++I) { + const auto *U = Unreachable[I]; + if (I >= AlreadyExcludedMark) + FPI.updateForBB(*U, -1); + for (const auto *Succ : successors(U)) + if (!DT.isReachableFromEntry(Succ)) + Unreachable.insert(Succ); + } + + const auto &LI = FAM.getResult<LoopAnalysis>(const_cast<Function &>(Caller)); + FPI.updateAggregateStats(Caller, LI); + assert(FPI == FunctionPropertiesInfo::getFunctionPropertiesInfo(Caller, FAM)); +}
diff --git a/src/llvm-project/llvm/lib/Analysis/GlobalsModRef.cpp b/src/llvm-project/llvm/lib/Analysis/GlobalsModRef.cpp index 6869530..e6ef1c7 100644 --- a/src/llvm-project/llvm/lib/Analysis/GlobalsModRef.cpp +++ b/src/llvm-project/llvm/lib/Analysis/GlobalsModRef.cpp
@@ -21,11 +21,11 @@ #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" @@ -68,8 +68,8 @@ /// should provide this much alignment at least, but this makes it clear we /// specifically rely on this amount of alignment. struct alignas(8) AlignedMap { - AlignedMap() {} - AlignedMap(const AlignedMap &Arg) : Map(Arg.Map) {} + AlignedMap() = default; + AlignedMap(const AlignedMap &Arg) = default; GlobalInfoMapType Map; }; @@ -102,7 +102,7 @@ "Insufficient low bits to store our flag and ModRef info."); public: - FunctionInfo() {} + FunctionInfo() = default; ~FunctionInfo() { delete Info.getPointer(); } @@ -256,22 +256,6 @@ return FunctionModRefBehavior(AAResultBase::getModRefBehavior(F) & Min); } -FunctionModRefBehavior -GlobalsAAResult::getModRefBehavior(const CallBase *Call) { - FunctionModRefBehavior Min = FMRB_UnknownModRefBehavior; - - if (!Call->hasOperandBundles()) - if (const Function *F = Call->getCalledFunction()) - if (FunctionInfo *FI = getFunctionInfo(F)) { - if (!isModOrRefSet(FI->getModRefInfo())) - Min = FMRB_DoesNotAccessMemory; - else if (!isModSet(FI->getModRefInfo())) - Min = FMRB_OnlyReadsMemory; - } - - return FunctionModRefBehavior(AAResultBase::getModRefBehavior(Call) & Min); -} - /// Returns the function info for the function, or null if we don't have /// anything useful to say about it. GlobalsAAResult::FunctionInfo * @@ -377,7 +361,7 @@ if (Call->isDataOperand(&U)) { // Detect calls to free. if (Call->isArgOperand(&U) && - isFreeCall(I, &GetTLI(*Call->getFunction()))) { + getFreedOperand(Call, &GetTLI(*Call->getFunction())) == U) { if (Writers) Writers->insert(Call->getParent()->getParent()); } else { @@ -511,6 +495,18 @@ Handles.front().I = Handles.begin(); bool KnowNothing = false; + // Intrinsics, like any other synchronizing function, can make effects + // of other threads visible. Without nosync we know nothing really. + // Similarly, if `nocallback` is missing the function, or intrinsic, + // can call into the module arbitrarily. If both are set the function + // has an effect but will not interact with accesses of internal + // globals inside the module. We are conservative here for optnone + // functions, might not be necessary. + auto MaySyncOrCallIntoModule = [](const Function &F) { + return !F.isDeclaration() || !F.hasNoSync() || + !F.hasFnAttribute(Attribute::NoCallback); + }; + // Collect the mod/ref properties due to called functions. We only compute // one mod-ref set. for (unsigned i = 0, e = SCC.size(); i != e && !KnowNothing; ++i) { @@ -525,7 +521,7 @@ // Can't do better than that! } else if (F->onlyReadsMemory()) { FI.addModRefInfo(ModRefInfo::Ref); - if (!F->isIntrinsic() && !F->onlyAccessesArgMemory()) + if (!F->onlyAccessesArgMemory() && MaySyncOrCallIntoModule(*F)) // This function might call back into the module and read a global - // consider every global as possibly being read by this function. FI.setMayReadAnyGlobal(); @@ -533,7 +529,7 @@ FI.addModRefInfo(ModRefInfo::ModRef); if (!F->onlyAccessesArgMemory()) FI.setMayReadAnyGlobal(); - if (!F->isIntrinsic()) { + if (MaySyncOrCallIntoModule(*F)) { KnowNothing = true; break; } @@ -585,12 +581,7 @@ // We handle calls specially because the graph-relevant aspects are // handled above. if (auto *Call = dyn_cast<CallBase>(&I)) { - auto &TLI = GetTLI(*Node->getFunction()); - if (isAllocationFn(Call, &TLI) || isFreeCall(Call, &TLI)) { - // FIXME: It is completely unclear why this is necessary and not - // handled by the above graph code. - FI.addModRefInfo(ModRefInfo::ModRef); - } else if (Function *Callee = Call->getCalledFunction()) { + if (Function *Callee = Call->getCalledFunction()) { // The callgraph doesn't include intrinsic calls. if (Callee->isIntrinsic()) { if (isa<DbgInfoIntrinsic>(Call)) @@ -915,7 +906,7 @@ // Iterate through all the arguments to the called function. If any argument // is based on GV, return the conservative result. - for (auto &A : Call->args()) { + for (const auto &A : Call->args()) { SmallVector<const Value*, 4> Objects; getUnderlyingObjects(A, Objects); @@ -979,7 +970,7 @@ } } -GlobalsAAResult::~GlobalsAAResult() {} +GlobalsAAResult::~GlobalsAAResult() = default; /*static*/ GlobalsAAResult GlobalsAAResult::analyzeModule( Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI, @@ -1010,6 +1001,24 @@ AM.getResult<CallGraphAnalysis>(M)); } +PreservedAnalyses RecomputeGlobalsAAPass::run(Module &M, + ModuleAnalysisManager &AM) { + if (auto *G = AM.getCachedResult<GlobalsAA>(M)) { + auto &CG = AM.getResult<CallGraphAnalysis>(M); + G->NonAddressTakenGlobals.clear(); + G->UnknownFunctionsWithLocalLinkage = false; + G->IndirectGlobals.clear(); + G->AllocsForIndirectGlobals.clear(); + G->FunctionInfos.clear(); + G->FunctionToSCCMap.clear(); + G->Handles.clear(); + G->CollectSCCMembership(CG); + G->AnalyzeGlobals(M); + G->AnalyzeCallGraph(CG, M); + } + return PreservedAnalyses::all(); +} + char GlobalsAAWrapperPass::ID = 0; INITIALIZE_PASS_BEGIN(GlobalsAAWrapperPass, "globals-aa", "Globals Alias Analysis", false, true)
diff --git a/src/llvm-project/llvm/lib/Analysis/IRSimilarityIdentifier.cpp b/src/llvm-project/llvm/lib/Analysis/IRSimilarityIdentifier.cpp index 01681c4..a681c52 100644 --- a/src/llvm-project/llvm/lib/Analysis/IRSimilarityIdentifier.cpp +++ b/src/llvm-project/llvm/lib/Analysis/IRSimilarityIdentifier.cpp
@@ -64,7 +64,7 @@ // Here we collect the operands and their types for determining whether // the structure of the operand use matches between two different candidates. for (Use &OI : Inst->operands()) { - if (isa<CmpInst>(Inst) && RevisedPredicate.hasValue()) { + if (isa<CmpInst>(Inst) && RevisedPredicate) { // If we have a CmpInst where the predicate is reversed, it means the // operands must be reversed as well. OperVals.insert(OperVals.begin(), OI.get()); @@ -183,9 +183,9 @@ assert(isa<CmpInst>(Inst) && "Can only get a predicate from a compare instruction"); - if (RevisedPredicate.hasValue()) - return RevisedPredicate.getValue(); - + if (RevisedPredicate) + return RevisedPredicate.value(); + return cast<CmpInst>(Inst)->getPredicate(); } @@ -193,7 +193,7 @@ assert(isa<CallInst>(Inst) && "Can only get a name from a call instruction"); - assert(CalleeName.hasValue() && "CalleeName has not been set"); + assert(CalleeName && "CalleeName has not been set"); return *CalleeName; } @@ -289,14 +289,12 @@ } } - if (HaveLegalRange) { - if (AddedIllegalLastTime) - mapToIllegalUnsigned(It, IntegerMappingForBB, InstrListForBB, true); - for (IRInstructionData *ID : InstrListForBB) - this->IDL->push_back(*ID); - llvm::append_range(InstrList, InstrListForBB); - llvm::append_range(IntegerMapping, IntegerMappingForBB); - } + if (AddedIllegalLastTime) + mapToIllegalUnsigned(It, IntegerMappingForBB, InstrListForBB, true); + for (IRInstructionData *ID : InstrListForBB) + this->IDL->push_back(*ID); + llvm::append_range(InstrList, InstrListForBB); + llvm::append_range(IntegerMapping, IntegerMappingForBB); } // TODO: This is the same as the MachineOutliner, and should be consolidated @@ -461,6 +459,18 @@ // that both of these instructions are not nullptrs. FirstInst = FirstInstIt; LastInst = LastInstIt; + + // Add the basic blocks contained in the set into the global value numbering. + DenseSet<BasicBlock *> BBSet; + getBasicBlocks(BBSet); + for (BasicBlock *BB : BBSet) { + if (ValueToNumber.find(BB) != ValueToNumber.end()) + continue; + + ValueToNumber.try_emplace(BB, LocalValNumber); + NumberToValue.try_emplace(LocalValNumber, BB); + LocalValNumber++; + } } bool IRSimilarityCandidate::isSimilar(const IRSimilarityCandidate &A, @@ -516,19 +526,13 @@ for (Value *V : SourceOperands) { ArgVal = SourceValueToNumberMapping.find(V)->second; + // Instead of finding a current mapping, we attempt to insert a set. std::tie(ValueMappingIt, WasInserted) = CurrentSrcTgtNumberMapping.insert( std::make_pair(ArgVal, TargetValueNumbers)); - // Instead of finding a current mapping, we inserted a set. This means a - // mapping did not exist for the source Instruction operand, it has no - // current constraints we need to check. - if (WasInserted) - continue; - - // If a mapping already exists for the source operand to the values in the - // other IRSimilarityCandidate we need to iterate over the items in other - // IRSimilarityCandidate's Instruction to determine whether there is a valid - // mapping of Value to Value. + // We need to iterate over the items in other IRSimilarityCandidate's + // Instruction to determine whether there is a valid mapping of + // Value to Value. DenseSet<unsigned> NewSet; for (unsigned &Curr : ValueMappingIt->second) // If we can find the value in the mapping, we add it to the new set. @@ -548,7 +552,6 @@ if (ValueMappingIt->second.size() != 1) continue; - unsigned ValToRemove = *ValueMappingIt->second.begin(); // When there is only one item left in the mapping for and operand, remove // the value from the other operands. If it results in there being no @@ -791,7 +794,8 @@ // We have different paths for commutative instructions and non-commutative // instructions since commutative instructions could allow multiple mappings // to certain values. - if (IA->isCommutative() && !isa<FPMathOperator>(IA)) { + if (IA->isCommutative() && !isa<FPMathOperator>(IA) && + !isa<IntrinsicInst>(IA)) { if (!compareCommutativeOperandMapping( {A, OperValsA, ValueNumberMappingA}, {B, OperValsB, ValueNumberMappingB})) @@ -1008,6 +1012,40 @@ CanonNumToNumber.insert(std::make_pair(CanonNum, SourceGVN)); NumberToCanonNum.insert(std::make_pair(SourceGVN, CanonNum)); } + + DenseSet<BasicBlock *> BBSet; + getBasicBlocks(BBSet); + // Find canonical numbers for the BasicBlocks in the current candidate. + // This is done by finding the corresponding value for the first instruction + // in the block in the current candidate, finding the matching value in the + // source candidate. Then by finding the parent of this value, use the + // canonical number of the block in the source candidate for the canonical + // number in the current candidate. + for (BasicBlock *BB : BBSet) { + unsigned BBGVNForCurrCand = ValueToNumber.find(BB)->second; + + // We can skip the BasicBlock if the canonical numbering has already been + // found in a separate instruction. + if (NumberToCanonNum.find(BBGVNForCurrCand) != NumberToCanonNum.end()) + continue; + + // If the basic block is the starting block, then the shared instruction may + // not be the first instruction in the block, it will be the first + // instruction in the similarity region. + Value *FirstOutlineInst = BB == getStartBB() + ? frontInstruction() + : &*BB->instructionsWithoutDebug().begin(); + + unsigned FirstInstGVN = *getGVN(FirstOutlineInst); + unsigned FirstInstCanonNum = *getCanonicalNum(FirstInstGVN); + unsigned SourceGVN = *SourceCand.fromCanonicalNum(FirstInstCanonNum); + Value *SourceV = *SourceCand.fromGVN(SourceGVN); + BasicBlock *SourceBB = cast<Instruction>(SourceV)->getParent(); + unsigned SourceBBGVN = *SourceCand.getGVN(SourceBB); + unsigned SourceCanonBBGVN = *SourceCand.getCanonicalNum(SourceBBGVN); + CanonNumToNumber.insert(std::make_pair(SourceCanonBBGVN, BBGVNForCurrCand)); + NumberToCanonNum.insert(std::make_pair(BBGVNForCurrCand, SourceCanonBBGVN)); + } } void IRSimilarityCandidate::createCanonicalMappingFor( @@ -1162,11 +1200,12 @@ Mapper.InstClassifier.EnableIndirectCalls = EnableIndirectCalls; Mapper.EnableMatchCallsByName = EnableMatchingCallsByName; Mapper.InstClassifier.EnableIntrinsics = EnableIntrinsics; + Mapper.InstClassifier.EnableMustTailCalls = EnableMustTailCalls; populateMapper(Modules, InstrList, IntegerMapping); findCandidates(InstrList, IntegerMapping); - return SimilarityCandidates.getValue(); + return *SimilarityCandidates; } SimilarityGroupList &IRSimilarityIdentifier::findSimilarity(Module &M) { @@ -1175,6 +1214,7 @@ Mapper.InstClassifier.EnableIndirectCalls = EnableIndirectCalls; Mapper.EnableMatchCallsByName = EnableMatchingCallsByName; Mapper.InstClassifier.EnableIntrinsics = EnableIntrinsics; + Mapper.InstClassifier.EnableMustTailCalls = EnableMustTailCalls; std::vector<IRInstructionData *> InstrList; std::vector<unsigned> IntegerMapping; @@ -1182,7 +1222,7 @@ populateMapper(M, InstrList, IntegerMapping); findCandidates(InstrList, IntegerMapping); - return SimilarityCandidates.getValue(); + return *SimilarityCandidates; } INITIALIZE_PASS(IRSimilarityIdentifierWrapperPass, "ir-similarity-identifier", @@ -1196,7 +1236,8 @@ bool IRSimilarityIdentifierWrapperPass::doInitialization(Module &M) { IRSI.reset(new IRSimilarityIdentifier(!DisableBranches, !DisableIndirectCalls, - MatchCallsByName, !DisableIntrinsics)); + MatchCallsByName, !DisableIntrinsics, + false)); return false; } @@ -1214,7 +1255,8 @@ IRSimilarityIdentifier IRSimilarityAnalysis::run(Module &M, ModuleAnalysisManager &) { auto IRSI = IRSimilarityIdentifier(!DisableBranches, !DisableIndirectCalls, - MatchCallsByName, !DisableIntrinsics); + MatchCallsByName, !DisableIntrinsics, + false); IRSI.findSimilarity(M); return IRSI; }
diff --git a/src/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp b/src/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp index 74b0d67..a51e974 100644 --- a/src/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp +++ b/src/llvm-project/llvm/lib/Analysis/IVDescriptors.cpp
@@ -11,26 +11,16 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/IVDescriptors.h" -#include "llvm/ADT/ScopeExit.h" -#include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/DemandedBits.h" -#include "llvm/Analysis/DomTreeUpdater.h" -#include "llvm/Analysis/GlobalsModRef.h" -#include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/LoopPass.h" -#include "llvm/Analysis/MustExecute.h" #include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" -#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/IR/PatternMatch.h" #include "llvm/IR/ValueHandle.h" -#include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/KnownBits.h" @@ -73,20 +63,6 @@ return (Kind != RecurKind::None) && !isIntegerRecurrenceKind(Kind); } -bool RecurrenceDescriptor::isArithmeticRecurrenceKind(RecurKind Kind) { - switch (Kind) { - default: - break; - case RecurKind::Add: - case RecurKind::Mul: - case RecurKind::FAdd: - case RecurKind::FMul: - case RecurKind::FMulAdd: - return true; - } - return false; -} - /// Determines if Phi may have been type-promoted. If Phi has a single user /// that ANDs the Phi with a type mask, return the user. RT is updated to /// account for the narrower bit width represented by the mask, and the AND @@ -237,12 +213,10 @@ return true; } -bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, - Loop *TheLoop, FastMathFlags FuncFMF, - RecurrenceDescriptor &RedDes, - DemandedBits *DB, - AssumptionCache *AC, - DominatorTree *DT) { +bool RecurrenceDescriptor::AddReductionVar( + PHINode *Phi, RecurKind Kind, Loop *TheLoop, FastMathFlags FuncFMF, + RecurrenceDescriptor &RedDes, DemandedBits *DB, AssumptionCache *AC, + DominatorTree *DT, ScalarEvolution *SE) { if (Phi->getNumIncomingValues() != 2) return false; @@ -259,6 +233,12 @@ // This includes users of the reduction, variables (which form a cycle // which ends in the phi node). Instruction *ExitInstruction = nullptr; + + // Variable to keep last visited store instruction. By the end of the + // algorithm this variable will be either empty or having intermediate + // reduction value stored in invariant address. + StoreInst *IntermediateStore = nullptr; + // Indicates that we found a reduction operation in our scan. bool FoundReduxOp = false; @@ -324,6 +304,10 @@ // - By instructions outside of the loop (safe). // * One value may have several outside users, but all outside // uses must be of the same value. + // - By store instructions with a loop invariant address (safe with + // the following restrictions): + // * If there are several stores, all must have the same address. + // * Final value should be stored in that loop invariant address. // - By an instruction that is not part of the reduction (not safe). // This is either: // * An instruction type other than PHI or the reduction operation. @@ -331,6 +315,43 @@ while (!Worklist.empty()) { Instruction *Cur = Worklist.pop_back_val(); + // Store instructions are allowed iff it is the store of the reduction + // value to the same loop invariant memory location. + if (auto *SI = dyn_cast<StoreInst>(Cur)) { + if (!SE) { + LLVM_DEBUG(dbgs() << "Store instructions are not processed without " + << "Scalar Evolution Analysis\n"); + return false; + } + + const SCEV *PtrScev = SE->getSCEV(SI->getPointerOperand()); + // Check it is the same address as previous stores + if (IntermediateStore) { + const SCEV *OtherScev = + SE->getSCEV(IntermediateStore->getPointerOperand()); + + if (OtherScev != PtrScev) { + LLVM_DEBUG(dbgs() << "Storing reduction value to different addresses " + << "inside the loop: " << *SI->getPointerOperand() + << " and " + << *IntermediateStore->getPointerOperand() << '\n'); + return false; + } + } + + // Check the pointer is loop invariant + if (!SE->isLoopInvariant(PtrScev, TheLoop)) { + LLVM_DEBUG(dbgs() << "Storing reduction value to non-uniform address " + << "inside the loop: " << *SI->getPointerOperand() + << '\n'); + return false; + } + + // IntermediateStore is always the last store in the loop. + IntermediateStore = SI; + continue; + } + // No Users. // If the instruction has no users then this is a broken chain and can't be // a reduction variable. @@ -453,10 +474,17 @@ // reductions which are represented as a cmp followed by a select. InstDesc IgnoredVal(false, nullptr); if (VisitedInsts.insert(UI).second) { - if (isa<PHINode>(UI)) + if (isa<PHINode>(UI)) { PHIs.push_back(UI); - else + } else { + StoreInst *SI = dyn_cast<StoreInst>(UI); + if (SI && SI->getPointerOperand() == Cur) { + // Reduction variable chain can only be stored somewhere but it + // can't be used as an address. + return false; + } NonPHIs.push_back(UI); + } } else if (!isa<PHINode>(UI) && ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) && !isa<SelectInst>(UI)) || @@ -476,7 +504,7 @@ // This means we have seen one but not the other instruction of the // pattern or more than just a select and cmp. Zero implies that we saw a - // llvm.min/max instrinsic, which is always OK. + // llvm.min/max intrinsic, which is always OK. if (isMinMaxRecurrenceKind(Kind) && NumCmpSelectPatternInst != 2 && NumCmpSelectPatternInst != 0) return false; @@ -484,6 +512,32 @@ if (isSelectCmpRecurrenceKind(Kind) && NumCmpSelectPatternInst != 1) return false; + if (IntermediateStore) { + // Check that stored value goes to the phi node again. This way we make sure + // that the value stored in IntermediateStore is indeed the final reduction + // value. + if (!is_contained(Phi->operands(), IntermediateStore->getValueOperand())) { + LLVM_DEBUG(dbgs() << "Not a final reduction value stored: " + << *IntermediateStore << '\n'); + return false; + } + + // If there is an exit instruction it's value should be stored in + // IntermediateStore + if (ExitInstruction && + IntermediateStore->getValueOperand() != ExitInstruction) { + LLVM_DEBUG(dbgs() << "Last store Instruction of reduction value does not " + "store last calculated value of the reduction: " + << *IntermediateStore << '\n'); + return false; + } + + // If all uses are inside the loop (intermediate stores), then the + // reduction value after the loop will be the one used in the last store. + if (!ExitInstruction) + ExitInstruction = cast<Instruction>(IntermediateStore->getValueOperand()); + } + if (!FoundStartPHI || !FoundReduxOp || !ExitInstruction) return false; @@ -545,9 +599,9 @@ // is saved as part of the RecurrenceDescriptor. // Save the description of this reduction variable. - RecurrenceDescriptor RD(RdxStart, ExitInstruction, Kind, FMF, ExactFPMathInst, - RecurrenceType, IsSigned, IsOrdered, CastInsts, - MinWidthCastToRecurrenceType); + RecurrenceDescriptor RD(RdxStart, ExitInstruction, IntermediateStore, Kind, + FMF, ExactFPMathInst, RecurrenceType, IsSigned, + IsOrdered, CastInsts, MinWidthCastToRecurrenceType); RedDes = RD; return true; @@ -771,7 +825,8 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RedDes, DemandedBits *DB, AssumptionCache *AC, - DominatorTree *DT) { + DominatorTree *DT, + ScalarEvolution *SE) { BasicBlock *Header = TheLoop->getHeader(); Function &F = *Header->getParent(); FastMathFlags FMF; @@ -780,72 +835,85 @@ FMF.setNoSignedZeros( F.getFnAttribute("no-signed-zeros-fp-math").getValueAsBool()); - if (AddReductionVar(Phi, RecurKind::Add, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::Add, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found a MUL reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::Or, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::Or, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found an OR reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::And, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::And, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found an AND reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::Xor, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::Xor, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::SMax, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::SMax, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found a SMAX reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::SMin, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::SMin, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found a SMIN reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::UMax, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::UMax, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found a UMAX reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::UMin, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::UMin, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n"); return true; } if (AddReductionVar(Phi, RecurKind::SelectICmp, TheLoop, FMF, RedDes, DB, AC, - DT)) { + DT, SE)) { LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found a float MAX reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FMin, TheLoop, FMF, RedDes, DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::FMin, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n"); return true; } if (AddReductionVar(Phi, RecurKind::SelectFCmp, TheLoop, FMF, RedDes, DB, AC, - DT)) { + DT, SE)) { LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI." << " PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, - DT)) { + if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT, + SE)) { LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n"); return true; } @@ -917,12 +985,37 @@ SinkCandidate->mayReadFromMemory() || SinkCandidate->isTerminator()) return false; - // Do not try to sink an instruction multiple times (if multiple operands - // are first order recurrences). - // TODO: We can support this case, by sinking the instruction after the - // 'deepest' previous instruction. - if (SinkAfter.find(SinkCandidate) != SinkAfter.end()) - return false; + // Avoid sinking an instruction multiple times (if multiple operands are + // first order recurrences) by sinking once - after the latest 'previous' + // instruction. + auto It = SinkAfter.find(SinkCandidate); + if (It != SinkAfter.end()) { + auto *OtherPrev = It->second; + // Find the earliest entry in the 'sink-after' chain. The last entry in + // the chain is the original 'Previous' for a recurrence handled earlier. + auto EarlierIt = SinkAfter.find(OtherPrev); + while (EarlierIt != SinkAfter.end()) { + Instruction *EarlierInst = EarlierIt->second; + EarlierIt = SinkAfter.find(EarlierInst); + // Bail out if order has not been preserved. + if (EarlierIt != SinkAfter.end() && + !DT->dominates(EarlierInst, OtherPrev)) + return false; + OtherPrev = EarlierInst; + } + // Bail out if order has not been preserved. + if (OtherPrev != It->second && !DT->dominates(It->second, OtherPrev)) + return false; + + // SinkCandidate is already being sunk after an instruction after + // Previous. Nothing left to do. + if (DT->dominates(Previous, OtherPrev) || Previous == OtherPrev) + return true; + // Otherwise, Previous comes after OtherPrev and SinkCandidate needs to be + // re-sunk to Previous, instead of sinking to OtherPrev. Remove + // SinkCandidate from SinkAfter to ensure it's insert position is updated. + SinkAfter.erase(SinkCandidate); + } // If we reach a PHI node that is not dominated by Previous, we reached a // header PHI. No need for sinking. @@ -1052,7 +1145,7 @@ // to check for a pair of icmp/select, for which we use getNextInstruction and // isCorrectOpcode functions to step the right number of instruction, and // check the icmp/select pair. - // FIXME: We also do not attempt to look through Phi/Select's yet, which might + // FIXME: We also do not attempt to look through Select's yet, which might // be part of the reduction chain, or attempt to looks through And's to find a // smaller bitwidth. Subs are also currently not allowed (which are usually // treated as part of a add reduction) as they are expected to generally be @@ -1062,16 +1155,21 @@ if (RedOp == Instruction::ICmp || RedOp == Instruction::FCmp) ExpectedUses = 2; - auto getNextInstruction = [&](Instruction *Cur) { - if (RedOp == Instruction::ICmp || RedOp == Instruction::FCmp) { - // We are expecting a icmp/select pair, which we go to the next select - // instruction if we can. We already know that Cur has 2 uses. - if (isa<SelectInst>(*Cur->user_begin())) - return cast<Instruction>(*Cur->user_begin()); - else - return cast<Instruction>(*std::next(Cur->user_begin())); + auto getNextInstruction = [&](Instruction *Cur) -> Instruction * { + for (auto *User : Cur->users()) { + Instruction *UI = cast<Instruction>(User); + if (isa<PHINode>(UI)) + continue; + if (RedOp == Instruction::ICmp || RedOp == Instruction::FCmp) { + // We are expecting a icmp/select pair, which we go to the next select + // instruction if we can. We already know that Cur has 2 uses. + if (isa<SelectInst>(UI)) + return UI; + continue; + } + return UI; } - return cast<Instruction>(*Cur->user_begin()); + return nullptr; }; auto isCorrectOpcode = [&](Instruction *Cur) { if (RedOp == Instruction::ICmp || RedOp == Instruction::FCmp) { @@ -1086,22 +1184,46 @@ return Cur->getOpcode() == RedOp; }; + // Attempt to look through Phis which are part of the reduction chain + unsigned ExtraPhiUses = 0; + Instruction *RdxInstr = LoopExitInstr; + if (auto ExitPhi = dyn_cast<PHINode>(LoopExitInstr)) { + if (ExitPhi->getNumIncomingValues() != 2) + return {}; + + Instruction *Inc0 = dyn_cast<Instruction>(ExitPhi->getIncomingValue(0)); + Instruction *Inc1 = dyn_cast<Instruction>(ExitPhi->getIncomingValue(1)); + + Instruction *Chain = nullptr; + if (Inc0 == Phi) + Chain = Inc1; + else if (Inc1 == Phi) + Chain = Inc0; + else + return {}; + + RdxInstr = Chain; + ExtraPhiUses = 1; + } + // The loop exit instruction we check first (as a quick test) but add last. We // check the opcode is correct (and dont allow them to be Subs) and that they // have expected to have the expected number of uses. They will have one use // from the phi and one from a LCSSA value, no matter the type. - if (!isCorrectOpcode(LoopExitInstr) || !LoopExitInstr->hasNUses(2)) + if (!isCorrectOpcode(RdxInstr) || !LoopExitInstr->hasNUses(2)) return {}; - // Check that the Phi has one (or two for min/max) uses. - if (!Phi->hasNUses(ExpectedUses)) + // Check that the Phi has one (or two for min/max) uses, plus an extra use + // for conditional reductions. + if (!Phi->hasNUses(ExpectedUses + ExtraPhiUses)) return {}; + Instruction *Cur = getNextInstruction(Phi); // Each other instruction in the chain should have the expected number of uses // and be the correct opcode. - while (Cur != LoopExitInstr) { - if (!isCorrectOpcode(Cur) || !Cur->hasNUses(ExpectedUses)) + while (Cur != RdxInstr) { + if (!Cur || !isCorrectOpcode(Cur) || !Cur->hasNUses(ExpectedUses)) return {}; ReductionOperations.push_back(Cur);
diff --git a/src/llvm-project/llvm/lib/Analysis/IVUsers.cpp b/src/llvm-project/llvm/lib/Analysis/IVUsers.cpp index 0f3929f..8302116 100644 --- a/src/llvm-project/llvm/lib/Analysis/IVUsers.cpp +++ b/src/llvm-project/llvm/lib/Analysis/IVUsers.cpp
@@ -12,25 +12,21 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/IVUsers.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/CodeMetrics.h" #include "llvm/Analysis/LoopAnalysisManager.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopPass.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Config/llvm-config.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" -#include "llvm/IR/Type.h" #include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> using namespace llvm; #define DEBUG_TYPE "iv-users" @@ -278,7 +274,7 @@ OS << " "; IVUse.getOperandValToReplace()->printAsOperand(OS, false); OS << " = " << *getReplacementExpr(IVUse); - for (auto PostIncLoop : IVUse.PostIncLoops) { + for (const auto *PostIncLoop : IVUse.PostIncLoops) { OS << " (post-inc with loop "; PostIncLoop->getHeader()->printAsOperand(OS, false); OS << ")";
diff --git a/src/llvm-project/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp index b112ed2..ebfa1c8 100644 --- a/src/llvm-project/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
@@ -13,12 +13,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/IndirectCallPromotionAnalysis.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/Analysis/IndirectCallVisitor.h" -#include "llvm/IR/InstIterator.h" -#include "llvm/IR/InstVisitor.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Instruction.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -31,7 +26,7 @@ // The percent threshold for the direct-call target (this call site vs the // remaining call count) for it to be considered as the promotion target. static cl::opt<unsigned> ICPRemainingPercentThreshold( - "icp-remaining-percent-threshold", cl::init(30), cl::Hidden, cl::ZeroOrMore, + "icp-remaining-percent-threshold", cl::init(30), cl::Hidden, cl::desc("The percentage threshold against remaining unpromoted indirect " "call count for the promotion")); @@ -39,14 +34,14 @@ // total call count) for it to be considered as the promotion target. static cl::opt<unsigned> ICPTotalPercentThreshold("icp-total-percent-threshold", cl::init(5), - cl::Hidden, cl::ZeroOrMore, + cl::Hidden, cl::desc("The percentage threshold against total " "count for the promotion")); // Set the maximum number of targets to promote for a single indirect-call // callsite. static cl::opt<unsigned> - MaxNumPromotions("icp-max-prom", cl::init(3), cl::Hidden, cl::ZeroOrMore, + MaxNumPromotions("icp-max-prom", cl::init(3), cl::Hidden, cl::desc("Max number of promotions for a single indirect " "call callsite"));
diff --git a/src/llvm-project/llvm/lib/Analysis/InlineAdvisor.cpp b/src/llvm-project/llvm/lib/Analysis/InlineAdvisor.cpp index f6e3dd3..3fafc30 100644 --- a/src/llvm-project/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/src/llvm-project/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -13,14 +13,15 @@ #include "llvm/Analysis/InlineAdvisor.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/InlineCost.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/ReplayInlineAdvisor.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" #include "llvm/IR/DebugInfoMetadata.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/raw_ostream.h" @@ -55,6 +56,11 @@ cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden); +static cl::opt<bool> + AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), + cl::desc("If true, annotate inline advisor remarks " + "with LTO and pass information.")); + extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats; namespace { @@ -80,7 +86,8 @@ void recordUnsuccessfulInliningImpl(const InlineResult &Result) override { if (IsInliningRecommended) ORE.emit([&]() { - return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) + return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), + "NotInlined", DLoc, Block) << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '" << NV("Caller", Caller) << "': " << NV("Reason", Result.getFailureReason()); @@ -99,7 +106,8 @@ llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) + "; " + inlineCostStr(*OIC)); ORE.emit([&]() { - return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) + return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), + "NotInlined", DLoc, Block) << "'" << NV("Callee", Callee) << "' is not inlined into '" << NV("Caller", Caller) << "': " << NV("Reason", Result.getFailureReason()); @@ -108,12 +116,16 @@ void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() { if (EmitRemarks) - emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC); + emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC, + /* ForProfileContext= */ false, + Advisor->getAnnotatedInlinePassName()); } void DefaultInlineAdvice::recordInliningImpl() { if (EmitRemarks) - emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC); + emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC, + /* ForProfileContext= */ false, + Advisor->getAnnotatedInlinePassName()); } llvm::Optional<llvm::InlineCost> static getDefaultInlineAdvice( @@ -146,7 +158,7 @@ }; return llvm::shouldInline( CB, GetInlineCost, ORE, - Params.EnableDeferral.getValueOr(EnableInlineDeferral)); + Params.EnableDeferral.value_or(EnableInlineDeferral)); } std::unique_ptr<InlineAdvice> @@ -185,18 +197,18 @@ bool InlineAdvisorAnalysis::Result::tryCreate( InlineParams Params, InliningAdvisorMode Mode, - const ReplayInlinerSettings &ReplaySettings) { + const ReplayInlinerSettings &ReplaySettings, InlineContext IC) { auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); switch (Mode) { case InliningAdvisorMode::Default: LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n"); - Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params)); + Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC)); // Restrict replay to default advisor, ML advisors are stateful so // replay will need augmentations to interleave with them correctly. if (!ReplaySettings.ReplayFile.empty()) { Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(), std::move(Advisor), ReplaySettings, - /* EmitRemarks =*/true); + /* EmitRemarks =*/true, IC); } break; case InliningAdvisorMode::Development: @@ -442,7 +454,7 @@ } void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) { - if (!DLoc.get()) { + if (!DLoc) { return; } @@ -499,8 +511,12 @@ PassName); } -InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM) - : M(M), FAM(FAM) { +InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + Optional<InlineContext> IC) + : M(M), FAM(FAM), IC(IC), + AnnotatedInlinePassName((IC && AnnotateInlinePhase) + ? llvm::AnnotateInlinePassName(*IC) + : DEBUG_TYPE) { if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) { ImportedFunctionsStats = std::make_unique<ImportedFunctionsInliningStatistics>(); @@ -522,6 +538,48 @@ Advice); } +static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) { + switch (LTOPhase) { + case (ThinOrFullLTOPhase::None): + return "main"; + case (ThinOrFullLTOPhase::ThinLTOPreLink): + case (ThinOrFullLTOPhase::FullLTOPreLink): + return "prelink"; + case (ThinOrFullLTOPhase::ThinLTOPostLink): + case (ThinOrFullLTOPhase::FullLTOPostLink): + return "postlink"; + } + llvm_unreachable("unreachable"); +} + +static inline const char *getInlineAdvisorContext(InlinePass IP) { + switch (IP) { + case (InlinePass::AlwaysInliner): + return "always-inline"; + case (InlinePass::CGSCCInliner): + return "cgscc-inline"; + case (InlinePass::EarlyInliner): + return "early-inline"; + case (InlinePass::MLInliner): + return "ml-inline"; + case (InlinePass::ModuleInliner): + return "module-inline"; + case (InlinePass::ReplayCGSCCInliner): + return "replay-cgscc-inline"; + case (InlinePass::ReplaySampleProfileInliner): + return "replay-sample-profile-inline"; + case (InlinePass::SampleProfileInliner): + return "sample-profile-inline"; + } + + llvm_unreachable("unreachable"); +} + +std::string llvm::AnnotateInlinePassName(InlineContext IC) { + return std::string(getLTOPhase(IC.LTOPhase)) + "-" + + std::string(getInlineAdvisorContext(IC.Pass)); +} + InlineAdvisor::MandatoryInliningKind InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE) { @@ -536,7 +594,7 @@ auto TrivialDecision = llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI); - if (TrivialDecision.hasValue()) { + if (TrivialDecision) { if (TrivialDecision->isSuccess()) return MandatoryInliningKind::Always; else @@ -568,3 +626,22 @@ IA->getAdvisor()->print(OS); return PreservedAnalyses::all(); } + +PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run( + LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG, + CGSCCUpdateResult &UR) { + const auto &MAMProxy = + AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG); + + if (InitialC.size() == 0) { + OS << "SCC is empty!\n"; + return PreservedAnalyses::all(); + } + Module &M = *InitialC.begin()->getFunction().getParent(); + const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M); + if (!IA) + OS << "No Inline Advisor\n"; + else + IA->getAdvisor()->print(OS); + return PreservedAnalyses::all(); +}
diff --git a/src/llvm-project/llvm/lib/Analysis/InlineCost.cpp b/src/llvm-project/llvm/lib/Analysis/InlineCost.cpp index cd5314e..8192ed5 100644 --- a/src/llvm-project/llvm/lib/Analysis/InlineCost.cpp +++ b/src/llvm-project/llvm/lib/Analysis/InlineCost.cpp
@@ -18,11 +18,11 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/CFG.h" #include "llvm/Analysis/CodeMetrics.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -42,6 +42,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/FormattedStream.h" #include "llvm/Support/raw_ostream.h" +#include <limits> using namespace llvm; @@ -51,24 +52,33 @@ static cl::opt<int> DefaultThreshold("inlinedefault-threshold", cl::Hidden, cl::init(225), - cl::ZeroOrMore, cl::desc("Default amount of inlining to perform")); +// We introduce this option since there is a minor compile-time win by avoiding +// addition of TTI attributes (target-features in particular) to inline +// candidates when they are guaranteed to be the same as top level methods in +// some use cases. If we avoid adding the attribute, we need an option to avoid +// checking these attributes. +static cl::opt<bool> IgnoreTTIInlineCompatible( + "ignore-tti-inline-compatible", cl::Hidden, cl::init(false), + cl::desc("Ignore TTI attributes compatibility check between callee/caller " + "during inline cost calculation")); + static cl::opt<bool> PrintInstructionComments( "print-instruction-comments", cl::Hidden, cl::init(false), cl::desc("Prints comments for instruction based on inline cost analysis")); static cl::opt<int> InlineThreshold( - "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, + "inline-threshold", cl::Hidden, cl::init(225), cl::desc("Control the amount of inlining to perform (default = 225)")); static cl::opt<int> HintThreshold( - "inlinehint-threshold", cl::Hidden, cl::init(325), cl::ZeroOrMore, + "inlinehint-threshold", cl::Hidden, cl::init(325), cl::desc("Threshold for inlining functions with inline hint")); static cl::opt<int> ColdCallSiteThreshold("inline-cold-callsite-threshold", cl::Hidden, - cl::init(45), cl::ZeroOrMore, + cl::init(45), cl::desc("Threshold for inlining cold callsites")); static cl::opt<bool> InlineEnableCostBenefitAnalysis( @@ -76,12 +86,11 @@ cl::desc("Enable the cost-benefit analysis for the inliner")); static cl::opt<int> InlineSavingsMultiplier( - "inline-savings-multiplier", cl::Hidden, cl::init(8), cl::ZeroOrMore, + "inline-savings-multiplier", cl::Hidden, cl::init(8), cl::desc("Multiplier to multiply cycle savings by during inlining")); static cl::opt<int> InlineSizeAllowance("inline-size-allowance", cl::Hidden, cl::init(100), - cl::ZeroOrMore, cl::desc("The maximum size of a callee that get's " "inlined without sufficient cycle savings")); @@ -89,26 +98,25 @@ // PGO before we actually hook up inliner with analysis passes such as BPI and // BFI. static cl::opt<int> ColdThreshold( - "inlinecold-threshold", cl::Hidden, cl::init(45), cl::ZeroOrMore, + "inlinecold-threshold", cl::Hidden, cl::init(45), cl::desc("Threshold for inlining functions with cold attribute")); static cl::opt<int> HotCallSiteThreshold("hot-callsite-threshold", cl::Hidden, cl::init(3000), - cl::ZeroOrMore, cl::desc("Threshold for hot callsites ")); static cl::opt<int> LocallyHotCallSiteThreshold( - "locally-hot-callsite-threshold", cl::Hidden, cl::init(525), cl::ZeroOrMore, + "locally-hot-callsite-threshold", cl::Hidden, cl::init(525), cl::desc("Threshold for locally hot callsites ")); static cl::opt<int> ColdCallSiteRelFreq( - "cold-callsite-rel-freq", cl::Hidden, cl::init(2), cl::ZeroOrMore, + "cold-callsite-rel-freq", cl::Hidden, cl::init(2), cl::desc("Maximum block frequency, expressed as a percentage of caller's " "entry frequency, for a callsite to be cold in the absence of " "profile information.")); static cl::opt<int> HotCallSiteRelFreq( - "hot-callsite-rel-freq", cl::Hidden, cl::init(60), cl::ZeroOrMore, + "hot-callsite-rel-freq", cl::Hidden, cl::init(60), cl::desc("Minimum block frequency, expressed as a multiple of caller's " "entry frequency, for a callsite to be hot in the absence of " "profile information.")); @@ -117,14 +125,25 @@ "inline-call-penalty", cl::Hidden, cl::init(25), cl::desc("Call penalty that is applied per callsite when inlining")); +static cl::opt<size_t> + StackSizeThreshold("inline-max-stacksize", cl::Hidden, + cl::init(std::numeric_limits<size_t>::max()), + cl::desc("Do not inline functions with a stack size " + "that exceeds the specified limit")); + +static cl::opt<size_t> + RecurStackSizeThreshold("recursive-inline-max-stacksize", cl::Hidden, + cl::init(InlineConstants::TotalAllocaSizeRecursiveCaller), + cl::desc("Do not inline recursive functions with a stack " + "size that exceeds the specified limit")); + static cl::opt<bool> OptComputeFullInlineCost( - "inline-cost-full", cl::Hidden, cl::init(false), cl::ZeroOrMore, + "inline-cost-full", cl::Hidden, cl::desc("Compute the full inline cost of a call site even when the cost " "exceeds the threshold.")); static cl::opt<bool> InlineCallerSupersetNoBuiltin( "inline-caller-superset-nobuiltin", cl::Hidden, cl::init(true), - cl::ZeroOrMore, cl::desc("Allow inlining when caller has a superset of callee's nobuiltin " "attributes.")); @@ -132,28 +151,9 @@ "disable-gep-const-evaluation", cl::Hidden, cl::init(false), cl::desc("Disables evaluation of GetElementPtr with constant operands")); -namespace { -/// This function behaves more like CallBase::hasFnAttr: when it looks for the -/// requested attribute, it check both the call instruction and the called -/// function (if it's available and operand bundles don't prohibit that). -Attribute getFnAttr(CallBase &CB, StringRef AttrKind) { - Attribute CallAttr = CB.getFnAttr(AttrKind); - if (CallAttr.isValid()) - return CallAttr; - - // Operand bundles override attributes on the called function, but don't - // override attributes directly present on the call instruction. - if (!CB.isFnAttrDisallowedByOpBundle(AttrKind)) - if (const Function *F = CB.getCalledFunction()) - return F->getFnAttribute(AttrKind); - - return {}; -} -} // namespace - namespace llvm { Optional<int> getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind) { - Attribute Attr = getFnAttr(CB, AttrKind); + Attribute Attr = CB.getFnAttr(AttrKind); int AttrValue; if (Attr.getValueAsString().getAsInteger(10, AttrValue)) return None; @@ -185,8 +185,8 @@ public: InlineCostAnnotationWriter(InlineCostCallAnalyzer *ICCA) : ICCA(ICCA) {} - virtual void emitInstructionAnnot(const Instruction *I, - formatted_raw_ostream &OS) override; + void emitInstructionAnnot(const Instruction *I, + formatted_raw_ostream &OS) override; }; /// Carry out call site analysis, in order to evaluate inlinability. @@ -202,7 +202,7 @@ friend class InstVisitor<CallAnalyzer, bool>; protected: - virtual ~CallAnalyzer() {} + virtual ~CallAnalyzer() = default; /// The TargetTransformInfo available for this compilation. const TargetTransformInfo &TTI; @@ -356,7 +356,7 @@ DenseMap<Value *, std::pair<Value *, APInt>> ConstantOffsetPtrs; /// Keep track of dead blocks due to the constant arguments. - SetVector<BasicBlock *> DeadBlocks; + SmallPtrSet<BasicBlock *, 16> DeadBlocks; /// The mapping of the blocks to their known unique successors due to the /// constant arguments. @@ -389,8 +389,7 @@ bool canFoldInboundsGEP(GetElementPtrInst &I); bool accumulateGEPOffset(GEPOperator &GEP, APInt &Offset); bool simplifyCallSite(Function *F, CallBase &Call); - template <typename Callable> - bool simplifyInstruction(Instruction &I, Callable Evaluate); + bool simplifyInstruction(Instruction &I); bool simplifyIntrinsicCallIsConstant(CallBase &CB); ConstantInt *stripAndComputeInBoundsConstantOffsets(Value *&V); @@ -708,8 +707,8 @@ BlockFrequencyInfo *BFI = &(GetBFI(F)); assert(BFI && "BFI must be available"); auto ProfileCount = BFI->getBlockProfileCount(BB); - assert(ProfileCount.hasValue()); - if (ProfileCount.getValue() == 0) + assert(ProfileCount); + if (ProfileCount.value() == 0) ColdSize += Cost - CostAtBBStart; } @@ -833,14 +832,14 @@ } auto ProfileCount = CalleeBFI->getBlockProfileCount(&BB); - assert(ProfileCount.hasValue()); - CurrentSavings *= ProfileCount.getValue(); + assert(ProfileCount); + CurrentSavings *= ProfileCount.value(); CycleSavings += CurrentSavings; } // Compute the cycle savings per call. auto EntryProfileCount = F.getEntryCount(); - assert(EntryProfileCount.hasValue() && EntryProfileCount->getCount()); + assert(EntryProfileCount && EntryProfileCount->getCount()); auto EntryCount = EntryProfileCount->getCount(); CycleSavings += EntryCount / 2; CycleSavings = CycleSavings.udiv(EntryCount); @@ -849,7 +848,7 @@ auto *CallerBB = CandidateCall.getParent(); BlockFrequencyInfo *CallerBFI = &(GetBFI(*(CallerBB->getParent()))); CycleSavings += getCallsiteCost(this->CandidateCall, DL); - CycleSavings *= CallerBFI->getBlockProfileCount(CallerBB).getValue(); + CycleSavings *= *CallerBFI->getBlockProfileCount(CallerBB); // Remove the cost of the cold basic blocks. int Size = Cost - ColdSize; @@ -919,7 +918,7 @@ if (auto Result = costBenefitAnalysis()) { DecidedByCostBenefit = true; - if (Result.getValue()) + if (*Result) return InlineResult::success(); else return InlineResult::failure("Cost over threshold."); @@ -987,6 +986,8 @@ if (F.getCallingConv() == CallingConv::Cold) Cost += InlineConstants::ColdccPenalty; + LLVM_DEBUG(dbgs() << " Initial cost: " << Cost << "\n"); + // Check if we're done. This can happen due to bonuses and penalties. if (Cost >= Threshold && !ComputeFullInlineCost) return InlineResult::failure("high cost"); @@ -1011,7 +1012,7 @@ BoostIndirectCalls(BoostIndirect), IgnoreThreshold(IgnoreThreshold), CostBenefitAnalysisEnabled(isCostBenefitAnalysisEnabled()), Writer(this) { - AllowRecursiveCall = Params.AllowRecursiveCall.getValue(); + AllowRecursiveCall = *Params.AllowRecursiveCall; } /// Annotation Writer for instruction details @@ -1029,7 +1030,7 @@ return None; } - virtual ~InlineCostCallAnalyzer() {} + virtual ~InlineCostCallAnalyzer() = default; int getThreshold() const { return Threshold; } int getCost() const { return Cost; } Optional<CostBenefitPair> getCostBenefitPair() { return CostBenefit; } @@ -1212,6 +1213,10 @@ set(InlineCostFeatureIndex::ColdCcPenalty, (F.getCallingConv() == CallingConv::Cold)); + set(InlineCostFeatureIndex::LastCallToStaticBonus, + (F.hasLocalLinkage() && F.hasOneLiveUse() && + &F == CandidateCall.getCalledFunction())); + // FIXME: we shouldn't repeat this logic in both the Features and Cost // analyzer - instead, we should abstract it to a common method in the // CallAnalyzer @@ -1271,7 +1276,7 @@ auto C = ICCA->getSimplifiedValue(const_cast<Instruction *>(I)); if (C) { OS << ", simplified to "; - C.getValue()->print(OS, true); + (*C)->print(OS, true); } OS << "\n"; } @@ -1510,13 +1515,7 @@ }; if (!DisableGEPConstOperand) - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - SmallVector<Constant *, 2> Indices; - for (unsigned int Index = 1; Index < COps.size(); ++Index) - Indices.push_back(COps[Index]); - return ConstantExpr::getGetElementPtr( - I.getSourceElementType(), COps[0], Indices, I.isInBounds()); - })) + if (simplifyInstruction(I)) return true; if ((I.isInBounds() && canFoldInboundsGEP(I)) || IsGEPOffsetConstant(I)) { @@ -1534,11 +1533,8 @@ } /// Simplify \p I if its operands are constants and update SimplifiedValues. -/// \p Evaluate is a callable specific to instruction type that evaluates the -/// instruction when all the operands are constants. -template <typename Callable> -bool CallAnalyzer::simplifyInstruction(Instruction &I, Callable Evaluate) { - SmallVector<Constant *, 2> COps; +bool CallAnalyzer::simplifyInstruction(Instruction &I) { + SmallVector<Constant *> COps; for (Value *Op : I.operands()) { Constant *COp = dyn_cast<Constant>(Op); if (!COp) @@ -1547,7 +1543,7 @@ return false; COps.push_back(COp); } - auto *C = Evaluate(COps); + auto *C = ConstantFoldInstOperands(&I, COps, DL); if (!C) return false; SimplifiedValues[&I] = C; @@ -1577,9 +1573,7 @@ bool CallAnalyzer::visitBitCast(BitCastInst &I) { // Propagate constants through bitcasts. - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - return ConstantExpr::getBitCast(COps[0], I.getType()); - })) + if (simplifyInstruction(I)) return true; // Track base/offsets through casts @@ -1599,9 +1593,7 @@ bool CallAnalyzer::visitPtrToInt(PtrToIntInst &I) { // Propagate constants through ptrtoint. - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - return ConstantExpr::getPtrToInt(COps[0], I.getType()); - })) + if (simplifyInstruction(I)) return true; // Track base/offset pairs when converted to a plain integer provided the @@ -1631,9 +1623,7 @@ bool CallAnalyzer::visitIntToPtr(IntToPtrInst &I) { // Propagate constants through ptrtoint. - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - return ConstantExpr::getIntToPtr(COps[0], I.getType()); - })) + if (simplifyInstruction(I)) return true; // Track base/offset pairs when round-tripped through a pointer without @@ -1656,9 +1646,7 @@ bool CallAnalyzer::visitCastInst(CastInst &I) { // Propagate constants through casts. - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - return ConstantExpr::getCast(I.getOpcode(), COps[0], I.getType()); - })) + if (simplifyInstruction(I)) return true; // Disable SROA in the face of arbitrary casts we don't explicitly list @@ -1799,12 +1787,12 @@ // return min(A, B) if B is valid. auto MinIfValid = [](int A, Optional<int> B) { - return B ? std::min(A, B.getValue()) : A; + return B ? std::min(A, B.value()) : A; }; // return max(A, B) if B is valid. auto MaxIfValid = [](int A, Optional<int> B) { - return B ? std::max(A, B.getValue()) : A; + return B ? std::max(A, B.value()) : A; }; // Various bonus percentages. These are multiplied by Threshold to get the @@ -1864,7 +1852,7 @@ // current threshold, but AutoFDO + ThinLTO currently relies on this // behavior to prevent inlining of hot callsites during ThinLTO // compile phase. - Threshold = HotCallSiteThreshold.getValue(); + Threshold = *HotCallSiteThreshold; } else if (isColdCallSite(Call, CallerBFI)) { LLVM_DEBUG(dbgs() << "Cold callsite.\n"); // Do not apply bonuses for a cold callsite including the @@ -1915,9 +1903,7 @@ bool CallAnalyzer::visitCmpInst(CmpInst &I) { Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); // First try to handle simplified comparisons. - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - return ConstantExpr::getCompare(I.getPredicate(), COps[0], COps[1]); - })) + if (simplifyInstruction(I)) return true; if (I.getOpcode() == Instruction::FCmp) @@ -1993,11 +1979,11 @@ Value *SimpleV = nullptr; if (auto FI = dyn_cast<FPMathOperator>(&I)) - SimpleV = SimplifyBinOp(I.getOpcode(), CLHS ? CLHS : LHS, CRHS ? CRHS : RHS, + SimpleV = simplifyBinOp(I.getOpcode(), CLHS ? CLHS : LHS, CRHS ? CRHS : RHS, FI->getFastMathFlags(), DL); else SimpleV = - SimplifyBinOp(I.getOpcode(), CLHS ? CLHS : LHS, CRHS ? CRHS : RHS, DL); + simplifyBinOp(I.getOpcode(), CLHS ? CLHS : LHS, CRHS ? CRHS : RHS, DL); if (Constant *C = dyn_cast_or_null<Constant>(SimpleV)) SimplifiedValues[&I] = C; @@ -2027,7 +2013,7 @@ if (!COp) COp = SimplifiedValues.lookup(Op); - Value *SimpleV = SimplifyFNegInst( + Value *SimpleV = simplifyFNegInst( COp ? COp : Op, cast<FPMathOperator>(I).getFastMathFlags(), DL); if (Constant *C = dyn_cast_or_null<Constant>(SimpleV)) @@ -2076,9 +2062,7 @@ bool CallAnalyzer::visitExtractValue(ExtractValueInst &I) { // Constant folding for extract value is trivial. - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - return ConstantExpr::getExtractValue(COps[0], I.getIndices()); - })) + if (simplifyInstruction(I)) return true; // SROA can't look through these, but they may be free. @@ -2087,11 +2071,7 @@ bool CallAnalyzer::visitInsertValue(InsertValueInst &I) { // Constant folding for insert value is trivial. - if (simplifyInstruction(I, [&](SmallVectorImpl<Constant *> &COps) { - return ConstantExpr::getInsertValue(/*AggregateOperand*/ COps[0], - /*InsertedValueOperand*/ COps[1], - I.getIndices()); - })) + if (simplifyInstruction(I)) return true; // SROA can't look through these, but they may be free. @@ -2145,14 +2125,14 @@ if (isa<CallInst>(Call) && cast<CallInst>(Call).cannotDuplicate()) ContainsNoDuplicateCall = true; - Value *Callee = Call.getCalledOperand(); - Function *F = dyn_cast_or_null<Function>(Callee); + Function *F = Call.getCalledFunction(); bool IsIndirectCall = !F; if (IsIndirectCall) { // Check if this happens to be an indirect function call to a known function // in this inline context. If not, we've done all we can. + Value *Callee = Call.getCalledOperand(); F = dyn_cast_or_null<Function>(SimplifiedValues.lookup(Callee)); - if (!F) { + if (!F || F->getFunctionType() != Call.getFunctionType()) { onCallArgumentSetup(Call); if (!Call.onlyReadsMemory()) @@ -2470,8 +2450,7 @@ // If the caller is a recursive function then we don't want to inline // functions which allocate a lot of stack space because it would increase // the caller stack usage dramatically. - if (IsCallerRecursive && - AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller) { + if (IsCallerRecursive && AllocatedSize > RecurStackSizeThreshold) { auto IR = InlineResult::failure("recursive and allocates too much stack space"); if (ORE) @@ -2561,7 +2540,7 @@ NewDead.push_back(Succ); while (!NewDead.empty()) { BasicBlock *Dead = NewDead.pop_back_val(); - if (DeadBlocks.insert(Dead)) + if (DeadBlocks.insert(Dead).second) // Continue growing the dead block lists. for (BasicBlock *S : successors(Dead)) if (IsNewlyDead(S)) @@ -2716,6 +2695,11 @@ if (!OnlyOneCallAndLocalLinkage && ContainsNoDuplicateCall) return InlineResult::failure("noduplicate"); + // If the callee's stack size exceeds the user-specified threshold, + // do not let it be inlined. + if (AllocatedSize > StackSizeThreshold) + return InlineResult::failure("stacksize"); + return finalizeAnalysis(); } @@ -2754,7 +2738,8 @@ // object, and always returns the same object (which is overwritten on each // GetTLI call). Therefore we copy the first result. auto CalleeTLI = GetTLI(*Callee); - return TTI.areInlineCompatible(Caller, Callee) && + return (IgnoreTTIInlineCompatible || + TTI.areInlineCompatible(Caller, Callee)) && GetTLI(*Caller).areInlineCompatible(CalleeTLI, InlineCallerSupersetNoBuiltin) && AttributeFuncs::areInlineCompatible(*Caller, *Callee); @@ -2873,6 +2858,9 @@ // Calls to functions with always-inline attributes should be inlined // whenever possible. if (Call.hasFnAttr(Attribute::AlwaysInline)) { + if (Call.getAttributes().hasFnAttr(Attribute::NoInline)) + return InlineResult::failure("noinline call site attribute"); + auto IsViable = isInlineViable(*Callee); if (IsViable.isSuccess()) return InlineResult::success(); @@ -2920,7 +2908,7 @@ auto UserDecision = llvm::getAttributeBasedInliningDecision(Call, Callee, CalleeTTI, GetTLI); - if (UserDecision.hasValue()) { + if (UserDecision) { if (UserDecision->isSuccess()) return llvm::InlineCost::getAlways("always inline attribute"); return llvm::InlineCost::getNever(UserDecision->getFailureReason());
diff --git a/src/llvm-project/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp index a2e231e..2371ecb 100644 --- a/src/llvm-project/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp
@@ -15,33 +15,32 @@ #ifdef LLVM_HAVE_TF_API #include "llvm/Analysis/Utils/TFUtils.h" #endif -#include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" -#include "llvm/MC/MCAsmLayout.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/CommandLine.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <deque> - using namespace llvm; AnalysisKey InlineSizeEstimatorAnalysis::Key; -#define DEBUG_TYPE "inline-size-estimator" - #ifdef LLVM_HAVE_TF_API +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Instructions.h" +#include "llvm/MC/MCAsmLayout.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include <algorithm> +#include <deque> + cl::opt<std::string> TFIR2NativeModelPath( "ml-inliner-ir2native-model", cl::Hidden, cl::desc("Path to saved model evaluating native size from IR.")); +#define DEBUG_TYPE "inline-size-estimator" namespace { unsigned getMaxInstructionID() { #define LAST_OTHER_INST(NR) return NR; @@ -261,10 +260,10 @@ namespace llvm { class TFModelEvaluator {}; } // namespace llvm -InlineSizeEstimatorAnalysis::InlineSizeEstimatorAnalysis() {} +InlineSizeEstimatorAnalysis::InlineSizeEstimatorAnalysis() = default; InlineSizeEstimatorAnalysis ::InlineSizeEstimatorAnalysis( InlineSizeEstimatorAnalysis &&) {} -InlineSizeEstimatorAnalysis::~InlineSizeEstimatorAnalysis() {} +InlineSizeEstimatorAnalysis::~InlineSizeEstimatorAnalysis() = default; InlineSizeEstimatorAnalysis::Result InlineSizeEstimatorAnalysis::run(const Function &F, FunctionAnalysisManager &FAM) {
diff --git a/src/llvm-project/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp b/src/llvm-project/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp index 9fee57c..78e7f45 100644 --- a/src/llvm-project/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp +++ b/src/llvm-project/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp
@@ -68,7 +68,7 @@ void InstructionPrecedenceTracking::fill(const BasicBlock *BB) { FirstSpecialInsts.erase(BB); - for (auto &I : *BB) { + for (const auto &I : *BB) { NumInstScanned++; if (isSpecialInstruction(&I)) { FirstSpecialInsts[BB] = &I; @@ -101,7 +101,7 @@ void InstructionPrecedenceTracking::validateAll() const { // Check that for every known block the cached value is correct. - for (auto &It : FirstSpecialInsts) + for (const auto &It : FirstSpecialInsts) validate(It.first); } #endif
diff --git a/src/llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp b/src/llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp index 60895d3..21fe448 100644 --- a/src/llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/src/llvm-project/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -20,7 +20,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AssumptionCache.h" @@ -36,13 +35,10 @@ #include "llvm/IR/ConstantRange.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" -#include "llvm/IR/GetElementPtrTypeIterator.h" -#include "llvm/IR/GlobalAlias.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PatternMatch.h" -#include "llvm/IR/ValueHandle.h" #include "llvm/Support/KnownBits.h" #include <algorithm> using namespace llvm; @@ -52,28 +48,30 @@ enum { RecursionLimit = 3 }; -STATISTIC(NumExpand, "Number of expansions"); +STATISTIC(NumExpand, "Number of expansions"); STATISTIC(NumReassoc, "Number of reassociations"); -static Value *SimplifyAndInst(Value *, Value *, const SimplifyQuery &, unsigned); +static Value *simplifyAndInst(Value *, Value *, const SimplifyQuery &, + unsigned); static Value *simplifyUnOp(unsigned, Value *, const SimplifyQuery &, unsigned); static Value *simplifyFPUnOp(unsigned, Value *, const FastMathFlags &, const SimplifyQuery &, unsigned); -static Value *SimplifyBinOp(unsigned, Value *, Value *, const SimplifyQuery &, +static Value *simplifyBinOp(unsigned, Value *, Value *, const SimplifyQuery &, unsigned); -static Value *SimplifyBinOp(unsigned, Value *, Value *, const FastMathFlags &, +static Value *simplifyBinOp(unsigned, Value *, Value *, const FastMathFlags &, const SimplifyQuery &, unsigned); -static Value *SimplifyCmpInst(unsigned, Value *, Value *, const SimplifyQuery &, +static Value *simplifyCmpInst(unsigned, Value *, Value *, const SimplifyQuery &, unsigned); -static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, +static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse); -static Value *SimplifyOrInst(Value *, Value *, const SimplifyQuery &, unsigned); -static Value *SimplifyXorInst(Value *, Value *, const SimplifyQuery &, unsigned); -static Value *SimplifyCastInst(unsigned, Value *, Type *, - const SimplifyQuery &, unsigned); -static Value *SimplifyGEPInst(Type *, Value *, ArrayRef<Value *>, bool, +static Value *simplifyOrInst(Value *, Value *, const SimplifyQuery &, unsigned); +static Value *simplifyXorInst(Value *, Value *, const SimplifyQuery &, + unsigned); +static Value *simplifyCastInst(unsigned, Value *, Type *, const SimplifyQuery &, + unsigned); +static Value *simplifyGEPInst(Type *, Value *, ArrayRef<Value *>, bool, const SimplifyQuery &, unsigned); -static Value *SimplifySelectInst(Value *, Value *, Value *, +static Value *simplifySelectInst(Value *, Value *, Value *, const SimplifyQuery &, unsigned); static Value *foldSelectWithBinaryOp(Value *Cond, Value *TrueVal, @@ -120,15 +118,11 @@ /// For a boolean type or a vector of boolean type, return false or a vector /// with every element false. -static Constant *getFalse(Type *Ty) { - return ConstantInt::getFalse(Ty); -} +static Constant *getFalse(Type *Ty) { return ConstantInt::getFalse(Ty); } /// For a boolean type or a vector of boolean type, return true or a vector /// with every element true. -static Constant *getTrue(Type *Ty) { - return ConstantInt::getTrue(Ty); -} +static Constant *getTrue(Type *Ty) { return ConstantInt::getTrue(Ty); } /// isSameCompare - Is V equivalent to the comparison "LHS Pred RHS"? static bool isSameCompare(Value *V, CmpInst::Predicate Pred, Value *LHS, @@ -141,7 +135,7 @@ if (CPred == Pred && CLHS == LHS && CRHS == RHS) return true; return CPred == CmpInst::getSwappedPredicate(Pred) && CLHS == RHS && - CRHS == LHS; + CRHS == LHS; } /// Simplify comparison with true or false branch of select: @@ -153,7 +147,7 @@ Value *RHS, Value *Cond, const SimplifyQuery &Q, unsigned MaxRecurse, Constant *TrueOrFalse) { - Value *SimplifiedCmp = SimplifyCmpInst(Pred, LHS, RHS, Q, MaxRecurse); + Value *SimplifiedCmp = simplifyCmpInst(Pred, LHS, RHS, Q, MaxRecurse); if (SimplifiedCmp == Cond) { // %cmp simplified to the select condition (%cond). return TrueOrFalse; @@ -196,17 +190,17 @@ // checks whether folding it does not convert a well-defined value into // poison. if (match(FCmp, m_Zero()) && impliesPoison(TCmp, Cond)) - if (Value *V = SimplifyAndInst(Cond, TCmp, Q, MaxRecurse)) + if (Value *V = simplifyAndInst(Cond, TCmp, Q, MaxRecurse)) return V; // If the true value simplified to true, then the result of the compare // is equal to "Cond || FCmp". if (match(TCmp, m_One()) && impliesPoison(FCmp, Cond)) - if (Value *V = SimplifyOrInst(Cond, FCmp, Q, MaxRecurse)) + if (Value *V = simplifyOrInst(Cond, FCmp, Q, MaxRecurse)) return V; // Finally, if the false value simplified to true and the true value to // false, then the result of the compare is equal to "!Cond". if (match(FCmp, m_One()) && match(TCmp, m_Zero())) - if (Value *V = SimplifyXorInst( + if (Value *V = simplifyXorInst( Cond, Constant::getAllOnesValue(Cond->getType()), Q, MaxRecurse)) return V; return nullptr; @@ -248,12 +242,12 @@ if (!B || B->getOpcode() != OpcodeToExpand) return nullptr; Value *B0 = B->getOperand(0), *B1 = B->getOperand(1); - Value *L = SimplifyBinOp(Opcode, B0, OtherOp, Q.getWithoutUndef(), - MaxRecurse); + Value *L = + simplifyBinOp(Opcode, B0, OtherOp, Q.getWithoutUndef(), MaxRecurse); if (!L) return nullptr; - Value *R = SimplifyBinOp(Opcode, B1, OtherOp, Q.getWithoutUndef(), - MaxRecurse); + Value *R = + simplifyBinOp(Opcode, B1, OtherOp, Q.getWithoutUndef(), MaxRecurse); if (!R) return nullptr; @@ -265,7 +259,7 @@ } // Otherwise, return "L op' R" if it simplifies. - Value *S = SimplifyBinOp(OpcodeToExpand, L, R, Q, MaxRecurse); + Value *S = simplifyBinOp(OpcodeToExpand, L, R, Q, MaxRecurse); if (!S) return nullptr; @@ -275,8 +269,8 @@ /// Try to simplify binops of form "A op (B op' C)" or the commuted variant by /// distributing op over op'. -static Value *expandCommutativeBinOp(Instruction::BinaryOps Opcode, - Value *L, Value *R, +static Value *expandCommutativeBinOp(Instruction::BinaryOps Opcode, Value *L, + Value *R, Instruction::BinaryOps OpcodeToExpand, const SimplifyQuery &Q, unsigned MaxRecurse) { @@ -293,7 +287,7 @@ /// Generic simplifications for associative binary operations. /// Returns the simpler value, or null if none was found. -static Value *SimplifyAssociativeBinOp(Instruction::BinaryOps Opcode, +static Value *simplifyAssociativeBinOp(Instruction::BinaryOps Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { @@ -313,12 +307,13 @@ Value *C = RHS; // Does "B op C" simplify? - if (Value *V = SimplifyBinOp(Opcode, B, C, Q, MaxRecurse)) { + if (Value *V = simplifyBinOp(Opcode, B, C, Q, MaxRecurse)) { // It does! Return "A op V" if it simplifies or is already available. // If V equals B then "A op V" is just the LHS. - if (V == B) return LHS; + if (V == B) + return LHS; // Otherwise return "A op V" if it simplifies. - if (Value *W = SimplifyBinOp(Opcode, A, V, Q, MaxRecurse)) { + if (Value *W = simplifyBinOp(Opcode, A, V, Q, MaxRecurse)) { ++NumReassoc; return W; } @@ -332,12 +327,13 @@ Value *C = Op1->getOperand(1); // Does "A op B" simplify? - if (Value *V = SimplifyBinOp(Opcode, A, B, Q, MaxRecurse)) { + if (Value *V = simplifyBinOp(Opcode, A, B, Q, MaxRecurse)) { // It does! Return "V op C" if it simplifies or is already available. // If V equals B then "V op C" is just the RHS. - if (V == B) return RHS; + if (V == B) + return RHS; // Otherwise return "V op C" if it simplifies. - if (Value *W = SimplifyBinOp(Opcode, V, C, Q, MaxRecurse)) { + if (Value *W = simplifyBinOp(Opcode, V, C, Q, MaxRecurse)) { ++NumReassoc; return W; } @@ -355,12 +351,13 @@ Value *C = RHS; // Does "C op A" simplify? - if (Value *V = SimplifyBinOp(Opcode, C, A, Q, MaxRecurse)) { + if (Value *V = simplifyBinOp(Opcode, C, A, Q, MaxRecurse)) { // It does! Return "V op B" if it simplifies or is already available. // If V equals A then "V op B" is just the LHS. - if (V == A) return LHS; + if (V == A) + return LHS; // Otherwise return "V op B" if it simplifies. - if (Value *W = SimplifyBinOp(Opcode, V, B, Q, MaxRecurse)) { + if (Value *W = simplifyBinOp(Opcode, V, B, Q, MaxRecurse)) { ++NumReassoc; return W; } @@ -374,12 +371,13 @@ Value *C = Op1->getOperand(1); // Does "C op A" simplify? - if (Value *V = SimplifyBinOp(Opcode, C, A, Q, MaxRecurse)) { + if (Value *V = simplifyBinOp(Opcode, C, A, Q, MaxRecurse)) { // It does! Return "B op V" if it simplifies or is already available. // If V equals C then "B op V" is just the RHS. - if (V == C) return RHS; + if (V == C) + return RHS; // Otherwise return "B op V" if it simplifies. - if (Value *W = SimplifyBinOp(Opcode, B, V, Q, MaxRecurse)) { + if (Value *W = simplifyBinOp(Opcode, B, V, Q, MaxRecurse)) { ++NumReassoc; return W; } @@ -393,7 +391,7 @@ /// try to simplify the binop by seeing whether evaluating it on both branches /// of the select results in the same value. Returns the common value if so, /// otherwise returns null. -static Value *ThreadBinOpOverSelect(Instruction::BinaryOps Opcode, Value *LHS, +static Value *threadBinOpOverSelect(Instruction::BinaryOps Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { // Recursion is always used, so bail out at once if we already hit the limit. @@ -412,11 +410,11 @@ Value *TV; Value *FV; if (SI == LHS) { - TV = SimplifyBinOp(Opcode, SI->getTrueValue(), RHS, Q, MaxRecurse); - FV = SimplifyBinOp(Opcode, SI->getFalseValue(), RHS, Q, MaxRecurse); + TV = simplifyBinOp(Opcode, SI->getTrueValue(), RHS, Q, MaxRecurse); + FV = simplifyBinOp(Opcode, SI->getFalseValue(), RHS, Q, MaxRecurse); } else { - TV = SimplifyBinOp(Opcode, LHS, SI->getTrueValue(), Q, MaxRecurse); - FV = SimplifyBinOp(Opcode, LHS, SI->getFalseValue(), Q, MaxRecurse); + TV = simplifyBinOp(Opcode, LHS, SI->getTrueValue(), Q, MaxRecurse); + FV = simplifyBinOp(Opcode, LHS, SI->getFalseValue(), Q, MaxRecurse); } // If they simplified to the same value, then return the common value. @@ -471,7 +469,7 @@ /// We can simplify %cmp1 to true, because both branches of select are /// less than 3. We compose new comparison by substituting %tmp with both /// branches of select and see if it can be simplified. -static Value *ThreadCmpOverSelect(CmpInst::Predicate Pred, Value *LHS, +static Value *threadCmpOverSelect(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { // Recursion is always used, so bail out at once if we already hit the limit. @@ -517,7 +515,7 @@ /// try to simplify the binop by seeing whether evaluating it on the incoming /// phi values yields the same result for every value. If so returns the common /// value, otherwise returns null. -static Value *ThreadBinOpOverPHI(Instruction::BinaryOps Opcode, Value *LHS, +static Value *threadBinOpOverPHI(Instruction::BinaryOps Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { // Recursion is always used, so bail out at once if we already hit the limit. @@ -542,10 +540,10 @@ Value *CommonValue = nullptr; for (Value *Incoming : PI->incoming_values()) { // If the incoming value is the phi node itself, it can safely be skipped. - if (Incoming == PI) continue; - Value *V = PI == LHS ? - SimplifyBinOp(Opcode, Incoming, RHS, Q, MaxRecurse) : - SimplifyBinOp(Opcode, LHS, Incoming, Q, MaxRecurse); + if (Incoming == PI) + continue; + Value *V = PI == LHS ? simplifyBinOp(Opcode, Incoming, RHS, Q, MaxRecurse) + : simplifyBinOp(Opcode, LHS, Incoming, Q, MaxRecurse); // If the operation failed to simplify, or simplified to a different value // to previously, then give up. if (!V || (CommonValue && V != CommonValue)) @@ -560,7 +558,7 @@ /// comparison by seeing whether comparing with all of the incoming phi values /// yields the same result every time. If so returns the common result, /// otherwise returns null. -static Value *ThreadCmpOverPHI(CmpInst::Predicate Pred, Value *LHS, Value *RHS, +static Value *threadCmpOverPHI(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { // Recursion is always used, so bail out at once if we already hit the limit. if (!MaxRecurse--) @@ -584,11 +582,12 @@ Value *Incoming = PI->getIncomingValue(u); Instruction *InTI = PI->getIncomingBlock(u)->getTerminator(); // If the incoming value is the phi node itself, it can safely be skipped. - if (Incoming == PI) continue; + if (Incoming == PI) + continue; // Change the context instruction to the "edge" that flows into the phi. // This is important because that is where incoming is actually "evaluated" // even though it is used later somewhere else. - Value *V = SimplifyCmpInst(Pred, Incoming, RHS, Q.getWithInstruction(InTI), + Value *V = simplifyCmpInst(Pred, Incoming, RHS, Q.getWithInstruction(InTI), MaxRecurse); // If the operation failed to simplify, or simplified to a different value // to previously, then give up. @@ -604,8 +603,20 @@ Value *&Op0, Value *&Op1, const SimplifyQuery &Q) { if (auto *CLHS = dyn_cast<Constant>(Op0)) { - if (auto *CRHS = dyn_cast<Constant>(Op1)) + if (auto *CRHS = dyn_cast<Constant>(Op1)) { + switch (Opcode) { + default: + break; + case Instruction::FAdd: + case Instruction::FSub: + case Instruction::FMul: + case Instruction::FDiv: + case Instruction::FRem: + if (Q.CxtI != nullptr) + return ConstantFoldFPInstOperands(Opcode, CLHS, CRHS, Q.DL, Q.CxtI); + } return ConstantFoldBinaryOpOperands(Opcode, CLHS, CRHS, Q.DL); + } // Canonicalize the constant to the RHS if this is a commutative operation. if (Instruction::isCommutative(Opcode)) @@ -616,7 +627,7 @@ /// Given operands for an Add, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, +static Value *simplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, const SimplifyQuery &Q, unsigned MaxRecurse) { if (Constant *C = foldOrCommuteConstant(Instruction::Add, Op0, Op1, Q)) return C; @@ -647,8 +658,7 @@ // X + ~X -> -1 since ~X = -X-1 Type *Ty = Op0->getType(); - if (match(Op0, m_Not(m_Specific(Op1))) || - match(Op1, m_Not(m_Specific(Op0)))) + if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0)))) return Constant::getAllOnesValue(Ty); // add nsw/nuw (xor Y, signmask), signmask --> Y @@ -664,12 +674,12 @@ /// i1 add -> xor. if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1)) - if (Value *V = SimplifyXorInst(Op0, Op1, Q, MaxRecurse-1)) + if (Value *V = simplifyXorInst(Op0, Op1, Q, MaxRecurse - 1)) return V; // Try some generic simplifications for associative operations. - if (Value *V = SimplifyAssociativeBinOp(Instruction::Add, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + simplifyAssociativeBinOp(Instruction::Add, Op0, Op1, Q, MaxRecurse)) return V; // Threading Add over selects and phi nodes is pointless, so don't bother. @@ -684,45 +694,37 @@ return nullptr; } -Value *llvm::SimplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, +Value *llvm::simplifyAddInst(Value *Op0, Value *Op1, bool IsNSW, bool IsNUW, const SimplifyQuery &Query) { - return ::SimplifyAddInst(Op0, Op1, IsNSW, IsNUW, Query, RecursionLimit); + return ::simplifyAddInst(Op0, Op1, IsNSW, IsNUW, Query, RecursionLimit); } /// Compute the base pointer and cumulative constant offsets for V. /// /// This strips all constant offsets off of V, leaving it the base pointer, and -/// accumulates the total constant offset applied in the returned constant. It -/// returns 0 if V is not a pointer, and returns the constant '0' if there are -/// no constant offsets applied. +/// accumulates the total constant offset applied in the returned constant. +/// It returns zero if there are no constant offsets applied. /// -/// This is very similar to GetPointerBaseWithConstantOffset except it doesn't -/// follow non-inbounds geps. This allows it to remain usable for icmp ult/etc. -/// folding. -static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V, - bool AllowNonInbounds = false) { +/// This is very similar to stripAndAccumulateConstantOffsets(), except it +/// normalizes the offset bitwidth to the stripped pointer type, not the +/// original pointer type. +static APInt stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V, + bool AllowNonInbounds = false) { assert(V->getType()->isPtrOrPtrVectorTy()); APInt Offset = APInt::getZero(DL.getIndexTypeSizeInBits(V->getType())); - V = V->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds); // As that strip may trace through `addrspacecast`, need to sext or trunc // the offset calculated. - Type *IntIdxTy = DL.getIndexType(V->getType())->getScalarType(); - Offset = Offset.sextOrTrunc(IntIdxTy->getIntegerBitWidth()); - - Constant *OffsetIntPtr = ConstantInt::get(IntIdxTy, Offset); - if (VectorType *VecTy = dyn_cast<VectorType>(V->getType())) - return ConstantVector::getSplat(VecTy->getElementCount(), OffsetIntPtr); - return OffsetIntPtr; + return Offset.sextOrTrunc(DL.getIndexTypeSizeInBits(V->getType())); } /// Compute the constant difference between two pointer values. /// If the difference is not a constant, returns zero. static Constant *computePointerDifference(const DataLayout &DL, Value *LHS, Value *RHS) { - Constant *LHSOffset = stripAndComputeConstantOffsets(DL, LHS); - Constant *RHSOffset = stripAndComputeConstantOffsets(DL, RHS); + APInt LHSOffset = stripAndComputeConstantOffsets(DL, LHS); + APInt RHSOffset = stripAndComputeConstantOffsets(DL, RHS); // If LHS and RHS are not related via constant offsets to the same base // value, there is nothing we can do here. @@ -733,12 +735,15 @@ // LHS - RHS // = (LHSOffset + Base) - (RHSOffset + Base) // = LHSOffset - RHSOffset - return ConstantExpr::getSub(LHSOffset, RHSOffset); + Constant *Res = ConstantInt::get(LHS->getContext(), LHSOffset - RHSOffset); + if (auto *VecTy = dyn_cast<VectorType>(LHS->getType())) + Res = ConstantVector::getSplat(VecTy->getElementCount(), Res); + return Res; } /// Given operands for a Sub, see if we can fold the result. /// If not, this returns null. -static Value *SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, +static Value *simplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, const SimplifyQuery &Q, unsigned MaxRecurse) { if (Constant *C = foldOrCommuteConstant(Instruction::Sub, Op0, Op1, Q)) return C; @@ -784,17 +789,17 @@ Value *X = nullptr, *Y = nullptr, *Z = Op1; if (MaxRecurse && match(Op0, m_Add(m_Value(X), m_Value(Y)))) { // (X + Y) - Z // See if "V === Y - Z" simplifies. - if (Value *V = SimplifyBinOp(Instruction::Sub, Y, Z, Q, MaxRecurse-1)) + if (Value *V = simplifyBinOp(Instruction::Sub, Y, Z, Q, MaxRecurse - 1)) // It does! Now see if "X + V" simplifies. - if (Value *W = SimplifyBinOp(Instruction::Add, X, V, Q, MaxRecurse-1)) { + if (Value *W = simplifyBinOp(Instruction::Add, X, V, Q, MaxRecurse - 1)) { // It does, we successfully reassociated! ++NumReassoc; return W; } // See if "V === X - Z" simplifies. - if (Value *V = SimplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse-1)) + if (Value *V = simplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse - 1)) // It does! Now see if "Y + V" simplifies. - if (Value *W = SimplifyBinOp(Instruction::Add, Y, V, Q, MaxRecurse-1)) { + if (Value *W = simplifyBinOp(Instruction::Add, Y, V, Q, MaxRecurse - 1)) { // It does, we successfully reassociated! ++NumReassoc; return W; @@ -806,17 +811,17 @@ X = Op0; if (MaxRecurse && match(Op1, m_Add(m_Value(Y), m_Value(Z)))) { // X - (Y + Z) // See if "V === X - Y" simplifies. - if (Value *V = SimplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse-1)) + if (Value *V = simplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse - 1)) // It does! Now see if "V - Z" simplifies. - if (Value *W = SimplifyBinOp(Instruction::Sub, V, Z, Q, MaxRecurse-1)) { + if (Value *W = simplifyBinOp(Instruction::Sub, V, Z, Q, MaxRecurse - 1)) { // It does, we successfully reassociated! ++NumReassoc; return W; } // See if "V === X - Z" simplifies. - if (Value *V = SimplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse-1)) + if (Value *V = simplifyBinOp(Instruction::Sub, X, Z, Q, MaxRecurse - 1)) // It does! Now see if "V - Y" simplifies. - if (Value *W = SimplifyBinOp(Instruction::Sub, V, Y, Q, MaxRecurse-1)) { + if (Value *W = simplifyBinOp(Instruction::Sub, V, Y, Q, MaxRecurse - 1)) { // It does, we successfully reassociated! ++NumReassoc; return W; @@ -828,9 +833,9 @@ Z = Op0; if (MaxRecurse && match(Op1, m_Sub(m_Value(X), m_Value(Y)))) // Z - (X - Y) // See if "V === Z - X" simplifies. - if (Value *V = SimplifyBinOp(Instruction::Sub, Z, X, Q, MaxRecurse-1)) + if (Value *V = simplifyBinOp(Instruction::Sub, Z, X, Q, MaxRecurse - 1)) // It does! Now see if "V + Y" simplifies. - if (Value *W = SimplifyBinOp(Instruction::Add, V, Y, Q, MaxRecurse-1)) { + if (Value *W = simplifyBinOp(Instruction::Add, V, Y, Q, MaxRecurse - 1)) { // It does, we successfully reassociated! ++NumReassoc; return W; @@ -841,22 +846,21 @@ match(Op1, m_Trunc(m_Value(Y)))) if (X->getType() == Y->getType()) // See if "V === X - Y" simplifies. - if (Value *V = SimplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse-1)) + if (Value *V = simplifyBinOp(Instruction::Sub, X, Y, Q, MaxRecurse - 1)) // It does! Now see if "trunc V" simplifies. - if (Value *W = SimplifyCastInst(Instruction::Trunc, V, Op0->getType(), + if (Value *W = simplifyCastInst(Instruction::Trunc, V, Op0->getType(), Q, MaxRecurse - 1)) // It does, return the simplified "trunc V". return W; // Variations on GEP(base, I, ...) - GEP(base, i, ...) -> GEP(null, I-i, ...). - if (match(Op0, m_PtrToInt(m_Value(X))) && - match(Op1, m_PtrToInt(m_Value(Y)))) + if (match(Op0, m_PtrToInt(m_Value(X))) && match(Op1, m_PtrToInt(m_Value(Y)))) if (Constant *Result = computePointerDifference(Q.DL, X, Y)) return ConstantExpr::getIntegerCast(Result, Op0->getType(), true); // i1 sub -> xor. if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1)) - if (Value *V = SimplifyXorInst(Op0, Op1, Q, MaxRecurse-1)) + if (Value *V = simplifyXorInst(Op0, Op1, Q, MaxRecurse - 1)) return V; // Threading Sub over selects and phi nodes is pointless, so don't bother. @@ -871,14 +875,14 @@ return nullptr; } -Value *llvm::SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, +Value *llvm::simplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, const SimplifyQuery &Q) { - return ::SimplifySubInst(Op0, Op1, isNSW, isNUW, Q, RecursionLimit); + return ::simplifySubInst(Op0, Op1, isNSW, isNUW, Q, RecursionLimit); } /// Given operands for a Mul, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyMulInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, +static Value *simplifyMulInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse) { if (Constant *C = foldOrCommuteConstant(Instruction::Mul, Op0, Op1, Q)) return C; @@ -906,12 +910,12 @@ // i1 mul -> and. if (MaxRecurse && Op0->getType()->isIntOrIntVectorTy(1)) - if (Value *V = SimplifyAndInst(Op0, Op1, Q, MaxRecurse-1)) + if (Value *V = simplifyAndInst(Op0, Op1, Q, MaxRecurse - 1)) return V; // Try some generic simplifications for associative operations. - if (Value *V = SimplifyAssociativeBinOp(Instruction::Mul, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + simplifyAssociativeBinOp(Instruction::Mul, Op0, Op1, Q, MaxRecurse)) return V; // Mul distributes over Add. Try some generic simplifications based on this. @@ -922,22 +926,22 @@ // If the operation is with the result of a select instruction, check whether // operating on either branch of the select always yields the same value. if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) - if (Value *V = ThreadBinOpOverSelect(Instruction::Mul, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + threadBinOpOverSelect(Instruction::Mul, Op0, Op1, Q, MaxRecurse)) return V; // If the operation is with the result of a phi instruction, check whether // operating on all incoming values of the phi always yields the same value. if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) - if (Value *V = ThreadBinOpOverPHI(Instruction::Mul, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + threadBinOpOverPHI(Instruction::Mul, Op0, Op1, Q, MaxRecurse)) return V; return nullptr; } -Value *llvm::SimplifyMulInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { - return ::SimplifyMulInst(Op0, Op1, Q, RecursionLimit); +Value *llvm::simplifyMulInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { + return ::simplifyMulInst(Op0, Op1, Q, RecursionLimit); } /// Check for common or similar folds of integer division or integer remainder. @@ -1026,7 +1030,7 @@ /// when we can prove a relationship between the operands. static bool isICmpTrue(ICmpInst::Predicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { - Value *V = SimplifyICmpInst(Pred, LHS, RHS, Q, MaxRecurse); + Value *V = simplifyICmpInst(Pred, LHS, RHS, Q, MaxRecurse); Constant *C = dyn_cast_or_null<Constant>(V); return (C && C->isAllOnesValue()); } @@ -1122,13 +1126,13 @@ // If the operation is with the result of a select instruction, check whether // operating on either branch of the select always yields the same value. if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) - if (Value *V = ThreadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) + if (Value *V = threadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) return V; // If the operation is with the result of a phi instruction, check whether // operating on all incoming values of the phi always yields the same value. if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) - if (Value *V = ThreadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) + if (Value *V = threadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) return V; if (isDivZero(Op0, Op1, Q, MaxRecurse, IsSigned)) @@ -1164,13 +1168,13 @@ // If the operation is with the result of a select instruction, check whether // operating on either branch of the select always yields the same value. if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) - if (Value *V = ThreadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) + if (Value *V = threadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) return V; // If the operation is with the result of a phi instruction, check whether // operating on all incoming values of the phi always yields the same value. if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) - if (Value *V = ThreadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) + if (Value *V = threadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) return V; // If X / Y == 0, then X % Y == X. @@ -1182,7 +1186,7 @@ /// Given operands for an SDiv, see if we can fold the result. /// If not, this returns null. -static Value *SimplifySDivInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, +static Value *simplifySDivInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse) { // If two operands are negated and no signed overflow, return -1. if (isKnownNegation(Op0, Op1, /*NeedNSW=*/true)) @@ -1191,24 +1195,24 @@ return simplifyDiv(Instruction::SDiv, Op0, Op1, Q, MaxRecurse); } -Value *llvm::SimplifySDivInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { - return ::SimplifySDivInst(Op0, Op1, Q, RecursionLimit); +Value *llvm::simplifySDivInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { + return ::simplifySDivInst(Op0, Op1, Q, RecursionLimit); } /// Given operands for a UDiv, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyUDivInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, +static Value *simplifyUDivInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse) { return simplifyDiv(Instruction::UDiv, Op0, Op1, Q, MaxRecurse); } -Value *llvm::SimplifyUDivInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { - return ::SimplifyUDivInst(Op0, Op1, Q, RecursionLimit); +Value *llvm::simplifyUDivInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { + return ::simplifyUDivInst(Op0, Op1, Q, RecursionLimit); } /// Given operands for an SRem, see if we can fold the result. /// If not, this returns null. -static Value *SimplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, +static Value *simplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse) { // If the divisor is 0, the result is undefined, so assume the divisor is -1. // srem Op0, (sext i1 X) --> srem Op0, -1 --> 0 @@ -1223,19 +1227,19 @@ return simplifyRem(Instruction::SRem, Op0, Op1, Q, MaxRecurse); } -Value *llvm::SimplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { - return ::SimplifySRemInst(Op0, Op1, Q, RecursionLimit); +Value *llvm::simplifySRemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { + return ::simplifySRemInst(Op0, Op1, Q, RecursionLimit); } /// Given operands for a URem, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, +static Value *simplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse) { return simplifyRem(Instruction::URem, Op0, Op1, Q, MaxRecurse); } -Value *llvm::SimplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { - return ::SimplifyURemInst(Op0, Op1, Q, RecursionLimit); +Value *llvm::simplifyURemInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { + return ::simplifyURemInst(Op0, Op1, Q, RecursionLimit); } /// Returns true if a shift by \c Amount always yields poison. @@ -1268,7 +1272,7 @@ /// Given operands for an Shl, LShr or AShr, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyShift(Instruction::BinaryOps Opcode, Value *Op0, +static Value *simplifyShift(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1, bool IsNSW, const SimplifyQuery &Q, unsigned MaxRecurse) { if (Constant *C = foldOrCommuteConstant(Opcode, Op0, Op1, Q)) @@ -1297,13 +1301,13 @@ // If the operation is with the result of a select instruction, check whether // operating on either branch of the select always yields the same value. if (isa<SelectInst>(Op0) || isa<SelectInst>(Op1)) - if (Value *V = ThreadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) + if (Value *V = threadBinOpOverSelect(Opcode, Op0, Op1, Q, MaxRecurse)) return V; // If the operation is with the result of a phi instruction, check whether // operating on all incoming values of the phi always yields the same value. if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) - if (Value *V = ThreadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) + if (Value *V = threadBinOpOverPHI(Opcode, Op0, Op1, Q, MaxRecurse)) return V; // If any bits in the shift amount make that value greater than or equal to @@ -1338,11 +1342,11 @@ /// Given operands for an Shl, LShr or AShr, see if we can /// fold the result. If not, this returns null. -static Value *SimplifyRightShift(Instruction::BinaryOps Opcode, Value *Op0, - Value *Op1, bool isExact, const SimplifyQuery &Q, - unsigned MaxRecurse) { +static Value *simplifyRightShift(Instruction::BinaryOps Opcode, Value *Op0, + Value *Op1, bool isExact, + const SimplifyQuery &Q, unsigned MaxRecurse) { if (Value *V = - SimplifyShift(Opcode, Op0, Op1, /*IsNSW*/ false, Q, MaxRecurse)) + simplifyShift(Opcode, Op0, Op1, /*IsNSW*/ false, Q, MaxRecurse)) return V; // X >> X -> 0 @@ -1356,7 +1360,8 @@ // The low bit cannot be shifted out of an exact shift if it is set. if (isExact) { - KnownBits Op0Known = computeKnownBits(Op0, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT); + KnownBits Op0Known = + computeKnownBits(Op0, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT); if (Op0Known.One[0]) return Op0; } @@ -1366,10 +1371,10 @@ /// Given operands for an Shl, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, +static Value *simplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, const SimplifyQuery &Q, unsigned MaxRecurse) { if (Value *V = - SimplifyShift(Instruction::Shl, Op0, Op1, isNSW, Q, MaxRecurse)) + simplifyShift(Instruction::Shl, Op0, Op1, isNSW, Q, MaxRecurse)) return V; // undef << X -> 0 @@ -1392,18 +1397,18 @@ return nullptr; } -Value *llvm::SimplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, +Value *llvm::simplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW, const SimplifyQuery &Q) { - return ::SimplifyShlInst(Op0, Op1, isNSW, isNUW, Q, RecursionLimit); + return ::simplifyShlInst(Op0, Op1, isNSW, isNUW, Q, RecursionLimit); } /// Given operands for an LShr, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact, +static Value *simplifyLShrInst(Value *Op0, Value *Op1, bool isExact, const SimplifyQuery &Q, unsigned MaxRecurse) { - if (Value *V = SimplifyRightShift(Instruction::LShr, Op0, Op1, isExact, Q, + if (Value *V = simplifyRightShift(Instruction::LShr, Op0, Op1, isExact, Q, MaxRecurse)) - return V; + return V; // (X << A) >> A -> X Value *X; @@ -1429,16 +1434,16 @@ return nullptr; } -Value *llvm::SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact, +Value *llvm::simplifyLShrInst(Value *Op0, Value *Op1, bool isExact, const SimplifyQuery &Q) { - return ::SimplifyLShrInst(Op0, Op1, isExact, Q, RecursionLimit); + return ::simplifyLShrInst(Op0, Op1, isExact, Q, RecursionLimit); } /// Given operands for an AShr, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact, +static Value *simplifyAShrInst(Value *Op0, Value *Op1, bool isExact, const SimplifyQuery &Q, unsigned MaxRecurse) { - if (Value *V = SimplifyRightShift(Instruction::AShr, Op0, Op1, isExact, Q, + if (Value *V = simplifyRightShift(Instruction::AShr, Op0, Op1, isExact, Q, MaxRecurse)) return V; @@ -1462,9 +1467,9 @@ return nullptr; } -Value *llvm::SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact, +Value *llvm::simplifyAShrInst(Value *Op0, Value *Op1, bool isExact, const SimplifyQuery &Q) { - return ::SimplifyAShrInst(Op0, Op1, isExact, Q, RecursionLimit); + return ::simplifyAShrInst(Op0, Op1, isExact, Q, RecursionLimit); } /// Commuted variants are assumed to be handled by calling this function again @@ -1581,17 +1586,11 @@ /// with the parameters swapped. static Value *simplifyAndOfICmpsWithSameOperands(ICmpInst *Op0, ICmpInst *Op1) { ICmpInst::Predicate Pred0, Pred1; - Value *A ,*B; + Value *A, *B; if (!match(Op0, m_ICmp(Pred0, m_Value(A), m_Value(B))) || !match(Op1, m_ICmp(Pred1, m_Specific(A), m_Specific(B)))) return nullptr; - // We have (icmp Pred0, A, B) & (icmp Pred1, A, B). - // If Op1 is always implied true by Op0, then Op0 is a subset of Op1, and we - // can eliminate Op1 from this 'and'. - if (ICmpInst::isImpliedTrueByMatchingCmp(Pred0, Pred1)) - return Op0; - // Check for any combination of predicates that are guaranteed to be disjoint. if ((Pred0 == ICmpInst::getInversePredicate(Pred1)) || (Pred0 == ICmpInst::ICMP_EQ && ICmpInst::isFalseWhenEqual(Pred1)) || @@ -1606,17 +1605,11 @@ /// with the parameters swapped. static Value *simplifyOrOfICmpsWithSameOperands(ICmpInst *Op0, ICmpInst *Op1) { ICmpInst::Predicate Pred0, Pred1; - Value *A ,*B; + Value *A, *B; if (!match(Op0, m_ICmp(Pred0, m_Value(A), m_Value(B))) || !match(Op1, m_ICmp(Pred1, m_Specific(A), m_Specific(B)))) return nullptr; - // We have (icmp Pred0, A, B) | (icmp Pred1, A, B). - // If Op1 is always implied true by Op0, then Op0 is a subset of Op1, and we - // can eliminate Op0 from this 'or'. - if (ICmpInst::isImpliedTrueByMatchingCmp(Pred0, Pred1)) - return Op1; - // Check for any combination of predicates that cover the entire range of // possibilities. if ((Pred0 == ICmpInst::getInversePredicate(Pred1)) || @@ -1812,6 +1805,27 @@ return nullptr; } +/// Try to simplify and/or of icmp with ctpop intrinsic. +static Value *simplifyAndOrOfICmpsWithCtpop(ICmpInst *Cmp0, ICmpInst *Cmp1, + bool IsAnd) { + ICmpInst::Predicate Pred0, Pred1; + Value *X; + const APInt *C; + if (!match(Cmp0, m_ICmp(Pred0, m_Intrinsic<Intrinsic::ctpop>(m_Value(X)), + m_APInt(C))) || + !match(Cmp1, m_ICmp(Pred1, m_Specific(X), m_ZeroInt())) || C->isZero()) + return nullptr; + + // (ctpop(X) == C) || (X != 0) --> X != 0 where C > 0 + if (!IsAnd && Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_NE) + return Cmp1; + // (ctpop(X) != C) && (X == 0) --> X == 0 where C > 0 + if (IsAnd && Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_EQ) + return Cmp1; + + return nullptr; +} + static Value *simplifyAndOfICmps(ICmpInst *Op0, ICmpInst *Op1, const SimplifyQuery &Q) { if (Value *X = simplifyUnsignedRangeCheck(Op0, Op1, /*IsAnd=*/true, Q)) @@ -1833,6 +1847,11 @@ if (Value *X = simplifyAndOrOfICmpsWithZero(Op0, Op1, true)) return X; + if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op0, Op1, true)) + return X; + if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op1, Op0, true)) + return X; + if (Value *X = simplifyAndOfICmpsWithAdd(Op0, Op1, Q.IIQ)) return X; if (Value *X = simplifyAndOfICmpsWithAdd(Op1, Op0, Q.IIQ)) @@ -1909,6 +1928,11 @@ if (Value *X = simplifyAndOrOfICmpsWithZero(Op0, Op1, false)) return X; + if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op0, Op1, false)) + return X; + if (Value *X = simplifyAndOrOfICmpsWithCtpop(Op1, Op0, false)) + return X; + if (Value *X = simplifyOrOfICmpsWithAdd(Op0, Op1, Q.IIQ)) return X; if (Value *X = simplifyOrOfICmpsWithAdd(Op1, Op0, Q.IIQ)) @@ -1917,8 +1941,8 @@ return nullptr; } -static Value *simplifyAndOrOfFCmps(const TargetLibraryInfo *TLI, - FCmpInst *LHS, FCmpInst *RHS, bool IsAnd) { +static Value *simplifyAndOrOfFCmps(const TargetLibraryInfo *TLI, FCmpInst *LHS, + FCmpInst *RHS, bool IsAnd) { Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1); Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1); if (LHS0->getType() != RHS0->getType()) @@ -1955,8 +1979,8 @@ return nullptr; } -static Value *simplifyAndOrOfCmps(const SimplifyQuery &Q, - Value *Op0, Value *Op1, bool IsAnd) { +static Value *simplifyAndOrOfCmps(const SimplifyQuery &Q, Value *Op0, + Value *Op1, bool IsAnd) { // Look through casts of the 'and' operands to find compares. auto *Cast0 = dyn_cast<CastInst>(Op0); auto *Cast1 = dyn_cast<CastInst>(Op1); @@ -2017,7 +2041,7 @@ /// Given operands for an And, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, +static Value *simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse) { if (Constant *C = foldOrCommuteConstant(Instruction::And, Op0, Op1, Q)) return C; @@ -2043,8 +2067,7 @@ return Op0; // A & ~A = ~A & A = 0 - if (match(Op0, m_Not(m_Specific(Op1))) || - match(Op1, m_Not(m_Specific(Op0)))) + if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0)))) return Constant::getNullValue(Op0->getType()); // (A | ?) & A = A @@ -2117,8 +2140,8 @@ return V; // Try some generic simplifications for associative operations. - if (Value *V = SimplifyAssociativeBinOp(Instruction::And, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + simplifyAssociativeBinOp(Instruction::And, Op0, Op1, Q, MaxRecurse)) return V; // And distributes over Or. Try some generic simplifications based on this. @@ -2142,16 +2165,16 @@ // If the operation is with the result of a select instruction, check // whether operating on either branch of the select always yields the same // value. - if (Value *V = ThreadBinOpOverSelect(Instruction::And, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + threadBinOpOverSelect(Instruction::And, Op0, Op1, Q, MaxRecurse)) return V; } // If the operation is with the result of a phi instruction, check whether // operating on all incoming values of the phi always yields the same value. if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) - if (Value *V = ThreadBinOpOverPHI(Instruction::And, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + threadBinOpOverPHI(Instruction::And, Op0, Op1, Q, MaxRecurse)) return V; // Assuming the effective width of Y is not larger than A, i.e. all bits @@ -2174,8 +2197,7 @@ const KnownBits YKnown = computeKnownBits(Y, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); const unsigned EffWidthY = YKnown.countMaxActiveBits(); if (EffWidthY <= ShftCnt) { - const KnownBits XKnown = computeKnownBits(X, Q.DL, 0, Q.AC, Q.CxtI, - Q.DT); + const KnownBits XKnown = computeKnownBits(X, Q.DL, 0, Q.AC, Q.CxtI, Q.DT); const unsigned EffWidthX = XKnown.countMaxActiveBits(); const APInt EffBitsY = APInt::getLowBitsSet(Width, EffWidthY); const APInt EffBitsX = APInt::getLowBitsSet(Width, EffWidthX) << ShftCnt; @@ -2197,11 +2219,20 @@ match(Op1, m_c_Xor(m_Specific(Or), m_Specific(Y)))) return Constant::getNullValue(Op0->getType()); + if (Op0->getType()->isIntOrIntVectorTy(1)) { + // Op0&Op1 -> Op0 where Op0 implies Op1 + if (isImpliedCondition(Op0, Op1, Q.DL).value_or(false)) + return Op0; + // Op0&Op1 -> Op1 where Op1 implies Op0 + if (isImpliedCondition(Op1, Op0, Q.DL).value_or(false)) + return Op1; + } + return nullptr; } -Value *llvm::SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { - return ::SimplifyAndInst(Op0, Op1, Q, RecursionLimit); +Value *llvm::simplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { + return ::simplifyAndInst(Op0, Op1, Q, RecursionLimit); } static Value *simplifyOrLogic(Value *X, Value *Y) { @@ -2289,7 +2320,7 @@ /// Given operands for an Or, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, +static Value *simplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse) { if (Constant *C = foldOrCommuteConstant(Instruction::Or, Op0, Op1, Q)) return C; @@ -2334,6 +2365,31 @@ } } + // A funnel shift (rotate) can be decomposed into simpler shifts. See if we + // are mixing in another shift that is redundant with the funnel shift. + + // (fshl X, ?, Y) | (shl X, Y) --> fshl X, ?, Y + // (shl X, Y) | (fshl X, ?, Y) --> fshl X, ?, Y + if (match(Op0, + m_Intrinsic<Intrinsic::fshl>(m_Value(X), m_Value(), m_Value(Y))) && + match(Op1, m_Shl(m_Specific(X), m_Specific(Y)))) + return Op0; + if (match(Op1, + m_Intrinsic<Intrinsic::fshl>(m_Value(X), m_Value(), m_Value(Y))) && + match(Op0, m_Shl(m_Specific(X), m_Specific(Y)))) + return Op1; + + // (fshr ?, X, Y) | (lshr X, Y) --> fshr ?, X, Y + // (lshr X, Y) | (fshr ?, X, Y) --> fshr ?, X, Y + if (match(Op0, + m_Intrinsic<Intrinsic::fshr>(m_Value(), m_Value(X), m_Value(Y))) && + match(Op1, m_LShr(m_Specific(X), m_Specific(Y)))) + return Op0; + if (match(Op1, + m_Intrinsic<Intrinsic::fshr>(m_Value(), m_Value(X), m_Value(Y))) && + match(Op0, m_LShr(m_Specific(X), m_Specific(Y)))) + return Op1; + if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, false)) return V; @@ -2346,8 +2402,8 @@ return Op0; // Try some generic simplifications for associative operations. - if (Value *V = SimplifyAssociativeBinOp(Instruction::Or, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + simplifyAssociativeBinOp(Instruction::Or, Op0, Op1, Q, MaxRecurse)) return V; // Or distributes over And. Try some generic simplifications based on this. @@ -2366,8 +2422,8 @@ // If the operation is with the result of a select instruction, check // whether operating on either branch of the select always yields the same // value. - if (Value *V = ThreadBinOpOverSelect(Instruction::Or, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + threadBinOpOverSelect(Instruction::Or, Op0, Op1, Q, MaxRecurse)) return V; } @@ -2389,8 +2445,7 @@ return A; } // Or commutes, try both ways. - if (C1->isMask() && - match(B, m_c_Add(m_Specific(A), m_Value(N)))) { + if (C1->isMask() && match(B, m_c_Add(m_Specific(A), m_Value(N)))) { // Add commutes, try both ways. if (MaskedValueIsZero(N, *C1, Q.DL, 0, Q.AC, Q.CxtI, Q.DT)) return B; @@ -2401,19 +2456,28 @@ // If the operation is with the result of a phi instruction, check whether // operating on all incoming values of the phi always yields the same value. if (isa<PHINode>(Op0) || isa<PHINode>(Op1)) - if (Value *V = ThreadBinOpOverPHI(Instruction::Or, Op0, Op1, Q, MaxRecurse)) + if (Value *V = threadBinOpOverPHI(Instruction::Or, Op0, Op1, Q, MaxRecurse)) return V; + if (Op0->getType()->isIntOrIntVectorTy(1)) { + // Op0|Op1 -> Op1 where Op0 implies Op1 + if (isImpliedCondition(Op0, Op1, Q.DL).value_or(false)) + return Op1; + // Op0|Op1 -> Op0 where Op1 implies Op0 + if (isImpliedCondition(Op1, Op0, Q.DL).value_or(false)) + return Op0; + } + return nullptr; } -Value *llvm::SimplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { - return ::SimplifyOrInst(Op0, Op1, Q, RecursionLimit); +Value *llvm::simplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { + return ::simplifyOrInst(Op0, Op1, Q, RecursionLimit); } /// Given operands for a Xor, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, +static Value *simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse) { if (Constant *C = foldOrCommuteConstant(Instruction::Xor, Op0, Op1, Q)) return C; @@ -2435,8 +2499,7 @@ return Constant::getNullValue(Op0->getType()); // A ^ ~A = ~A ^ A = -1 - if (match(Op0, m_Not(m_Specific(Op1))) || - match(Op1, m_Not(m_Specific(Op0)))) + if (match(Op0, m_Not(m_Specific(Op1))) || match(Op1, m_Not(m_Specific(Op0)))) return Constant::getAllOnesValue(Op0->getType()); auto foldAndOrNot = [](Value *X, Value *Y) -> Value * { @@ -2467,8 +2530,8 @@ return V; // Try some generic simplifications for associative operations. - if (Value *V = SimplifyAssociativeBinOp(Instruction::Xor, Op0, Op1, Q, - MaxRecurse)) + if (Value *V = + simplifyAssociativeBinOp(Instruction::Xor, Op0, Op1, Q, MaxRecurse)) return V; // Threading Xor over selects and phi nodes is pointless, so don't bother. @@ -2483,19 +2546,18 @@ return nullptr; } -Value *llvm::SimplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { - return ::SimplifyXorInst(Op0, Op1, Q, RecursionLimit); +Value *llvm::simplifyXorInst(Value *Op0, Value *Op1, const SimplifyQuery &Q) { + return ::simplifyXorInst(Op0, Op1, Q, RecursionLimit); } - -static Type *GetCompareTy(Value *Op) { +static Type *getCompareTy(Value *Op) { return CmpInst::makeCmpResultType(Op->getType()); } /// Rummage around inside V looking for something equivalent to the comparison /// "LHS Pred RHS". Return such a value if found, otherwise return null. /// Helper function for analyzing max/min idioms. -static Value *ExtractEquivalentCondition(Value *V, CmpInst::Predicate Pred, +static Value *extractEquivalentCondition(Value *V, CmpInst::Predicate Pred, Value *LHS, Value *RHS) { SelectInst *SI = dyn_cast<SelectInst>(V); if (!SI) @@ -2512,6 +2574,70 @@ return nullptr; } +/// Return true if the underlying object (storage) must be disjoint from +/// storage returned by any noalias return call. +static bool isAllocDisjoint(const Value *V) { + // For allocas, we consider only static ones (dynamic + // allocas might be transformed into calls to malloc not simultaneously + // live with the compared-to allocation). For globals, we exclude symbols + // that might be resolve lazily to symbols in another dynamically-loaded + // library (and, thus, could be malloc'ed by the implementation). + if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) + return AI->getParent() && AI->getFunction() && AI->isStaticAlloca(); + if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) + return (GV->hasLocalLinkage() || GV->hasHiddenVisibility() || + GV->hasProtectedVisibility() || GV->hasGlobalUnnamedAddr()) && + !GV->isThreadLocal(); + if (const Argument *A = dyn_cast<Argument>(V)) + return A->hasByValAttr(); + return false; +} + +/// Return true if V1 and V2 are each the base of some distict storage region +/// [V, object_size(V)] which do not overlap. Note that zero sized regions +/// *are* possible, and that zero sized regions do not overlap with any other. +static bool haveNonOverlappingStorage(const Value *V1, const Value *V2) { + // Global variables always exist, so they always exist during the lifetime + // of each other and all allocas. Global variables themselves usually have + // non-overlapping storage, but since their addresses are constants, the + // case involving two globals does not reach here and is instead handled in + // constant folding. + // + // Two different allocas usually have different addresses... + // + // However, if there's an @llvm.stackrestore dynamically in between two + // allocas, they may have the same address. It's tempting to reduce the + // scope of the problem by only looking at *static* allocas here. That would + // cover the majority of allocas while significantly reducing the likelihood + // of having an @llvm.stackrestore pop up in the middle. However, it's not + // actually impossible for an @llvm.stackrestore to pop up in the middle of + // an entry block. Also, if we have a block that's not attached to a + // function, we can't tell if it's "static" under the current definition. + // Theoretically, this problem could be fixed by creating a new kind of + // instruction kind specifically for static allocas. Such a new instruction + // could be required to be at the top of the entry block, thus preventing it + // from being subject to a @llvm.stackrestore. Instcombine could even + // convert regular allocas into these special allocas. It'd be nifty. + // However, until then, this problem remains open. + // + // So, we'll assume that two non-empty allocas have different addresses + // for now. + auto isByValArg = [](const Value *V) { + const Argument *A = dyn_cast<Argument>(V); + return A && A->hasByValAttr(); + }; + + // Byval args are backed by store which does not overlap with each other, + // allocas, or globals. + if (isByValArg(V1)) + return isa<AllocaInst>(V2) || isa<GlobalVariable>(V2) || isByValArg(V2); + if (isByValArg(V2)) + return isa<AllocaInst>(V1) || isa<GlobalVariable>(V1) || isByValArg(V1); + + return isa<AllocaInst>(V1) && + (isa<AllocaInst>(V2) || isa<GlobalVariable>(V2)); +} + // A significant optimization not implemented here is assuming that alloca // addresses are not equal to incoming argument values. They don't *alias*, // as we say, but that doesn't mean they aren't equal, so we take a @@ -2540,9 +2666,8 @@ // If the C and C++ standards are ever made sufficiently restrictive in this // area, it may be possible to update LLVM's semantics accordingly and reinstate // this optimization. -static Constant * -computePointerICmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, - const SimplifyQuery &Q) { +static Constant *computePointerICmp(CmpInst::Predicate Pred, Value *LHS, + Value *RHS, const SimplifyQuery &Q) { const DataLayout &DL = Q.DL; const TargetLibraryInfo *TLI = Q.TLI; const DominatorTree *DT = Q.DT; @@ -2557,8 +2682,7 @@ if (isa<ConstantPointerNull>(RHS) && ICmpInst::isEquality(Pred) && llvm::isKnownNonZero(LHS, DL, 0, nullptr, nullptr, nullptr, IIQ.UseInstrInfo)) - return ConstantInt::get(GetCompareTy(LHS), - !CmpInst::isTrueWhenEqual(Pred)); + return ConstantInt::get(getCompareTy(LHS), !CmpInst::isTrueWhenEqual(Pred)); // We can only fold certain predicates on pointer comparisons. switch (Pred) { @@ -2592,77 +2716,41 @@ // Even if an non-inbounds GEP occurs along the path we can still optimize // equality comparisons concerning the result. bool AllowNonInbounds = ICmpInst::isEquality(Pred); - Constant *LHSOffset = - stripAndComputeConstantOffsets(DL, LHS, AllowNonInbounds); - Constant *RHSOffset = - stripAndComputeConstantOffsets(DL, RHS, AllowNonInbounds); + APInt LHSOffset = stripAndComputeConstantOffsets(DL, LHS, AllowNonInbounds); + APInt RHSOffset = stripAndComputeConstantOffsets(DL, RHS, AllowNonInbounds); // If LHS and RHS are related via constant offsets to the same base // value, we can replace it with an icmp which just compares the offsets. if (LHS == RHS) - return ConstantExpr::getICmp(Pred, LHSOffset, RHSOffset); + return ConstantInt::get(getCompareTy(LHS), + ICmpInst::compare(LHSOffset, RHSOffset, Pred)); // Various optimizations for (in)equality comparisons. if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) { // Different non-empty allocations that exist at the same time have - // different addresses (if the program can tell). Global variables always - // exist, so they always exist during the lifetime of each other and all - // allocas. Two different allocas usually have different addresses... - // - // However, if there's an @llvm.stackrestore dynamically in between two - // allocas, they may have the same address. It's tempting to reduce the - // scope of the problem by only looking at *static* allocas here. That would - // cover the majority of allocas while significantly reducing the likelihood - // of having an @llvm.stackrestore pop up in the middle. However, it's not - // actually impossible for an @llvm.stackrestore to pop up in the middle of - // an entry block. Also, if we have a block that's not attached to a - // function, we can't tell if it's "static" under the current definition. - // Theoretically, this problem could be fixed by creating a new kind of - // instruction kind specifically for static allocas. Such a new instruction - // could be required to be at the top of the entry block, thus preventing it - // from being subject to a @llvm.stackrestore. Instcombine could even - // convert regular allocas into these special allocas. It'd be nifty. - // However, until then, this problem remains open. - // - // So, we'll assume that two non-empty allocas have different addresses - // for now. - // - // With all that, if the offsets are within the bounds of their allocations - // (and not one-past-the-end! so we can't use inbounds!), and their - // allocations aren't the same, the pointers are not equal. - // - // Note that it's not necessary to check for LHS being a global variable - // address, due to canonicalization and constant folding. - if (isa<AllocaInst>(LHS) && - (isa<AllocaInst>(RHS) || isa<GlobalVariable>(RHS))) { - ConstantInt *LHSOffsetCI = dyn_cast<ConstantInt>(LHSOffset); - ConstantInt *RHSOffsetCI = dyn_cast<ConstantInt>(RHSOffset); + // different addresses (if the program can tell). If the offsets are + // within the bounds of their allocations (and not one-past-the-end! + // so we can't use inbounds!), and their allocations aren't the same, + // the pointers are not equal. + if (haveNonOverlappingStorage(LHS, RHS)) { uint64_t LHSSize, RHSSize; ObjectSizeOpts Opts; - Opts.NullIsUnknownSize = - NullPointerIsDefined(cast<AllocaInst>(LHS)->getFunction()); - if (LHSOffsetCI && RHSOffsetCI && - getObjectSize(LHS, LHSSize, DL, TLI, Opts) && - getObjectSize(RHS, RHSSize, DL, TLI, Opts)) { - const APInt &LHSOffsetValue = LHSOffsetCI->getValue(); - const APInt &RHSOffsetValue = RHSOffsetCI->getValue(); - if (!LHSOffsetValue.isNegative() && - !RHSOffsetValue.isNegative() && - LHSOffsetValue.ult(LHSSize) && - RHSOffsetValue.ult(RHSSize)) { - return ConstantInt::get(GetCompareTy(LHS), - !CmpInst::isTrueWhenEqual(Pred)); - } - } - - // Repeat the above check but this time without depending on DataLayout - // or being able to compute a precise size. - if (!cast<PointerType>(LHS->getType())->isEmptyTy() && - !cast<PointerType>(RHS->getType())->isEmptyTy() && - LHSOffset->isNullValue() && - RHSOffset->isNullValue()) - return ConstantInt::get(GetCompareTy(LHS), + Opts.EvalMode = ObjectSizeOpts::Mode::Min; + auto *F = [](Value *V) -> Function * { + if (auto *I = dyn_cast<Instruction>(V)) + return I->getFunction(); + if (auto *A = dyn_cast<Argument>(V)) + return A->getParent(); + return nullptr; + }(LHS); + Opts.NullIsUnknownSize = F ? NullPointerIsDefined(F) : true; + if (getObjectSize(LHS, LHSSize, DL, TLI, Opts) && + getObjectSize(RHS, RHSSize, DL, TLI, Opts) && + !LHSOffset.isNegative() && !RHSOffset.isNegative() && + LHSOffset.ult(LHSSize) && RHSOffset.ult(RHSSize)) { + return ConstantInt::get(getCompareTy(LHS), !CmpInst::isTrueWhenEqual(Pred)); + } } // If one side of the equality comparison must come from a noalias call @@ -2680,29 +2768,16 @@ }; // Is the set of underlying objects all things which must be disjoint from - // noalias calls. For allocas, we consider only static ones (dynamic - // allocas might be transformed into calls to malloc not simultaneously - // live with the compared-to allocation). For globals, we exclude symbols - // that might be resolve lazily to symbols in another dynamically-loaded - // library (and, thus, could be malloc'ed by the implementation). + // noalias calls. We assume that indexing from such disjoint storage + // into the heap is undefined, and thus offsets can be safely ignored. auto IsAllocDisjoint = [](ArrayRef<const Value *> Objects) { - return all_of(Objects, [](const Value *V) { - if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) - return AI->getParent() && AI->getFunction() && AI->isStaticAlloca(); - if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) - return (GV->hasLocalLinkage() || GV->hasHiddenVisibility() || - GV->hasProtectedVisibility() || GV->hasGlobalUnnamedAddr()) && - !GV->isThreadLocal(); - if (const Argument *A = dyn_cast<Argument>(V)) - return A->hasByValAttr(); - return false; - }); + return all_of(Objects, ::isAllocDisjoint); }; if ((IsNAC(LHSUObjs) && IsAllocDisjoint(RHSUObjs)) || (IsNAC(RHSUObjs) && IsAllocDisjoint(LHSUObjs))) - return ConstantInt::get(GetCompareTy(LHS), - !CmpInst::isTrueWhenEqual(Pred)); + return ConstantInt::get(getCompareTy(LHS), + !CmpInst::isTrueWhenEqual(Pred)); // Fold comparisons for non-escaping pointer even if the allocation call // cannot be elided. We cannot fold malloc comparison to null. Also, the @@ -2719,7 +2794,7 @@ // FIXME: We should also fold the compare when the pointer escapes, but the // compare dominates the pointer escape if (MI && !PointerMayBeCaptured(MI, true, true)) - return ConstantInt::get(GetCompareTy(LHS), + return ConstantInt::get(getCompareTy(LHS), CmpInst::isFalseWhenEqual(Pred)); } @@ -2730,7 +2805,7 @@ /// Fold an icmp when its operands have i1 scalar type. static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q) { - Type *ITy = GetCompareTy(LHS); // The return type. + Type *ITy = getCompareTy(LHS); // The return type. Type *OpTy = LHS->getType(); // The operand type. if (!OpTy->isIntOrIntVectorTy(1)) return nullptr; @@ -2768,7 +2843,8 @@ case CmpInst::ICMP_SLE: // X <=s 0 -> true return getTrue(ITy); - default: break; + default: + break; } } else if (match(RHS, m_One())) { switch (Pred) { @@ -2792,7 +2868,8 @@ case CmpInst::ICMP_SGE: // X >=s -1 -> true return getTrue(ITy); - default: break; + default: + break; } } @@ -2800,7 +2877,7 @@ default: break; case ICmpInst::ICMP_UGE: - if (isImpliedCondition(RHS, LHS, Q.DL).getValueOr(false)) + if (isImpliedCondition(RHS, LHS, Q.DL).value_or(false)) return getTrue(ITy); break; case ICmpInst::ICMP_SGE: @@ -2811,11 +2888,11 @@ /// 0 | 1 | 1 (0 >= -1) | 1 /// 1 | 0 | 0 (-1 >= 0) | 0 /// 1 | 1 | 1 (-1 >= -1) | 1 - if (isImpliedCondition(LHS, RHS, Q.DL).getValueOr(false)) + if (isImpliedCondition(LHS, RHS, Q.DL).value_or(false)) return getTrue(ITy); break; case ICmpInst::ICMP_ULE: - if (isImpliedCondition(LHS, RHS, Q.DL).getValueOr(false)) + if (isImpliedCondition(LHS, RHS, Q.DL).value_or(false)) return getTrue(ITy); break; } @@ -2829,7 +2906,7 @@ if (!match(RHS, m_Zero())) return nullptr; - Type *ITy = GetCompareTy(LHS); // The return type. + Type *ITy = getCompareTy(LHS); // The return type. switch (Pred) { default: llvm_unreachable("Unknown ICmp predicate!"); @@ -2888,7 +2965,7 @@ static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const InstrInfoQuery &IIQ) { - Type *ITy = GetCompareTy(RHS); // The return type. + Type *ITy = getCompareTy(RHS); // The return type. Value *X; // Sign-bit checks can be optimized to true/false after unsigned @@ -2935,10 +3012,11 @@ return nullptr; } -static Value *simplifyICmpWithBinOpOnLHS( - CmpInst::Predicate Pred, BinaryOperator *LBO, Value *RHS, - const SimplifyQuery &Q, unsigned MaxRecurse) { - Type *ITy = GetCompareTy(RHS); // The return type. +static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred, + BinaryOperator *LBO, Value *RHS, + const SimplifyQuery &Q, + unsigned MaxRecurse) { + Type *ITy = getCompareTy(RHS); // The return type. Value *Y = nullptr; // icmp pred (or X, Y), X @@ -3073,7 +3151,6 @@ return nullptr; } - // If only one of the icmp's operands has NSW flags, try to prove that: // // icmp slt (x + C1), (x +nsw C2) @@ -3108,7 +3185,6 @@ (C2->slt(*C1) && C1->isNonPositive()); } - /// TODO: A large part of this logic is duplicated in InstCombine's /// foldICmpBinOp(). We should be able to share that and avoid the code /// duplication. @@ -3145,7 +3221,7 @@ // icmp (X+Y), X -> icmp Y, 0 for equalities or if there is no overflow. if ((A == RHS || B == RHS) && NoLHSWrapProblem) - if (Value *V = SimplifyICmpInst(Pred, A == RHS ? B : A, + if (Value *V = simplifyICmpInst(Pred, A == RHS ? B : A, Constant::getNullValue(RHS->getType()), Q, MaxRecurse - 1)) return V; @@ -3153,7 +3229,7 @@ // icmp X, (X+Y) -> icmp 0, Y for equalities or if there is no overflow. if ((C == LHS || D == LHS) && NoRHSWrapProblem) if (Value *V = - SimplifyICmpInst(Pred, Constant::getNullValue(LHS->getType()), + simplifyICmpInst(Pred, Constant::getNullValue(LHS->getType()), C == LHS ? D : C, Q, MaxRecurse - 1)) return V; @@ -3181,7 +3257,7 @@ Y = A; Z = C; } - if (Value *V = SimplifyICmpInst(Pred, Y, Z, Q, MaxRecurse - 1)) + if (Value *V = simplifyICmpInst(Pred, Y, Z, Q, MaxRecurse - 1)) return V; } } @@ -3201,15 +3277,15 @@ if (match(RHS, m_APInt(C))) { if (C->isStrictlyPositive()) { if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_NE) - return ConstantInt::getTrue(GetCompareTy(RHS)); + return ConstantInt::getTrue(getCompareTy(RHS)); if (Pred == ICmpInst::ICMP_SGE || Pred == ICmpInst::ICMP_EQ) - return ConstantInt::getFalse(GetCompareTy(RHS)); + return ConstantInt::getFalse(getCompareTy(RHS)); } if (C->isNonNegative()) { if (Pred == ICmpInst::ICMP_SLE) - return ConstantInt::getTrue(GetCompareTy(RHS)); + return ConstantInt::getTrue(getCompareTy(RHS)); if (Pred == ICmpInst::ICMP_SGT) - return ConstantInt::getFalse(GetCompareTy(RHS)); + return ConstantInt::getFalse(getCompareTy(RHS)); } } } @@ -3232,9 +3308,9 @@ Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(LBO)) || match(LHS, m_Shl(m_One(), m_Value())) || !C->isZero()) { if (Pred == ICmpInst::ICMP_EQ) - return ConstantInt::getFalse(GetCompareTy(RHS)); + return ConstantInt::getFalse(getCompareTy(RHS)); if (Pred == ICmpInst::ICMP_NE) - return ConstantInt::getTrue(GetCompareTy(RHS)); + return ConstantInt::getTrue(getCompareTy(RHS)); } } @@ -3243,9 +3319,9 @@ // (1 << X) <=u 0x8000 --> true if (match(LHS, m_Shl(m_One(), m_Value())) && match(RHS, m_SignMask())) { if (Pred == ICmpInst::ICMP_UGT) - return ConstantInt::getFalse(GetCompareTy(RHS)); + return ConstantInt::getFalse(getCompareTy(RHS)); if (Pred == ICmpInst::ICMP_ULE) - return ConstantInt::getTrue(GetCompareTy(RHS)); + return ConstantInt::getTrue(getCompareTy(RHS)); } if (MaxRecurse && LBO && RBO && LBO->getOpcode() == RBO->getOpcode() && @@ -3258,22 +3334,22 @@ if (ICmpInst::isSigned(Pred) || !Q.IIQ.isExact(LBO) || !Q.IIQ.isExact(RBO)) break; - if (Value *V = SimplifyICmpInst(Pred, LBO->getOperand(0), + if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0), RBO->getOperand(0), Q, MaxRecurse - 1)) - return V; + return V; break; case Instruction::SDiv: if (!ICmpInst::isEquality(Pred) || !Q.IIQ.isExact(LBO) || !Q.IIQ.isExact(RBO)) break; - if (Value *V = SimplifyICmpInst(Pred, LBO->getOperand(0), + if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0), RBO->getOperand(0), Q, MaxRecurse - 1)) return V; break; case Instruction::AShr: if (!Q.IIQ.isExact(LBO) || !Q.IIQ.isExact(RBO)) break; - if (Value *V = SimplifyICmpInst(Pred, LBO->getOperand(0), + if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0), RBO->getOperand(0), Q, MaxRecurse - 1)) return V; break; @@ -3284,7 +3360,7 @@ break; if (!NSW && ICmpInst::isSigned(Pred)) break; - if (Value *V = SimplifyICmpInst(Pred, LBO->getOperand(0), + if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(0), RBO->getOperand(0), Q, MaxRecurse - 1)) return V; break; @@ -3294,12 +3370,12 @@ return nullptr; } -/// Simplify integer comparisons where at least one operand of the compare +/// simplify integer comparisons where at least one operand of the compare /// matches an integer min/max idiom. static Value *simplifyICmpWithMinMax(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { - Type *ITy = GetCompareTy(LHS); // The return type. + Type *ITy = getCompareTy(LHS); // The return type. Value *A, *B; CmpInst::Predicate P = CmpInst::BAD_ICMP_PREDICATE; CmpInst::Predicate EqP; // Chosen so that "A == max/min(A,B)" iff "A EqP B". @@ -3344,13 +3420,13 @@ case CmpInst::ICMP_SLE: // Equivalent to "A EqP B". This may be the same as the condition tested // in the max/min; if so, we can just return that. - if (Value *V = ExtractEquivalentCondition(LHS, EqP, A, B)) + if (Value *V = extractEquivalentCondition(LHS, EqP, A, B)) return V; - if (Value *V = ExtractEquivalentCondition(RHS, EqP, A, B)) + if (Value *V = extractEquivalentCondition(RHS, EqP, A, B)) return V; // Otherwise, see if "A EqP B" simplifies. if (MaxRecurse) - if (Value *V = SimplifyICmpInst(EqP, A, B, Q, MaxRecurse - 1)) + if (Value *V = simplifyICmpInst(EqP, A, B, Q, MaxRecurse - 1)) return V; break; case CmpInst::ICMP_NE: @@ -3358,13 +3434,13 @@ CmpInst::Predicate InvEqP = CmpInst::getInversePredicate(EqP); // Equivalent to "A InvEqP B". This may be the same as the condition // tested in the max/min; if so, we can just return that. - if (Value *V = ExtractEquivalentCondition(LHS, InvEqP, A, B)) + if (Value *V = extractEquivalentCondition(LHS, InvEqP, A, B)) return V; - if (Value *V = ExtractEquivalentCondition(RHS, InvEqP, A, B)) + if (Value *V = extractEquivalentCondition(RHS, InvEqP, A, B)) return V; // Otherwise, see if "A InvEqP B" simplifies. if (MaxRecurse) - if (Value *V = SimplifyICmpInst(InvEqP, A, B, Q, MaxRecurse - 1)) + if (Value *V = simplifyICmpInst(InvEqP, A, B, Q, MaxRecurse - 1)) return V; break; } @@ -3418,13 +3494,13 @@ case CmpInst::ICMP_ULE: // Equivalent to "A EqP B". This may be the same as the condition tested // in the max/min; if so, we can just return that. - if (Value *V = ExtractEquivalentCondition(LHS, EqP, A, B)) + if (Value *V = extractEquivalentCondition(LHS, EqP, A, B)) return V; - if (Value *V = ExtractEquivalentCondition(RHS, EqP, A, B)) + if (Value *V = extractEquivalentCondition(RHS, EqP, A, B)) return V; // Otherwise, see if "A EqP B" simplifies. if (MaxRecurse) - if (Value *V = SimplifyICmpInst(EqP, A, B, Q, MaxRecurse - 1)) + if (Value *V = simplifyICmpInst(EqP, A, B, Q, MaxRecurse - 1)) return V; break; case CmpInst::ICMP_NE: @@ -3432,13 +3508,13 @@ CmpInst::Predicate InvEqP = CmpInst::getInversePredicate(EqP); // Equivalent to "A InvEqP B". This may be the same as the condition // tested in the max/min; if so, we can just return that. - if (Value *V = ExtractEquivalentCondition(LHS, InvEqP, A, B)) + if (Value *V = extractEquivalentCondition(LHS, InvEqP, A, B)) return V; - if (Value *V = ExtractEquivalentCondition(RHS, InvEqP, A, B)) + if (Value *V = extractEquivalentCondition(RHS, InvEqP, A, B)) return V; // Otherwise, see if "A InvEqP B" simplifies. if (MaxRecurse) - if (Value *V = SimplifyICmpInst(InvEqP, A, B, Q, MaxRecurse - 1)) + if (Value *V = simplifyICmpInst(InvEqP, A, B, Q, MaxRecurse - 1)) return V; break; } @@ -3494,11 +3570,10 @@ continue; CallInst *Assume = cast<CallInst>(AssumeVH); - if (Optional<bool> Imp = - isImpliedCondition(Assume->getArgOperand(0), Predicate, LHS, RHS, - Q.DL)) + if (Optional<bool> Imp = isImpliedCondition(Assume->getArgOperand(0), + Predicate, LHS, RHS, Q.DL)) if (isValidAssumeForContext(Assume, Q.CxtI, Q.DT)) - return ConstantInt::get(GetCompareTy(LHS), *Imp); + return ConstantInt::get(getCompareTy(LHS), *Imp); } } @@ -3507,7 +3582,7 @@ /// Given operands for an ICmpInst, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, +static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate; assert(CmpInst::isIntPredicate(Pred) && "Not an integer compare!"); @@ -3522,7 +3597,7 @@ } assert(!isa<UndefValue>(LHS) && "Unexpected icmp undef,%X"); - Type *ITy = GetCompareTy(LHS); // The return type. + Type *ITy = getCompareTy(LHS); // The return type. // icmp poison, X -> poison if (isa<PoisonValue>(RHS)) @@ -3584,15 +3659,15 @@ Q.DL.getTypeSizeInBits(SrcTy) == DstTy->getPrimitiveSizeInBits()) { if (Constant *RHSC = dyn_cast<Constant>(RHS)) { // Transfer the cast to the constant. - if (Value *V = SimplifyICmpInst(Pred, SrcOp, + if (Value *V = simplifyICmpInst(Pred, SrcOp, ConstantExpr::getIntToPtr(RHSC, SrcTy), - Q, MaxRecurse-1)) + Q, MaxRecurse - 1)) return V; } else if (PtrToIntInst *RI = dyn_cast<PtrToIntInst>(RHS)) { if (RI->getOperand(0)->getType() == SrcTy) // Compare without the cast. - if (Value *V = SimplifyICmpInst(Pred, SrcOp, RI->getOperand(0), - Q, MaxRecurse-1)) + if (Value *V = simplifyICmpInst(Pred, SrcOp, RI->getOperand(0), Q, + MaxRecurse - 1)) return V; } } @@ -3603,9 +3678,9 @@ if (ZExtInst *RI = dyn_cast<ZExtInst>(RHS)) { if (MaxRecurse && SrcTy == RI->getOperand(0)->getType()) // Compare X and Y. Note that signed predicates become unsigned. - if (Value *V = SimplifyICmpInst(ICmpInst::getUnsignedPredicate(Pred), - SrcOp, RI->getOperand(0), Q, - MaxRecurse-1)) + if (Value *V = + simplifyICmpInst(ICmpInst::getUnsignedPredicate(Pred), SrcOp, + RI->getOperand(0), Q, MaxRecurse - 1)) return V; } // Fold (zext X) ule (sext X), (zext X) sge (sext X) to true. @@ -3628,15 +3703,16 @@ // If the re-extended constant didn't change then this is effectively // also a case of comparing two zero-extended values. if (RExt == CI && MaxRecurse) - if (Value *V = SimplifyICmpInst(ICmpInst::getUnsignedPredicate(Pred), - SrcOp, Trunc, Q, MaxRecurse-1)) + if (Value *V = simplifyICmpInst(ICmpInst::getUnsignedPredicate(Pred), + SrcOp, Trunc, Q, MaxRecurse - 1)) return V; // Otherwise the upper bits of LHS are zero while RHS has a non-zero bit // there. Use this to work out the result of the comparison. if (RExt != CI) { switch (Pred) { - default: llvm_unreachable("Unknown ICmp predicate!"); + default: + llvm_unreachable("Unknown ICmp predicate!"); // LHS <u RHS. case ICmpInst::ICMP_EQ: case ICmpInst::ICMP_UGT: @@ -3652,15 +3728,15 @@ // is non-negative then LHS <s RHS. case ICmpInst::ICMP_SGT: case ICmpInst::ICMP_SGE: - return CI->getValue().isNegative() ? - ConstantInt::getTrue(CI->getContext()) : - ConstantInt::getFalse(CI->getContext()); + return CI->getValue().isNegative() + ? ConstantInt::getTrue(CI->getContext()) + : ConstantInt::getFalse(CI->getContext()); case ICmpInst::ICMP_SLT: case ICmpInst::ICMP_SLE: - return CI->getValue().isNegative() ? - ConstantInt::getFalse(CI->getContext()) : - ConstantInt::getTrue(CI->getContext()); + return CI->getValue().isNegative() + ? ConstantInt::getFalse(CI->getContext()) + : ConstantInt::getTrue(CI->getContext()); } } } @@ -3672,8 +3748,8 @@ if (SExtInst *RI = dyn_cast<SExtInst>(RHS)) { if (MaxRecurse && SrcTy == RI->getOperand(0)->getType()) // Compare X and Y. Note that the predicate does not change. - if (Value *V = SimplifyICmpInst(Pred, SrcOp, RI->getOperand(0), - Q, MaxRecurse-1)) + if (Value *V = simplifyICmpInst(Pred, SrcOp, RI->getOperand(0), Q, + MaxRecurse - 1)) return V; } // Fold (sext X) uge (zext X), (sext X) sle (zext X) to true. @@ -3696,14 +3772,16 @@ // If the re-extended constant didn't change then this is effectively // also a case of comparing two sign-extended values. if (RExt == CI && MaxRecurse) - if (Value *V = SimplifyICmpInst(Pred, SrcOp, Trunc, Q, MaxRecurse-1)) + if (Value *V = + simplifyICmpInst(Pred, SrcOp, Trunc, Q, MaxRecurse - 1)) return V; // Otherwise the upper bits of LHS are all equal, while RHS has varying // bits there. Use this to work out the result of the comparison. if (RExt != CI) { switch (Pred) { - default: llvm_unreachable("Unknown ICmp predicate!"); + default: + llvm_unreachable("Unknown ICmp predicate!"); case ICmpInst::ICMP_EQ: return ConstantInt::getFalse(CI->getContext()); case ICmpInst::ICMP_NE: @@ -3713,14 +3791,14 @@ // LHS >s RHS. case ICmpInst::ICMP_SGT: case ICmpInst::ICMP_SGE: - return CI->getValue().isNegative() ? - ConstantInt::getTrue(CI->getContext()) : - ConstantInt::getFalse(CI->getContext()); + return CI->getValue().isNegative() + ? ConstantInt::getTrue(CI->getContext()) + : ConstantInt::getFalse(CI->getContext()); case ICmpInst::ICMP_SLT: case ICmpInst::ICMP_SLE: - return CI->getValue().isNegative() ? - ConstantInt::getFalse(CI->getContext()) : - ConstantInt::getTrue(CI->getContext()); + return CI->getValue().isNegative() + ? ConstantInt::getFalse(CI->getContext()) + : ConstantInt::getTrue(CI->getContext()); // If LHS is non-negative then LHS <u RHS. If LHS is negative then // LHS >u RHS. @@ -3728,18 +3806,18 @@ case ICmpInst::ICMP_UGE: // Comparison is true iff the LHS <s 0. if (MaxRecurse) - if (Value *V = SimplifyICmpInst(ICmpInst::ICMP_SLT, SrcOp, - Constant::getNullValue(SrcTy), - Q, MaxRecurse-1)) + if (Value *V = simplifyICmpInst(ICmpInst::ICMP_SLT, SrcOp, + Constant::getNullValue(SrcTy), Q, + MaxRecurse - 1)) return V; break; case ICmpInst::ICMP_ULT: case ICmpInst::ICMP_ULE: // Comparison is true iff the LHS >=s 0. if (MaxRecurse) - if (Value *V = SimplifyICmpInst(ICmpInst::ICMP_SGE, SrcOp, - Constant::getNullValue(SrcTy), - Q, MaxRecurse-1)) + if (Value *V = simplifyICmpInst(ICmpInst::ICMP_SGE, SrcOp, + Constant::getNullValue(SrcTy), Q, + MaxRecurse - 1)) return V; break; } @@ -3783,26 +3861,26 @@ // If the comparison is with the result of a select instruction, check whether // comparing with either branch of the select always yields the same value. if (isa<SelectInst>(LHS) || isa<SelectInst>(RHS)) - if (Value *V = ThreadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse)) + if (Value *V = threadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse)) return V; // If the comparison is with the result of a phi instruction, check whether // doing the compare with each incoming phi value yields a common result. if (isa<PHINode>(LHS) || isa<PHINode>(RHS)) - if (Value *V = ThreadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse)) + if (Value *V = threadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse)) return V; return nullptr; } -Value *llvm::SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, +Value *llvm::simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q) { - return ::SimplifyICmpInst(Predicate, LHS, RHS, Q, RecursionLimit); + return ::simplifyICmpInst(Predicate, LHS, RHS, Q, RecursionLimit); } /// Given operands for an FCmpInst, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, +static Value *simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, unsigned MaxRecurse) { CmpInst::Predicate Pred = (CmpInst::Predicate)Predicate; @@ -3810,7 +3888,8 @@ if (Constant *CLHS = dyn_cast<Constant>(LHS)) { if (Constant *CRHS = dyn_cast<Constant>(RHS)) - return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI); + return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI, + Q.CxtI); // If we have a constant, make sure it is on the RHS. std::swap(LHS, RHS); @@ -3818,7 +3897,7 @@ } // Fold trivial predicates. - Type *RetTy = GetCompareTy(LHS); + Type *RetTy = getCompareTy(LHS); if (Pred == FCmpInst::FCMP_FALSE) return getFalse(RetTy); if (Pred == FCmpInst::FCMP_TRUE) @@ -3938,23 +4017,29 @@ // The ordered relationship and minnum/maxnum guarantee that we do not // have NaN constants, so ordered/unordered preds are handled the same. switch (Pred) { - case FCmpInst::FCMP_OEQ: case FCmpInst::FCMP_UEQ: + case FCmpInst::FCMP_OEQ: + case FCmpInst::FCMP_UEQ: // minnum(X, LesserC) == C --> false // maxnum(X, GreaterC) == C --> false return getFalse(RetTy); - case FCmpInst::FCMP_ONE: case FCmpInst::FCMP_UNE: + case FCmpInst::FCMP_ONE: + case FCmpInst::FCMP_UNE: // minnum(X, LesserC) != C --> true // maxnum(X, GreaterC) != C --> true return getTrue(RetTy); - case FCmpInst::FCMP_OGE: case FCmpInst::FCMP_UGE: - case FCmpInst::FCMP_OGT: case FCmpInst::FCMP_UGT: + case FCmpInst::FCMP_OGE: + case FCmpInst::FCMP_UGE: + case FCmpInst::FCMP_OGT: + case FCmpInst::FCMP_UGT: // minnum(X, LesserC) >= C --> false // minnum(X, LesserC) > C --> false // maxnum(X, GreaterC) >= C --> true // maxnum(X, GreaterC) > C --> true return ConstantInt::get(RetTy, IsMaxNum); - case FCmpInst::FCMP_OLE: case FCmpInst::FCMP_ULE: - case FCmpInst::FCMP_OLT: case FCmpInst::FCMP_ULT: + case FCmpInst::FCMP_OLE: + case FCmpInst::FCMP_ULE: + case FCmpInst::FCMP_OLT: + case FCmpInst::FCMP_ULT: // minnum(X, LesserC) <= C --> true // minnum(X, LesserC) < C --> true // maxnum(X, GreaterC) <= C --> false @@ -3992,21 +4077,21 @@ // If the comparison is with the result of a select instruction, check whether // comparing with either branch of the select always yields the same value. if (isa<SelectInst>(LHS) || isa<SelectInst>(RHS)) - if (Value *V = ThreadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse)) + if (Value *V = threadCmpOverSelect(Pred, LHS, RHS, Q, MaxRecurse)) return V; // If the comparison is with the result of a phi instruction, check whether // doing the compare with each incoming phi value yields a common result. if (isa<PHINode>(LHS) || isa<PHINode>(RHS)) - if (Value *V = ThreadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse)) + if (Value *V = threadCmpOverPHI(Pred, LHS, RHS, Q, MaxRecurse)) return V; return nullptr; } -Value *llvm::SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, +Value *llvm::simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q) { - return ::SimplifyFCmpInst(Predicate, LHS, RHS, FMF, Q, RecursionLimit); + return ::simplifyFCmpInst(Predicate, LHS, RHS, FMF, Q, RecursionLimit); } static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp, @@ -4073,22 +4158,21 @@ }; if (auto *B = dyn_cast<BinaryOperator>(I)) - return PreventSelfSimplify(SimplifyBinOp(B->getOpcode(), NewOps[0], + return PreventSelfSimplify(simplifyBinOp(B->getOpcode(), NewOps[0], NewOps[1], Q, MaxRecurse - 1)); if (CmpInst *C = dyn_cast<CmpInst>(I)) - return PreventSelfSimplify(SimplifyCmpInst(C->getPredicate(), NewOps[0], + return PreventSelfSimplify(simplifyCmpInst(C->getPredicate(), NewOps[0], NewOps[1], Q, MaxRecurse - 1)); if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) - return PreventSelfSimplify(SimplifyGEPInst( + return PreventSelfSimplify(simplifyGEPInst( GEP->getSourceElementType(), NewOps[0], makeArrayRef(NewOps).slice(1), GEP->isInBounds(), Q, MaxRecurse - 1)); if (isa<SelectInst>(I)) - return PreventSelfSimplify( - SimplifySelectInst(NewOps[0], NewOps[1], NewOps[2], Q, - MaxRecurse - 1)); + return PreventSelfSimplify(simplifySelectInst( + NewOps[0], NewOps[1], NewOps[2], Q, MaxRecurse - 1)); // TODO: We could hand off more cases to instsimplify here. } @@ -4114,14 +4198,6 @@ if (!AllowRefinement && canCreatePoison(cast<Operator>(I))) return nullptr; - if (CmpInst *C = dyn_cast<CmpInst>(I)) - return ConstantFoldCompareInstOperands(C->getPredicate(), ConstOps[0], - ConstOps[1], Q.DL, Q.TLI); - - if (LoadInst *LI = dyn_cast<LoadInst>(I)) - if (!LI->isVolatile()) - return ConstantFoldLoadFromConstPtr(ConstOps[0], LI->getType(), Q.DL); - return ConstantFoldInstOperands(I, ConstOps, Q.DL, Q.TLI); } @@ -4184,7 +4260,8 @@ /// Try to simplify a select instruction when its condition operand is an /// integer comparison. static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal, - Value *FalseVal, const SimplifyQuery &Q, + Value *FalseVal, + const SimplifyQuery &Q, unsigned MaxRecurse) { ICmpInst::Predicate Pred; Value *CmpLHS, *CmpRHS; @@ -4204,7 +4281,8 @@ Value *X, *Y; SelectPatternFlavor SPF = matchDecomposedSelectPattern(cast<ICmpInst>(CondVal), TrueVal, FalseVal, - X, Y).Flavor; + X, Y) + .Flavor; if (SelectPatternResult::isMinOrMax(SPF) && Pred == getMinMaxPred(SPF)) { APInt LimitC = getMinMaxLimit(getInverseMinMaxFlavor(SPF), X->getType()->getScalarSizeInBits()); @@ -4256,8 +4334,8 @@ } // Check for other compares that behave like bit test. - if (Value *V = simplifySelectWithFakeICmpEq(CmpLHS, CmpRHS, Pred, - TrueVal, FalseVal)) + if (Value *V = + simplifySelectWithFakeICmpEq(CmpLHS, CmpRHS, Pred, TrueVal, FalseVal)) return V; // If we have a scalar equality comparison, then we know the value in one of @@ -4267,18 +4345,18 @@ // because each element of a vector select is chosen independently. if (Pred == ICmpInst::ICMP_EQ && !CondVal->getType()->isVectorTy()) { if (simplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, Q, - /* AllowRefinement */ false, MaxRecurse) == - TrueVal || + /* AllowRefinement */ false, + MaxRecurse) == TrueVal || simplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, Q, - /* AllowRefinement */ false, MaxRecurse) == - TrueVal) + /* AllowRefinement */ false, + MaxRecurse) == TrueVal) return FalseVal; if (simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, Q, - /* AllowRefinement */ true, MaxRecurse) == - FalseVal || + /* AllowRefinement */ true, + MaxRecurse) == FalseVal || simplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, Q, - /* AllowRefinement */ true, MaxRecurse) == - FalseVal) + /* AllowRefinement */ true, + MaxRecurse) == FalseVal) return FalseVal; } @@ -4297,11 +4375,11 @@ // This transform is safe if we do not have (do not care about) -0.0 or if // at least one operand is known to not be -0.0. Otherwise, the select can // change the sign of a zero operand. - bool HasNoSignedZeros = Q.CxtI && isa<FPMathOperator>(Q.CxtI) && - Q.CxtI->hasNoSignedZeros(); + bool HasNoSignedZeros = + Q.CxtI && isa<FPMathOperator>(Q.CxtI) && Q.CxtI->hasNoSignedZeros(); const APFloat *C; if (HasNoSignedZeros || (match(T, m_APFloat(C)) && C->isNonZero()) || - (match(F, m_APFloat(C)) && C->isNonZero())) { + (match(F, m_APFloat(C)) && C->isNonZero())) { // (T == F) ? T : F --> F // (F == T) ? T : F --> F if (Pred == FCmpInst::FCMP_OEQ) @@ -4318,7 +4396,7 @@ /// Given operands for a SelectInst, see if we can fold the result. /// If not, this returns null. -static Value *SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, +static Value *simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, const SimplifyQuery &Q, unsigned MaxRecurse) { if (auto *CondC = dyn_cast<Constant>(Cond)) { if (auto *TrueC = dyn_cast<Constant>(TrueVal)) @@ -4434,14 +4512,14 @@ return nullptr; } -Value *llvm::SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, +Value *llvm::simplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, const SimplifyQuery &Q) { - return ::SimplifySelectInst(Cond, TrueVal, FalseVal, Q, RecursionLimit); + return ::simplifySelectInst(Cond, TrueVal, FalseVal, Q, RecursionLimit); } /// Given operands for an GetElementPtrInst, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyGEPInst(Type *SrcTy, Value *Ptr, +static Value *simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices, bool InBounds, const SimplifyQuery &Q, unsigned) { // The type of the GEP pointer operand. @@ -4468,6 +4546,13 @@ } } + // For opaque pointers an all-zero GEP is a no-op. For typed pointers, + // it may be equivalent to a bitcast. + if (Ptr->getType()->getScalarType()->isOpaquePointerTy() && + Ptr->getType() == GEPTy && + all_of(Indices, [](const auto *V) { return match(V, m_Zero()); })) + return Ptr; + // getelementptr poison, idx -> poison // getelementptr baseptr, poison -> poison if (isa<PoisonValue>(Ptr) || @@ -4572,16 +4657,16 @@ return ConstantFoldConstant(CE, Q.DL); } -Value *llvm::SimplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices, +Value *llvm::simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices, bool InBounds, const SimplifyQuery &Q) { - return ::SimplifyGEPInst(SrcTy, Ptr, Indices, InBounds, Q, RecursionLimit); + return ::simplifyGEPInst(SrcTy, Ptr, Indices, InBounds, Q, RecursionLimit); } /// Given operands for an InsertValueInst, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyInsertValueInst(Value *Agg, Value *Val, - ArrayRef<unsigned> Idxs, const SimplifyQuery &Q, - unsigned) { +static Value *simplifyInsertValueInst(Value *Agg, Value *Val, + ArrayRef<unsigned> Idxs, + const SimplifyQuery &Q, unsigned) { if (Constant *CAgg = dyn_cast<Constant>(Agg)) if (Constant *CVal = dyn_cast<Constant>(Val)) return ConstantFoldInsertValueInstruction(CAgg, CVal, Idxs); @@ -4606,13 +4691,13 @@ return nullptr; } -Value *llvm::SimplifyInsertValueInst(Value *Agg, Value *Val, +Value *llvm::simplifyInsertValueInst(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs, const SimplifyQuery &Q) { - return ::SimplifyInsertValueInst(Agg, Val, Idxs, Q, RecursionLimit); + return ::simplifyInsertValueInst(Agg, Val, Idxs, Q, RecursionLimit); } -Value *llvm::SimplifyInsertElementInst(Value *Vec, Value *Val, Value *Idx, +Value *llvm::simplifyInsertElementInst(Value *Vec, Value *Val, Value *Idx, const SimplifyQuery &Q) { // Try to constant fold. auto *VecC = dyn_cast<Constant>(Vec); @@ -4649,7 +4734,7 @@ /// Given operands for an ExtractValueInst, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, +static Value *simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, const SimplifyQuery &, unsigned) { if (auto *CAgg = dyn_cast<Constant>(Agg)) return ConstantFoldExtractValueInstruction(CAgg, Idxs); @@ -4672,14 +4757,14 @@ return nullptr; } -Value *llvm::SimplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, +Value *llvm::simplifyExtractValueInst(Value *Agg, ArrayRef<unsigned> Idxs, const SimplifyQuery &Q) { - return ::SimplifyExtractValueInst(Agg, Idxs, Q, RecursionLimit); + return ::simplifyExtractValueInst(Agg, Idxs, Q, RecursionLimit); } /// Given operands for an ExtractElementInst, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyExtractElementInst(Value *Vec, Value *Idx, +static Value *simplifyExtractElementInst(Value *Vec, Value *Idx, const SimplifyQuery &Q, unsigned) { auto *VecVTy = cast<VectorType>(Vec->getType()); if (auto *CVec = dyn_cast<Constant>(Vec)) { @@ -4716,13 +4801,13 @@ return nullptr; } -Value *llvm::SimplifyExtractElementInst(Value *Vec, Value *Idx, +Value *llvm::simplifyExtractElementInst(Value *Vec, Value *Idx, const SimplifyQuery &Q) { - return ::SimplifyExtractElementInst(Vec, Idx, Q, RecursionLimit); + return ::simplifyExtractElementInst(Vec, Idx, Q, RecursionLimit); } /// See if we can fold the given phi. If not, returns null. -static Value *SimplifyPHINode(PHINode *PN, ArrayRef<Value *> IncomingValues, +static Value *simplifyPHINode(PHINode *PN, ArrayRef<Value *> IncomingValues, const SimplifyQuery &Q) { // WARNING: no matter how worthwhile it may seem, we can not perform PHI CSE // here, because the PHI we may succeed simplifying to was not @@ -4734,14 +4819,15 @@ bool HasUndefInput = false; for (Value *Incoming : IncomingValues) { // If the incoming value is the phi node itself, it can safely be skipped. - if (Incoming == PN) continue; + if (Incoming == PN) + continue; if (Q.isUndefValue(Incoming)) { // Remember that we saw an undef value, but otherwise ignore them. HasUndefInput = true; continue; } if (CommonValue && Incoming != CommonValue) - return nullptr; // Not the same, bail out. + return nullptr; // Not the same, bail out. CommonValue = Incoming; } @@ -4750,17 +4836,18 @@ if (!CommonValue) return UndefValue::get(PN->getType()); - // If we have a PHI node like phi(X, undef, X), where X is defined by some - // instruction, we cannot return X as the result of the PHI node unless it - // dominates the PHI block. - if (HasUndefInput) + if (HasUndefInput) { + // If we have a PHI node like phi(X, undef, X), where X is defined by some + // instruction, we cannot return X as the result of the PHI node unless it + // dominates the PHI block. return valueDominatesPHI(CommonValue, PN, Q.DT) ? CommonValue : nullptr; + } return CommonValue; } -static Value *SimplifyCastInst(unsigned CastOpc, Value *Op, - Type *Ty, const SimplifyQuery &Q, unsigned MaxRecurse) { +static Value *simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, + const SimplifyQuery &Q, unsigned MaxRecurse) { if (auto *C = dyn_cast<Constant>(Op)) return ConstantFoldCastOperand(CastOpc, C, Ty, Q.DL); @@ -4793,9 +4880,9 @@ return nullptr; } -Value *llvm::SimplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, +Value *llvm::simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, const SimplifyQuery &Q) { - return ::SimplifyCastInst(CastOpc, Op, Ty, Q, RecursionLimit); + return ::simplifyCastInst(CastOpc, Op, Ty, Q, RecursionLimit); } /// For the given destination element of a shuffle, peek through shuffles to @@ -4849,7 +4936,7 @@ return RootVec; } -static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, +static Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask, Type *RetTy, const SimplifyQuery &Q, unsigned MaxRecurse) { @@ -4965,14 +5052,14 @@ } /// Given operands for a ShuffleVectorInst, fold the result or return null. -Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, +Value *llvm::simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask, Type *RetTy, const SimplifyQuery &Q) { - return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit); + return ::simplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit); } -static Constant *foldConstant(Instruction::UnaryOps Opcode, - Value *&Op, const SimplifyQuery &Q) { +static Constant *foldConstant(Instruction::UnaryOps Opcode, Value *&Op, + const SimplifyQuery &Q) { if (auto *C = dyn_cast<Constant>(Op)) return ConstantFoldUnaryOpOperand(Opcode, C, Q.DL); return nullptr; @@ -4993,7 +5080,7 @@ return nullptr; } -Value *llvm::SimplifyFNegInst(Value *Op, FastMathFlags FMF, +Value *llvm::simplifyFNegInst(Value *Op, FastMathFlags FMF, const SimplifyQuery &Q) { return ::simplifyFNegInst(Op, FMF, Q, RecursionLimit); } @@ -5044,15 +5131,10 @@ return nullptr; } -// TODO: Move this out to a header file: -static inline bool canIgnoreSNaN(fp::ExceptionBehavior EB, FastMathFlags FMF) { - return (EB == fp::ebIgnore || FMF.noNaNs()); -} - /// Given operands for an FAdd, see if we can fold the result. If not, this /// returns null. static Value * -SimplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, +simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, unsigned MaxRecurse, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven) { @@ -5114,7 +5196,7 @@ /// Given operands for an FSub, see if we can fold the result. If not, this /// returns null. static Value * -SimplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, +simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, unsigned MaxRecurse, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven) { @@ -5125,24 +5207,28 @@ if (Constant *C = simplifyFPOp({Op0, Op1}, FMF, Q, ExBehavior, Rounding)) return C; - if (!isDefaultFPEnvironment(ExBehavior, Rounding)) - return nullptr; - // fsub X, +0 ==> X - if (match(Op1, m_PosZeroFP())) - return Op0; + if (canIgnoreSNaN(ExBehavior, FMF) && + (!canRoundingModeBe(Rounding, RoundingMode::TowardNegative) || + FMF.noSignedZeros())) + if (match(Op1, m_PosZeroFP())) + return Op0; // fsub X, -0 ==> X, when we know X is not -0 - if (match(Op1, m_NegZeroFP()) && - (FMF.noSignedZeros() || CannotBeNegativeZero(Op0, Q.TLI))) - return Op0; + if (canIgnoreSNaN(ExBehavior, FMF)) + if (match(Op1, m_NegZeroFP()) && + (FMF.noSignedZeros() || CannotBeNegativeZero(Op0, Q.TLI))) + return Op0; // fsub -0.0, (fsub -0.0, X) ==> X // fsub -0.0, (fneg X) ==> X Value *X; - if (match(Op0, m_NegZeroFP()) && - match(Op1, m_FNeg(m_Value(X)))) - return X; + if (canIgnoreSNaN(ExBehavior, FMF)) + if (match(Op0, m_NegZeroFP()) && match(Op1, m_FNeg(m_Value(X)))) + return X; + + if (!isDefaultFPEnvironment(ExBehavior, Rounding)) + return nullptr; // fsub 0.0, (fsub 0.0, X) ==> X if signed zeros are ignored. // fsub 0.0, (fneg X) ==> X if signed zeros are ignored. @@ -5165,7 +5251,7 @@ return nullptr; } -static Value *SimplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF, +static Value *simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, unsigned MaxRecurse, fp::ExceptionBehavior ExBehavior, RoundingMode Rounding) { @@ -5196,8 +5282,8 @@ // 2. Ignore non-zero negative numbers because sqrt would produce NAN. // 3. Ignore -0.0 because sqrt(-0.0) == -0.0, but -0.0 * -0.0 == 0.0. Value *X; - if (Op0 == Op1 && match(Op0, m_Intrinsic<Intrinsic::sqrt>(m_Value(X))) && - FMF.allowReassoc() && FMF.noNaNs() && FMF.noSignedZeros()) + if (Op0 == Op1 && match(Op0, m_Sqrt(m_Value(X))) && FMF.allowReassoc() && + FMF.noNaNs() && FMF.noSignedZeros()) return X; return nullptr; @@ -5205,7 +5291,7 @@ /// Given the operands for an FMul, see if we can fold the result static Value * -SimplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF, +simplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, unsigned MaxRecurse, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven) { @@ -5214,43 +5300,43 @@ return C; // Now apply simplifications that do not require rounding. - return SimplifyFMAFMul(Op0, Op1, FMF, Q, MaxRecurse, ExBehavior, Rounding); + return simplifyFMAFMul(Op0, Op1, FMF, Q, MaxRecurse, ExBehavior, Rounding); } -Value *llvm::SimplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, +Value *llvm::simplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior, RoundingMode Rounding) { - return ::SimplifyFAddInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, + return ::simplifyFAddInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, Rounding); } -Value *llvm::SimplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, +Value *llvm::simplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior, RoundingMode Rounding) { - return ::SimplifyFSubInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, + return ::simplifyFSubInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, Rounding); } -Value *llvm::SimplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF, +Value *llvm::simplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior, RoundingMode Rounding) { - return ::SimplifyFMulInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, + return ::simplifyFMulInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, Rounding); } -Value *llvm::SimplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF, +Value *llvm::simplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior, RoundingMode Rounding) { - return ::SimplifyFMAFMul(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, + return ::simplifyFMAFMul(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, Rounding); } static Value * -SimplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF, +simplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, unsigned, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven) { @@ -5296,16 +5382,16 @@ return nullptr; } -Value *llvm::SimplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF, +Value *llvm::simplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior, RoundingMode Rounding) { - return ::SimplifyFDivInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, + return ::simplifyFDivInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, Rounding); } static Value * -SimplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, +simplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, unsigned, fp::ExceptionBehavior ExBehavior = fp::ebIgnore, RoundingMode Rounding = RoundingMode::NearestTiesToEven) { @@ -5334,11 +5420,11 @@ return nullptr; } -Value *llvm::SimplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, +Value *llvm::simplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior, RoundingMode Rounding) { - return ::SimplifyFRemInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, + return ::simplifyFRemInst(Op0, Op1, FMF, Q, RecursionLimit, ExBehavior, Rounding); } @@ -5360,8 +5446,8 @@ /// If not, this returns null. /// Try to use FastMathFlags when folding the result. static Value *simplifyFPUnOp(unsigned Opcode, Value *Op, - const FastMathFlags &FMF, - const SimplifyQuery &Q, unsigned MaxRecurse) { + const FastMathFlags &FMF, const SimplifyQuery &Q, + unsigned MaxRecurse) { switch (Opcode) { case Instruction::FNeg: return simplifyFNegInst(Op, FMF, Q, MaxRecurse); @@ -5370,56 +5456,56 @@ } } -Value *llvm::SimplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q) { +Value *llvm::simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q) { return ::simplifyUnOp(Opcode, Op, Q, RecursionLimit); } -Value *llvm::SimplifyUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, +Value *llvm::simplifyUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, const SimplifyQuery &Q) { return ::simplifyFPUnOp(Opcode, Op, FMF, Q, RecursionLimit); } /// Given operands for a BinaryOperator, see if we can fold the result. /// If not, this returns null. -static Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, +static Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { switch (Opcode) { case Instruction::Add: - return SimplifyAddInst(LHS, RHS, false, false, Q, MaxRecurse); + return simplifyAddInst(LHS, RHS, false, false, Q, MaxRecurse); case Instruction::Sub: - return SimplifySubInst(LHS, RHS, false, false, Q, MaxRecurse); + return simplifySubInst(LHS, RHS, false, false, Q, MaxRecurse); case Instruction::Mul: - return SimplifyMulInst(LHS, RHS, Q, MaxRecurse); + return simplifyMulInst(LHS, RHS, Q, MaxRecurse); case Instruction::SDiv: - return SimplifySDivInst(LHS, RHS, Q, MaxRecurse); + return simplifySDivInst(LHS, RHS, Q, MaxRecurse); case Instruction::UDiv: - return SimplifyUDivInst(LHS, RHS, Q, MaxRecurse); + return simplifyUDivInst(LHS, RHS, Q, MaxRecurse); case Instruction::SRem: - return SimplifySRemInst(LHS, RHS, Q, MaxRecurse); + return simplifySRemInst(LHS, RHS, Q, MaxRecurse); case Instruction::URem: - return SimplifyURemInst(LHS, RHS, Q, MaxRecurse); + return simplifyURemInst(LHS, RHS, Q, MaxRecurse); case Instruction::Shl: - return SimplifyShlInst(LHS, RHS, false, false, Q, MaxRecurse); + return simplifyShlInst(LHS, RHS, false, false, Q, MaxRecurse); case Instruction::LShr: - return SimplifyLShrInst(LHS, RHS, false, Q, MaxRecurse); + return simplifyLShrInst(LHS, RHS, false, Q, MaxRecurse); case Instruction::AShr: - return SimplifyAShrInst(LHS, RHS, false, Q, MaxRecurse); + return simplifyAShrInst(LHS, RHS, false, Q, MaxRecurse); case Instruction::And: - return SimplifyAndInst(LHS, RHS, Q, MaxRecurse); + return simplifyAndInst(LHS, RHS, Q, MaxRecurse); case Instruction::Or: - return SimplifyOrInst(LHS, RHS, Q, MaxRecurse); + return simplifyOrInst(LHS, RHS, Q, MaxRecurse); case Instruction::Xor: - return SimplifyXorInst(LHS, RHS, Q, MaxRecurse); + return simplifyXorInst(LHS, RHS, Q, MaxRecurse); case Instruction::FAdd: - return SimplifyFAddInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); + return simplifyFAddInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); case Instruction::FSub: - return SimplifyFSubInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); + return simplifyFSubInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); case Instruction::FMul: - return SimplifyFMulInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); + return simplifyFMulInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); case Instruction::FDiv: - return SimplifyFDivInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); + return simplifyFDivInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); case Instruction::FRem: - return SimplifyFRemInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); + return simplifyFRemInst(LHS, RHS, FastMathFlags(), Q, MaxRecurse); default: llvm_unreachable("Unexpected opcode"); } @@ -5428,49 +5514,50 @@ /// Given operands for a BinaryOperator, see if we can fold the result. /// If not, this returns null. /// Try to use FastMathFlags when folding the result. -static Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, +static Value *simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const FastMathFlags &FMF, const SimplifyQuery &Q, unsigned MaxRecurse) { switch (Opcode) { case Instruction::FAdd: - return SimplifyFAddInst(LHS, RHS, FMF, Q, MaxRecurse); + return simplifyFAddInst(LHS, RHS, FMF, Q, MaxRecurse); case Instruction::FSub: - return SimplifyFSubInst(LHS, RHS, FMF, Q, MaxRecurse); + return simplifyFSubInst(LHS, RHS, FMF, Q, MaxRecurse); case Instruction::FMul: - return SimplifyFMulInst(LHS, RHS, FMF, Q, MaxRecurse); + return simplifyFMulInst(LHS, RHS, FMF, Q, MaxRecurse); case Instruction::FDiv: - return SimplifyFDivInst(LHS, RHS, FMF, Q, MaxRecurse); + return simplifyFDivInst(LHS, RHS, FMF, Q, MaxRecurse); default: - return SimplifyBinOp(Opcode, LHS, RHS, Q, MaxRecurse); + return simplifyBinOp(Opcode, LHS, RHS, Q, MaxRecurse); } } -Value *llvm::SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, +Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q) { - return ::SimplifyBinOp(Opcode, LHS, RHS, Q, RecursionLimit); + return ::simplifyBinOp(Opcode, LHS, RHS, Q, RecursionLimit); } -Value *llvm::SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, +Value *llvm::simplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q) { - return ::SimplifyBinOp(Opcode, LHS, RHS, FMF, Q, RecursionLimit); + return ::simplifyBinOp(Opcode, LHS, RHS, FMF, Q, RecursionLimit); } /// Given operands for a CmpInst, see if we can fold the result. -static Value *SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, +static Value *simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q, unsigned MaxRecurse) { if (CmpInst::isIntPredicate((CmpInst::Predicate)Predicate)) - return SimplifyICmpInst(Predicate, LHS, RHS, Q, MaxRecurse); - return SimplifyFCmpInst(Predicate, LHS, RHS, FastMathFlags(), Q, MaxRecurse); + return simplifyICmpInst(Predicate, LHS, RHS, Q, MaxRecurse); + return simplifyFCmpInst(Predicate, LHS, RHS, FastMathFlags(), Q, MaxRecurse); } -Value *llvm::SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, +Value *llvm::simplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q) { - return ::SimplifyCmpInst(Predicate, LHS, RHS, Q, RecursionLimit); + return ::simplifyCmpInst(Predicate, LHS, RHS, Q, RecursionLimit); } -static bool IsIdempotent(Intrinsic::ID ID) { +static bool isIdempotent(Intrinsic::ID ID) { switch (ID) { - default: return false; + default: + return false; // Unary idempotent: f(f(x)) = f(x) case Intrinsic::fabs: @@ -5486,7 +5573,7 @@ } } -static Value *SimplifyRelativeLoad(Constant *Ptr, Constant *Offset, +static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset, const DataLayout &DL) { GlobalValue *PtrSym; APInt PtrOffset; @@ -5546,7 +5633,7 @@ const SimplifyQuery &Q) { // Idempotent functions return the same result when called repeatedly. Intrinsic::ID IID = F->getIntrinsicID(); - if (IsIdempotent(IID)) + if (isIdempotent(IID)) if (auto *II = dyn_cast<IntrinsicInst>(Op0)) if (II->getIntrinsicID() == IID) return II; @@ -5554,15 +5641,18 @@ Value *X; switch (IID) { case Intrinsic::fabs: - if (SignBitMustBeZero(Op0, Q.TLI)) return Op0; + if (SignBitMustBeZero(Op0, Q.TLI)) + return Op0; break; case Intrinsic::bswap: // bswap(bswap(x)) -> x - if (match(Op0, m_BSwap(m_Value(X)))) return X; + if (match(Op0, m_BSwap(m_Value(X)))) + return X; break; case Intrinsic::bitreverse: // bitreverse(bitreverse(x)) -> x - if (match(Op0, m_BitReverse(m_Value(X)))) return X; + if (match(Op0, m_BitReverse(m_Value(X)))) + return X; break; case Intrinsic::ctpop: { // If everything but the lowest bit is zero, that bit is the pop-count. Ex: @@ -5576,30 +5666,34 @@ case Intrinsic::exp: // exp(log(x)) -> x if (Q.CxtI->hasAllowReassoc() && - match(Op0, m_Intrinsic<Intrinsic::log>(m_Value(X)))) return X; + match(Op0, m_Intrinsic<Intrinsic::log>(m_Value(X)))) + return X; break; case Intrinsic::exp2: // exp2(log2(x)) -> x if (Q.CxtI->hasAllowReassoc() && - match(Op0, m_Intrinsic<Intrinsic::log2>(m_Value(X)))) return X; + match(Op0, m_Intrinsic<Intrinsic::log2>(m_Value(X)))) + return X; break; case Intrinsic::log: // log(exp(x)) -> x if (Q.CxtI->hasAllowReassoc() && - match(Op0, m_Intrinsic<Intrinsic::exp>(m_Value(X)))) return X; + match(Op0, m_Intrinsic<Intrinsic::exp>(m_Value(X)))) + return X; break; case Intrinsic::log2: // log2(exp2(x)) -> x if (Q.CxtI->hasAllowReassoc() && (match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X))) || - match(Op0, m_Intrinsic<Intrinsic::pow>(m_SpecificFP(2.0), - m_Value(X))))) return X; + match(Op0, + m_Intrinsic<Intrinsic::pow>(m_SpecificFP(2.0), m_Value(X))))) + return X; break; case Intrinsic::log10: // log10(pow(10.0, x)) -> x if (Q.CxtI->hasAllowReassoc() && - match(Op0, m_Intrinsic<Intrinsic::pow>(m_SpecificFP(10.0), - m_Value(X)))) return X; + match(Op0, m_Intrinsic<Intrinsic::pow>(m_SpecificFP(10.0), m_Value(X)))) + return X; break; case Intrinsic::floor: case Intrinsic::trunc: @@ -5821,7 +5915,7 @@ case Intrinsic::load_relative: if (auto *C0 = dyn_cast<Constant>(Op0)) if (auto *C1 = dyn_cast<Constant>(Op1)) - return SimplifyRelativeLoad(C0, C1, Q.DL); + return simplifyRelativeLoad(C0, C1, Q.DL); break; case Intrinsic::powi: if (auto *Power = dyn_cast<ConstantInt>(Op1)) { @@ -5848,7 +5942,8 @@ case Intrinsic::maximum: case Intrinsic::minimum: { // If the arguments are the same, this is a no-op. - if (Op0 == Op1) return Op0; + if (Op0 == Op1) + return Op0; // Canonicalize constant operand as Op1. if (isa<Constant>(Op0)) @@ -5901,14 +5996,14 @@ break; } - case Intrinsic::experimental_vector_extract: { + case Intrinsic::vector_extract: { Type *ReturnType = F->getReturnType(); // (extract_vector (insert_vector _, X, 0), 0) -> X unsigned IdxN = cast<ConstantInt>(Op1)->getZExtValue(); Value *X = nullptr; - if (match(Op0, m_Intrinsic<Intrinsic::experimental_vector_insert>( - m_Value(), m_Value(X), m_Zero())) && + if (match(Op0, m_Intrinsic<Intrinsic::vector_insert>(m_Value(), m_Value(X), + m_Zero())) && IdxN == 0 && X->getType() == ReturnType) return X; @@ -6004,8 +6099,8 @@ Value *Op2 = Call->getArgOperand(2); auto *FPI = cast<ConstrainedFPIntrinsic>(Call); if (Value *V = simplifyFPOp({Op0, Op1, Op2}, {}, Q, - FPI->getExceptionBehavior().getValue(), - FPI->getRoundingMode().getValue())) + FPI->getExceptionBehavior().value(), + FPI->getRoundingMode().value())) return V; return nullptr; } @@ -6049,7 +6144,7 @@ return nullptr; } - case Intrinsic::experimental_vector_insert: { + case Intrinsic::vector_insert: { Value *Vec = Call->getArgOperand(0); Value *SubVec = Call->getArgOperand(1); Value *Idx = Call->getArgOperand(2); @@ -6059,8 +6154,8 @@ // where: Y is X, or Y is undef unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue(); Value *X = nullptr; - if (match(SubVec, m_Intrinsic<Intrinsic::experimental_vector_extract>( - m_Value(X), m_Zero())) && + if (match(SubVec, + m_Intrinsic<Intrinsic::vector_extract>(m_Value(X), m_Zero())) && (Q.isUndefValue(Vec) || Vec == X) && IdxN == 0 && X->getType() == ReturnType) return X; @@ -6069,43 +6164,33 @@ } case Intrinsic::experimental_constrained_fadd: { auto *FPI = cast<ConstrainedFPIntrinsic>(Call); - return SimplifyFAddInst(FPI->getArgOperand(0), FPI->getArgOperand(1), - FPI->getFastMathFlags(), Q, - FPI->getExceptionBehavior().getValue(), - FPI->getRoundingMode().getValue()); - break; + return simplifyFAddInst( + FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), + Q, FPI->getExceptionBehavior().value(), FPI->getRoundingMode().value()); } case Intrinsic::experimental_constrained_fsub: { auto *FPI = cast<ConstrainedFPIntrinsic>(Call); - return SimplifyFSubInst(FPI->getArgOperand(0), FPI->getArgOperand(1), - FPI->getFastMathFlags(), Q, - FPI->getExceptionBehavior().getValue(), - FPI->getRoundingMode().getValue()); - break; + return simplifyFSubInst( + FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), + Q, FPI->getExceptionBehavior().value(), FPI->getRoundingMode().value()); } case Intrinsic::experimental_constrained_fmul: { auto *FPI = cast<ConstrainedFPIntrinsic>(Call); - return SimplifyFMulInst(FPI->getArgOperand(0), FPI->getArgOperand(1), - FPI->getFastMathFlags(), Q, - FPI->getExceptionBehavior().getValue(), - FPI->getRoundingMode().getValue()); - break; + return simplifyFMulInst( + FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), + Q, FPI->getExceptionBehavior().value(), FPI->getRoundingMode().value()); } case Intrinsic::experimental_constrained_fdiv: { auto *FPI = cast<ConstrainedFPIntrinsic>(Call); - return SimplifyFDivInst(FPI->getArgOperand(0), FPI->getArgOperand(1), - FPI->getFastMathFlags(), Q, - FPI->getExceptionBehavior().getValue(), - FPI->getRoundingMode().getValue()); - break; + return simplifyFDivInst( + FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), + Q, FPI->getExceptionBehavior().value(), FPI->getRoundingMode().value()); } case Intrinsic::experimental_constrained_frem: { auto *FPI = cast<ConstrainedFPIntrinsic>(Call); - return SimplifyFRemInst(FPI->getArgOperand(0), FPI->getArgOperand(1), - FPI->getFastMathFlags(), Q, - FPI->getExceptionBehavior().getValue(), - FPI->getRoundingMode().getValue()); - break; + return simplifyFRemInst( + FPI->getArgOperand(0), FPI->getArgOperand(1), FPI->getFastMathFlags(), + Q, FPI->getExceptionBehavior().value(), FPI->getRoundingMode().value()); } default: return nullptr; @@ -6133,7 +6218,7 @@ return ConstantFoldCall(Call, F, ConstantArgs, Q.TLI); } -Value *llvm::SimplifyCall(CallBase *Call, const SimplifyQuery &Q) { +Value *llvm::simplifyCall(CallBase *Call, const SimplifyQuery &Q) { // musttail calls can only be simplified if they are also DCEd. // As we can't guarantee this here, don't simplify them. if (Call->isMustTailCall()) @@ -6156,8 +6241,17 @@ return nullptr; } +Value *llvm::simplifyConstrainedFPCall(CallBase *Call, const SimplifyQuery &Q) { + assert(isa<ConstrainedFPIntrinsic>(Call)); + if (Value *V = tryConstantFoldCall(Call, Q)) + return V; + if (Value *Ret = simplifyIntrinsic(Call, Q)) + return Ret; + return nullptr; +} + /// Given operands for a Freeze, see if we can fold the result. -static Value *SimplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) { +static Value *simplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) { // Use a utility function defined in ValueTracking. if (llvm::isGuaranteedNotToBeUndefOrPoison(Op0, Q.AC, Q.CxtI, Q.DT)) return Op0; @@ -6165,11 +6259,11 @@ return nullptr; } -Value *llvm::SimplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) { - return ::SimplifyFreezeInst(Op0, Q); +Value *llvm::simplifyFreezeInst(Value *Op0, const SimplifyQuery &Q) { + return ::simplifyFreezeInst(Op0, Q); } -static Value *SimplifyLoadInst(LoadInst *LI, Value *PtrOp, +static Value *simplifyLoadInst(LoadInst *LI, Value *PtrOp, const SimplifyQuery &Q) { if (LI->isVolatile()) return nullptr; @@ -6213,134 +6307,134 @@ } break; case Instruction::FNeg: - Result = SimplifyFNegInst(NewOps[0], I->getFastMathFlags(), Q); + Result = simplifyFNegInst(NewOps[0], I->getFastMathFlags(), Q); break; case Instruction::FAdd: - Result = SimplifyFAddInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); + Result = simplifyFAddInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); break; case Instruction::Add: - Result = SimplifyAddInst( + Result = simplifyAddInst( NewOps[0], NewOps[1], Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)), Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q); break; case Instruction::FSub: - Result = SimplifyFSubInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); + Result = simplifyFSubInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); break; case Instruction::Sub: - Result = SimplifySubInst( + Result = simplifySubInst( NewOps[0], NewOps[1], Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)), Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q); break; case Instruction::FMul: - Result = SimplifyFMulInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); + Result = simplifyFMulInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); break; case Instruction::Mul: - Result = SimplifyMulInst(NewOps[0], NewOps[1], Q); + Result = simplifyMulInst(NewOps[0], NewOps[1], Q); break; case Instruction::SDiv: - Result = SimplifySDivInst(NewOps[0], NewOps[1], Q); + Result = simplifySDivInst(NewOps[0], NewOps[1], Q); break; case Instruction::UDiv: - Result = SimplifyUDivInst(NewOps[0], NewOps[1], Q); + Result = simplifyUDivInst(NewOps[0], NewOps[1], Q); break; case Instruction::FDiv: - Result = SimplifyFDivInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); + Result = simplifyFDivInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); break; case Instruction::SRem: - Result = SimplifySRemInst(NewOps[0], NewOps[1], Q); + Result = simplifySRemInst(NewOps[0], NewOps[1], Q); break; case Instruction::URem: - Result = SimplifyURemInst(NewOps[0], NewOps[1], Q); + Result = simplifyURemInst(NewOps[0], NewOps[1], Q); break; case Instruction::FRem: - Result = SimplifyFRemInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); + Result = simplifyFRemInst(NewOps[0], NewOps[1], I->getFastMathFlags(), Q); break; case Instruction::Shl: - Result = SimplifyShlInst( + Result = simplifyShlInst( NewOps[0], NewOps[1], Q.IIQ.hasNoSignedWrap(cast<BinaryOperator>(I)), Q.IIQ.hasNoUnsignedWrap(cast<BinaryOperator>(I)), Q); break; case Instruction::LShr: - Result = SimplifyLShrInst(NewOps[0], NewOps[1], + Result = simplifyLShrInst(NewOps[0], NewOps[1], Q.IIQ.isExact(cast<BinaryOperator>(I)), Q); break; case Instruction::AShr: - Result = SimplifyAShrInst(NewOps[0], NewOps[1], + Result = simplifyAShrInst(NewOps[0], NewOps[1], Q.IIQ.isExact(cast<BinaryOperator>(I)), Q); break; case Instruction::And: - Result = SimplifyAndInst(NewOps[0], NewOps[1], Q); + Result = simplifyAndInst(NewOps[0], NewOps[1], Q); break; case Instruction::Or: - Result = SimplifyOrInst(NewOps[0], NewOps[1], Q); + Result = simplifyOrInst(NewOps[0], NewOps[1], Q); break; case Instruction::Xor: - Result = SimplifyXorInst(NewOps[0], NewOps[1], Q); + Result = simplifyXorInst(NewOps[0], NewOps[1], Q); break; case Instruction::ICmp: - Result = SimplifyICmpInst(cast<ICmpInst>(I)->getPredicate(), NewOps[0], + Result = simplifyICmpInst(cast<ICmpInst>(I)->getPredicate(), NewOps[0], NewOps[1], Q); break; case Instruction::FCmp: - Result = SimplifyFCmpInst(cast<FCmpInst>(I)->getPredicate(), NewOps[0], + Result = simplifyFCmpInst(cast<FCmpInst>(I)->getPredicate(), NewOps[0], NewOps[1], I->getFastMathFlags(), Q); break; case Instruction::Select: - Result = SimplifySelectInst(NewOps[0], NewOps[1], NewOps[2], Q); + Result = simplifySelectInst(NewOps[0], NewOps[1], NewOps[2], Q); break; case Instruction::GetElementPtr: { auto *GEPI = cast<GetElementPtrInst>(I); Result = - SimplifyGEPInst(GEPI->getSourceElementType(), NewOps[0], + simplifyGEPInst(GEPI->getSourceElementType(), NewOps[0], makeArrayRef(NewOps).slice(1), GEPI->isInBounds(), Q); break; } case Instruction::InsertValue: { InsertValueInst *IV = cast<InsertValueInst>(I); - Result = SimplifyInsertValueInst(NewOps[0], NewOps[1], IV->getIndices(), Q); + Result = simplifyInsertValueInst(NewOps[0], NewOps[1], IV->getIndices(), Q); break; } case Instruction::InsertElement: { - Result = SimplifyInsertElementInst(NewOps[0], NewOps[1], NewOps[2], Q); + Result = simplifyInsertElementInst(NewOps[0], NewOps[1], NewOps[2], Q); break; } case Instruction::ExtractValue: { auto *EVI = cast<ExtractValueInst>(I); - Result = SimplifyExtractValueInst(NewOps[0], EVI->getIndices(), Q); + Result = simplifyExtractValueInst(NewOps[0], EVI->getIndices(), Q); break; } case Instruction::ExtractElement: { - Result = SimplifyExtractElementInst(NewOps[0], NewOps[1], Q); + Result = simplifyExtractElementInst(NewOps[0], NewOps[1], Q); break; } case Instruction::ShuffleVector: { auto *SVI = cast<ShuffleVectorInst>(I); - Result = SimplifyShuffleVectorInst( + Result = simplifyShuffleVectorInst( NewOps[0], NewOps[1], SVI->getShuffleMask(), SVI->getType(), Q); break; } case Instruction::PHI: - Result = SimplifyPHINode(cast<PHINode>(I), NewOps, Q); + Result = simplifyPHINode(cast<PHINode>(I), NewOps, Q); break; case Instruction::Call: { // TODO: Use NewOps - Result = SimplifyCall(cast<CallInst>(I), Q); + Result = simplifyCall(cast<CallInst>(I), Q); break; } case Instruction::Freeze: - Result = llvm::SimplifyFreezeInst(NewOps[0], Q); + Result = llvm::simplifyFreezeInst(NewOps[0], Q); break; #define HANDLE_CAST_INST(num, opc, clas) case Instruction::opc: #include "llvm/IR/Instruction.def" #undef HANDLE_CAST_INST - Result = SimplifyCastInst(I->getOpcode(), NewOps[0], I->getType(), Q); + Result = simplifyCastInst(I->getOpcode(), NewOps[0], I->getType(), Q); break; case Instruction::Alloca: // No simplifications for Alloca and it can't be constant folded. Result = nullptr; break; case Instruction::Load: - Result = SimplifyLoadInst(cast<LoadInst>(I), NewOps[0], Q); + Result = simplifyLoadInst(cast<LoadInst>(I), NewOps[0], Q); break; } @@ -6350,7 +6444,7 @@ return Result == I ? UndefValue::get(I->getType()) : Result; } -Value *llvm::SimplifyInstructionWithOperands(Instruction *I, +Value *llvm::simplifyInstructionWithOperands(Instruction *I, ArrayRef<Value *> NewOps, const SimplifyQuery &SQ, OptimizationRemarkEmitter *ORE) { @@ -6359,7 +6453,7 @@ return ::simplifyInstructionWithOperands(I, NewOps, SQ, ORE); } -Value *llvm::SimplifyInstruction(Instruction *I, const SimplifyQuery &SQ, +Value *llvm::simplifyInstruction(Instruction *I, const SimplifyQuery &SQ, OptimizationRemarkEmitter *ORE) { SmallVector<Value *, 8> Ops(I->operands()); return ::simplifyInstructionWithOperands(I, Ops, SQ, ORE); @@ -6410,7 +6504,7 @@ I = Worklist[Idx]; // See if this instruction simplifies. - SimpleV = SimplifyInstruction(I, {DL, TLI, DT, AC}); + SimpleV = simplifyInstruction(I, {DL, TLI, DT, AC}); if (!SimpleV) { if (UnsimplifiedUsers) UnsimplifiedUsers->insert(I); @@ -6473,6 +6567,6 @@ } template const SimplifyQuery getBestSimplifyQuery(AnalysisManager<Function> &, Function &); -} +} // namespace llvm void InstSimplifyFolder::anchor() {}
diff --git a/src/llvm-project/llvm/lib/Analysis/Interval.cpp b/src/llvm-project/llvm/lib/Analysis/Interval.cpp index e228ec4..f7fffcb 100644 --- a/src/llvm-project/llvm/lib/Analysis/Interval.cpp +++ b/src/llvm-project/llvm/lib/Analysis/Interval.cpp
@@ -13,7 +13,6 @@ #include "llvm/Analysis/Interval.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CFG.h" #include "llvm/Support/raw_ostream.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/LazyCallGraph.cpp b/src/llvm-project/llvm/lib/Analysis/LazyCallGraph.cpp index e8e9593..20a905e 100644 --- a/src/llvm-project/llvm/lib/Analysis/LazyCallGraph.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LazyCallGraph.cpp
@@ -9,14 +9,13 @@ #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Sequence.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/VectorUtils.h" #include "llvm/Config/llvm-config.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/InstIterator.h" @@ -30,12 +29,15 @@ #include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> -#include <cstddef> #include <iterator> #include <string> #include <tuple> #include <utility> +#ifdef EXPENSIVE_CHECKS +#include "llvm/ADT/ScopeExit.h" +#endif + using namespace llvm; #define DEBUG_TYPE "lcg"
diff --git a/src/llvm-project/llvm/lib/Analysis/LazyValueInfo.cpp b/src/llvm-project/llvm/lib/Analysis/LazyValueInfo.cpp index e311b40..2fae260 100644 --- a/src/llvm-project/llvm/lib/Analysis/LazyValueInfo.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -38,7 +38,6 @@ #include "llvm/Support/FormattedStream.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/raw_ostream.h" -#include <map> using namespace llvm; using namespace PatternMatch; @@ -919,10 +918,10 @@ // transfer rule on the full set since we may be able to locally infer // interesting facts. Optional<ConstantRange> LHSRes = getRangeFor(CI->getOperand(0), CI, BB); - if (!LHSRes.hasValue()) + if (!LHSRes) // More work to do before applying this transfer rule. return None; - const ConstantRange &LHSRange = LHSRes.getValue(); + const ConstantRange &LHSRange = LHSRes.value(); const unsigned ResultBitWidth = CI->getType()->getIntegerBitWidth(); @@ -943,12 +942,12 @@ // @foo()), 32" Optional<ConstantRange> LHSRes = getRangeFor(I->getOperand(0), I, BB); Optional<ConstantRange> RHSRes = getRangeFor(I->getOperand(1), I, BB); - if (!LHSRes.hasValue() || !RHSRes.hasValue()) + if (!LHSRes || !RHSRes) // More work to do before applying this transfer rule. return None; - const ConstantRange &LHSRange = LHSRes.getValue(); - const ConstantRange &RHSRange = RHSRes.getValue(); + const ConstantRange &LHSRange = LHSRes.value(); + const ConstantRange &RHSRange = RHSRes.value(); return ValueLatticeElement::getRange(OpFn(LHSRange, RHSRange)); } @@ -956,13 +955,6 @@ BinaryOperator *BO, BasicBlock *BB) { assert(BO->getOperand(0)->getType()->isSized() && "all operands to binary operators are sized"); - if (BO->getOpcode() == Instruction::Xor) { - // Xor is the only operation not supported by ConstantRange::binaryOp(). - LLVM_DEBUG(dbgs() << " compute BB '" << BB->getName() - << "' - overdefined (unknown binary operator).\n"); - return ValueLatticeElement::getOverdefined(); - } - if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(BO)) { unsigned NoWrapKind = 0; if (OBO->hasNoUnsignedWrap()) @@ -1020,7 +1012,7 @@ // Handle extractvalue of insertvalue to allow further simplification // based on replaced with.overflow intrinsics. - if (Value *V = SimplifyExtractValueInst( + if (Value *V = simplifyExtractValueInst( EVI->getAggregateOperand(), EVI->getIndices(), EVI->getModule()->getDataLayout())) return getBlockValue(V, BB, EVI); @@ -1141,7 +1133,7 @@ ConstantRange CR = ConstantRange::makeExactICmpRegion(EdgePred, *C); if (!CR.isEmptySet()) return ValueLatticeElement::getRange(ConstantRange::getNonEmpty( - CR.getUnsignedMin().zextOrSelf(BitWidth), APInt(BitWidth, 0))); + CR.getUnsignedMin().zext(BitWidth), APInt(BitWidth, 0))); } return ValueLatticeElement::getOverdefined(); @@ -1278,7 +1270,7 @@ if (auto *CI = dyn_cast<CastInst>(Usr)) { assert(CI->getOperand(0) == Op && "Operand 0 isn't Op"); if (auto *C = dyn_cast_or_null<ConstantInt>( - SimplifyCastInst(CI->getOpcode(), OpConst, + simplifyCastInst(CI->getOpcode(), OpConst, CI->getDestTy(), DL))) { return ValueLatticeElement::getRange(ConstantRange(C->getValue())); } @@ -1290,7 +1282,7 @@ Value *LHS = Op0Match ? OpConst : BO->getOperand(0); Value *RHS = Op1Match ? OpConst : BO->getOperand(1); if (auto *C = dyn_cast_or_null<ConstantInt>( - SimplifyBinOp(BO->getOpcode(), LHS, RHS, DL))) { + simplifyBinOp(BO->getOpcode(), LHS, RHS, DL))) { return ValueLatticeElement::getRange(ConstantRange(C->getValue())); } } else if (isa<FreezeInst>(Usr)) { @@ -1361,7 +1353,7 @@ ValueLatticeElement OpLatticeVal = getValueFromCondition(Op, Condition, isTrueDest); if (Optional<APInt> OpConst = OpLatticeVal.asConstantInteger()) { - Result = constantFoldUser(Usr, Op, OpConst.getValue(), DL); + Result = constantFoldUser(Usr, Op, *OpConst, DL); break; } } @@ -1432,8 +1424,9 @@ if (Constant *VC = dyn_cast<Constant>(Val)) return ValueLatticeElement::get(VC); - ValueLatticeElement LocalResult = getEdgeValueLocal(Val, BBFrom, BBTo) - .getValueOr(ValueLatticeElement::getOverdefined()); + ValueLatticeElement LocalResult = + getEdgeValueLocal(Val, BBFrom, BBTo) + .value_or(ValueLatticeElement::getOverdefined()); if (hasSingleValue(LocalResult)) // Can't get any more precise here return LocalResult; @@ -1886,6 +1879,11 @@ } } +void LazyValueInfo::clear(const Module *M) { + if (PImpl) { + getImpl(PImpl, AC, M).clear(); + } +} void LazyValueInfo::printLVI(Function &F, DominatorTree &DTree, raw_ostream &OS) { if (PImpl) { @@ -1898,7 +1896,7 @@ const BasicBlock *BB, formatted_raw_ostream &OS) { // Find if there are latticevalues defined for arguments of the function. auto *F = BB->getParent(); - for (auto &Arg : F->args()) { + for (const auto &Arg : F->args()) { ValueLatticeElement Result = LVIImpl->getValueInBlock( const_cast<Argument *>(&Arg), const_cast<BasicBlock *>(BB)); if (Result.isUnknown()) @@ -1934,12 +1932,12 @@ printResult(ParentBB); // Print the LVI analysis results for the immediate successor blocks, that // are dominated by `ParentBB`. - for (auto *BBSucc : successors(ParentBB)) + for (const auto *BBSucc : successors(ParentBB)) if (DT.dominates(ParentBB, BBSucc)) printResult(BBSucc); // Print LVI in blocks where `I` is used. - for (auto *U : I->users()) + for (const auto *U : I->users()) if (auto *UseI = dyn_cast<Instruction>(U)) if (!isa<PHINode>(UseI) || DT.dominates(ParentBB, UseI->getParent())) printResult(UseI->getParent());
diff --git a/src/llvm-project/llvm/lib/Analysis/LegacyDivergenceAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/LegacyDivergenceAnalysis.cpp index 031bf3b..381d62a 100644 --- a/src/llvm-project/llvm/lib/Analysis/LegacyDivergenceAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LegacyDivergenceAnalysis.cpp
@@ -68,6 +68,7 @@ #include "llvm/ADT/PostOrderIterator.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/DivergenceAnalysis.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/Passes.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -392,14 +393,14 @@ return; // Dumps all divergent values in F, arguments and then instructions. - for (auto &Arg : F->args()) { + for (const auto &Arg : F->args()) { OS << (isDivergent(&Arg) ? "DIVERGENT: " : " "); OS << Arg << "\n"; } // Iterate instructions using instructions() to ensure a deterministic order. for (const BasicBlock &BB : *F) { OS << "\n " << BB.getName() << ":\n"; - for (auto &I : BB.instructionsWithoutDebug()) { + for (const auto &I : BB.instructionsWithoutDebug()) { OS << (isDivergent(&I) ? "DIVERGENT: " : " "); OS << I << "\n"; }
diff --git a/src/llvm-project/llvm/lib/Analysis/Lint.cpp b/src/llvm-project/llvm/lib/Analysis/Lint.cpp index f9a7a5b..8b0f2a8 100644 --- a/src/llvm-project/llvm/lib/Analysis/Lint.cpp +++ b/src/llvm-project/llvm/lib/Analysis/Lint.cpp
@@ -44,7 +44,6 @@ #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/MemoryLocation.h" -#include "llvm/Analysis/Passes.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Argument.h" @@ -69,9 +68,7 @@ #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/KnownBits.h" -#include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include <cassert> #include <cstdint> @@ -169,8 +166,8 @@ }; } // end anonymous namespace -// Assert - We know that cond should be true, if not print an error message. -#define Assert(C, ...) \ +// Check - We know that cond should be true, if not print an error message. +#define Check(C, ...) \ do { \ if (!(C)) { \ CheckFailed(__VA_ARGS__); \ @@ -181,8 +178,8 @@ void Lint::visitFunction(Function &F) { // This isn't undefined behavior, it's just a little unusual, and it's a // fairly common mistake to neglect to name a function. - Assert(F.hasName() || F.hasLocalLinkage(), - "Unusual: Unnamed function with non-local linkage", &F); + Check(F.hasName() || F.hasLocalLinkage(), + "Unusual: Unnamed function with non-local linkage", &F); // TODO: Check for irreducible control flow. } @@ -195,23 +192,23 @@ if (Function *F = dyn_cast<Function>(findValue(Callee, /*OffsetOk=*/false))) { - Assert(I.getCallingConv() == F->getCallingConv(), - "Undefined behavior: Caller and callee calling convention differ", - &I); + Check(I.getCallingConv() == F->getCallingConv(), + "Undefined behavior: Caller and callee calling convention differ", + &I); FunctionType *FT = F->getFunctionType(); unsigned NumActualArgs = I.arg_size(); - Assert(FT->isVarArg() ? FT->getNumParams() <= NumActualArgs - : FT->getNumParams() == NumActualArgs, - "Undefined behavior: Call argument count mismatches callee " - "argument count", - &I); + Check(FT->isVarArg() ? FT->getNumParams() <= NumActualArgs + : FT->getNumParams() == NumActualArgs, + "Undefined behavior: Call argument count mismatches callee " + "argument count", + &I); - Assert(FT->getReturnType() == I.getType(), - "Undefined behavior: Call return type mismatches " - "callee return type", - &I); + Check(FT->getReturnType() == I.getType(), + "Undefined behavior: Call return type mismatches " + "callee return type", + &I); // Check argument types (in case the callee was casted) and attributes. // TODO: Verify that caller and callee attributes are compatible. @@ -221,10 +218,10 @@ Value *Actual = *AI; if (PI != PE) { Argument *Formal = &*PI++; - Assert(Formal->getType() == Actual->getType(), - "Undefined behavior: Call argument type mismatches " - "callee parameter type", - &I); + Check(Formal->getType() == Actual->getType(), + "Undefined behavior: Call argument type mismatches " + "callee parameter type", + &I); // Check that noalias arguments don't alias other arguments. This is // not fully precise because we don't know the sizes of the dereferenced @@ -232,7 +229,7 @@ if (Formal->hasNoAliasAttr() && Actual->getType()->isPointerTy()) { AttributeList PAL = I.getAttributes(); unsigned ArgNo = 0; - for (auto BI = I.arg_begin(); BI != AE; ++BI, ++ArgNo) { + for (auto *BI = I.arg_begin(); BI != AE; ++BI, ++ArgNo) { // Skip ByVal arguments since they will be memcpy'd to the callee's // stack so we're not really passing the pointer anyway. if (PAL.hasParamAttr(ArgNo, Attribute::ByVal)) @@ -242,9 +239,9 @@ continue; if (AI != BI && (*BI)->getType()->isPointerTy()) { AliasResult Result = AA->alias(*AI, *BI); - Assert(Result != AliasResult::MustAlias && - Result != AliasResult::PartialAlias, - "Unusual: noalias argument aliases another argument", &I); + Check(Result != AliasResult::MustAlias && + Result != AliasResult::PartialAlias, + "Unusual: noalias argument aliases another argument", &I); } } } @@ -271,10 +268,10 @@ if (PAL.hasParamAttr(ArgNo++, Attribute::ByVal)) continue; Value *Obj = findValue(Arg, /*OffsetOk=*/true); - Assert(!isa<AllocaInst>(Obj), - "Undefined behavior: Call with \"tail\" keyword references " - "alloca", - &I); + Check(!isa<AllocaInst>(Obj), + "Undefined behavior: Call with \"tail\" keyword references " + "alloca", + &I); } } } @@ -302,9 +299,9 @@ /*OffsetOk=*/false))) if (Len->getValue().isIntN(32)) Size = LocationSize::precise(Len->getValue().getZExtValue()); - Assert(AA->alias(MCI->getSource(), Size, MCI->getDest(), Size) != - AliasResult::MustAlias, - "Undefined behavior: memcpy source and destination overlap", &I); + Check(AA->alias(MCI->getSource(), Size, MCI->getDest(), Size) != + AliasResult::MustAlias, + "Undefined behavior: memcpy source and destination overlap", &I); break; } case Intrinsic::memcpy_inline: { @@ -319,9 +316,9 @@ // isn't expressive enough for what we really want to do. Known partial // overlap is not distinguished from the case where nothing is known. const LocationSize LS = LocationSize::precise(Size); - Assert(AA->alias(MCII->getSource(), LS, MCII->getDest(), LS) != - AliasResult::MustAlias, - "Undefined behavior: memcpy source and destination overlap", &I); + Check(AA->alias(MCII->getSource(), LS, MCII->getDest(), LS) != + AliasResult::MustAlias, + "Undefined behavior: memcpy source and destination overlap", &I); break; } case Intrinsic::memmove: { @@ -338,11 +335,17 @@ MSI->getDestAlign(), nullptr, MemRef::Write); break; } + case Intrinsic::memset_inline: { + MemSetInlineInst *MSII = cast<MemSetInlineInst>(&I); + visitMemoryReference(I, MemoryLocation::getForDest(MSII), + MSII->getDestAlign(), nullptr, MemRef::Write); + break; + } case Intrinsic::vastart: - Assert(I.getParent()->getParent()->isVarArg(), - "Undefined behavior: va_start called in a non-varargs function", - &I); + Check(I.getParent()->getParent()->isVarArg(), + "Undefined behavior: va_start called in a non-varargs function", + &I); visitMemoryReference(I, MemoryLocation::getForArgument(&I, 0, TLI), None, nullptr, MemRef::Read | MemRef::Write); @@ -367,20 +370,22 @@ break; case Intrinsic::get_active_lane_mask: if (auto *TripCount = dyn_cast<ConstantInt>(I.getArgOperand(1))) - Assert(!TripCount->isZero(), "get_active_lane_mask: operand #2 " - "must be greater than 0", &I); + Check(!TripCount->isZero(), + "get_active_lane_mask: operand #2 " + "must be greater than 0", + &I); break; } } void Lint::visitReturnInst(ReturnInst &I) { Function *F = I.getParent()->getParent(); - Assert(!F->doesNotReturn(), - "Unusual: Return statement in function with noreturn attribute", &I); + Check(!F->doesNotReturn(), + "Unusual: Return statement in function with noreturn attribute", &I); if (Value *V = I.getReturnValue()) { Value *Obj = findValue(V, /*OffsetOk=*/true); - Assert(!isa<AllocaInst>(Obj), "Unusual: Returning alloca value", &I); + Check(!isa<AllocaInst>(Obj), "Unusual: Returning alloca value", &I); } } @@ -395,39 +400,39 @@ Value *Ptr = const_cast<Value *>(Loc.Ptr); Value *UnderlyingObject = findValue(Ptr, /*OffsetOk=*/true); - Assert(!isa<ConstantPointerNull>(UnderlyingObject), - "Undefined behavior: Null pointer dereference", &I); - Assert(!isa<UndefValue>(UnderlyingObject), - "Undefined behavior: Undef pointer dereference", &I); - Assert(!isa<ConstantInt>(UnderlyingObject) || - !cast<ConstantInt>(UnderlyingObject)->isMinusOne(), - "Unusual: All-ones pointer dereference", &I); - Assert(!isa<ConstantInt>(UnderlyingObject) || - !cast<ConstantInt>(UnderlyingObject)->isOne(), - "Unusual: Address one pointer dereference", &I); + Check(!isa<ConstantPointerNull>(UnderlyingObject), + "Undefined behavior: Null pointer dereference", &I); + Check(!isa<UndefValue>(UnderlyingObject), + "Undefined behavior: Undef pointer dereference", &I); + Check(!isa<ConstantInt>(UnderlyingObject) || + !cast<ConstantInt>(UnderlyingObject)->isMinusOne(), + "Unusual: All-ones pointer dereference", &I); + Check(!isa<ConstantInt>(UnderlyingObject) || + !cast<ConstantInt>(UnderlyingObject)->isOne(), + "Unusual: Address one pointer dereference", &I); if (Flags & MemRef::Write) { if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(UnderlyingObject)) - Assert(!GV->isConstant(), "Undefined behavior: Write to read-only memory", - &I); - Assert(!isa<Function>(UnderlyingObject) && - !isa<BlockAddress>(UnderlyingObject), - "Undefined behavior: Write to text section", &I); + Check(!GV->isConstant(), "Undefined behavior: Write to read-only memory", + &I); + Check(!isa<Function>(UnderlyingObject) && + !isa<BlockAddress>(UnderlyingObject), + "Undefined behavior: Write to text section", &I); } if (Flags & MemRef::Read) { - Assert(!isa<Function>(UnderlyingObject), "Unusual: Load from function body", - &I); - Assert(!isa<BlockAddress>(UnderlyingObject), - "Undefined behavior: Load from block address", &I); + Check(!isa<Function>(UnderlyingObject), "Unusual: Load from function body", + &I); + Check(!isa<BlockAddress>(UnderlyingObject), + "Undefined behavior: Load from block address", &I); } if (Flags & MemRef::Callee) { - Assert(!isa<BlockAddress>(UnderlyingObject), - "Undefined behavior: Call to block address", &I); + Check(!isa<BlockAddress>(UnderlyingObject), + "Undefined behavior: Call to block address", &I); } if (Flags & MemRef::Branchee) { - Assert(!isa<Constant>(UnderlyingObject) || - isa<BlockAddress>(UnderlyingObject), - "Undefined behavior: Branch to non-blockaddress", &I); + Check(!isa<Constant>(UnderlyingObject) || + isa<BlockAddress>(UnderlyingObject), + "Undefined behavior: Branch to non-blockaddress", &I); } // Check for buffer overflows and misalignment. @@ -461,17 +466,17 @@ // Accesses from before the start or after the end of the object are not // defined. - Assert(!Loc.Size.hasValue() || BaseSize == MemoryLocation::UnknownSize || - (Offset >= 0 && Offset + Loc.Size.getValue() <= BaseSize), - "Undefined behavior: Buffer overflow", &I); + Check(!Loc.Size.hasValue() || BaseSize == MemoryLocation::UnknownSize || + (Offset >= 0 && Offset + Loc.Size.getValue() <= BaseSize), + "Undefined behavior: Buffer overflow", &I); // Accesses that say that the memory is more aligned than it is are not // defined. if (!Align && Ty && Ty->isSized()) Align = DL->getABITypeAlign(Ty); if (BaseAlign && Align) - Assert(*Align <= commonAlignment(*BaseAlign, Offset), - "Undefined behavior: Memory reference address is misaligned", &I); + Check(*Align <= commonAlignment(*BaseAlign, Offset), + "Undefined behavior: Memory reference address is misaligned", &I); } } @@ -486,34 +491,34 @@ } void Lint::visitXor(BinaryOperator &I) { - Assert(!isa<UndefValue>(I.getOperand(0)) || !isa<UndefValue>(I.getOperand(1)), - "Undefined result: xor(undef, undef)", &I); + Check(!isa<UndefValue>(I.getOperand(0)) || !isa<UndefValue>(I.getOperand(1)), + "Undefined result: xor(undef, undef)", &I); } void Lint::visitSub(BinaryOperator &I) { - Assert(!isa<UndefValue>(I.getOperand(0)) || !isa<UndefValue>(I.getOperand(1)), - "Undefined result: sub(undef, undef)", &I); + Check(!isa<UndefValue>(I.getOperand(0)) || !isa<UndefValue>(I.getOperand(1)), + "Undefined result: sub(undef, undef)", &I); } void Lint::visitLShr(BinaryOperator &I) { if (ConstantInt *CI = dyn_cast<ConstantInt>(findValue(I.getOperand(1), /*OffsetOk=*/false))) - Assert(CI->getValue().ult(cast<IntegerType>(I.getType())->getBitWidth()), - "Undefined result: Shift count out of range", &I); + Check(CI->getValue().ult(cast<IntegerType>(I.getType())->getBitWidth()), + "Undefined result: Shift count out of range", &I); } void Lint::visitAShr(BinaryOperator &I) { if (ConstantInt *CI = dyn_cast<ConstantInt>(findValue(I.getOperand(1), /*OffsetOk=*/false))) - Assert(CI->getValue().ult(cast<IntegerType>(I.getType())->getBitWidth()), - "Undefined result: Shift count out of range", &I); + Check(CI->getValue().ult(cast<IntegerType>(I.getType())->getBitWidth()), + "Undefined result: Shift count out of range", &I); } void Lint::visitShl(BinaryOperator &I) { if (ConstantInt *CI = dyn_cast<ConstantInt>(findValue(I.getOperand(1), /*OffsetOk=*/false))) - Assert(CI->getValue().ult(cast<IntegerType>(I.getType())->getBitWidth()), - "Undefined result: Shift count out of range", &I); + Check(CI->getValue().ult(cast<IntegerType>(I.getType())->getBitWidth()), + "Undefined result: Shift count out of range", &I); } static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, @@ -554,30 +559,30 @@ } void Lint::visitSDiv(BinaryOperator &I) { - Assert(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), - "Undefined behavior: Division by zero", &I); + Check(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), + "Undefined behavior: Division by zero", &I); } void Lint::visitUDiv(BinaryOperator &I) { - Assert(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), - "Undefined behavior: Division by zero", &I); + Check(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), + "Undefined behavior: Division by zero", &I); } void Lint::visitSRem(BinaryOperator &I) { - Assert(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), - "Undefined behavior: Division by zero", &I); + Check(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), + "Undefined behavior: Division by zero", &I); } void Lint::visitURem(BinaryOperator &I) { - Assert(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), - "Undefined behavior: Division by zero", &I); + Check(!isZero(I.getOperand(1), I.getModule()->getDataLayout(), DT, AC), + "Undefined behavior: Division by zero", &I); } void Lint::visitAllocaInst(AllocaInst &I) { if (isa<ConstantInt>(I.getArraySize())) // This isn't undefined behavior, it's just an obvious pessimization. - Assert(&I.getParent()->getParent()->getEntryBlock() == I.getParent(), - "Pessimization: Static alloca outside of entry block", &I); + Check(&I.getParent()->getParent()->getEntryBlock() == I.getParent(), + "Pessimization: Static alloca outside of entry block", &I); // TODO: Check for an unusual size (MSB set?) } @@ -591,14 +596,14 @@ visitMemoryReference(I, MemoryLocation::getAfter(I.getAddress()), None, nullptr, MemRef::Branchee); - Assert(I.getNumDestinations() != 0, - "Undefined behavior: indirectbr with no destinations", &I); + Check(I.getNumDestinations() != 0, + "Undefined behavior: indirectbr with no destinations", &I); } void Lint::visitExtractElementInst(ExtractElementInst &I) { if (ConstantInt *CI = dyn_cast<ConstantInt>(findValue(I.getIndexOperand(), /*OffsetOk=*/false))) - Assert( + Check( CI->getValue().ult( cast<FixedVectorType>(I.getVectorOperandType())->getNumElements()), "Undefined result: extractelement index out of range", &I); @@ -607,18 +612,18 @@ void Lint::visitInsertElementInst(InsertElementInst &I) { if (ConstantInt *CI = dyn_cast<ConstantInt>(findValue(I.getOperand(2), /*OffsetOk=*/false))) - Assert(CI->getValue().ult( - cast<FixedVectorType>(I.getType())->getNumElements()), - "Undefined result: insertelement index out of range", &I); + Check(CI->getValue().ult( + cast<FixedVectorType>(I.getType())->getNumElements()), + "Undefined result: insertelement index out of range", &I); } void Lint::visitUnreachableInst(UnreachableInst &I) { // This isn't undefined behavior, it's merely suspicious. - Assert(&I == &I.getParent()->front() || - std::prev(I.getIterator())->mayHaveSideEffects(), - "Unusual: unreachable immediately preceded by instruction without " - "side effects", - &I); + Check(&I == &I.getParent()->front() || + std::prev(I.getIterator())->mayHaveSideEffects(), + "Unusual: unreachable immediately preceded by instruction without " + "side effects", + &I); } /// findValue - Look through bitcasts and simple memory reference patterns @@ -681,17 +686,12 @@ CE->getOperand(0)->getType(), CE->getType(), *DL)) return findValueImpl(CE->getOperand(0), OffsetOk, Visited); - } else if (CE->getOpcode() == Instruction::ExtractValue) { - ArrayRef<unsigned> Indices = CE->getIndices(); - if (Value *W = FindInsertedValue(CE->getOperand(0), Indices)) - if (W != V) - return findValueImpl(W, OffsetOk, Visited); } } // As a last resort, try SimplifyInstruction or constant folding. if (Instruction *Inst = dyn_cast<Instruction>(V)) { - if (Value *W = SimplifyInstruction(Inst, {*DL, TLI, DT, AC})) + if (Value *W = simplifyInstruction(Inst, {*DL, TLI, DT, AC})) return findValueImpl(W, OffsetOk, Visited); } else if (auto *C = dyn_cast<Constant>(V)) { Value *W = ConstantFoldConstant(C, *DL, TLI);
diff --git a/src/llvm-project/llvm/lib/Analysis/Loads.cpp b/src/llvm-project/llvm/lib/Analysis/Loads.cpp index cd0d4d6..938d950 100644 --- a/src/llvm-project/llvm/lib/Analysis/Loads.cpp +++ b/src/llvm-project/llvm/lib/Analysis/Loads.cpp
@@ -13,19 +13,14 @@ #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AssumeBundleQueries.h" -#include "llvm/Analysis/CaptureTracking.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" -#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/GlobalAlias.h" -#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" @@ -410,7 +405,10 @@ Instruction *ScanFrom, const DominatorTree *DT, const TargetLibraryInfo *TLI) { - APInt Size(DL.getIndexTypeSizeInBits(V->getType()), DL.getTypeStoreSize(Ty)); + TypeSize TySize = DL.getTypeStoreSize(Ty); + if (TySize.isScalable()) + return false; + APInt Size(DL.getIndexTypeSizeInBits(V->getType()), TySize.getFixedValue()); return isSafeToLoadUnconditionally(V, Alignment, Size, DL, ScanFrom, DT, TLI); } @@ -509,8 +507,8 @@ if (CastInst::isBitOrNoopPointerCastable(Val->getType(), AccessTy, DL)) return Val; - TypeSize StoreSize = DL.getTypeStoreSize(Val->getType()); - TypeSize LoadSize = DL.getTypeStoreSize(AccessTy); + TypeSize StoreSize = DL.getTypeSizeInBits(Val->getType()); + TypeSize LoadSize = DL.getTypeSizeInBits(AccessTy); if (TypeSize::isKnownLE(LoadSize, StoreSize)) if (auto *C = dyn_cast<Constant>(Val)) return ConstantFoldLoadFromConst(C, AccessTy, DL);
diff --git a/src/llvm-project/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 2ab78d2..8311b48 100644 --- a/src/llvm-project/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -47,6 +47,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/IR/ValueHandle.h" @@ -60,12 +61,12 @@ #include <algorithm> #include <cassert> #include <cstdint> -#include <cstdlib> #include <iterator> #include <utility> #include <vector> using namespace llvm; +using namespace llvm::PatternMatch; #define DEBUG_TYPE "loop-accesses" @@ -129,6 +130,11 @@ cl::desc("Enable conflict detection in loop-access analysis"), cl::init(true)); +static cl::opt<unsigned> MaxForkedSCEVDepth( + "max-forked-scev-depth", cl::Hidden, + cl::desc("Maximum recursion depth when finding forked SCEVs (default = 5)"), + cl::init(5)); + bool VectorizerParams::isInterleaveForced() { return ::VectorizationInterleave.getNumOccurrences() > 0; } @@ -172,7 +178,8 @@ : High(RtCheck.Pointers[Index].End), Low(RtCheck.Pointers[Index].Start), AddressSpace(RtCheck.Pointers[Index] .PointerValue->getType() - ->getPointerAddressSpace()) { + ->getPointerAddressSpace()), + NeedsFreeze(RtCheck.Pointers[Index].NeedsFreeze) { Members.push_back(Index); } @@ -189,21 +196,20 @@ /// /// There is no conflict when the intervals are disjoint: /// NoConflict = (P2.Start >= P1.End) || (P1.Start >= P2.End) -void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, bool WritePtr, +void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr, + Type *AccessTy, bool WritePtr, unsigned DepSetId, unsigned ASId, - const ValueToValueMap &Strides, - PredicatedScalarEvolution &PSE) { - // Get the stride replaced scev. - const SCEV *Sc = replaceSymbolicStrideSCEV(PSE, Strides, Ptr); + PredicatedScalarEvolution &PSE, + bool NeedsFreeze) { ScalarEvolution *SE = PSE.getSE(); const SCEV *ScStart; const SCEV *ScEnd; - if (SE->isLoopInvariant(Sc, Lp)) { - ScStart = ScEnd = Sc; + if (SE->isLoopInvariant(PtrExpr, Lp)) { + ScStart = ScEnd = PtrExpr; } else { - const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Sc); + const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrExpr); assert(AR && "Invalid addrec expression"); const SCEV *Ex = PSE.getBackedgeTakenCount(); @@ -227,15 +233,103 @@ // Add the size of the pointed element to ScEnd. auto &DL = Lp->getHeader()->getModule()->getDataLayout(); Type *IdxTy = DL.getIndexType(Ptr->getType()); - const SCEV *EltSizeSCEV = - SE->getStoreSizeOfExpr(IdxTy, Ptr->getType()->getPointerElementType()); + const SCEV *EltSizeSCEV = SE->getStoreSizeOfExpr(IdxTy, AccessTy); ScEnd = SE->getAddExpr(ScEnd, EltSizeSCEV); - Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, Sc); + Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, PtrExpr, + NeedsFreeze); } -SmallVector<RuntimePointerCheck, 4> -RuntimePointerChecking::generateChecks() const { +void RuntimePointerChecking::tryToCreateDiffCheck( + const RuntimeCheckingPtrGroup &CGI, const RuntimeCheckingPtrGroup &CGJ) { + if (!CanUseDiffCheck) + return; + + // If either group contains multiple different pointers, bail out. + // TODO: Support multiple pointers by using the minimum or maximum pointer, + // depending on src & sink. + if (CGI.Members.size() != 1 || CGJ.Members.size() != 1) { + CanUseDiffCheck = false; + return; + } + + PointerInfo *Src = &Pointers[CGI.Members[0]]; + PointerInfo *Sink = &Pointers[CGJ.Members[0]]; + + // If either pointer is read and written, multiple checks may be needed. Bail + // out. + if (!DC.getOrderForAccess(Src->PointerValue, !Src->IsWritePtr).empty() || + !DC.getOrderForAccess(Sink->PointerValue, !Sink->IsWritePtr).empty()) { + CanUseDiffCheck = false; + return; + } + + ArrayRef<unsigned> AccSrc = + DC.getOrderForAccess(Src->PointerValue, Src->IsWritePtr); + ArrayRef<unsigned> AccSink = + DC.getOrderForAccess(Sink->PointerValue, Sink->IsWritePtr); + // If either pointer is accessed multiple times, there may not be a clear + // src/sink relation. Bail out for now. + if (AccSrc.size() != 1 || AccSink.size() != 1) { + CanUseDiffCheck = false; + return; + } + // If the sink is accessed before src, swap src/sink. + if (AccSink[0] < AccSrc[0]) + std::swap(Src, Sink); + + auto *SrcAR = dyn_cast<SCEVAddRecExpr>(Src->Expr); + auto *SinkAR = dyn_cast<SCEVAddRecExpr>(Sink->Expr); + if (!SrcAR || !SinkAR || SrcAR->getLoop() != DC.getInnermostLoop() || + SinkAR->getLoop() != DC.getInnermostLoop()) { + CanUseDiffCheck = false; + return; + } + + const DataLayout &DL = + SinkAR->getLoop()->getHeader()->getModule()->getDataLayout(); + SmallVector<Instruction *, 4> SrcInsts = + DC.getInstructionsForAccess(Src->PointerValue, Src->IsWritePtr); + SmallVector<Instruction *, 4> SinkInsts = + DC.getInstructionsForAccess(Sink->PointerValue, Sink->IsWritePtr); + Type *SrcTy = getLoadStoreType(SrcInsts[0]); + Type *DstTy = getLoadStoreType(SinkInsts[0]); + if (isa<ScalableVectorType>(SrcTy) || isa<ScalableVectorType>(DstTy)) { + CanUseDiffCheck = false; + return; + } + unsigned AllocSize = + std::max(DL.getTypeAllocSize(SrcTy), DL.getTypeAllocSize(DstTy)); + IntegerType *IntTy = + IntegerType::get(Src->PointerValue->getContext(), + DL.getPointerSizeInBits(CGI.AddressSpace)); + + // Only matching constant steps matching the AllocSize are supported at the + // moment. This simplifies the difference computation. Can be extended in the + // future. + auto *Step = dyn_cast<SCEVConstant>(SinkAR->getStepRecurrence(*SE)); + if (!Step || Step != SrcAR->getStepRecurrence(*SE) || + Step->getAPInt().abs() != AllocSize) { + CanUseDiffCheck = false; + return; + } + + // When counting down, the dependence distance needs to be swapped. + if (Step->getValue()->isNegative()) + std::swap(SinkAR, SrcAR); + + const SCEV *SinkStartInt = SE->getPtrToIntExpr(SinkAR->getStart(), IntTy); + const SCEV *SrcStartInt = SE->getPtrToIntExpr(SrcAR->getStart(), IntTy); + if (isa<SCEVCouldNotCompute>(SinkStartInt) || + isa<SCEVCouldNotCompute>(SrcStartInt)) { + CanUseDiffCheck = false; + return; + } + DiffChecks.emplace_back(SrcStartInt, SinkStartInt, AllocSize, + Src->NeedsFreeze || Sink->NeedsFreeze); +} + +SmallVector<RuntimePointerCheck, 4> RuntimePointerChecking::generateChecks() { SmallVector<RuntimePointerCheck, 4> Checks; for (unsigned I = 0; I < CheckingGroups.size(); ++I) { @@ -243,8 +337,10 @@ const RuntimeCheckingPtrGroup &CGI = CheckingGroups[I]; const RuntimeCheckingPtrGroup &CGJ = CheckingGroups[J]; - if (needsChecking(CGI, CGJ)) + if (needsChecking(CGI, CGJ)) { + tryToCreateDiffCheck(CGI, CGJ); Checks.push_back(std::make_pair(&CGI, &CGJ)); + } } } return Checks; @@ -285,11 +381,12 @@ return addPointer( Index, RtCheck.Pointers[Index].Start, RtCheck.Pointers[Index].End, RtCheck.Pointers[Index].PointerValue->getType()->getPointerAddressSpace(), - *RtCheck.SE); + RtCheck.Pointers[Index].NeedsFreeze, *RtCheck.SE); } bool RuntimeCheckingPtrGroup::addPointer(unsigned Index, const SCEV *Start, const SCEV *End, unsigned AS, + bool NeedsFreeze, ScalarEvolution &SE) { assert(AddressSpace == AS && "all pointers in a checking group must be in the same address space"); @@ -314,6 +411,7 @@ High = End; Members.push_back(Index); + this->NeedsFreeze |= NeedsFreeze; return true; } @@ -371,9 +469,11 @@ unsigned TotalComparisons = 0; - DenseMap<Value *, unsigned> PositionMap; - for (unsigned Index = 0; Index < Pointers.size(); ++Index) - PositionMap[Pointers[Index].PointerValue] = Index; + DenseMap<Value *, SmallVector<unsigned>> PositionMap; + for (unsigned Index = 0; Index < Pointers.size(); ++Index) { + auto Iter = PositionMap.insert({Pointers[Index].PointerValue, {}}); + Iter.first->second.push_back(Index); + } // We need to keep track of what pointers we've already seen so we // don't process them twice. @@ -404,34 +504,35 @@ auto PointerI = PositionMap.find(MI->getPointer()); assert(PointerI != PositionMap.end() && "pointer in equivalence class not found in PositionMap"); - unsigned Pointer = PointerI->second; - bool Merged = false; - // Mark this pointer as seen. - Seen.insert(Pointer); + for (unsigned Pointer : PointerI->second) { + bool Merged = false; + // Mark this pointer as seen. + Seen.insert(Pointer); - // Go through all the existing sets and see if we can find one - // which can include this pointer. - for (RuntimeCheckingPtrGroup &Group : Groups) { - // Don't perform more than a certain amount of comparisons. - // This should limit the cost of grouping the pointers to something - // reasonable. If we do end up hitting this threshold, the algorithm - // will create separate groups for all remaining pointers. - if (TotalComparisons > MemoryCheckMergeThreshold) - break; + // Go through all the existing sets and see if we can find one + // which can include this pointer. + for (RuntimeCheckingPtrGroup &Group : Groups) { + // Don't perform more than a certain amount of comparisons. + // This should limit the cost of grouping the pointers to something + // reasonable. If we do end up hitting this threshold, the algorithm + // will create separate groups for all remaining pointers. + if (TotalComparisons > MemoryCheckMergeThreshold) + break; - TotalComparisons++; + TotalComparisons++; - if (Group.addPointer(Pointer, *this)) { - Merged = true; - break; + if (Group.addPointer(Pointer, *this)) { + Merged = true; + break; + } } - } - if (!Merged) - // We couldn't add this pointer to any existing set or the threshold - // for the number of comparisons has been reached. Create a new group - // to hold the current pointer. - Groups.push_back(RuntimeCheckingPtrGroup(Pointer, *this)); + if (!Merged) + // We couldn't add this pointer to any existing set or the threshold + // for the number of comparisons has been reached. Create a new group + // to hold the current pointer. + Groups.push_back(RuntimeCheckingPtrGroup(Pointer, *this)); + } } // We've computed the grouped checks for this partition. @@ -522,19 +623,19 @@ : TheLoop(TheLoop), AST(*AA), LI(LI), DepCands(DA), PSE(PSE) {} /// Register a load and whether it is only read from. - void addLoad(MemoryLocation &Loc, bool IsReadOnly) { + void addLoad(MemoryLocation &Loc, Type *AccessTy, bool IsReadOnly) { Value *Ptr = const_cast<Value*>(Loc.Ptr); AST.add(Ptr, LocationSize::beforeOrAfterPointer(), Loc.AATags); - Accesses.insert(MemAccessInfo(Ptr, false)); + Accesses[MemAccessInfo(Ptr, false)].insert(AccessTy); if (IsReadOnly) ReadOnlyPtr.insert(Ptr); } /// Register a store. - void addStore(MemoryLocation &Loc) { + void addStore(MemoryLocation &Loc, Type *AccessTy) { Value *Ptr = const_cast<Value*>(Loc.Ptr); AST.add(Ptr, LocationSize::beforeOrAfterPointer(), Loc.AATags); - Accesses.insert(MemAccessInfo(Ptr, true)); + Accesses[MemAccessInfo(Ptr, true)].insert(AccessTy); } /// Check if we can emit a run-time no-alias check for \p Access. @@ -545,12 +646,11 @@ /// we will attempt to use additional run-time checks in order to get /// the bounds of the pointer. bool createCheckForAccess(RuntimePointerChecking &RtCheck, - MemAccessInfo Access, + MemAccessInfo Access, Type *AccessTy, const ValueToValueMap &Strides, DenseMap<Value *, unsigned> &DepSetId, Loop *TheLoop, unsigned &RunningDepId, - unsigned ASId, bool ShouldCheckStride, - bool Assume); + unsigned ASId, bool ShouldCheckStride, bool Assume); /// Check whether we can check the pointers at runtime for /// non-intersection. @@ -559,7 +659,7 @@ /// (i.e. the pointers have computable bounds). bool canCheckPtrAtRT(RuntimePointerChecking &RtCheck, ScalarEvolution *SE, Loop *TheLoop, const ValueToValueMap &Strides, - bool ShouldCheckWrap = false); + Value *&UncomputablePtr, bool ShouldCheckWrap = false); /// Goes over all memory accesses, checks whether a RT check is needed /// and builds sets of dependent accesses. @@ -583,14 +683,15 @@ MemAccessInfoList &getDependenciesToCheck() { return CheckDeps; } private: - typedef SetVector<MemAccessInfo> PtrAccessSet; + typedef MapVector<MemAccessInfo, SmallSetVector<Type *, 1>> PtrAccessMap; /// Go over all memory access and check whether runtime pointer checks /// are needed and build sets of dependency check candidates. void processMemAccesses(); - /// Set of all accesses. - PtrAccessSet Accesses; + /// Map of all accesses. Values are the types used to access memory pointed to + /// by the pointer. + PtrAccessMap Accesses; /// The loop being checked. const Loop *TheLoop; @@ -630,11 +731,8 @@ /// Check whether a pointer can participate in a runtime bounds check. /// If \p Assume, try harder to prove that we can compute the bounds of \p Ptr /// by adding run-time checks (overflow checks) if necessary. -static bool hasComputableBounds(PredicatedScalarEvolution &PSE, - const ValueToValueMap &Strides, Value *Ptr, - Loop *L, bool Assume) { - const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr); - +static bool hasComputableBounds(PredicatedScalarEvolution &PSE, Value *Ptr, + const SCEV *PtrScev, Loop *L, bool Assume) { // The bounds for loop-invariant pointer is trivial. if (PSE.getSE()->isLoopInvariant(PtrScev, L)) return true; @@ -652,12 +750,12 @@ /// Check whether a pointer address cannot wrap. static bool isNoWrap(PredicatedScalarEvolution &PSE, - const ValueToValueMap &Strides, Value *Ptr, Loop *L) { + const ValueToValueMap &Strides, Value *Ptr, Type *AccessTy, + Loop *L) { const SCEV *PtrScev = PSE.getSCEV(Ptr); if (PSE.getSE()->isLoopInvariant(PtrScev, L)) return true; - Type *AccessTy = Ptr->getType()->getPointerElementType(); int64_t Stride = getPtrStride(PSE, AccessTy, Ptr, L, Strides); if (Stride == 1 || PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW)) return true; @@ -688,8 +786,142 @@ } } +// Walk back through the IR for a pointer, looking for a select like the +// following: +// +// %offset = select i1 %cmp, i64 %a, i64 %b +// %addr = getelementptr double, double* %base, i64 %offset +// %ld = load double, double* %addr, align 8 +// +// We won't be able to form a single SCEVAddRecExpr from this since the +// address for each loop iteration depends on %cmp. We could potentially +// produce multiple valid SCEVAddRecExprs, though, and check all of them for +// memory safety/aliasing if needed. +// +// If we encounter some IR we don't yet handle, or something obviously fine +// like a constant, then we just add the SCEV for that term to the list passed +// in by the caller. If we have a node that may potentially yield a valid +// SCEVAddRecExpr then we decompose it into parts and build the SCEV terms +// ourselves before adding to the list. +static void +findForkedSCEVs(ScalarEvolution *SE, const Loop *L, Value *Ptr, + SmallVectorImpl<std::pair<const SCEV *, bool>> &ScevList, + unsigned Depth) { + // If our Value is a SCEVAddRecExpr, loop invariant, not an instruction, or + // we've exceeded our limit on recursion, just return whatever we have + // regardless of whether it can be used for a forked pointer or not, along + // with an indication of whether it might be a poison or undef value. + const SCEV *Scev = SE->getSCEV(Ptr); + if (isa<SCEVAddRecExpr>(Scev) || L->isLoopInvariant(Ptr) || + !isa<Instruction>(Ptr) || Depth == 0) { + ScevList.push_back( + std::make_pair(Scev, !isGuaranteedNotToBeUndefOrPoison(Ptr))); + return; + } + + Depth--; + + auto UndefPoisonCheck = [](std::pair<const SCEV *, bool> S) -> bool { + return S.second; + }; + + Instruction *I = cast<Instruction>(Ptr); + unsigned Opcode = I->getOpcode(); + switch (Opcode) { + case Instruction::GetElementPtr: { + GetElementPtrInst *GEP = cast<GetElementPtrInst>(I); + Type *SourceTy = GEP->getSourceElementType(); + // We only handle base + single offset GEPs here for now. + // Not dealing with preexisting gathers yet, so no vectors. + if (I->getNumOperands() != 2 || SourceTy->isVectorTy()) { + ScevList.push_back( + std::make_pair(Scev, !isGuaranteedNotToBeUndefOrPoison(GEP))); + break; + } + SmallVector<std::pair<const SCEV *, bool>, 2> BaseScevs; + SmallVector<std::pair<const SCEV *, bool>, 2> OffsetScevs; + findForkedSCEVs(SE, L, I->getOperand(0), BaseScevs, Depth); + findForkedSCEVs(SE, L, I->getOperand(1), OffsetScevs, Depth); + + // See if we need to freeze our fork... + bool NeedsFreeze = any_of(BaseScevs, UndefPoisonCheck) || + any_of(OffsetScevs, UndefPoisonCheck); + + // Check that we only have a single fork, on either the base or the offset. + // Copy the SCEV across for the one without a fork in order to generate + // the full SCEV for both sides of the GEP. + if (OffsetScevs.size() == 2 && BaseScevs.size() == 1) + BaseScevs.push_back(BaseScevs[0]); + else if (BaseScevs.size() == 2 && OffsetScevs.size() == 1) + OffsetScevs.push_back(OffsetScevs[0]); + else { + ScevList.push_back(std::make_pair(Scev, NeedsFreeze)); + break; + } + + // Find the pointer type we need to extend to. + Type *IntPtrTy = SE->getEffectiveSCEVType( + SE->getSCEV(GEP->getPointerOperand())->getType()); + + // Find the size of the type being pointed to. We only have a single + // index term (guarded above) so we don't need to index into arrays or + // structures, just get the size of the scalar value. + const SCEV *Size = SE->getSizeOfExpr(IntPtrTy, SourceTy); + + // Scale up the offsets by the size of the type, then add to the bases. + const SCEV *Scaled1 = SE->getMulExpr( + Size, SE->getTruncateOrSignExtend(OffsetScevs[0].first, IntPtrTy)); + const SCEV *Scaled2 = SE->getMulExpr( + Size, SE->getTruncateOrSignExtend(OffsetScevs[1].first, IntPtrTy)); + ScevList.push_back(std::make_pair( + SE->getAddExpr(BaseScevs[0].first, Scaled1), NeedsFreeze)); + ScevList.push_back(std::make_pair( + SE->getAddExpr(BaseScevs[1].first, Scaled2), NeedsFreeze)); + break; + } + case Instruction::Select: { + SmallVector<std::pair<const SCEV *, bool>, 2> ChildScevs; + // A select means we've found a forked pointer, but we currently only + // support a single select per pointer so if there's another behind this + // then we just bail out and return the generic SCEV. + findForkedSCEVs(SE, L, I->getOperand(1), ChildScevs, Depth); + findForkedSCEVs(SE, L, I->getOperand(2), ChildScevs, Depth); + if (ChildScevs.size() == 2) { + ScevList.push_back(ChildScevs[0]); + ScevList.push_back(ChildScevs[1]); + } else + ScevList.push_back( + std::make_pair(Scev, !isGuaranteedNotToBeUndefOrPoison(Ptr))); + break; + } + default: + // Just return the current SCEV if we haven't handled the instruction yet. + LLVM_DEBUG(dbgs() << "ForkedPtr unhandled instruction: " << *I << "\n"); + ScevList.push_back( + std::make_pair(Scev, !isGuaranteedNotToBeUndefOrPoison(Ptr))); + break; + } +} + +static SmallVector<std::pair<const SCEV *, bool>> +findForkedPointer(PredicatedScalarEvolution &PSE, + const ValueToValueMap &StridesMap, Value *Ptr, + const Loop *L) { + ScalarEvolution *SE = PSE.getSE(); + assert(SE->isSCEVable(Ptr->getType()) && "Value is not SCEVable!"); + SmallVector<std::pair<const SCEV *, bool>> Scevs; + findForkedSCEVs(SE, L, Ptr, Scevs, MaxForkedSCEVDepth); + + // For now, we will only accept a forked pointer with two possible SCEVs. + if (Scevs.size() == 2) + return Scevs; + + return { + std::make_pair(replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr), false)}; +} + bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck, - MemAccessInfo Access, + MemAccessInfo Access, Type *AccessTy, const ValueToValueMap &StridesMap, DenseMap<Value *, unsigned> &DepSetId, Loop *TheLoop, unsigned &RunningDepId, @@ -697,42 +929,64 @@ bool Assume) { Value *Ptr = Access.getPointer(); - if (!hasComputableBounds(PSE, StridesMap, Ptr, TheLoop, Assume)) - return false; + SmallVector<std::pair<const SCEV *, bool>> TranslatedPtrs = + findForkedPointer(PSE, StridesMap, Ptr, TheLoop); - // When we run after a failing dependency check we have to make sure - // we don't have wrapping pointers. - if (ShouldCheckWrap && !isNoWrap(PSE, StridesMap, Ptr, TheLoop)) { - auto *Expr = PSE.getSCEV(Ptr); - if (!Assume || !isa<SCEVAddRecExpr>(Expr)) + for (auto &P : TranslatedPtrs) { + const SCEV *PtrExpr = P.first; + if (!hasComputableBounds(PSE, Ptr, PtrExpr, TheLoop, Assume)) return false; - PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW); + + // When we run after a failing dependency check we have to make sure + // we don't have wrapping pointers. + if (ShouldCheckWrap) { + // Skip wrap checking when translating pointers. + if (TranslatedPtrs.size() > 1) + return false; + + if (!isNoWrap(PSE, StridesMap, Ptr, AccessTy, TheLoop)) { + auto *Expr = PSE.getSCEV(Ptr); + if (!Assume || !isa<SCEVAddRecExpr>(Expr)) + return false; + PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW); + } + } + // If there's only one option for Ptr, look it up after bounds and wrap + // checking, because assumptions might have been added to PSE. + if (TranslatedPtrs.size() == 1) + TranslatedPtrs[0] = std::make_pair( + replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr), false); } - // The id of the dependence set. - unsigned DepId; + for (auto &P : TranslatedPtrs) { + const SCEV *PtrExpr = P.first; - if (isDependencyCheckNeeded()) { - Value *Leader = DepCands.getLeaderValue(Access).getPointer(); - unsigned &LeaderId = DepSetId[Leader]; - if (!LeaderId) - LeaderId = RunningDepId++; - DepId = LeaderId; - } else - // Each access has its own dependence set. - DepId = RunningDepId++; + // The id of the dependence set. + unsigned DepId; - bool IsWrite = Access.getInt(); - RtCheck.insert(TheLoop, Ptr, IsWrite, DepId, ASId, StridesMap, PSE); - LLVM_DEBUG(dbgs() << "LAA: Found a runtime check ptr:" << *Ptr << '\n'); + if (isDependencyCheckNeeded()) { + Value *Leader = DepCands.getLeaderValue(Access).getPointer(); + unsigned &LeaderId = DepSetId[Leader]; + if (!LeaderId) + LeaderId = RunningDepId++; + DepId = LeaderId; + } else + // Each access has its own dependence set. + DepId = RunningDepId++; + + bool IsWrite = Access.getInt(); + RtCheck.insert(TheLoop, Ptr, PtrExpr, AccessTy, IsWrite, DepId, ASId, PSE, + P.second); + LLVM_DEBUG(dbgs() << "LAA: Found a runtime check ptr:" << *Ptr << '\n'); + } return true; - } +} bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck, ScalarEvolution *SE, Loop *TheLoop, const ValueToValueMap &StridesMap, - bool ShouldCheckWrap) { + Value *&UncomputablePtr, bool ShouldCheckWrap) { // Find pointers with computable bounds. We are going to use this information // to place a runtime bound check. bool CanDoRT = true; @@ -756,7 +1010,7 @@ unsigned RunningDepId = 1; DenseMap<Value *, unsigned> DepSetId; - SmallVector<MemAccessInfo, 4> Retries; + SmallVector<std::pair<MemAccessInfo, Type *>, 4> Retries; // First, count how many write and read accesses are in the alias set. Also // collect MemAccessInfos for later. @@ -788,12 +1042,15 @@ } for (auto &Access : AccessInfos) { - if (!createCheckForAccess(RtCheck, Access, StridesMap, DepSetId, TheLoop, - RunningDepId, ASId, ShouldCheckWrap, false)) { - LLVM_DEBUG(dbgs() << "LAA: Can't find bounds for ptr:" - << *Access.getPointer() << '\n'); - Retries.push_back(Access); - CanDoAliasSetRT = false; + for (const auto &AccessTy : Accesses[Access]) { + if (!createCheckForAccess(RtCheck, Access, AccessTy, StridesMap, + DepSetId, TheLoop, RunningDepId, ASId, + ShouldCheckWrap, false)) { + LLVM_DEBUG(dbgs() << "LAA: Can't find bounds for ptr:" + << *Access.getPointer() << '\n'); + Retries.push_back({Access, AccessTy}); + CanDoAliasSetRT = false; + } } } @@ -815,13 +1072,17 @@ // We know that we need these checks, so we can now be more aggressive // and add further checks if required (overflow checks). CanDoAliasSetRT = true; - for (auto Access : Retries) - if (!createCheckForAccess(RtCheck, Access, StridesMap, DepSetId, - TheLoop, RunningDepId, ASId, + for (auto Retry : Retries) { + MemAccessInfo Access = Retry.first; + Type *AccessTy = Retry.second; + if (!createCheckForAccess(RtCheck, Access, AccessTy, StridesMap, + DepSetId, TheLoop, RunningDepId, ASId, ShouldCheckWrap, /*Assume=*/true)) { CanDoAliasSetRT = false; + UncomputablePtr = Access.getPointer(); break; } + } } CanDoRT &= CanDoAliasSetRT; @@ -886,9 +1147,12 @@ LLVM_DEBUG(dbgs() << "LAA: Accesses(" << Accesses.size() << "):\n"); LLVM_DEBUG({ for (auto A : Accesses) - dbgs() << "\t" << *A.getPointer() << " (" << - (A.getInt() ? "write" : (ReadOnlyPtr.count(A.getPointer()) ? - "read-only" : "read")) << ")\n"; + dbgs() << "\t" << *A.first.getPointer() << " (" + << (A.first.getInt() + ? "write" + : (ReadOnlyPtr.count(A.first.getPointer()) ? "read-only" + : "read")) + << ")\n"; }); // The AliasSetTracker has nicely partitioned our pointers by metadata @@ -907,13 +1171,13 @@ UnderlyingObjToAccessMap ObjToLastAccess; // Set of access to check after all writes have been processed. - PtrAccessSet DeferredAccesses; + PtrAccessMap DeferredAccesses; // Iterate over each alias set twice, once to process read/write pointers, // and then to process read-only pointers. for (int SetIteration = 0; SetIteration < 2; ++SetIteration) { bool UseDeferred = SetIteration > 0; - PtrAccessSet &S = UseDeferred ? DeferredAccesses : Accesses; + PtrAccessMap &S = UseDeferred ? DeferredAccesses : Accesses; for (const auto &AV : AS) { Value *Ptr = AV.getValue(); @@ -921,10 +1185,10 @@ // For a single memory access in AliasSetTracker, Accesses may contain // both read and write, and they both need to be handled for CheckDeps. for (const auto &AC : S) { - if (AC.getPointer() != Ptr) + if (AC.first.getPointer() != Ptr) continue; - bool IsWrite = AC.getInt(); + bool IsWrite = AC.first.getInt(); // If we're using the deferred access set, then it contains only // reads. @@ -946,7 +1210,9 @@ // consecutive as "read-only" pointers (so that we check // "a[b[i]] +="). Hence, we need the second check for "!IsWrite". if (!UseDeferred && IsReadOnlyPtr) { - DeferredAccesses.insert(Access); + // We only use the pointer keys, the types vector values don't + // matter. + DeferredAccesses.insert({Access, {}}); continue; } @@ -1235,9 +1501,7 @@ Value *Ptr0 = VL[0]; using DistOrdPair = std::pair<int64_t, int>; - auto Compare = [](const DistOrdPair &L, const DistOrdPair &R) { - return L.first < R.first; - }; + auto Compare = llvm::less_first(); std::set<DistOrdPair, decltype(Compare)> Offsets(Compare); Offsets.emplace(0, 0); int Cnt = 1; @@ -1445,13 +1709,13 @@ const SCEV *CastedDist = &Dist; const SCEV *CastedProduct = Product; - uint64_t DistTypeSize = DL.getTypeAllocSize(Dist.getType()); - uint64_t ProductTypeSize = DL.getTypeAllocSize(Product->getType()); + uint64_t DistTypeSizeBits = DL.getTypeSizeInBits(Dist.getType()); + uint64_t ProductTypeSizeBits = DL.getTypeSizeInBits(Product->getType()); // The dependence distance can be positive/negative, so we sign extend Dist; // The multiplication of the absolute stride in bytes and the // backedgeTakenCount is non-negative, so we zero extend Product. - if (DistTypeSize > ProductTypeSize) + if (DistTypeSizeBits > ProductTypeSizeBits) CastedProduct = SE.getZeroExtendExpr(Product, Dist.getType()); else CastedDist = SE.getNoopOrSignExtend(&Dist, Product->getType()); @@ -1518,8 +1782,8 @@ Value *BPtr = B.getPointer(); bool AIsWrite = A.getInt(); bool BIsWrite = B.getInt(); - Type *ATy = APtr->getType()->getPointerElementType(); - Type *BTy = BPtr->getType()->getPointerElementType(); + Type *ATy = getLoadStoreType(InstMap[AIdx]); + Type *BTy = getLoadStoreType(InstMap[BIdx]); // Two reads are independent. if (!AIsWrite && !BIsWrite) @@ -1842,8 +2106,6 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI, const TargetLibraryInfo *TLI, DominatorTree *DT) { - typedef SmallPtrSet<Value*, 16> ValueSet; - // Holds the Load and Store instructions. SmallVector<LoadInst *, 16> Loads; SmallVector<StoreInst *, 16> Stores; @@ -1975,22 +2237,26 @@ // for read and once for write, it will only appear once (on the write // list). This is okay, since we are going to check for conflicts between // writes and between reads and writes, but not between reads and reads. - ValueSet Seen; + SmallSet<std::pair<Value *, Type *>, 16> Seen; // Record uniform store addresses to identify if we have multiple stores // to the same address. - ValueSet UniformStores; + SmallPtrSet<Value *, 16> UniformStores; for (StoreInst *ST : Stores) { Value *Ptr = ST->getPointerOperand(); - if (isUniform(Ptr)) + if (isUniform(Ptr)) { + // Record store instructions to loop invariant addresses + StoresToInvariantAddresses.push_back(ST); HasDependenceInvolvingLoopInvariantAddress |= !UniformStores.insert(Ptr).second; + } // If we did *not* see this pointer before, insert it to the read-write // list. At this phase it is only a 'write' list. - if (Seen.insert(Ptr).second) { + Type *AccessTy = getLoadStoreType(ST); + if (Seen.insert({Ptr, AccessTy}).second) { ++NumReadWrites; MemoryLocation Loc = MemoryLocation::get(ST); @@ -2001,9 +2267,9 @@ Loc.AATags.TBAA = nullptr; visitPointers(const_cast<Value *>(Loc.Ptr), *TheLoop, - [&Accesses, Loc](Value *Ptr) { + [&Accesses, AccessTy, Loc](Value *Ptr) { MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr); - Accesses.addStore(NewLoc); + Accesses.addStore(NewLoc, AccessTy); }); } } @@ -2027,7 +2293,8 @@ // read a few words, modify, and write a few words, and some of the // words may be written to the same address. bool IsReadOnlyPtr = false; - if (Seen.insert(Ptr).second || + Type *AccessTy = getLoadStoreType(LD); + if (Seen.insert({Ptr, AccessTy}).second || !getPtrStride(*PSE, LD->getType(), Ptr, TheLoop, SymbolicStrides)) { ++NumReads; IsReadOnlyPtr = true; @@ -2049,9 +2316,9 @@ Loc.AATags.TBAA = nullptr; visitPointers(const_cast<Value *>(Loc.Ptr), *TheLoop, - [&Accesses, Loc, IsReadOnlyPtr](Value *Ptr) { + [&Accesses, AccessTy, Loc, IsReadOnlyPtr](Value *Ptr) { MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr); - Accesses.addLoad(NewLoc, IsReadOnlyPtr); + Accesses.addLoad(NewLoc, AccessTy, IsReadOnlyPtr); }); } @@ -2069,10 +2336,14 @@ // Find pointers with computable bounds. We are going to use this information // to place a runtime bound check. - bool CanDoRTIfNeeded = Accesses.canCheckPtrAtRT(*PtrRtChecking, PSE->getSE(), - TheLoop, SymbolicStrides); + Value *UncomputablePtr = nullptr; + bool CanDoRTIfNeeded = + Accesses.canCheckPtrAtRT(*PtrRtChecking, PSE->getSE(), TheLoop, + SymbolicStrides, UncomputablePtr, false); if (!CanDoRTIfNeeded) { - recordAnalysis("CantIdentifyArrayBounds") << "cannot identify array bounds"; + auto *I = dyn_cast_or_null<Instruction>(UncomputablePtr); + recordAnalysis("CantIdentifyArrayBounds", I) + << "cannot identify array bounds"; LLVM_DEBUG(dbgs() << "LAA: We can't vectorize because we can't find " << "the array bounds.\n"); CanVecMem = false; @@ -2099,12 +2370,14 @@ PtrRtChecking->Need = true; auto *SE = PSE->getSE(); - CanDoRTIfNeeded = Accesses.canCheckPtrAtRT(*PtrRtChecking, SE, TheLoop, - SymbolicStrides, true); + UncomputablePtr = nullptr; + CanDoRTIfNeeded = Accesses.canCheckPtrAtRT( + *PtrRtChecking, SE, TheLoop, SymbolicStrides, UncomputablePtr, true); // Check that we found the bounds for the pointer. if (!CanDoRTIfNeeded) { - recordAnalysis("CantCheckMemDepsAtRunTime") + auto *I = dyn_cast_or_null<Instruction>(UncomputablePtr); + recordAnalysis("CantCheckMemDepsAtRunTime", I) << "cannot check memory dependencies at runtime"; LLVM_DEBUG(dbgs() << "LAA: Can't vectorize with memory checks\n"); CanVecMem = false; @@ -2129,13 +2402,61 @@ dbgs() << "LAA: No unsafe dependent memory operations in loop. We" << (PtrRtChecking->Need ? "" : " don't") << " need runtime memory checks.\n"); - else { - recordAnalysis("UnsafeMemDep") - << "unsafe dependent memory operations in loop. Use " - "#pragma loop distribute(enable) to allow loop distribution " - "to attempt to isolate the offending operations into a separate " - "loop"; - LLVM_DEBUG(dbgs() << "LAA: unsafe dependent memory operations in loop\n"); + else + emitUnsafeDependenceRemark(); +} + +void LoopAccessInfo::emitUnsafeDependenceRemark() { + auto Deps = getDepChecker().getDependences(); + if (!Deps) + return; + auto Found = std::find_if( + Deps->begin(), Deps->end(), [](const MemoryDepChecker::Dependence &D) { + return MemoryDepChecker::Dependence::isSafeForVectorization(D.Type) != + MemoryDepChecker::VectorizationSafetyStatus::Safe; + }); + if (Found == Deps->end()) + return; + MemoryDepChecker::Dependence Dep = *Found; + + LLVM_DEBUG(dbgs() << "LAA: unsafe dependent memory operations in loop\n"); + + // Emit remark for first unsafe dependence + OptimizationRemarkAnalysis &R = + recordAnalysis("UnsafeDep", Dep.getDestination(*this)) + << "unsafe dependent memory operations in loop. Use " + "#pragma loop distribute(enable) to allow loop distribution " + "to attempt to isolate the offending operations into a separate " + "loop"; + + switch (Dep.Type) { + case MemoryDepChecker::Dependence::NoDep: + case MemoryDepChecker::Dependence::Forward: + case MemoryDepChecker::Dependence::BackwardVectorizable: + llvm_unreachable("Unexpected dependence"); + case MemoryDepChecker::Dependence::Backward: + R << "\nBackward loop carried data dependence."; + break; + case MemoryDepChecker::Dependence::ForwardButPreventsForwarding: + R << "\nForward loop carried data dependence that prevents " + "store-to-load forwarding."; + break; + case MemoryDepChecker::Dependence::BackwardVectorizableButPreventsForwarding: + R << "\nBackward loop carried data dependence that prevents " + "store-to-load forwarding."; + break; + case MemoryDepChecker::Dependence::Unknown: + R << "\nUnknown data dependence."; + break; + } + + if (Instruction *I = Dep.getSource(*this)) { + DebugLoc SourceLoc = I->getDebugLoc(); + if (auto *DD = dyn_cast_or_null<Instruction>(getPointerOperand(I))) + SourceLoc = DD->getDebugLoc(); + if (SourceLoc) + R << " Memory location is the same as accessed at " + << ore::NV("Location", SourceLoc); } } @@ -2212,12 +2533,12 @@ // The Stride can be positive/negative, so we sign extend Stride; // The backedgeTakenCount is non-negative, so we zero extend BETakenCount. const DataLayout &DL = TheLoop->getHeader()->getModule()->getDataLayout(); - uint64_t StrideTypeSize = DL.getTypeAllocSize(StrideExpr->getType()); - uint64_t BETypeSize = DL.getTypeAllocSize(BETakenCount->getType()); + uint64_t StrideTypeSizeBits = DL.getTypeSizeInBits(StrideExpr->getType()); + uint64_t BETypeSizeBits = DL.getTypeSizeInBits(BETakenCount->getType()); const SCEV *CastedStride = StrideExpr; const SCEV *CastedBECount = BETakenCount; ScalarEvolution *SE = PSE->getSE(); - if (BETypeSize >= StrideTypeSize) + if (BETypeSizeBits >= StrideTypeSizeBits) CastedStride = SE->getNoopOrSignExtend(StrideExpr, BETakenCount->getType()); else CastedBECount = SE->getZeroExtendExpr(BETakenCount, StrideExpr->getType()); @@ -2232,7 +2553,7 @@ "at most once.\n"); return; } - LLVM_DEBUG(dbgs() << "LAA: Found a strided access that we can version."); + LLVM_DEBUG(dbgs() << "LAA: Found a strided access that we can version.\n"); SymbolicStrides[Ptr] = Stride; StrideSet.insert(Stride); @@ -2242,10 +2563,12 @@ const TargetLibraryInfo *TLI, AAResults *AA, DominatorTree *DT, LoopInfo *LI) : PSE(std::make_unique<PredicatedScalarEvolution>(*SE, *L)), - PtrRtChecking(std::make_unique<RuntimePointerChecking>(SE)), + PtrRtChecking(nullptr), DepChecker(std::make_unique<MemoryDepChecker>(*PSE, L)), TheLoop(L) { - if (canAnalyzeLoop()) + PtrRtChecking = std::make_unique<RuntimePointerChecking>(*DepChecker, SE); + if (canAnalyzeLoop()) { analyzeLoop(AA, LI, TLI, DT); + } } void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const { @@ -2267,7 +2590,7 @@ if (auto *Dependences = DepChecker->getDependences()) { OS.indent(Depth) << "Dependences:\n"; - for (auto &Dep : *Dependences) { + for (const auto &Dep : *Dependences) { Dep.print(OS, Depth + 2, DepChecker->getMemoryInstructions()); OS << "\n"; } @@ -2283,7 +2606,7 @@ << "found in loop.\n"; OS.indent(Depth) << "SCEV assumptions:\n"; - PSE->getUnionPredicate().print(OS, Depth); + PSE->getPredicate().print(OS, Depth); OS << "\n"; @@ -2301,7 +2624,7 @@ if (!LAI) LAI = std::make_unique<LoopAccessInfo>(L, SE, TLI, AA, DT, LI); - return *LAI.get(); + return *LAI; } void LoopAccessLegacyAnalysis::print(raw_ostream &OS, const Module *M) const {
diff --git a/src/llvm-project/llvm/lib/Analysis/LoopAnalysisManager.cpp b/src/llvm-project/llvm/lib/Analysis/LoopAnalysisManager.cpp index 4d6f8a6..8d71b31 100644 --- a/src/llvm-project/llvm/lib/Analysis/LoopAnalysisManager.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LoopAnalysisManager.cpp
@@ -8,12 +8,9 @@ #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/BasicAliasAnalysis.h" -#include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/PassManagerImpl.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/LoopCacheAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/LoopCacheAnalysis.cpp index ba014bd..85f2dad 100644 --- a/src/llvm-project/llvm/lib/Analysis/LoopCacheAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LoopCacheAnalysis.cpp
@@ -103,14 +103,24 @@ return StepRec == &ElemSize; } -/// Compute the trip count for the given loop \p L. Return the SCEV expression -/// for the trip count or nullptr if it cannot be computed. -static const SCEV *computeTripCount(const Loop &L, ScalarEvolution &SE) { +/// Compute the trip count for the given loop \p L or assume a default value if +/// it is not a compile time constant. Return the SCEV expression for the trip +/// count. +static const SCEV *computeTripCount(const Loop &L, const SCEV &ElemSize, + ScalarEvolution &SE) { const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(&L); - if (isa<SCEVCouldNotCompute>(BackedgeTakenCount) || - !isa<SCEVConstant>(BackedgeTakenCount)) - return nullptr; - return SE.getTripCountFromExitCount(BackedgeTakenCount); + const SCEV *TripCount = (!isa<SCEVCouldNotCompute>(BackedgeTakenCount) && + isa<SCEVConstant>(BackedgeTakenCount)) + ? SE.getTripCountFromExitCount(BackedgeTakenCount) + : nullptr; + + if (!TripCount) { + LLVM_DEBUG(dbgs() << "Trip count of loop " << L.getName() + << " could not be computed, using DefaultTripCount\n"); + TripCount = SE.getConstant(ElemSize.getType(), DefaultTripCount); + } + + return TripCount; } //===----------------------------------------------------------------------===// @@ -274,27 +284,19 @@ return 1; } - const SCEV *TripCount = computeTripCount(L, SE); - if (!TripCount) { - LLVM_DEBUG(dbgs() << "Trip count of loop " << L.getName() - << " could not be computed, using DefaultTripCount\n"); - const SCEV *ElemSize = Sizes.back(); - TripCount = SE.getConstant(ElemSize->getType(), DefaultTripCount); - } + const SCEV *TripCount = computeTripCount(L, *Sizes.back(), SE); + assert(TripCount && "Expecting valid TripCount"); LLVM_DEBUG(dbgs() << "TripCount=" << *TripCount << "\n"); - // If the indexed reference is 'consecutive' the cost is - // (TripCount*Stride)/CLS, otherwise the cost is TripCount. - const SCEV *RefCost = TripCount; - - if (isConsecutive(L, CLS)) { - const SCEV *Coeff = getLastCoefficient(); - const SCEV *ElemSize = Sizes.back(); - const SCEV *Stride = SE.getMulExpr(Coeff, ElemSize); + const SCEV *RefCost = nullptr; + const SCEV *Stride = nullptr; + if (isConsecutive(L, Stride, CLS)) { + // If the indexed reference is 'consecutive' the cost is + // (TripCount*Stride)/CLS. + assert(Stride != nullptr && + "Stride should not be null for consecutive access!"); Type *WiderType = SE.getWiderType(Stride->getType(), TripCount->getType()); const SCEV *CacheLineSize = SE.getConstant(WiderType, CLS); - if (SE.isKnownNegative(Stride)) - Stride = SE.getNegativeSCEV(Stride); Stride = SE.getNoopOrAnyExtend(Stride, WiderType); TripCount = SE.getNoopOrAnyExtend(TripCount, WiderType); const SCEV *Numerator = SE.getMulExpr(Stride, TripCount); @@ -303,10 +305,33 @@ LLVM_DEBUG(dbgs().indent(4) << "Access is consecutive: RefCost=(TripCount*Stride)/CLS=" << *RefCost << "\n"); - } else + } else { + // If the indexed reference is not 'consecutive' the cost is proportional to + // the trip count and the depth of the dimension which the subject loop + // subscript is accessing. We try to estimate this by multiplying the cost + // by the trip counts of loops corresponding to the inner dimensions. For + // example, given the indexed reference 'A[i][j][k]', and assuming the + // i-loop is in the innermost position, the cost would be equal to the + // iterations of the i-loop multiplied by iterations of the j-loop. + RefCost = TripCount; + + int Index = getSubscriptIndex(L); + assert(Index >= 0 && "Cound not locate a valid Index"); + + for (unsigned I = Index + 1; I < getNumSubscripts() - 1; ++I) { + const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(getSubscript(I)); + assert(AR && AR->getLoop() && "Expecting valid loop"); + const SCEV *TripCount = + computeTripCount(*AR->getLoop(), *Sizes.back(), SE); + Type *WiderType = SE.getWiderType(RefCost->getType(), TripCount->getType()); + RefCost = SE.getMulExpr(SE.getNoopOrAnyExtend(RefCost, WiderType), + SE.getNoopOrAnyExtend(TripCount, WiderType)); + } + LLVM_DEBUG(dbgs().indent(4) - << "Access is not consecutive: RefCost=TripCount=" << *RefCost - << "\n"); + << "Access is not consecutive: RefCost=" << *RefCost << "\n"); + } + assert(RefCost && "Expecting a valid RefCost"); // Attempt to fold RefCost into a constant. if (auto ConstantCost = dyn_cast<SCEVConstant>(RefCost)) @@ -319,6 +344,26 @@ return CacheCost::InvalidCost; } +bool IndexedReference::tryDelinearizeFixedSize( + const SCEV *AccessFn, SmallVectorImpl<const SCEV *> &Subscripts) { + SmallVector<int, 4> ArraySizes; + if (!tryDelinearizeFixedSizeImpl(&SE, &StoreOrLoadInst, AccessFn, Subscripts, + ArraySizes)) + return false; + + // Populate Sizes with scev expressions to be used in calculations later. + for (auto Idx : seq<unsigned>(1, Subscripts.size())) + Sizes.push_back( + SE.getConstant(Subscripts[Idx]->getType(), ArraySizes[Idx - 1])); + + LLVM_DEBUG({ + dbgs() << "Delinearized subscripts of fixed-size array\n" + << "GEP:" << *getLoadStorePointerOperand(&StoreOrLoadInst) + << "\n"; + }); + return true; +} + bool IndexedReference::delinearize(const LoopInfo &LI) { assert(Subscripts.empty() && "Subscripts should be empty"); assert(Sizes.empty() && "Sizes should be empty"); @@ -340,13 +385,25 @@ return false; } + bool IsFixedSize = false; + // Try to delinearize fixed-size arrays. + if (tryDelinearizeFixedSize(AccessFn, Subscripts)) { + IsFixedSize = true; + // The last element of Sizes is the element size. + Sizes.push_back(ElemSize); + LLVM_DEBUG(dbgs().indent(2) << "In Loop '" << L->getName() + << "', AccessFn: " << *AccessFn << "\n"); + } + AccessFn = SE.getMinusSCEV(AccessFn, BasePointer); - LLVM_DEBUG(dbgs().indent(2) << "In Loop '" << L->getName() - << "', AccessFn: " << *AccessFn << "\n"); - - llvm::delinearize(SE, AccessFn, Subscripts, Sizes, - SE.getElementSize(&StoreOrLoadInst)); + // Try to delinearize parametric-size arrays. + if (!IsFixedSize) { + LLVM_DEBUG(dbgs().indent(2) << "In Loop '" << L->getName() + << "', AccessFn: " << *AccessFn << "\n"); + llvm::delinearize(SE, AccessFn, Subscripts, Sizes, + SE.getElementSize(&StoreOrLoadInst)); + } if (Subscripts.empty() || Sizes.empty() || Subscripts.size() != Sizes.size()) { @@ -403,7 +460,8 @@ return allCoeffForLoopAreZero; } -bool IndexedReference::isConsecutive(const Loop &L, unsigned CLS) const { +bool IndexedReference::isConsecutive(const Loop &L, const SCEV *&Stride, + unsigned CLS) const { // The indexed reference is 'consecutive' if the only coefficient that uses // the loop induction variable is the last one... const SCEV *LastSubscript = Subscripts.back(); @@ -417,13 +475,35 @@ // ...and the access stride is less than the cache line size. const SCEV *Coeff = getLastCoefficient(); const SCEV *ElemSize = Sizes.back(); - const SCEV *Stride = SE.getMulExpr(Coeff, ElemSize); + Type *WiderType = SE.getWiderType(Coeff->getType(), ElemSize->getType()); + // FIXME: This assumes that all values are signed integers which may + // be incorrect in unusual codes and incorrectly use sext instead of zext. + // for (uint32_t i = 0; i < 512; ++i) { + // uint8_t trunc = i; + // A[trunc] = 42; + // } + // This consecutively iterates twice over A. If `trunc` is sign-extended, + // we would conclude that this may iterate backwards over the array. + // However, LoopCacheAnalysis is heuristic anyway and transformations must + // not result in wrong optimizations if the heuristic was incorrect. + Stride = SE.getMulExpr(SE.getNoopOrSignExtend(Coeff, WiderType), + SE.getNoopOrSignExtend(ElemSize, WiderType)); const SCEV *CacheLineSize = SE.getConstant(Stride->getType(), CLS); Stride = SE.isKnownNegative(Stride) ? SE.getNegativeSCEV(Stride) : Stride; return SE.isKnownPredicate(ICmpInst::ICMP_ULT, Stride, CacheLineSize); } +int IndexedReference::getSubscriptIndex(const Loop &L) const { + for (auto Idx : seq<int>(0, getNumSubscripts())) { + const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(getSubscript(Idx)); + if (AR && AR->getLoop() == &L) { + return Idx; + } + } + return -1; +} + const SCEV *IndexedReference::getLastCoefficient() const { const SCEV *LastSubscript = getLastSubscript(); auto *AR = cast<SCEVAddRecExpr>(LastSubscript); @@ -550,7 +630,7 @@ bool Added = false; for (ReferenceGroupTy &RefGroup : RefGroups) { - const IndexedReference &Representative = *RefGroup.front().get(); + const IndexedReference &Representative = *RefGroup.front(); LLVM_DEBUG({ dbgs() << "References:\n"; dbgs().indent(2) << *R << "\n"; @@ -574,8 +654,8 @@ Optional<bool> HasSpacialReuse = R->hasSpacialReuse(Representative, CLS, AA); - if ((HasTemporalReuse.hasValue() && *HasTemporalReuse) || - (HasSpacialReuse.hasValue() && *HasSpacialReuse)) { + if ((HasTemporalReuse && *HasTemporalReuse) || + (HasSpacialReuse && *HasSpacialReuse)) { RefGroup.push_back(std::move(R)); Added = true; break;
diff --git a/src/llvm-project/llvm/lib/Analysis/LoopInfo.cpp b/src/llvm-project/llvm/lib/Analysis/LoopInfo.cpp index b161c49..693b9eb 100644 --- a/src/llvm-project/llvm/lib/Analysis/LoopInfo.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LoopInfo.cpp
@@ -14,7 +14,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/LoopInfo.h" -#include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/IVDescriptors.h" @@ -30,7 +29,6 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Dominators.h" -#include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" @@ -38,9 +36,7 @@ #include "llvm/IR/PrintPasses.h" #include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include <algorithm> using namespace llvm; // Explicitly instantiate methods in LoopInfoImpl.h for IR-level Loops. @@ -429,12 +425,12 @@ // Check that 'BB' doesn't have any uses outside of the 'L' static bool isBlockInLCSSAForm(const Loop &L, const BasicBlock &BB, - const DominatorTree &DT) { + const DominatorTree &DT, bool IgnoreTokens) { for (const Instruction &I : BB) { // Tokens can't be used in PHI nodes and live-out tokens prevent loop // optimizations, so for the purposes of considered LCSSA form, we // can ignore them. - if (I.getType()->isTokenTy()) + if (IgnoreTokens && I.getType()->isTokenTy()) continue; for (const Use &U : I.uses()) { @@ -459,20 +455,20 @@ return true; } -bool Loop::isLCSSAForm(const DominatorTree &DT) const { +bool Loop::isLCSSAForm(const DominatorTree &DT, bool IgnoreTokens) const { // For each block we check that it doesn't have any uses outside of this loop. return all_of(this->blocks(), [&](const BasicBlock *BB) { - return isBlockInLCSSAForm(*this, *BB, DT); + return isBlockInLCSSAForm(*this, *BB, DT, IgnoreTokens); }); } -bool Loop::isRecursivelyLCSSAForm(const DominatorTree &DT, - const LoopInfo &LI) const { +bool Loop::isRecursivelyLCSSAForm(const DominatorTree &DT, const LoopInfo &LI, + bool IgnoreTokens) const { // For each block we check that it doesn't have any uses outside of its // innermost loop. This process will transitively guarantee that the current // loop and all of the nested loops are in LCSSA form. return all_of(this->blocks(), [&](const BasicBlock *BB) { - return isBlockInLCSSAForm(*LI.getLoopFor(BB), *BB, DT); + return isBlockInLCSSAForm(*LI.getLoopFor(BB), *BB, DT, IgnoreTokens); }); } @@ -486,11 +482,8 @@ bool Loop::isSafeToClone() const { // Return false if any loop blocks contain indirectbrs, or there are any calls // to noduplicate functions. - // FIXME: it should be ok to clone CallBrInst's if we correctly update the - // operand list to reflect the newly cloned labels. for (BasicBlock *BB : this->blocks()) { - if (isa<IndirectBrInst>(BB->getTerminator()) || - isa<CallBrInst>(BB->getTerminator())) + if (isa<IndirectBrInst>(BB->getTerminator())) return false; for (Instruction &I : *BB) @@ -740,6 +733,7 @@ bool Changed = FoundIB; for (unsigned NIters = 0; Changed; ++NIters) { assert(NIters < Unloop.getNumBlocks() && "runaway iterative algorithm"); + (void) NIters; // Iterate over the postorder list of blocks, propagating the nearest loop // from successors to predecessors as before. @@ -1085,13 +1079,13 @@ } bool llvm::getBooleanLoopAttribute(const Loop *TheLoop, StringRef Name) { - return getOptionalBoolLoopAttribute(TheLoop, Name).getValueOr(false); + return getOptionalBoolLoopAttribute(TheLoop, Name).value_or(false); } llvm::Optional<int> llvm::getOptionalIntLoopAttribute(const Loop *TheLoop, StringRef Name) { const MDOperand *AttrMD = - findStringMetadataForLoop(TheLoop, Name).getValueOr(nullptr); + findStringMetadataForLoop(TheLoop, Name).value_or(nullptr); if (!AttrMD) return None; @@ -1104,7 +1098,7 @@ int llvm::getIntLoopAttribute(const Loop *TheLoop, StringRef Name, int Default) { - return getOptionalIntLoopAttribute(TheLoop, Name).getValueOr(Default); + return getOptionalIntLoopAttribute(TheLoop, Name).value_or(Default); } bool llvm::isFinite(const Loop *L) {
diff --git a/src/llvm-project/llvm/lib/Analysis/LoopNestAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/LoopNestAnalysis.cpp index 675bb7a..bff796f 100644 --- a/src/llvm-project/llvm/lib/Analysis/LoopNestAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LoopNestAnalysis.cpp
@@ -13,8 +13,7 @@ #include "llvm/Analysis/LoopNestAnalysis.h" #include "llvm/ADT/BreadthFirstIterator.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/Analysis/PostDominators.h" +#include "llvm/ADT/DepthFirstIterator.h" #include "llvm/Analysis/ValueTracking.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/LoopPass.cpp b/src/llvm-project/llvm/lib/Analysis/LoopPass.cpp index b720bab..5d824ae 100644 --- a/src/llvm-project/llvm/lib/Analysis/LoopPass.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LoopPass.cpp
@@ -13,14 +13,12 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/LoopPass.h" -#include "llvm/Analysis/LoopAnalysisManager.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/OptBisect.h" -#include "llvm/IR/PassManager.h" #include "llvm/IR/PassTimingInfo.h" #include "llvm/IR/PrintPasses.h" -#include "llvm/IR/StructuralHash.h" #include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" #include "llvm/Support/TimeProfiler.h" @@ -192,12 +190,12 @@ PassManagerPrettyStackEntry X(P, *CurrentLoop->getHeader()); TimeRegion PassTimer(getPassTimer(P)); #ifdef EXPENSIVE_CHECKS - uint64_t RefHash = StructuralHash(F); + uint64_t RefHash = P->structuralHash(F); #endif LocalChanged = P->runOnLoop(CurrentLoop, *this); #ifdef EXPENSIVE_CHECKS - if (!LocalChanged && (RefHash != StructuralHash(F))) { + if (!LocalChanged && (RefHash != P->structuralHash(F))) { llvm::errs() << "Pass modifies its input and doesn't report it: " << P->getPassName() << "\n"; llvm_unreachable("Pass modifies its input and doesn't report it");
diff --git a/src/llvm-project/llvm/lib/Analysis/LoopUnrollAnalyzer.cpp b/src/llvm-project/llvm/lib/Analysis/LoopUnrollAnalyzer.cpp index 15095d6..84f1eff 100644 --- a/src/llvm-project/llvm/lib/Analysis/LoopUnrollAnalyzer.cpp +++ b/src/llvm-project/llvm/lib/Analysis/LoopUnrollAnalyzer.cpp
@@ -13,7 +13,10 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/LoopUnrollAnalyzer.h" +#include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/IR/Operator.h" using namespace llvm; @@ -84,9 +87,9 @@ const DataLayout &DL = I.getModule()->getDataLayout(); if (auto FI = dyn_cast<FPMathOperator>(&I)) SimpleV = - SimplifyBinOp(I.getOpcode(), LHS, RHS, FI->getFastMathFlags(), DL); + simplifyBinOp(I.getOpcode(), LHS, RHS, FI->getFastMathFlags(), DL); else - SimpleV = SimplifyBinOp(I.getOpcode(), LHS, RHS, DL); + SimpleV = simplifyBinOp(I.getOpcode(), LHS, RHS, DL); if (SimpleV) { SimplifiedValues[&I] = SimpleV; @@ -155,7 +158,7 @@ // i32 0). if (CastInst::castIsValid(I.getOpcode(), Op, I.getType())) { const DataLayout &DL = I.getModule()->getDataLayout(); - if (Value *V = SimplifyCastInst(I.getOpcode(), Op, I.getType(), DL)) { + if (Value *V = simplifyCastInst(I.getOpcode(), Op, I.getType(), DL)) { SimplifiedValues[&I] = V; return true; } @@ -192,7 +195,7 @@ } const DataLayout &DL = I.getModule()->getDataLayout(); - if (Value *V = SimplifyCmpInst(I.getPredicate(), LHS, RHS, DL)) { + if (Value *V = simplifyCmpInst(I.getPredicate(), LHS, RHS, DL)) { SimplifiedValues[&I] = V; return true; }
diff --git a/src/llvm-project/llvm/lib/Analysis/MLInlineAdvisor.cpp b/src/llvm-project/llvm/lib/Analysis/MLInlineAdvisor.cpp index 0480c1c..f55de71 100644 --- a/src/llvm-project/llvm/lib/Analysis/MLInlineAdvisor.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MLInlineAdvisor.cpp
@@ -13,30 +13,25 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/MLInlineAdvisor.h" #include "llvm/ADT/SCCIterator.h" +#include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/FunctionPropertiesAnalysis.h" #include "llvm/Analysis/InlineCost.h" #include "llvm/Analysis/InlineModelFeatureMaps.h" #include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MLModelRunner.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" -#include "llvm/Analysis/ReleaseModeModelRunner.h" -#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/Config/config.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/InstIterator.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Path.h" - -#include <limits> -#include <unordered_map> -#include <unordered_set> using namespace llvm; #if defined(LLVM_HAVE_TF_AOT_INLINERSIZEMODEL) +#include "llvm/Analysis/ReleaseModeModelRunner.h" // codegen-ed file #include "InlinerSizeModel.h" // NOLINT @@ -44,7 +39,7 @@ llvm::getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM) { auto AOTRunner = std::make_unique<ReleaseModeModelRunner<llvm::InlinerSizeModel>>( - M.getContext(), FeatureNameMap, DecisionName); + M.getContext(), FeatureMap, DecisionName); return std::make_unique<MLInlineAdvisor>(M, MAM, std::move(AOTRunner)); } #endif @@ -57,15 +52,21 @@ "blocking any further inlining."), cl::init(2.0)); +static cl::opt<bool> KeepFPICache( + "ml-advisor-keep-fpi-cache", cl::Hidden, + cl::desc( + "For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"), + cl::init(false)); + // clang-format off -const std::array<std::string, NumberOfFeatures> llvm::FeatureNameMap{ +const std::array<TensorSpec, NumberOfFeatures> llvm::FeatureMap{ +#define POPULATE_NAMES(_, NAME) TensorSpec::createSpec<int64_t>(NAME, {1} ), // InlineCost features - these must come first -#define POPULATE_NAMES(INDEX_NAME, NAME) NAME, INLINE_COST_FEATURE_ITERATOR(POPULATE_NAMES) #undef POPULATE_NAMES // Non-cost features -#define POPULATE_NAMES(INDEX_NAME, NAME, COMMENT) NAME, +#define POPULATE_NAMES(_, NAME, __) TensorSpec::createSpec<int64_t>(NAME, {1} ), INLINE_FEATURE_ITERATOR(POPULATE_NAMES) #undef POPULATE_NAMES }; @@ -138,7 +139,10 @@ return CG.lookup(F) ? FunctionLevels.at(CG.lookup(F)) : 0; } -void MLInlineAdvisor::onPassEntry() { +void MLInlineAdvisor::onPassEntry(LazyCallGraph::SCC *LastSCC) { + if (!LastSCC || ForceStop) + return; + FPICache.clear(); // Function passes executed between InlinerPass runs may have changed the // module-wide features. // The cgscc pass manager rules are such that: @@ -154,8 +158,8 @@ // care about the nature of the Edge (call or ref). NodeCount -= static_cast<int64_t>(NodesInLastSCC.size()); while (!NodesInLastSCC.empty()) { - const auto *N = NodesInLastSCC.front(); - NodesInLastSCC.pop_front(); + const auto *N = *NodesInLastSCC.begin(); + NodesInLastSCC.erase(N); // The Function wrapped by N could have been deleted since we last saw it. if (N->isDead()) { assert(!N->getFunction().isDeclaration()); @@ -168,34 +172,52 @@ assert(!AdjNode->isDead() && !AdjNode->getFunction().isDeclaration()); auto I = AllNodes.insert(AdjNode); if (I.second) - NodesInLastSCC.push_back(AdjNode); + NodesInLastSCC.insert(AdjNode); } } EdgeCount -= EdgesOfLastSeenNodes; EdgesOfLastSeenNodes = 0; + + // (Re)use NodesInLastSCC to remember the nodes in the SCC right now, + // in case the SCC is split before onPassExit and some nodes are split out + assert(NodesInLastSCC.empty()); + for (const auto &N : *LastSCC) + NodesInLastSCC.insert(&N); } void MLInlineAdvisor::onPassExit(LazyCallGraph::SCC *LastSCC) { - if (!LastSCC) + // No need to keep this around - function passes will invalidate it. + if (!KeepFPICache) + FPICache.clear(); + if (!LastSCC || ForceStop) return; // Keep track of the nodes and edges we last saw. Then, in onPassEntry, // we update the node count and edge count from the subset of these nodes that // survived. - assert(NodesInLastSCC.empty()); - assert(NodeCount >= LastSCC->size()); EdgesOfLastSeenNodes = 0; + + // Check on nodes that were in SCC onPassEntry + for (auto I = NodesInLastSCC.begin(); I != NodesInLastSCC.end();) { + if ((*I)->isDead()) + NodesInLastSCC.erase(*I++); + else + EdgesOfLastSeenNodes += getLocalCalls((*I++)->getFunction()); + } + + // Check on nodes that may have got added to SCC for (const auto &N : *LastSCC) { assert(!N.isDead()); - EdgesOfLastSeenNodes += getLocalCalls(N.getFunction()); - NodesInLastSCC.push_back(&N); + auto I = NodesInLastSCC.insert(&N); + if (I.second) + EdgesOfLastSeenNodes += getLocalCalls(N.getFunction()); } + assert(NodeCount >= NodesInLastSCC.size()); assert(EdgeCount >= EdgesOfLastSeenNodes); } int64_t MLInlineAdvisor::getLocalCalls(Function &F) { - return FAM.getResult<FunctionPropertiesAnalysis>(F) - .DirectCallsToDefinedFunctions; + return getCachedFPI(F).DirectCallsToDefinedFunctions; } // Update the internal state of the advisor, and force invalidate feature @@ -208,13 +230,15 @@ assert(!ForceStop); Function *Caller = Advice.getCaller(); Function *Callee = Advice.getCallee(); - // The caller features aren't valid anymore. { PreservedAnalyses PA = PreservedAnalyses::all(); PA.abandon<FunctionPropertiesAnalysis>(); + PA.abandon<DominatorTreeAnalysis>(); + PA.abandon<LoopAnalysis>(); FAM.invalidate(*Caller, PA); } + Advice.updateCachedCallerFPI(FAM); int64_t IRSizeAfter = getIRSize(*Caller) + (CalleeWasDeleted ? 0 : Advice.CalleeIRSize); CurrentIRSize += IRSizeAfter - (Advice.CallerIRSize + Advice.CalleeIRSize); @@ -227,15 +251,13 @@ // For edges, we 'forget' the edges that the caller and callee used to have // before inlining, and add back what they currently have together. int64_t NewCallerAndCalleeEdges = - FAM.getResult<FunctionPropertiesAnalysis>(*Caller) - .DirectCallsToDefinedFunctions; + getCachedFPI(*Caller).DirectCallsToDefinedFunctions; if (CalleeWasDeleted) --NodeCount; else NewCallerAndCalleeEdges += - FAM.getResult<FunctionPropertiesAnalysis>(*Callee) - .DirectCallsToDefinedFunctions; + getCachedFPI(*Callee).DirectCallsToDefinedFunctions; EdgeCount += (NewCallerAndCalleeEdges - Advice.CallerAndCalleeEdges); assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0); } @@ -248,7 +270,19 @@ return Ret; } +FunctionPropertiesInfo &MLInlineAdvisor::getCachedFPI(Function &F) const { + auto InsertPair = + FPICache.insert(std::make_pair(&F, FunctionPropertiesInfo())); + if (!InsertPair.second) + return InsertPair.first->second; + InsertPair.first->second = FAM.getResult<FunctionPropertiesAnalysis>(F); + return InsertPair.first->second; +} + std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) { + if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB)) + return Skip; + auto &Caller = *CB.getCaller(); auto &Callee = *CB.getCalledFunction(); @@ -307,8 +341,8 @@ NrCtantParams += (isa<Constant>(*I)); } - auto &CallerBefore = FAM.getResult<FunctionPropertiesAnalysis>(Caller); - auto &CalleeBefore = FAM.getResult<FunctionPropertiesAnalysis>(Callee); + auto &CallerBefore = getCachedFPI(Caller); + auto &CalleeBefore = getCachedFPI(Callee); *ModelRunner->getTensor<int64_t>(FeatureIndex::CalleeBasicBlockCount) = CalleeBefore.BasicBlockCount; @@ -348,9 +382,19 @@ this, CB, ORE, static_cast<bool>(ModelRunner->evaluate<int64_t>())); } +std::unique_ptr<InlineAdvice> +MLInlineAdvisor::getSkipAdviceIfUnreachableCallsite(CallBase &CB) { + if (!FAM.getResult<DominatorTreeAnalysis>(*CB.getCaller()) + .isReachableFromEntry(CB.getParent())) + return std::make_unique<InlineAdvice>(this, CB, getCallerORE(CB), false); + return nullptr; +} + std::unique_ptr<InlineAdvice> MLInlineAdvisor::getMandatoryAdvice(CallBase &CB, bool Advice) { // Make sure we track inlinings in all cases - mandatory or not. + if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB)) + return Skip; if (Advice && !ForceStop) return getMandatoryAdviceImpl(CB); @@ -366,16 +410,47 @@ return std::make_unique<MLInlineAdvice>(this, CB, getCallerORE(CB), true); } +void MLInlineAdvisor::print(raw_ostream &OS) const { + OS << "[MLInlineAdvisor] Nodes: " << NodeCount << " Edges: " << EdgeCount + << " EdgesOfLastSeenNodes: " << EdgesOfLastSeenNodes << "\n"; + OS << "[MLInlineAdvisor] FPI:\n"; + for (auto I : FPICache) { + OS << I.getFirst()->getName() << ":\n"; + I.getSecond().print(OS); + OS << "\n"; + } + OS << "\n"; +} + +MLInlineAdvice::MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB, + OptimizationRemarkEmitter &ORE, + bool Recommendation) + : InlineAdvice(Advisor, CB, ORE, Recommendation), + CallerIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*Caller)), + CalleeIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*Callee)), + CallerAndCalleeEdges(Advisor->isForcedToStop() + ? 0 + : (Advisor->getLocalCalls(*Caller) + + Advisor->getLocalCalls(*Callee))), + PreInlineCallerFPI(Advisor->getCachedFPI(*Caller)) { + if (Recommendation) + FPU.emplace(Advisor->getCachedFPI(*getCaller()), CB); +} + void MLInlineAdvice::reportContextForRemark( DiagnosticInfoOptimizationBase &OR) { using namespace ore; OR << NV("Callee", Callee->getName()); for (size_t I = 0; I < NumberOfFeatures; ++I) - OR << NV(FeatureNameMap[I], + OR << NV(FeatureMap[I].name(), *getAdvisor()->getModelRunner().getTensor<int64_t>(I)); OR << NV("ShouldInline", isInliningRecommended()); } +void MLInlineAdvice::updateCachedCallerFPI(FunctionAnalysisManager &FAM) const { + FPU->finish(FAM); +} + void MLInlineAdvice::recordInliningImpl() { ORE.emit([&]() { OptimizationRemark R(DEBUG_TYPE, "InliningSuccess", DLoc, Block); @@ -397,6 +472,7 @@ void MLInlineAdvice::recordUnsuccessfulInliningImpl( const InlineResult &Result) { + getAdvisor()->getCachedFPI(*Caller) = PreInlineCallerFPI; ORE.emit([&]() { OptimizationRemarkMissed R(DEBUG_TYPE, "InliningAttemptedAndUnsuccessful", DLoc, Block); @@ -405,6 +481,7 @@ }); } void MLInlineAdvice::recordUnattemptedInliningImpl() { + assert(!FPU); ORE.emit([&]() { OptimizationRemarkMissed R(DEBUG_TYPE, "IniningNotAttempted", DLoc, Block); reportContextForRemark(R);
diff --git a/src/llvm-project/llvm/lib/Analysis/MemDepPrinter.cpp b/src/llvm-project/llvm/lib/Analysis/MemDepPrinter.cpp index 0064234..305ae3e 100644 --- a/src/llvm-project/llvm/lib/Analysis/MemDepPrinter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MemDepPrinter.cpp
@@ -15,7 +15,6 @@ #include "llvm/Analysis/Passes.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/InitializePasses.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/MemDerefPrinter.cpp b/src/llvm-project/llvm/lib/Analysis/MemDerefPrinter.cpp index 82617c7..4dd5c76 100644 --- a/src/llvm-project/llvm/lib/Analysis/MemDerefPrinter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MemDerefPrinter.cpp
@@ -9,14 +9,11 @@ #include "llvm/Analysis/MemDerefPrinter.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/Passes.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/MemoryBuiltins.cpp b/src/llvm-project/llvm/lib/Analysis/MemoryBuiltins.cpp index 4b08e64..413ec6d 100644 --- a/src/llvm-project/llvm/lib/Analysis/MemoryBuiltins.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -17,7 +17,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/TargetFolder.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/Utils/Local.h" @@ -43,6 +43,8 @@ #include <cassert> #include <cstdint> #include <iterator> +#include <numeric> +#include <type_traits> #include <utility> using namespace llvm; @@ -62,6 +64,42 @@ AnyAlloc = AllocLike | ReallocLike }; +enum class MallocFamily { + Malloc, + CPPNew, // new(unsigned int) + CPPNewAligned, // new(unsigned int, align_val_t) + CPPNewArray, // new[](unsigned int) + CPPNewArrayAligned, // new[](unsigned long, align_val_t) + MSVCNew, // new(unsigned int) + MSVCArrayNew, // new[](unsigned int) + VecMalloc, + KmpcAllocShared, +}; + +StringRef mangledNameForMallocFamily(const MallocFamily &Family) { + switch (Family) { + case MallocFamily::Malloc: + return "malloc"; + case MallocFamily::CPPNew: + return "_Znwm"; + case MallocFamily::CPPNewAligned: + return "_ZnwmSt11align_val_t"; + case MallocFamily::CPPNewArray: + return "_Znam"; + case MallocFamily::CPPNewArrayAligned: + return "_ZnamSt11align_val_t"; + case MallocFamily::MSVCNew: + return "??2@YAPAXI@Z"; + case MallocFamily::MSVCArrayNew: + return "??_U@YAPAXI@Z"; + case MallocFamily::VecMalloc: + return "vec_malloc"; + case MallocFamily::KmpcAllocShared: + return "__kmpc_alloc_shared"; + } + llvm_unreachable("missing an alloc family"); +} + struct AllocFnsTy { AllocType AllocTy; unsigned NumParams; @@ -69,53 +107,49 @@ int FstParam, SndParam; // Alignment parameter for aligned_alloc and aligned new int AlignParam; + // Name of default allocator function to group malloc/free calls by family + MallocFamily Family; }; +// clang-format off // FIXME: certain users need more information. E.g., SimplifyLibCalls needs to // know which functions are nounwind, noalias, nocapture parameters, etc. static const std::pair<LibFunc, AllocFnsTy> AllocationFnData[] = { - {LibFunc_malloc, {MallocLike, 1, 0, -1, -1}}, - {LibFunc_vec_malloc, {MallocLike, 1, 0, -1, -1}}, - {LibFunc_valloc, {MallocLike, 1, 0, -1, -1}}, - {LibFunc_Znwj, {OpNewLike, 1, 0, -1, -1}}, // new(unsigned int) - {LibFunc_ZnwjRKSt9nothrow_t, {MallocLike, 2, 0, -1, -1}}, // new(unsigned int, nothrow) - {LibFunc_ZnwjSt11align_val_t, {OpNewLike, 2, 0, -1, 1}}, // new(unsigned int, align_val_t) - {LibFunc_ZnwjSt11align_val_tRKSt9nothrow_t, {MallocLike, 3, 0, -1, 1}}, // new(unsigned int, align_val_t, nothrow) - {LibFunc_Znwm, {OpNewLike, 1, 0, -1, -1}}, // new(unsigned long) - {LibFunc_ZnwmRKSt9nothrow_t, {MallocLike, 2, 0, -1, -1}}, // new(unsigned long, nothrow) - {LibFunc_ZnwmSt11align_val_t, {OpNewLike, 2, 0, -1, 1}}, // new(unsigned long, align_val_t) - {LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t, {MallocLike, 3, 0, -1, 1}}, // new(unsigned long, align_val_t, nothrow) - {LibFunc_Znaj, {OpNewLike, 1, 0, -1, -1}}, // new[](unsigned int) - {LibFunc_ZnajRKSt9nothrow_t, {MallocLike, 2, 0, -1, -1}}, // new[](unsigned int, nothrow) - {LibFunc_ZnajSt11align_val_t, {OpNewLike, 2, 0, -1, 1}}, // new[](unsigned int, align_val_t) - {LibFunc_ZnajSt11align_val_tRKSt9nothrow_t, {MallocLike, 3, 0, -1, 1}}, // new[](unsigned int, align_val_t, nothrow) - {LibFunc_Znam, {OpNewLike, 1, 0, -1, -1}}, // new[](unsigned long) - {LibFunc_ZnamRKSt9nothrow_t, {MallocLike, 2, 0, -1, -1}}, // new[](unsigned long, nothrow) - {LibFunc_ZnamSt11align_val_t, {OpNewLike, 2, 0, -1, 1}}, // new[](unsigned long, align_val_t) - {LibFunc_ZnamSt11align_val_tRKSt9nothrow_t, {MallocLike, 3, 0, -1, 1}}, // new[](unsigned long, align_val_t, nothrow) - {LibFunc_msvc_new_int, {OpNewLike, 1, 0, -1, -1}}, // new(unsigned int) - {LibFunc_msvc_new_int_nothrow, {MallocLike, 2, 0, -1, -1}}, // new(unsigned int, nothrow) - {LibFunc_msvc_new_longlong, {OpNewLike, 1, 0, -1, -1}}, // new(unsigned long long) - {LibFunc_msvc_new_longlong_nothrow, {MallocLike, 2, 0, -1, -1}}, // new(unsigned long long, nothrow) - {LibFunc_msvc_new_array_int, {OpNewLike, 1, 0, -1, -1}}, // new[](unsigned int) - {LibFunc_msvc_new_array_int_nothrow, {MallocLike, 2, 0, -1, -1}}, // new[](unsigned int, nothrow) - {LibFunc_msvc_new_array_longlong, {OpNewLike, 1, 0, -1, -1}}, // new[](unsigned long long) - {LibFunc_msvc_new_array_longlong_nothrow, {MallocLike, 2, 0, -1, -1}}, // new[](unsigned long long, nothrow) - {LibFunc_aligned_alloc, {AlignedAllocLike, 2, 1, -1, 0}}, - {LibFunc_memalign, {AlignedAllocLike, 2, 1, -1, 0}}, - {LibFunc_calloc, {CallocLike, 2, 0, 1, -1}}, - {LibFunc_vec_calloc, {CallocLike, 2, 0, 1, -1}}, - {LibFunc_realloc, {ReallocLike, 2, 1, -1, -1}}, - {LibFunc_vec_realloc, {ReallocLike, 2, 1, -1, -1}}, - {LibFunc_reallocf, {ReallocLike, 2, 1, -1, -1}}, - {LibFunc_strdup, {StrDupLike, 1, -1, -1, -1}}, - {LibFunc_strndup, {StrDupLike, 2, 1, -1, -1}}, - {LibFunc___kmpc_alloc_shared, {MallocLike, 1, 0, -1, -1}}, - {LibFunc_rust_alloc, {MallocLike, 2, 0, -1, 1}}, - {LibFunc_rust_alloc_zeroed, {CallocLike, 2, 0, -1, 1}}, - {LibFunc_rust_realloc, {ReallocLike, 4, 3, -1, 2}}, - // TODO: Handle "int posix_memalign(void **, size_t, size_t)" + {LibFunc_vec_malloc, {MallocLike, 1, 0, -1, -1, MallocFamily::VecMalloc}}, + {LibFunc_Znwj, {OpNewLike, 1, 0, -1, -1, MallocFamily::CPPNew}}, // new(unsigned int) + {LibFunc_ZnwjRKSt9nothrow_t, {MallocLike, 2, 0, -1, -1, MallocFamily::CPPNew}}, // new(unsigned int, nothrow) + {LibFunc_ZnwjSt11align_val_t, {OpNewLike, 2, 0, -1, 1, MallocFamily::CPPNewAligned}}, // new(unsigned int, align_val_t) + {LibFunc_ZnwjSt11align_val_tRKSt9nothrow_t, {MallocLike, 3, 0, -1, 1, MallocFamily::CPPNewAligned}}, // new(unsigned int, align_val_t, nothrow) + {LibFunc_Znwm, {OpNewLike, 1, 0, -1, -1, MallocFamily::CPPNew}}, // new(unsigned long) + {LibFunc_ZnwmRKSt9nothrow_t, {MallocLike, 2, 0, -1, -1, MallocFamily::CPPNew}}, // new(unsigned long, nothrow) + {LibFunc_ZnwmSt11align_val_t, {OpNewLike, 2, 0, -1, 1, MallocFamily::CPPNewAligned}}, // new(unsigned long, align_val_t) + {LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t, {MallocLike, 3, 0, -1, 1, MallocFamily::CPPNewAligned}}, // new(unsigned long, align_val_t, nothrow) + {LibFunc_Znaj, {OpNewLike, 1, 0, -1, -1, MallocFamily::CPPNewArray}}, // new[](unsigned int) + {LibFunc_ZnajRKSt9nothrow_t, {MallocLike, 2, 0, -1, -1, MallocFamily::CPPNewArray}}, // new[](unsigned int, nothrow) + {LibFunc_ZnajSt11align_val_t, {OpNewLike, 2, 0, -1, 1, MallocFamily::CPPNewArrayAligned}}, // new[](unsigned int, align_val_t) + {LibFunc_ZnajSt11align_val_tRKSt9nothrow_t, {MallocLike, 3, 0, -1, 1, MallocFamily::CPPNewArrayAligned}}, // new[](unsigned int, align_val_t, nothrow) + {LibFunc_Znam, {OpNewLike, 1, 0, -1, -1, MallocFamily::CPPNewArray}}, // new[](unsigned long) + {LibFunc_ZnamRKSt9nothrow_t, {MallocLike, 2, 0, -1, -1, MallocFamily::CPPNewArray}}, // new[](unsigned long, nothrow) + {LibFunc_ZnamSt11align_val_t, {OpNewLike, 2, 0, -1, 1, MallocFamily::CPPNewArrayAligned}}, // new[](unsigned long, align_val_t) + {LibFunc_ZnamSt11align_val_tRKSt9nothrow_t, {MallocLike, 3, 0, -1, 1, MallocFamily::CPPNewArrayAligned}}, // new[](unsigned long, align_val_t, nothrow) + {LibFunc_msvc_new_int, {OpNewLike, 1, 0, -1, -1, MallocFamily::MSVCNew}}, // new(unsigned int) + {LibFunc_msvc_new_int_nothrow, {MallocLike, 2, 0, -1, -1, MallocFamily::MSVCNew}}, // new(unsigned int, nothrow) + {LibFunc_msvc_new_longlong, {OpNewLike, 1, 0, -1, -1, MallocFamily::MSVCNew}}, // new(unsigned long long) + {LibFunc_msvc_new_longlong_nothrow, {MallocLike, 2, 0, -1, -1, MallocFamily::MSVCNew}}, // new(unsigned long long, nothrow) + {LibFunc_msvc_new_array_int, {OpNewLike, 1, 0, -1, -1, MallocFamily::MSVCArrayNew}}, // new[](unsigned int) + {LibFunc_msvc_new_array_int_nothrow, {MallocLike, 2, 0, -1, -1, MallocFamily::MSVCArrayNew}}, // new[](unsigned int, nothrow) + {LibFunc_msvc_new_array_longlong, {OpNewLike, 1, 0, -1, -1, MallocFamily::MSVCArrayNew}}, // new[](unsigned long long) + {LibFunc_msvc_new_array_longlong_nothrow, {MallocLike, 2, 0, -1, -1, MallocFamily::MSVCArrayNew}}, // new[](unsigned long long, nothrow) + {LibFunc_memalign, {AlignedAllocLike, 2, 1, -1, 0, MallocFamily::Malloc}}, + {LibFunc_vec_calloc, {CallocLike, 2, 0, 1, -1, MallocFamily::VecMalloc}}, + {LibFunc_vec_realloc, {ReallocLike, 2, 1, -1, -1, MallocFamily::VecMalloc}}, + {LibFunc_strdup, {StrDupLike, 1, -1, -1, -1, MallocFamily::Malloc}}, + {LibFunc_dunder_strdup, {StrDupLike, 1, -1, -1, -1, MallocFamily::Malloc}}, + {LibFunc_strndup, {StrDupLike, 2, 1, -1, -1, MallocFamily::Malloc}}, + {LibFunc_dunder_strndup, {StrDupLike, 2, 1, -1, -1, MallocFamily::Malloc}}, + {LibFunc___kmpc_alloc_shared, {MallocLike, 1, 0, -1, -1, MallocFamily::KmpcAllocShared}}, }; +// clang-format on static const Function *getCalledFunction(const Value *V, bool &IsNoBuiltin) { @@ -139,6 +173,11 @@ static Optional<AllocFnsTy> getAllocationDataForFunction(const Function *Callee, AllocType AllocTy, const TargetLibraryInfo *TLI) { + // Don't perform a slow TLI lookup, if this function doesn't return a pointer + // and thus can't be an allocation function. + if (!Callee->getReturnType()->isPointerTy()) + return None; + // Make sure that the function is available. LibFunc TLIFn; if (!TLI || !TLI->getLibFunc(*Callee, TLIFn) || !TLI->has(TLIFn)) @@ -220,87 +259,117 @@ Result.AllocTy = MallocLike; Result.NumParams = Callee->getNumOperands(); Result.FstParam = Args.first; - Result.SndParam = Args.second.getValueOr(-1); + Result.SndParam = Args.second.value_or(-1); // Allocsize has no way to specify an alignment argument Result.AlignParam = -1; return Result; } +static AllocFnKind getAllocFnKind(const Value *V) { + if (const auto *CB = dyn_cast<CallBase>(V)) { + Attribute Attr = CB->getFnAttr(Attribute::AllocKind); + if (Attr.isValid()) + return AllocFnKind(Attr.getValueAsInt()); + } + return AllocFnKind::Unknown; +} + +static AllocFnKind getAllocFnKind(const Function *F) { + Attribute Attr = F->getFnAttribute(Attribute::AllocKind); + if (Attr.isValid()) + return AllocFnKind(Attr.getValueAsInt()); + return AllocFnKind::Unknown; +} + +static bool checkFnAllocKind(const Value *V, AllocFnKind Wanted) { + return (getAllocFnKind(V) & Wanted) != AllocFnKind::Unknown; +} + +static bool checkFnAllocKind(const Function *F, AllocFnKind Wanted) { + return (getAllocFnKind(F) & Wanted) != AllocFnKind::Unknown; +} + /// Tests if a value is a call or invoke to a library function that /// allocates or reallocates memory (either malloc, calloc, realloc, or strdup /// like). bool llvm::isAllocationFn(const Value *V, const TargetLibraryInfo *TLI) { - return getAllocationData(V, AnyAlloc, TLI).hasValue(); + return getAllocationData(V, AnyAlloc, TLI).has_value() || + checkFnAllocKind(V, AllocFnKind::Alloc | AllocFnKind::Realloc); } bool llvm::isAllocationFn( - const Value *V, function_ref<const TargetLibraryInfo &(Function &)> GetTLI) { - return getAllocationData(V, AnyAlloc, GetTLI).hasValue(); + const Value *V, + function_ref<const TargetLibraryInfo &(Function &)> GetTLI) { + return getAllocationData(V, AnyAlloc, GetTLI).has_value() || + checkFnAllocKind(V, AllocFnKind::Alloc | AllocFnKind::Realloc); } /// Tests if a value is a call or invoke to a library function that /// allocates uninitialized memory (such as malloc). static bool isMallocLikeFn(const Value *V, const TargetLibraryInfo *TLI) { - return getAllocationData(V, MallocOrOpNewLike, TLI).hasValue(); + return getAllocationData(V, MallocOrOpNewLike, TLI).has_value(); } /// Tests if a value is a call or invoke to a library function that /// allocates uninitialized memory with alignment (such as aligned_alloc). static bool isAlignedAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI) { - return getAllocationData(V, AlignedAllocLike, TLI) - .hasValue(); + return getAllocationData(V, AlignedAllocLike, TLI).has_value(); } /// Tests if a value is a call or invoke to a library function that /// allocates zero-filled memory (such as calloc). static bool isCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI) { - return getAllocationData(V, CallocLike, TLI).hasValue(); + return getAllocationData(V, CallocLike, TLI).has_value(); } /// Tests if a value is a call or invoke to a library function that /// allocates memory similar to malloc or calloc. bool llvm::isMallocOrCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI) { - return getAllocationData(V, MallocOrCallocLike, TLI).hasValue(); + return getAllocationData(V, MallocOrCallocLike, TLI).has_value(); } /// Tests if a value is a call or invoke to a library function that /// allocates memory (either malloc, calloc, or strdup like). bool llvm::isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI) { - return getAllocationData(V, AllocLike, TLI).hasValue(); -} - -/// Tests if a value is a call or invoke to a library function that -/// reallocates memory (e.g., realloc). -bool llvm::isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI) { - return getAllocationData(V, ReallocLike, TLI).hasValue(); + return getAllocationData(V, AllocLike, TLI).has_value() || + checkFnAllocKind(V, AllocFnKind::Alloc); } /// Tests if a functions is a call or invoke to a library function that /// reallocates memory (e.g., realloc). bool llvm::isReallocLikeFn(const Function *F, const TargetLibraryInfo *TLI) { - return getAllocationDataForFunction(F, ReallocLike, TLI).hasValue(); + return getAllocationDataForFunction(F, ReallocLike, TLI).has_value() || + checkFnAllocKind(F, AllocFnKind::Realloc); } -bool llvm::isAllocRemovable(const CallBase *CB, const TargetLibraryInfo *TLI) { - assert(isAllocationFn(CB, TLI)); +Value *llvm::getReallocatedOperand(const CallBase *CB, + const TargetLibraryInfo *TLI) { + if (getAllocationData(CB, ReallocLike, TLI).has_value()) { + // All currently supported realloc functions reallocate the first argument. + return CB->getArgOperand(0); + } + if (checkFnAllocKind(CB, AllocFnKind::Realloc)) + return CB->getArgOperandWithAttribute(Attribute::AllocatedPointer); + return nullptr; +} +bool llvm::isRemovableAlloc(const CallBase *CB, const TargetLibraryInfo *TLI) { // Note: Removability is highly dependent on the source language. For // example, recent C++ requires direct calls to the global allocation // [basic.stc.dynamic.allocation] to be observable unless part of a new // expression [expr.new paragraph 13]. - // Historically we've treated the C family allocation routines as removable + // Historically we've treated the C family allocation routines and operator + // new as removable return isAllocLikeFn(CB, TLI); } Value *llvm::getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI) { - assert(isAllocationFn(V, TLI)); - const Optional<AllocFnsTy> FnData = getAllocationData(V, AnyAlloc, TLI); - if (!FnData.hasValue() || FnData->AlignParam < 0) { - return nullptr; + if (FnData && FnData->AlignParam >= 0) { + return V->getOperand(FnData->AlignParam); } - return V->getOperand(FnData->AlignParam); + return V->getArgOperandWithAttribute(Attribute::AllocAlign); } /// When we're compiling N-bit code, and the user uses parameters that are @@ -320,9 +389,8 @@ } Optional<APInt> -llvm::getAllocSize(const CallBase *CB, - const TargetLibraryInfo *TLI, - std::function<const Value*(const Value*)> Mapper) { +llvm::getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI, + function_ref<const Value *(const Value *)> Mapper) { // Note: This handles both explicitly listed allocation functions and // allocsize. The code structure could stand to be cleaned up a bit. Optional<AllocFnsTy> FnData = getAllocationSize(CB, TLI); @@ -347,7 +415,7 @@ if (!Arg) return None; - APInt MaxSize = Arg->getValue().zextOrSelf(IntTyBits); + APInt MaxSize = Arg->getValue().zext(IntTyBits); if (Size.ugt(MaxSize)) Size = MaxSize + 1; } @@ -382,10 +450,12 @@ return Size; } -Constant *llvm::getInitialValueOfAllocation(const CallBase *Alloc, +Constant *llvm::getInitialValueOfAllocation(const Value *V, const TargetLibraryInfo *TLI, Type *Ty) { - assert(isAllocationFn(Alloc, TLI)); + auto *Alloc = dyn_cast<CallBase>(V); + if (!Alloc) + return nullptr; // malloc and aligned_alloc are uninitialized (undef) if (isMallocLikeFn(Alloc, TLI) || isAlignedAllocLikeFn(Alloc, TLI)) @@ -395,48 +465,99 @@ if (isCallocLikeFn(Alloc, TLI)) return Constant::getNullValue(Ty); + AllocFnKind AK = getAllocFnKind(Alloc); + if ((AK & AllocFnKind::Uninitialized) != AllocFnKind::Unknown) + return UndefValue::get(Ty); + if ((AK & AllocFnKind::Zeroed) != AllocFnKind::Unknown) + return Constant::getNullValue(Ty); + return nullptr; } +struct FreeFnsTy { + unsigned NumParams; + // Name of default allocator function to group malloc/free calls by family + MallocFamily Family; +}; + +// clang-format off +static const std::pair<LibFunc, FreeFnsTy> FreeFnData[] = { + {LibFunc_vec_free, {1, MallocFamily::VecMalloc}}, + {LibFunc_ZdlPv, {1, MallocFamily::CPPNew}}, // operator delete(void*) + {LibFunc_ZdaPv, {1, MallocFamily::CPPNewArray}}, // operator delete[](void*) + {LibFunc_msvc_delete_ptr32, {1, MallocFamily::MSVCNew}}, // operator delete(void*) + {LibFunc_msvc_delete_ptr64, {1, MallocFamily::MSVCNew}}, // operator delete(void*) + {LibFunc_msvc_delete_array_ptr32, {1, MallocFamily::MSVCArrayNew}}, // operator delete[](void*) + {LibFunc_msvc_delete_array_ptr64, {1, MallocFamily::MSVCArrayNew}}, // operator delete[](void*) + {LibFunc_ZdlPvj, {2, MallocFamily::CPPNew}}, // delete(void*, uint) + {LibFunc_ZdlPvm, {2, MallocFamily::CPPNew}}, // delete(void*, ulong) + {LibFunc_ZdlPvRKSt9nothrow_t, {2, MallocFamily::CPPNew}}, // delete(void*, nothrow) + {LibFunc_ZdlPvSt11align_val_t, {2, MallocFamily::CPPNewAligned}}, // delete(void*, align_val_t) + {LibFunc_ZdaPvj, {2, MallocFamily::CPPNewArray}}, // delete[](void*, uint) + {LibFunc_ZdaPvm, {2, MallocFamily::CPPNewArray}}, // delete[](void*, ulong) + {LibFunc_ZdaPvRKSt9nothrow_t, {2, MallocFamily::CPPNewArray}}, // delete[](void*, nothrow) + {LibFunc_ZdaPvSt11align_val_t, {2, MallocFamily::CPPNewArrayAligned}}, // delete[](void*, align_val_t) + {LibFunc_msvc_delete_ptr32_int, {2, MallocFamily::MSVCNew}}, // delete(void*, uint) + {LibFunc_msvc_delete_ptr64_longlong, {2, MallocFamily::MSVCNew}}, // delete(void*, ulonglong) + {LibFunc_msvc_delete_ptr32_nothrow, {2, MallocFamily::MSVCNew}}, // delete(void*, nothrow) + {LibFunc_msvc_delete_ptr64_nothrow, {2, MallocFamily::MSVCNew}}, // delete(void*, nothrow) + {LibFunc_msvc_delete_array_ptr32_int, {2, MallocFamily::MSVCArrayNew}}, // delete[](void*, uint) + {LibFunc_msvc_delete_array_ptr64_longlong, {2, MallocFamily::MSVCArrayNew}}, // delete[](void*, ulonglong) + {LibFunc_msvc_delete_array_ptr32_nothrow, {2, MallocFamily::MSVCArrayNew}}, // delete[](void*, nothrow) + {LibFunc_msvc_delete_array_ptr64_nothrow, {2, MallocFamily::MSVCArrayNew}}, // delete[](void*, nothrow) + {LibFunc___kmpc_free_shared, {2, MallocFamily::KmpcAllocShared}}, // OpenMP Offloading RTL free + {LibFunc_ZdlPvSt11align_val_tRKSt9nothrow_t, {3, MallocFamily::CPPNewAligned}}, // delete(void*, align_val_t, nothrow) + {LibFunc_ZdaPvSt11align_val_tRKSt9nothrow_t, {3, MallocFamily::CPPNewArrayAligned}}, // delete[](void*, align_val_t, nothrow) + {LibFunc_ZdlPvjSt11align_val_t, {3, MallocFamily::CPPNewAligned}}, // delete(void*, unsigned int, align_val_t) + {LibFunc_ZdlPvmSt11align_val_t, {3, MallocFamily::CPPNewAligned}}, // delete(void*, unsigned long, align_val_t) + {LibFunc_ZdaPvjSt11align_val_t, {3, MallocFamily::CPPNewArrayAligned}}, // delete[](void*, unsigned int, align_val_t) + {LibFunc_ZdaPvmSt11align_val_t, {3, MallocFamily::CPPNewArrayAligned}}, // delete[](void*, unsigned long, align_val_t) +}; +// clang-format on + +Optional<FreeFnsTy> getFreeFunctionDataForFunction(const Function *Callee, + const LibFunc TLIFn) { + const auto *Iter = + find_if(FreeFnData, [TLIFn](const std::pair<LibFunc, FreeFnsTy> &P) { + return P.first == TLIFn; + }); + if (Iter == std::end(FreeFnData)) + return None; + return Iter->second; +} + +Optional<StringRef> llvm::getAllocationFamily(const Value *I, + const TargetLibraryInfo *TLI) { + bool IsNoBuiltin; + const Function *Callee = getCalledFunction(I, IsNoBuiltin); + if (Callee == nullptr || IsNoBuiltin) + return None; + LibFunc TLIFn; + + if (TLI && TLI->getLibFunc(*Callee, TLIFn) && TLI->has(TLIFn)) { + // Callee is some known library function. + const auto AllocData = getAllocationDataForFunction(Callee, AnyAlloc, TLI); + if (AllocData) + return mangledNameForMallocFamily(AllocData.value().Family); + const auto FreeData = getFreeFunctionDataForFunction(Callee, TLIFn); + if (FreeData) + return mangledNameForMallocFamily(FreeData.value().Family); + } + // Callee isn't a known library function, still check attributes. + if (checkFnAllocKind(I, AllocFnKind::Free | AllocFnKind::Alloc | + AllocFnKind::Realloc)) { + Attribute Attr = cast<CallBase>(I)->getFnAttr("alloc-family"); + if (Attr.isValid()) + return Attr.getValueAsString(); + } + return None; +} + /// isLibFreeFunction - Returns true if the function is a builtin free() bool llvm::isLibFreeFunction(const Function *F, const LibFunc TLIFn) { - unsigned ExpectedNumParams; - if (TLIFn == LibFunc_free || - TLIFn == LibFunc_ZdlPv || // operator delete(void*) - TLIFn == LibFunc_ZdaPv || // operator delete[](void*) - TLIFn == LibFunc_msvc_delete_ptr32 || // operator delete(void*) - TLIFn == LibFunc_msvc_delete_ptr64 || // operator delete(void*) - TLIFn == LibFunc_msvc_delete_array_ptr32 || // operator delete[](void*) - TLIFn == LibFunc_msvc_delete_array_ptr64) // operator delete[](void*) - ExpectedNumParams = 1; - else if (TLIFn == LibFunc_ZdlPvj || // delete(void*, uint) - TLIFn == LibFunc_ZdlPvm || // delete(void*, ulong) - TLIFn == LibFunc_ZdlPvRKSt9nothrow_t || // delete(void*, nothrow) - TLIFn == LibFunc_ZdlPvSt11align_val_t || // delete(void*, align_val_t) - TLIFn == LibFunc_ZdaPvj || // delete[](void*, uint) - TLIFn == LibFunc_ZdaPvm || // delete[](void*, ulong) - TLIFn == LibFunc_ZdaPvRKSt9nothrow_t || // delete[](void*, nothrow) - TLIFn == LibFunc_ZdaPvSt11align_val_t || // delete[](void*, align_val_t) - TLIFn == LibFunc_msvc_delete_ptr32_int || // delete(void*, uint) - TLIFn == LibFunc_msvc_delete_ptr64_longlong || // delete(void*, ulonglong) - TLIFn == LibFunc_msvc_delete_ptr32_nothrow || // delete(void*, nothrow) - TLIFn == LibFunc_msvc_delete_ptr64_nothrow || // delete(void*, nothrow) - TLIFn == LibFunc_msvc_delete_array_ptr32_int || // delete[](void*, uint) - TLIFn == LibFunc_msvc_delete_array_ptr64_longlong || // delete[](void*, ulonglong) - TLIFn == LibFunc_msvc_delete_array_ptr32_nothrow || // delete[](void*, nothrow) - TLIFn == LibFunc_msvc_delete_array_ptr64_nothrow || // delete[](void*, nothrow) - TLIFn == LibFunc___kmpc_free_shared) // OpenMP Offloading RTL free - ExpectedNumParams = 2; - else if (TLIFn == LibFunc_ZdaPvSt11align_val_tRKSt9nothrow_t || // delete(void*, align_val_t, nothrow) - TLIFn == LibFunc_ZdlPvSt11align_val_tRKSt9nothrow_t || // delete[](void*, align_val_t, nothrow) - TLIFn == LibFunc_ZdlPvjSt11align_val_t || // delete(void*, unsigned long, align_val_t) - TLIFn == LibFunc_ZdlPvmSt11align_val_t || // delete(void*, unsigned long, align_val_t) - TLIFn == LibFunc_ZdaPvjSt11align_val_t || // delete[](void*, unsigned int, align_val_t) - TLIFn == LibFunc_ZdaPvmSt11align_val_t || // delete[](void*, unsigned long, align_val_t) - TLIFn == LibFunc_rust_dealloc) - ExpectedNumParams = 3; - else - return false; + Optional<FreeFnsTy> FnData = getFreeFunctionDataForFunction(F, TLIFn); + if (!FnData) + return checkFnAllocKind(F, AllocFnKind::Free); // Check free prototype. // FIXME: workaround for PR5130, this will be obsolete when a nobuiltin @@ -444,7 +565,7 @@ FunctionType *FTy = F->getFunctionType(); if (!FTy->getReturnType()->isVoidTy()) return false; - if (FTy->getNumParams() != ExpectedNumParams) + if (FTy->getNumParams() != FnData->NumParams) return false; if (FTy->getParamType(0) != Type::getInt8PtrTy(F->getContext())) return false; @@ -452,21 +573,25 @@ return true; } -/// isFreeCall - Returns non-null if the value is a call to the builtin free() -const CallInst *llvm::isFreeCall(const Value *I, const TargetLibraryInfo *TLI) { +Value *llvm::getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI) { bool IsNoBuiltinCall; - const Function *Callee = getCalledFunction(I, IsNoBuiltinCall); + const Function *Callee = getCalledFunction(CB, IsNoBuiltinCall); if (Callee == nullptr || IsNoBuiltinCall) return nullptr; LibFunc TLIFn; - if (!TLI || !TLI->getLibFunc(*Callee, TLIFn) || !TLI->has(TLIFn)) - return nullptr; + if (TLI && TLI->getLibFunc(*Callee, TLIFn) && TLI->has(TLIFn) && + isLibFreeFunction(Callee, TLIFn)) { + // All currently supported free functions free the first argument. + return CB->getArgOperand(0); + } - return isLibFreeFunction(Callee, TLIFn) ? dyn_cast<CallInst>(I) : nullptr; + if (checkFnAllocKind(CB, AllocFnKind::Free)) + return CB->getArgOperandWithAttribute(Attribute::AllocatedPointer); + + return nullptr; } - //===----------------------------------------------------------------------===// // Utility functions to compute size of objects. // @@ -495,11 +620,21 @@ const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed) { + return lowerObjectSizeCall(ObjectSize, DL, TLI, /*AAResults=*/nullptr, + MustSucceed); +} + +Value *llvm::lowerObjectSizeCall(IntrinsicInst *ObjectSize, + const DataLayout &DL, + const TargetLibraryInfo *TLI, AAResults *AA, + bool MustSucceed) { assert(ObjectSize->getIntrinsicID() == Intrinsic::objectsize && "ObjectSize must be a call to llvm.objectsize!"); bool MaxVal = cast<ConstantInt>(ObjectSize->getArgOperand(1))->isZero(); ObjectSizeOpts EvalOptions; + EvalOptions.AA = AA; + // Unless we have to fold this to something, try to be as accurate as // possible. if (MustSucceed) @@ -563,7 +698,7 @@ APInt ObjectSizeOffsetVisitor::align(APInt Size, MaybeAlign Alignment) { if (Options.RoundToAlign && Alignment) - return APInt(IntTyBits, alignTo(Size.getZExtValue(), Alignment)); + return APInt(IntTyBits, alignTo(Size.getZExtValue(), *Alignment)); return Size; } @@ -645,10 +780,10 @@ if (!I.getAllocatedType()->isSized()) return unknown(); - if (isa<ScalableVectorType>(I.getAllocatedType())) + TypeSize ElemSize = DL.getTypeAllocSize(I.getAllocatedType()); + if (ElemSize.isScalable() && Options.EvalMode != ObjectSizeOpts::Mode::Min) return unknown(); - - APInt Size(IntTyBits, DL.getTypeAllocSize(I.getAllocatedType())); + APInt Size(IntTyBits, ElemSize.getKnownMinSize()); if (!I.isArrayAllocation()) return std::make_pair(align(Size, I.getAlign()), Zero); @@ -679,8 +814,7 @@ } SizeOffsetType ObjectSizeOffsetVisitor::visitCallBase(CallBase &CB) { - auto Mapper = [](const Value *V) { return V; }; - if (Optional<APInt> Size = getAllocSize(&CB, TLI, Mapper)) + if (Optional<APInt> Size = getAllocSize(&CB, TLI)) return std::make_pair(*Size, Zero); return unknown(); } @@ -729,42 +863,161 @@ return unknown(); } -SizeOffsetType ObjectSizeOffsetVisitor::visitLoadInst(LoadInst&) { - ++ObjectVisitorLoad; - return unknown(); +SizeOffsetType ObjectSizeOffsetVisitor::findLoadSizeOffset( + LoadInst &Load, BasicBlock &BB, BasicBlock::iterator From, + SmallDenseMap<BasicBlock *, SizeOffsetType, 8> &VisitedBlocks, + unsigned &ScannedInstCount) { + constexpr unsigned MaxInstsToScan = 128; + + auto Where = VisitedBlocks.find(&BB); + if (Where != VisitedBlocks.end()) + return Where->second; + + auto Unknown = [this, &BB, &VisitedBlocks]() { + return VisitedBlocks[&BB] = unknown(); + }; + auto Known = [&BB, &VisitedBlocks](SizeOffsetType SO) { + return VisitedBlocks[&BB] = SO; + }; + + do { + Instruction &I = *From; + + if (I.isDebugOrPseudoInst()) + continue; + + if (++ScannedInstCount > MaxInstsToScan) + return Unknown(); + + if (!I.mayWriteToMemory()) + continue; + + if (auto *SI = dyn_cast<StoreInst>(&I)) { + AliasResult AR = + Options.AA->alias(SI->getPointerOperand(), Load.getPointerOperand()); + switch ((AliasResult::Kind)AR) { + case AliasResult::NoAlias: + continue; + case AliasResult::MustAlias: + if (SI->getValueOperand()->getType()->isPointerTy()) + return Known(compute(SI->getValueOperand())); + else + return Unknown(); // No handling of non-pointer values by `compute`. + default: + return Unknown(); + } + } + + if (auto *CB = dyn_cast<CallBase>(&I)) { + Function *Callee = CB->getCalledFunction(); + // Bail out on indirect call. + if (!Callee) + return Unknown(); + + LibFunc TLIFn; + if (!TLI || !TLI->getLibFunc(*CB->getCalledFunction(), TLIFn) || + !TLI->has(TLIFn)) + return Unknown(); + + // TODO: There's probably more interesting case to support here. + if (TLIFn != LibFunc_posix_memalign) + return Unknown(); + + AliasResult AR = + Options.AA->alias(CB->getOperand(0), Load.getPointerOperand()); + switch ((AliasResult::Kind)AR) { + case AliasResult::NoAlias: + continue; + case AliasResult::MustAlias: + break; + default: + return Unknown(); + } + + // Is the error status of posix_memalign correctly checked? If not it + // would be incorrect to assume it succeeds and load doesn't see the + // previous value. + Optional<bool> Checked = isImpliedByDomCondition( + ICmpInst::ICMP_EQ, CB, ConstantInt::get(CB->getType(), 0), &Load, DL); + if (!Checked || !*Checked) + return Unknown(); + + Value *Size = CB->getOperand(2); + auto *C = dyn_cast<ConstantInt>(Size); + if (!C) + return Unknown(); + + return Known({C->getValue(), APInt(C->getValue().getBitWidth(), 0)}); + } + + return Unknown(); + } while (From-- != BB.begin()); + + SmallVector<SizeOffsetType> PredecessorSizeOffsets; + for (auto *PredBB : predecessors(&BB)) { + PredecessorSizeOffsets.push_back(findLoadSizeOffset( + Load, *PredBB, BasicBlock::iterator(PredBB->getTerminator()), + VisitedBlocks, ScannedInstCount)); + if (!bothKnown(PredecessorSizeOffsets.back())) + return Unknown(); + } + + if (PredecessorSizeOffsets.empty()) + return Unknown(); + + return Known(std::accumulate(PredecessorSizeOffsets.begin() + 1, + PredecessorSizeOffsets.end(), + PredecessorSizeOffsets.front(), + [this](SizeOffsetType LHS, SizeOffsetType RHS) { + return combineSizeOffset(LHS, RHS); + })); } -SizeOffsetType ObjectSizeOffsetVisitor::visitPHINode(PHINode&) { - // too complex to analyze statically. - return unknown(); +SizeOffsetType ObjectSizeOffsetVisitor::visitLoadInst(LoadInst &LI) { + if (!Options.AA) { + ++ObjectVisitorLoad; + return unknown(); + } + + SmallDenseMap<BasicBlock *, SizeOffsetType, 8> VisitedBlocks; + unsigned ScannedInstCount = 0; + SizeOffsetType SO = + findLoadSizeOffset(LI, *LI.getParent(), BasicBlock::iterator(LI), + VisitedBlocks, ScannedInstCount); + if (!bothKnown(SO)) + ++ObjectVisitorLoad; + return SO; +} + +SizeOffsetType ObjectSizeOffsetVisitor::combineSizeOffset(SizeOffsetType LHS, + SizeOffsetType RHS) { + if (!bothKnown(LHS) || !bothKnown(RHS)) + return unknown(); + + switch (Options.EvalMode) { + case ObjectSizeOpts::Mode::Min: + return (getSizeWithOverflow(LHS).slt(getSizeWithOverflow(RHS))) ? LHS : RHS; + case ObjectSizeOpts::Mode::Max: + return (getSizeWithOverflow(LHS).sgt(getSizeWithOverflow(RHS))) ? LHS : RHS; + case ObjectSizeOpts::Mode::Exact: + return (getSizeWithOverflow(LHS).eq(getSizeWithOverflow(RHS))) ? LHS + : unknown(); + } + llvm_unreachable("missing an eval mode"); +} + +SizeOffsetType ObjectSizeOffsetVisitor::visitPHINode(PHINode &PN) { + auto IncomingValues = PN.incoming_values(); + return std::accumulate(IncomingValues.begin() + 1, IncomingValues.end(), + compute(*IncomingValues.begin()), + [this](SizeOffsetType LHS, Value *VRHS) { + return combineSizeOffset(LHS, compute(VRHS)); + }); } SizeOffsetType ObjectSizeOffsetVisitor::visitSelectInst(SelectInst &I) { - SizeOffsetType TrueSide = compute(I.getTrueValue()); - SizeOffsetType FalseSide = compute(I.getFalseValue()); - if (bothKnown(TrueSide) && bothKnown(FalseSide)) { - if (TrueSide == FalseSide) { - return TrueSide; - } - - APInt TrueResult = getSizeWithOverflow(TrueSide); - APInt FalseResult = getSizeWithOverflow(FalseSide); - - if (TrueResult == FalseResult) { - return TrueSide; - } - if (Options.EvalMode == ObjectSizeOpts::Mode::Min) { - if (TrueResult.slt(FalseResult)) - return TrueSide; - return FalseSide; - } - if (Options.EvalMode == ObjectSizeOpts::Mode::Max) { - if (TrueResult.sgt(FalseResult)) - return TrueSide; - return FalseSide; - } - } - return unknown(); + return combineSizeOffset(compute(I.getTrueValue()), + compute(I.getFalseValue())); } SizeOffsetType ObjectSizeOffsetVisitor::visitUndefValue(UndefValue&) { @@ -809,7 +1062,7 @@ // Erase any instructions we inserted as part of the traversal. for (Instruction *I : InsertedInstructions) { - I->replaceAllUsesWith(UndefValue::get(I->getType())); + I->replaceAllUsesWith(PoisonValue::get(I->getType())); I->eraseFromParent(); } } @@ -938,7 +1191,7 @@ return unknown(); } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitLoadInst(LoadInst&) { +SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitLoadInst(LoadInst &LI) { return unknown(); } @@ -956,10 +1209,10 @@ SizeOffsetEvalType EdgeData = compute_(PHI.getIncomingValue(i)); if (!bothKnown(EdgeData)) { - OffsetPHI->replaceAllUsesWith(UndefValue::get(IntTy)); + OffsetPHI->replaceAllUsesWith(PoisonValue::get(IntTy)); OffsetPHI->eraseFromParent(); InsertedInstructions.erase(OffsetPHI); - SizePHI->replaceAllUsesWith(UndefValue::get(IntTy)); + SizePHI->replaceAllUsesWith(PoisonValue::get(IntTy)); SizePHI->eraseFromParent(); InsertedInstructions.erase(SizePHI); return unknown();
diff --git a/src/llvm-project/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp index 36df462..fce9d5b 100644 --- a/src/llvm-project/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -27,11 +27,7 @@ #include "llvm/Analysis/PhiValues.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" @@ -44,7 +40,6 @@ #include "llvm/IR/PredIteratorCache.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" -#include "llvm/IR/User.h" #include "llvm/IR/Value.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" @@ -53,10 +48,8 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/MathExtras.h" #include <algorithm> #include <cassert> -#include <cstdint> #include <iterator> #include <utility> @@ -146,10 +139,12 @@ return ModRefInfo::ModRef; } - if (const CallInst *CI = isFreeCall(Inst, &TLI)) { - // calls to free() deallocate the entire structure - Loc = MemoryLocation::getAfter(CI->getArgOperand(0)); - return ModRefInfo::Mod; + if (const CallBase *CB = dyn_cast<CallBase>(Inst)) { + if (Value *FreedOp = getFreedOperand(CB, &TLI)) { + // calls to free() deallocate the entire structure + Loc = MemoryLocation::getAfter(FreedOp); + return ModRefInfo::Mod; + } } if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { @@ -414,20 +409,17 @@ isInvariantLoad = true; } - // Return "true" if and only if the instruction I is either a non-simple - // load or a non-simple store. - auto isNonSimpleLoadOrStore = [](Instruction *I) -> bool { + // True for volatile instruction. + // For Load/Store return true if atomic ordering is stronger than AO, + // for other instruction just true if it can read or write to memory. + auto isComplexForReordering = [](Instruction * I, AtomicOrdering AO)->bool { + if (I->isVolatile()) + return true; if (auto *LI = dyn_cast<LoadInst>(I)) - return !LI->isSimple(); + return isStrongerThan(LI->getOrdering(), AO); if (auto *SI = dyn_cast<StoreInst>(I)) - return !SI->isSimple(); - return false; - }; - - // Return "true" if I is not a load and not a store, but it does access - // memory. - auto isOtherMemAccess = [](Instruction *I) -> bool { - return !isa<LoadInst>(I) && !isa<StoreInst>(I) && I->mayReadOrWriteMemory(); + return isStrongerThan(SI->getOrdering(), AO); + return I->mayReadOrWriteMemory(); }; // Walk backwards through the basic block, looking for dependencies. @@ -500,8 +492,8 @@ // atomic. // FIXME: This is overly conservative. if (LI->isAtomic() && isStrongerThanUnordered(LI->getOrdering())) { - if (!QueryInst || isNonSimpleLoadOrStore(QueryInst) || - isOtherMemAccess(QueryInst)) + if (!QueryInst || + isComplexForReordering(QueryInst, AtomicOrdering::NotAtomic)) return MemDepResult::getClobber(LI); if (LI->getOrdering() != AtomicOrdering::Monotonic) return MemDepResult::getClobber(LI); @@ -512,10 +504,10 @@ // If we found a pointer, check if it could be the same as our pointer. AliasResult R = BatchAA.alias(LoadLoc, MemLoc); - if (isLoad) { - if (R == AliasResult::NoAlias) - continue; + if (R == AliasResult::NoAlias) + continue; + if (isLoad) { // Must aliased loads are defs of each other. if (R == AliasResult::MustAlias) return MemDepResult::getDef(Inst); @@ -532,10 +524,6 @@ continue; } - // Stores don't depend on other no-aliased accesses. - if (R == AliasResult::NoAlias) - continue; - // Stores don't alias loads from read-only memory. if (BatchAA.pointsToConstantMemory(LoadLoc)) continue; @@ -549,20 +537,25 @@ // A Monotonic store is OK if the query inst is itself not atomic. // FIXME: This is overly conservative. if (!SI->isUnordered() && SI->isAtomic()) { - if (!QueryInst || isNonSimpleLoadOrStore(QueryInst) || - isOtherMemAccess(QueryInst)) + if (!QueryInst || + isComplexForReordering(QueryInst, AtomicOrdering::Unordered)) return MemDepResult::getClobber(SI); - if (SI->getOrdering() != AtomicOrdering::Monotonic) - return MemDepResult::getClobber(SI); + // Ok, if we are here the guard above guarantee us that + // QueryInst is a non-atomic or unordered load/store. + // SI is atomic with monotonic or release semantic (seq_cst for store + // is actually a release semantic plus total order over other seq_cst + // instructions, as soon as QueryInst is not seq_cst we can consider it + // as simple release semantic). + // Monotonic and Release semantic allows re-ordering before store + // so we are safe to go further and check the aliasing. It will prohibit + // re-ordering in case locations are may or must alias. } - // FIXME: this is overly conservative. // While volatile access cannot be eliminated, they do not have to clobber // non-aliasing locations, as normal accesses can for example be reordered // with volatile accesses. if (SI->isVolatile()) - if (!QueryInst || isNonSimpleLoadOrStore(QueryInst) || - isOtherMemAccess(QueryInst)) + if (!QueryInst || QueryInst->isVolatile()) return MemDepResult::getClobber(SI); // If alias analysis can tell that this store is guaranteed to not modify @@ -743,8 +736,6 @@ llvm::sort(Cache); ++NumCacheDirtyNonLocal; - // cerr << "CACHED CASE: " << DirtyBlocks.size() << " dirty: " - // << Cache.size() << " cached: " << *QueryInst; } else { // Seed DirtyBlocks with each of the preds of QueryInst's block. BasicBlock *QueryBB = QueryCall->getParent(); @@ -1204,7 +1195,6 @@ // If we do process a large number of blocks it becomes very expensive and // likely it isn't worth worrying about if (Result.size() > NumResultsLimit) { - Worklist.clear(); // Sort it now (if needed) so that recursive invocations of // getNonLocalPointerDepFromBB and other routines that could reuse the // cache value will only see properly sorted cache arrays.
diff --git a/src/llvm-project/llvm/lib/Analysis/MemoryLocation.cpp b/src/llvm-project/llvm/lib/Analysis/MemoryLocation.cpp index a877b19..2ed3222 100644 --- a/src/llvm-project/llvm/lib/Analysis/MemoryLocation.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MemoryLocation.cpp
@@ -8,12 +8,10 @@ #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/IR/BasicBlock.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicsARM.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/MemoryProfileInfo.cpp b/src/llvm-project/llvm/lib/Analysis/MemoryProfileInfo.cpp new file mode 100644 index 0000000..3d11cb8 --- /dev/null +++ b/src/llvm-project/llvm/lib/Analysis/MemoryProfileInfo.cpp
@@ -0,0 +1,226 @@ +//===-- MemoryProfileInfo.cpp - memory profile info ------------------------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains utilities to analyze memory profile information. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/MemoryProfileInfo.h" +#include "llvm/Support/CommandLine.h" + +using namespace llvm; +using namespace llvm::memprof; + +#define DEBUG_TYPE "memory-profile-info" + +// Upper bound on accesses per byte for marking an allocation cold. +cl::opt<float> MemProfAccessesPerByteColdThreshold( + "memprof-accesses-per-byte-cold-threshold", cl::init(10.0), cl::Hidden, + cl::desc("The threshold the accesses per byte must be under to consider " + "an allocation cold")); + +// Lower bound on lifetime to mark an allocation cold (in addition to accesses +// per byte above). This is to avoid pessimizing short lived objects. +cl::opt<unsigned> MemProfMinLifetimeColdThreshold( + "memprof-min-lifetime-cold-threshold", cl::init(200), cl::Hidden, + cl::desc("The minimum lifetime (s) for an allocation to be considered " + "cold")); + +AllocationType llvm::memprof::getAllocType(uint64_t MaxAccessCount, + uint64_t MinSize, + uint64_t MinLifetime) { + if (((float)MaxAccessCount) / MinSize < MemProfAccessesPerByteColdThreshold && + // MinLifetime is expected to be in ms, so convert the threshold to ms. + MinLifetime >= MemProfMinLifetimeColdThreshold * 1000) + return AllocationType::Cold; + return AllocationType::NotCold; +} + +MDNode *llvm::memprof::buildCallstackMetadata(ArrayRef<uint64_t> CallStack, + LLVMContext &Ctx) { + std::vector<Metadata *> StackVals; + for (auto Id : CallStack) { + auto *StackValMD = + ValueAsMetadata::get(ConstantInt::get(Type::getInt64Ty(Ctx), Id)); + StackVals.push_back(StackValMD); + } + return MDNode::get(Ctx, StackVals); +} + +MDNode *llvm::memprof::getMIBStackNode(const MDNode *MIB) { + assert(MIB->getNumOperands() == 2); + // The stack metadata is the first operand of each memprof MIB metadata. + return cast<MDNode>(MIB->getOperand(0)); +} + +AllocationType llvm::memprof::getMIBAllocType(const MDNode *MIB) { + assert(MIB->getNumOperands() == 2); + // The allocation type is currently the second operand of each memprof + // MIB metadata. This will need to change as we add additional allocation + // types that can be applied based on the allocation profile data. + auto *MDS = dyn_cast<MDString>(MIB->getOperand(1)); + assert(MDS); + if (MDS->getString().equals("cold")) + return AllocationType::Cold; + return AllocationType::NotCold; +} + +static std::string getAllocTypeAttributeString(AllocationType Type) { + switch (Type) { + case AllocationType::NotCold: + return "notcold"; + break; + case AllocationType::Cold: + return "cold"; + break; + default: + assert(false && "Unexpected alloc type"); + } + llvm_unreachable("invalid alloc type"); +} + +static void addAllocTypeAttribute(LLVMContext &Ctx, CallBase *CI, + AllocationType AllocType) { + auto AllocTypeString = getAllocTypeAttributeString(AllocType); + auto A = llvm::Attribute::get(Ctx, "memprof", AllocTypeString); + CI->addFnAttr(A); +} + +static bool hasSingleAllocType(uint8_t AllocTypes) { + const unsigned NumAllocTypes = countPopulation(AllocTypes); + assert(NumAllocTypes != 0); + return NumAllocTypes == 1; +} + +void CallStackTrie::addCallStack(AllocationType AllocType, + ArrayRef<uint64_t> StackIds) { + bool First = true; + CallStackTrieNode *Curr = nullptr; + for (auto StackId : StackIds) { + // If this is the first stack frame, add or update alloc node. + if (First) { + First = false; + if (Alloc) { + assert(AllocStackId == StackId); + Alloc->AllocTypes |= static_cast<uint8_t>(AllocType); + } else { + AllocStackId = StackId; + Alloc = new CallStackTrieNode(AllocType); + } + Curr = Alloc; + continue; + } + // Update existing caller node if it exists. + auto Next = Curr->Callers.find(StackId); + if (Next != Curr->Callers.end()) { + Curr = Next->second; + Curr->AllocTypes |= static_cast<uint8_t>(AllocType); + continue; + } + // Otherwise add a new caller node. + auto *New = new CallStackTrieNode(AllocType); + Curr->Callers[StackId] = New; + Curr = New; + } + assert(Curr); +} + +void CallStackTrie::addCallStack(MDNode *MIB) { + MDNode *StackMD = getMIBStackNode(MIB); + assert(StackMD); + std::vector<uint64_t> CallStack; + CallStack.reserve(StackMD->getNumOperands()); + for (auto &MIBStackIter : StackMD->operands()) { + auto *StackId = mdconst::dyn_extract<ConstantInt>(MIBStackIter); + assert(StackId); + CallStack.push_back(StackId->getZExtValue()); + } + addCallStack(getMIBAllocType(MIB), CallStack); +} + +static MDNode *createMIBNode(LLVMContext &Ctx, + std::vector<uint64_t> &MIBCallStack, + AllocationType AllocType) { + std::vector<Metadata *> MIBPayload( + {buildCallstackMetadata(MIBCallStack, Ctx)}); + MIBPayload.push_back( + MDString::get(Ctx, getAllocTypeAttributeString(AllocType))); + return MDNode::get(Ctx, MIBPayload); +} + +// Recursive helper to trim contexts and create metadata nodes. +// Caller should have pushed Node's loc to MIBCallStack. Doing this in the +// caller makes it simpler to handle the many early returns in this method. +bool CallStackTrie::buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx, + std::vector<uint64_t> &MIBCallStack, + std::vector<Metadata *> &MIBNodes, + bool CalleeHasAmbiguousCallerContext) { + // Trim context below the first node in a prefix with a single alloc type. + // Add an MIB record for the current call stack prefix. + if (hasSingleAllocType(Node->AllocTypes)) { + MIBNodes.push_back( + createMIBNode(Ctx, MIBCallStack, (AllocationType)Node->AllocTypes)); + return true; + } + + // We don't have a single allocation for all the contexts sharing this prefix, + // so recursively descend into callers in trie. + if (!Node->Callers.empty()) { + bool NodeHasAmbiguousCallerContext = Node->Callers.size() > 1; + bool AddedMIBNodesForAllCallerContexts = true; + for (auto &Caller : Node->Callers) { + MIBCallStack.push_back(Caller.first); + AddedMIBNodesForAllCallerContexts &= + buildMIBNodes(Caller.second, Ctx, MIBCallStack, MIBNodes, + NodeHasAmbiguousCallerContext); + // Remove Caller. + MIBCallStack.pop_back(); + } + if (AddedMIBNodesForAllCallerContexts) + return true; + // We expect that the callers should be forced to add MIBs to disambiguate + // the context in this case (see below). + assert(!NodeHasAmbiguousCallerContext); + } + + // If we reached here, then this node does not have a single allocation type, + // and we didn't add metadata for a longer call stack prefix including any of + // Node's callers. That means we never hit a single allocation type along all + // call stacks with this prefix. This can happen due to recursion collapsing + // or the stack being deeper than tracked by the profiler runtime, leading to + // contexts with different allocation types being merged. In that case, we + // trim the context just below the deepest context split, which is this + // node if the callee has an ambiguous caller context (multiple callers), + // since the recursive calls above returned false. Conservatively give it + // non-cold allocation type. + if (!CalleeHasAmbiguousCallerContext) + return false; + MIBNodes.push_back(createMIBNode(Ctx, MIBCallStack, AllocationType::NotCold)); + return true; +} + +// Build and attach the minimal necessary MIB metadata. If the alloc has a +// single allocation type, add a function attribute instead. Returns true if +// memprof metadata attached, false if not (attribute added). +bool CallStackTrie::buildAndAttachMIBMetadata(CallBase *CI) { + auto &Ctx = CI->getContext(); + if (hasSingleAllocType(Alloc->AllocTypes)) { + addAllocTypeAttribute(Ctx, CI, (AllocationType)Alloc->AllocTypes); + return false; + } + std::vector<uint64_t> MIBCallStack; + MIBCallStack.push_back(AllocStackId); + std::vector<Metadata *> MIBNodes; + assert(!Alloc->Callers.empty() && "addCallStack has not been called yet"); + buildMIBNodes(Alloc, Ctx, MIBCallStack, MIBNodes, + /*CalleeHasAmbiguousCallerContext=*/true); + assert(MIBCallStack.size() == 1 && + "Should only be left with Alloc's location in stack"); + CI->setMetadata(LLVMContext::MD_memprof, MDNode::get(Ctx, MIBNodes)); + return true; +}
diff --git a/src/llvm-project/llvm/lib/Analysis/MemorySSA.cpp b/src/llvm-project/llvm/lib/Analysis/MemorySSA.cpp index 57f431e..76371b8 100644 --- a/src/llvm-project/llvm/lib/Analysis/MemorySSA.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MemorySSA.cpp
@@ -36,8 +36,8 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Operator.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Use.h" #include "llvm/InitializePasses.h" @@ -49,10 +49,10 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormattedStream.h" +#include "llvm/Support/GraphWriter.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> #include <cassert> -#include <cstdlib> #include <iterator> #include <memory> #include <utility> @@ -130,6 +130,12 @@ MemorySSAWalkerAnnotatedWriter(MemorySSA *M) : MSSA(M), Walker(M->getWalker()) {} + void emitBasicBlockStartAnnot(const BasicBlock *BB, + formatted_raw_ostream &OS) override { + if (MemoryAccess *MA = MSSA->getMemoryAccess(BB)) + OS << "; " << *MA << "\n"; + } + void emitInstructionAnnot(const Instruction *I, formatted_raw_ostream &OS) override { if (MemoryAccess *MA = MSSA->getMemoryAccess(I)) { @@ -732,7 +738,7 @@ struct generic_def_path_iterator : public iterator_facade_base<generic_def_path_iterator<T, Walker>, std::forward_iterator_tag, T *> { - generic_def_path_iterator() {} + generic_def_path_iterator() = default; generic_def_path_iterator(Walker *W, ListIndex N) : W(W), N(N) {} T &operator*() const { return curNode(); } @@ -743,9 +749,9 @@ } bool operator==(const generic_def_path_iterator &O) const { - if (N.hasValue() != O.N.hasValue()) + if (N.has_value() != O.N.has_value()) return false; - return !N.hasValue() || *N == *O.N; + return !N || *N == *O.N; } private: @@ -1397,6 +1403,9 @@ continue; } + if (MU->isOptimized()) + continue; + if (isUseTriviallyOptimizableToLiveOnEntry(*AA, MU->getMemoryInst())) { MU->setDefiningAccess(MSSA->getLiveOnEntryDef(), true, None); continue; @@ -1585,10 +1594,6 @@ SmallPtrSet<BasicBlock *, 16> Visited; renamePass(DT->getRootNode(), LiveOnEntryDef.get(), Visited); - ClobberWalkerBase<BatchAAResults> WalkerBase(this, &BAA, DT); - CachingWalker<BatchAAResults> WalkerLocal(this, &WalkerBase); - OptimizeUses(this, &WalkerLocal, &BAA, DT).optimizeUses(); - // Mark the uses in unreachable blocks as live on entry, so that they go // somewhere. for (auto &BB : F) @@ -2178,6 +2183,17 @@ return dominates(Dominator, cast<MemoryAccess>(Dominatee.getUser())); } +void MemorySSA::ensureOptimizedUses() { + if (IsOptimized) + return; + + BatchAAResults BatchAA(*AA); + ClobberWalkerBase<BatchAAResults> WalkerBase(this, &BatchAA, DT); + CachingWalker<BatchAAResults> WalkerLocal(this, &WalkerBase); + OptimizeUses(this, &WalkerLocal, &BatchAA, DT).optimizeUses(); + IsOptimized = true; +} + void MemoryAccess::print(raw_ostream &OS) const { switch (getValueID()) { case MemoryPhiVal: return static_cast<const MemoryPhi *>(this)->print(OS); @@ -2350,6 +2366,7 @@ bool MemorySSAPrinterLegacyPass::runOnFunction(Function &F) { auto &MSSA = getAnalysis<MemorySSAWrapperPass>().getMSSA(); + MSSA.ensureOptimizedUses(); if (DotCFGMSSA != "") { DOTFuncMSSAInfo CFGInfo(F, MSSA); WriteGraph(&CFGInfo, "", false, "MSSA", DotCFGMSSA); @@ -2382,6 +2399,7 @@ PreservedAnalyses MemorySSAPrinterPass::run(Function &F, FunctionAnalysisManager &AM) { auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA(); + MSSA.ensureOptimizedUses(); if (DotCFGMSSA != "") { DOTFuncMSSAInfo CFGInfo(F, MSSA); WriteGraph(&CFGInfo, "", false, "MSSA", DotCFGMSSA);
diff --git a/src/llvm-project/llvm/lib/Analysis/MemorySSAUpdater.cpp b/src/llvm-project/llvm/lib/Analysis/MemorySSAUpdater.cpp index 9c84188..9ad60f7 100644 --- a/src/llvm-project/llvm/lib/Analysis/MemorySSAUpdater.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MemorySSAUpdater.cpp
@@ -10,22 +10,15 @@ // //===----------------------------------------------------------------===// #include "llvm/Analysis/MemorySSAUpdater.h" -#include "llvm/Analysis/LoopIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/IteratedDominanceFrontier.h" +#include "llvm/Analysis/LoopIterator.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" -#include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Metadata.h" -#include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/FormattedStream.h" #include <algorithm> #define DEBUG_TYPE "memoryssa" @@ -243,6 +236,7 @@ } void MemorySSAUpdater::insertUse(MemoryUse *MU, bool RenameUses) { + VisitedBlocks.clear(); InsertedPHIs.clear(); MU->setDefiningAccess(getPreviousDef(MU)); @@ -311,6 +305,13 @@ // point to the correct new defs, to ensure we only have one variable, and no // disconnected stores. void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) { + // Don't bother updating dead code. + if (!MSSA->DT->isReachableFromEntry(MD->getBlock())) { + MD->setDefiningAccess(MSSA->getLiveOnEntryDef()); + return; + } + + VisitedBlocks.clear(); InsertedPHIs.clear(); // See if we had a local def, and if not, go hunting. @@ -427,10 +428,10 @@ if (NewPhiSize) tryRemoveTrivialPhis(ArrayRef<WeakVH>(&InsertedPHIs[NewPhiIndex], NewPhiSize)); - // Now that all fixups are done, rename all uses if we are asked. Skip - // renaming for defs in unreachable blocks. + // Now that all fixups are done, rename all uses if we are asked. The defs are + // guaranteed to be in reachable code due to the check at the method entry. BasicBlock *StartBlock = MD->getBlock(); - if (RenameUses && MSSA->getDomTree().getNode(StartBlock)) { + if (RenameUses) { SmallPtrSet<BasicBlock *, 16> Visited; // We are guaranteed there is a def in the block, because we just got it // handed to us in this function. @@ -450,7 +451,7 @@ } // Existing Phi blocks may need renaming too, if an access was previously // optimized and the inserted Defs "covers" the Optimized value. - for (auto &MP : ExistingPhis) { + for (const auto &MP : ExistingPhis) { MemoryPhi *Phi = dyn_cast_or_null<MemoryPhi>(MP); if (Phi) MSSA->renamePass(Phi->getBlock(), nullptr, Visited); @@ -461,7 +462,7 @@ void MemorySSAUpdater::fixupDefs(const SmallVectorImpl<WeakVH> &Vars) { SmallPtrSet<const BasicBlock *, 8> Seen; SmallVector<const BasicBlock *, 16> Worklist; - for (auto &Var : Vars) { + for (const auto &Var : Vars) { MemoryAccess *NewDef = dyn_cast_or_null<MemoryAccess>(Var); if (!NewDef) continue; @@ -743,10 +744,10 @@ cloneUsesAndDefs(BB, NewBlock, VMap, MPhiMap); }; - for (auto BB : llvm::concat<BasicBlock *const>(LoopBlocks, ExitBlocks)) + for (auto *BB : llvm::concat<BasicBlock *const>(LoopBlocks, ExitBlocks)) ProcessBlock(BB); - for (auto BB : llvm::concat<BasicBlock *const>(LoopBlocks, ExitBlocks)) + for (auto *BB : llvm::concat<BasicBlock *const>(LoopBlocks, ExitBlocks)) if (MemoryPhi *MPhi = MSSA->getMemoryAccess(BB)) if (MemoryAccess *NewPhi = MPhiMap.lookup(MPhi)) FixPhiIncomingValues(MPhi, cast<MemoryPhi>(NewPhi)); @@ -810,7 +811,7 @@ SmallVector<CFGUpdate, 4> DeleteUpdates; SmallVector<CFGUpdate, 4> RevDeleteUpdates; SmallVector<CFGUpdate, 4> InsertUpdates; - for (auto &Update : Updates) { + for (const auto &Update : Updates) { if (Update.getKind() == DT.Insert) InsertUpdates.push_back({DT.Insert, Update.getFrom(), Update.getTo()}); else { @@ -957,7 +958,7 @@ }; SmallDenseMap<BasicBlock *, PredInfo> PredMap; - for (auto &Edge : Updates) { + for (const auto &Edge : Updates) { BasicBlock *BB = Edge.getTo(); auto &AddedBlockSet = PredMap[BB].Added; AddedBlockSet.insert(Edge.getFrom()); @@ -1002,7 +1003,7 @@ // First create MemoryPhis in all blocks that don't have one. Create in the // order found in Updates, not in PredMap, to get deterministic numbering. - for (auto &Edge : Updates) { + for (const auto &Edge : Updates) { BasicBlock *BB = Edge.getTo(); if (PredMap.count(BB) && !MSSA->getMemoryAccess(BB)) InsertedPhis.push_back(MSSA->createMemoryPhi(BB)); @@ -1399,7 +1400,7 @@ } void MemorySSAUpdater::tryRemoveTrivialPhis(ArrayRef<WeakVH> UpdatedPHIs) { - for (auto &VH : UpdatedPHIs) + for (const auto &VH : UpdatedPHIs) if (auto *MPhi = cast_or_null<MemoryPhi>(VH)) tryRemoveTrivialPhi(MPhi); }
diff --git a/src/llvm-project/llvm/lib/Analysis/ModelUnderTrainingRunner.cpp b/src/llvm-project/llvm/lib/Analysis/ModelUnderTrainingRunner.cpp index fab51d6..dc149f3 100644 --- a/src/llvm-project/llvm/lib/Analysis/ModelUnderTrainingRunner.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ModelUnderTrainingRunner.cpp
@@ -22,7 +22,7 @@ LLVMContext &Ctx, const std::string &ModelPath, const std::vector<TensorSpec> &InputSpecs, const std::vector<LoggedFeatureSpec> &OutputSpecs) - : MLModelRunner(Ctx, MLModelRunner::Kind::Development), + : MLModelRunner(Ctx, MLModelRunner::Kind::Development, InputSpecs.size()), OutputSpecs(OutputSpecs) { Evaluator = std::make_unique<TFModelEvaluator>( ModelPath, InputSpecs, [&](size_t I) { return OutputSpecs[I].Spec; }, @@ -32,6 +32,10 @@ Evaluator.reset(); return; } + + for (size_t I = 0, E = InputSpecs.size(); I < E; ++I) { + setUpBufferForTensor(I, InputSpecs[I], Evaluator->getUntypedInput(I)); + } } void *ModelUnderTrainingRunner::evaluateUntyped() { @@ -43,24 +47,31 @@ return LastEvaluationResult->getUntypedTensorValue(0); } -void *ModelUnderTrainingRunner::getTensorUntyped(size_t Index) { - return Evaluator->getUntypedInput(Index); +std::unique_ptr<ModelUnderTrainingRunner> +ModelUnderTrainingRunner::createAndEnsureValid( + LLVMContext &Ctx, const std::string &ModelPath, StringRef DecisionName, + const std::vector<TensorSpec> &InputSpecs, + StringRef OutputSpecsPathOverride) { + if (auto MaybeOutputSpecs = loadOutputSpecs(Ctx, DecisionName, ModelPath, + OutputSpecsPathOverride)) + return createAndEnsureValid(Ctx, ModelPath, DecisionName, InputSpecs, + *MaybeOutputSpecs); + Ctx.emitError("Could not load the policy model from the provided path"); + return nullptr; } std::unique_ptr<ModelUnderTrainingRunner> ModelUnderTrainingRunner::createAndEnsureValid( LLVMContext &Ctx, const std::string &ModelPath, StringRef DecisionName, const std::vector<TensorSpec> &InputSpecs, - StringRef OutputSpecsPathOverride) { + const std::vector<LoggedFeatureSpec> &OutputSpecs) { std::unique_ptr<ModelUnderTrainingRunner> MUTR; - if (auto MaybeOutputSpecs = loadOutputSpecs(Ctx, DecisionName, ModelPath, - OutputSpecsPathOverride)) - MUTR.reset(new ModelUnderTrainingRunner(Ctx, ModelPath, InputSpecs, - *MaybeOutputSpecs)); + MUTR.reset( + new ModelUnderTrainingRunner(Ctx, ModelPath, InputSpecs, OutputSpecs)); if (MUTR && MUTR->isValid()) return MUTR; - Ctx.emitError("Could not load the policy model from the provided path"); + Ctx.emitError("Could not load or create model evaluator."); return nullptr; }
diff --git a/src/llvm-project/llvm/lib/Analysis/ModuleDebugInfoPrinter.cpp b/src/llvm-project/llvm/lib/Analysis/ModuleDebugInfoPrinter.cpp index 64fd5eb..756f92e 100644 --- a/src/llvm-project/llvm/lib/Analysis/ModuleDebugInfoPrinter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ModuleDebugInfoPrinter.cpp
@@ -15,8 +15,8 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/ModuleDebugInfoPrinter.h" -#include "llvm/ADT/Statistic.h" #include "llvm/Analysis/Passes.h" +#include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/PassManager.h" #include "llvm/InitializePasses.h" @@ -95,7 +95,7 @@ O << '\n'; } - for (auto GVU : Finder.global_variables()) { + for (auto *GVU : Finder.global_variables()) { const auto *GV = GVU->getVariable(); O << "Global variable: " << GV->getName(); printFile(O, GV->getFilename(), GV->getDirectory(), GV->getLine());
diff --git a/src/llvm-project/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index 2880ca6..efe6058 100644 --- a/src/llvm-project/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -38,7 +38,6 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSummaryIndex.h" @@ -165,7 +164,8 @@ SetVector<FunctionSummary::ConstVCall> &TypeCheckedLoadConstVCalls, DominatorTree &DT) { switch (CI->getCalledFunction()->getIntrinsicID()) { - case Intrinsic::type_test: { + case Intrinsic::type_test: + case Intrinsic::public_type_test: { auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1)); auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata()); if (!TypeId) @@ -368,7 +368,7 @@ // We should have named any anonymous globals assert(CalledFunction->hasName()); auto ScaledCount = PSI->getProfileCount(*CB, BFI); - auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI) + auto Hotness = ScaledCount ? getHotness(*ScaledCount, PSI) : CalleeInfo::HotnessType::Unknown; if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None) Hotness = CalleeInfo::HotnessType::Cold; @@ -401,7 +401,7 @@ // to enable importing for subsequent indirect call promotion and // inlining. if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) { - for (auto &Op : MD->operands()) { + for (const auto &Op : MD->operands()) { Function *Callee = mdconst::extract_or_null<Function>(Op); if (Callee) CallGraphEdges[Index.getOrInsertValueInfo(Callee)]; @@ -413,7 +413,7 @@ auto CandidateProfileData = ICallAnalysis.getPromotionCandidatesForInstruction( &I, NumVals, TotalCount, NumCandidates); - for (auto &Candidate : CandidateProfileData) + for (const auto &Candidate : CandidateProfileData) CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)] .updateHotness(getHotness(Candidate.Count, PSI)); } @@ -452,7 +452,7 @@ // If both load and store instruction reference the same variable // we won't be able to optimize it. Add all such reference edges // to RefEdges set. - for (auto &VI : StoreRefEdges) + for (const auto &VI : StoreRefEdges) if (LoadRefEdges.remove(VI)) RefEdges.insert(VI); @@ -460,11 +460,11 @@ // All new reference edges inserted in two loops below are either // read or write only. They will be grouped in the end of RefEdges // vector, so we can use a single integer value to identify them. - for (auto &VI : LoadRefEdges) + for (const auto &VI : LoadRefEdges) RefEdges.insert(VI); unsigned FirstWORef = RefEdges.size(); - for (auto &VI : StoreRefEdges) + for (const auto &VI : StoreRefEdges) RefEdges.insert(VI); Refs = RefEdges.takeVector(); @@ -490,8 +490,7 @@ HasIndirBranchToBlockAddress; GlobalValueSummary::GVFlags Flags( F.getLinkage(), F.getVisibility(), NotEligibleForImport, - /* Live = */ false, F.isDSOLocal(), - F.hasLinkOnceODRLinkage() && F.hasGlobalUnnamedAddr()); + /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable()); FunctionSummary::FFlags FunFlags{ F.hasFnAttribute(Attribute::ReadNone), F.hasFnAttribute(Attribute::ReadOnly), @@ -612,8 +611,7 @@ bool NonRenamableLocal = isNonRenamableLocal(V); GlobalValueSummary::GVFlags Flags( V.getLinkage(), V.getVisibility(), NonRenamableLocal, - /* Live = */ false, V.isDSOLocal(), - V.hasLinkOnceODRLinkage() && V.hasGlobalUnnamedAddr()); + /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable()); VTableFuncList VTableFuncs; // If splitting is not enabled, then we compute the summary information @@ -649,16 +647,18 @@ Index.addGlobalValueSummary(V, std::move(GVarSummary)); } -static void -computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, - DenseSet<GlobalValue::GUID> &CantBePromoted) { +static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, + DenseSet<GlobalValue::GUID> &CantBePromoted) { + // Skip summary for indirect function aliases as summary for aliasee will not + // be emitted. + const GlobalObject *Aliasee = A.getAliaseeObject(); + if (isa<GlobalIFunc>(Aliasee)) + return; bool NonRenamableLocal = isNonRenamableLocal(A); GlobalValueSummary::GVFlags Flags( A.getLinkage(), A.getVisibility(), NonRenamableLocal, - /* Live = */ false, A.isDSOLocal(), - A.hasLinkOnceODRLinkage() && A.hasGlobalUnnamedAddr()); + /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable()); auto AS = std::make_unique<AliasSummary>(Flags); - auto *Aliasee = A.getAliaseeObject(); auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID()); assert(AliaseeVI && "Alias expects aliasee summary to be available"); assert(AliaseeVI.getSummaryList().size() == 1 && @@ -672,7 +672,7 @@ // Set LiveRoot flag on entries matching the given value name. static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) { if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name))) - for (auto &Summary : VI.getSummaryList()) + for (const auto &Summary : VI.getSummaryList()) Summary->setLive(true); } @@ -733,8 +733,7 @@ GlobalValue::InternalLinkage, GlobalValue::DefaultVisibility, /* NotEligibleToImport = */ true, /* Live = */ true, - /* Local */ GV->isDSOLocal(), - GV->hasLinkOnceODRLinkage() && GV->hasGlobalUnnamedAddr()); + /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable()); CantBePromoted.insert(GV->getGUID()); // Create the appropriate summary type. if (Function *F = dyn_cast<Function>(GV)) { @@ -781,7 +780,7 @@ // Compute summaries for all functions defined in module, and save in the // index. - for (auto &F : M) { + for (const auto &F : M) { if (F.isDeclaration()) continue; @@ -816,6 +815,13 @@ for (const GlobalAlias &A : M.aliases()) computeAliasSummary(Index, A, CantBePromoted); + // Iterate through ifuncs, set their resolvers all alive. + for (const GlobalIFunc &I : M.ifuncs()) { + I.applyAlongResolverPath([&Index](const GlobalValue &GV) { + Index.getGlobalValueSummary(GV)->setLive(true); + }); + } + for (auto *V : LocalsUsed) { auto *Summary = Index.getGlobalValueSummary(*V); assert(Summary && "Missing summary for global value");
diff --git a/src/llvm-project/llvm/lib/Analysis/MustExecute.cpp b/src/llvm-project/llvm/lib/Analysis/MustExecute.cpp index 5ca72f5..ac6590c 100644 --- a/src/llvm-project/llvm/lib/Analysis/MustExecute.cpp +++ b/src/llvm-project/llvm/lib/Analysis/MustExecute.cpp
@@ -16,14 +16,11 @@ #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/AssemblyAnnotationWriter.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/InstIterator.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/InitializePasses.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormattedStream.h" #include "llvm/Support/raw_ostream.h" @@ -84,7 +81,7 @@ MW.clear(); MayThrow = false; // Figure out the fact that at least one block may throw. - for (auto &BB : CurLoop->blocks()) + for (const auto &BB : CurLoop->blocks()) if (ICF.hasICF(&*BB)) { MayThrow = true; break; @@ -143,7 +140,7 @@ return false; auto DL = ExitBlock->getModule()->getDataLayout(); auto *IVStart = LHS->getIncomingValueForBlock(CurLoop->getLoopPreheader()); - auto *SimpleValOrNull = SimplifyCmpInst(Cond->getPredicate(), + auto *SimpleValOrNull = simplifyCmpInst(Cond->getPredicate(), IVStart, RHS, {DL, /*TLI*/ nullptr, DT, /*AC*/ nullptr, BI}); @@ -167,7 +164,7 @@ if (BB == CurLoop->getHeader()) return; SmallVector<const BasicBlock *, 4> WorkList; - for (auto *Pred : predecessors(BB)) { + for (const auto *Pred : predecessors(BB)) { Predecessors.insert(Pred); WorkList.push_back(Pred); } @@ -183,7 +180,7 @@ // @nested and @nested_no_throw in test/Analysis/MustExecute/loop-header.ll. // We can ignore backedge of all loops containing BB to get a sligtly more // optimistic result. - for (auto *PredPred : predecessors(Pred)) + for (const auto *PredPred : predecessors(Pred)) if (Predecessors.insert(PredPred).second) WorkList.push_back(PredPred); } @@ -210,7 +207,7 @@ // 3) Exit blocks which are not taken on 1st iteration. // Memoize blocks we've already checked. SmallPtrSet<const BasicBlock *, 4> CheckedSuccessors; - for (auto *Pred : Predecessors) { + for (const auto *Pred : Predecessors) { // Predecessor block may throw, so it has a side exit. if (blockMayThrow(Pred)) return false; @@ -220,7 +217,7 @@ if (DT->dominates(BB, Pred)) continue; - for (auto *Succ : successors(Pred)) + for (const auto *Succ : successors(Pred)) if (CheckedSuccessors.insert(Succ).second && Succ != BB && !Predecessors.count(Succ)) // By discharging conditions that are not executed on the 1st iteration, @@ -288,7 +285,7 @@ collectTransitivePredecessors(CurLoop, BB, Predecessors); // Find if there any instruction in either predecessor that could write // to memory. - for (auto *Pred : Predecessors) + for (const auto *Pred : Predecessors) if (MW.mayWriteToMemory(Pred)) return false; return true; @@ -416,7 +413,7 @@ public: MustExecuteAnnotatedWriter(const Function &F, DominatorTree &DT, LoopInfo &LI) { - for (auto &I: instructions(F)) { + for (const auto &I: instructions(F)) { Loop *L = LI.getLoopFor(I.getParent()); while (L) { if (isMustExecuteIn(I, L, &DT)) { @@ -428,8 +425,8 @@ } MustExecuteAnnotatedWriter(const Module &M, DominatorTree &DT, LoopInfo &LI) { - for (auto &F : M) - for (auto &I: instructions(F)) { + for (const auto &F : M) + for (const auto &I: instructions(F)) { Loop *L = LI.getLoopFor(I.getParent()); while (L) { if (isMustExecuteIn(I, L, &DT)) { @@ -494,9 +491,9 @@ static V getOrCreateCachedOptional(K Key, DenseMap<K, Optional<V>> &Map, FnTy &&Fn, ArgsTy&&... args) { Optional<V> &OptVal = Map[Key]; - if (!OptVal.hasValue()) + if (!OptVal) OptVal = Fn(std::forward<ArgsTy>(args)...); - return OptVal.getValue(); + return OptVal.value(); } const BasicBlock *
diff --git a/src/llvm-project/llvm/lib/Analysis/NoInferenceModelRunner.cpp b/src/llvm-project/llvm/lib/Analysis/NoInferenceModelRunner.cpp index 7178120..1914b22 100644 --- a/src/llvm-project/llvm/lib/Analysis/NoInferenceModelRunner.cpp +++ b/src/llvm-project/llvm/lib/Analysis/NoInferenceModelRunner.cpp
@@ -10,24 +10,14 @@ // logs for the default policy, in 'development' mode, but never ask it to // 'run'. //===----------------------------------------------------------------------===// -#include "llvm/Config/config.h" -#if defined(LLVM_HAVE_TF_API) - #include "llvm/Analysis/NoInferenceModelRunner.h" -#include "llvm/Analysis/Utils/TFUtils.h" using namespace llvm; NoInferenceModelRunner::NoInferenceModelRunner( LLVMContext &Ctx, const std::vector<TensorSpec> &Inputs) - : MLModelRunner(Ctx, MLModelRunner::Kind::NoOp) { - ValuesBuffer.reserve(Inputs.size()); + : MLModelRunner(Ctx, MLModelRunner::Kind::NoOp, Inputs.size()) { + size_t Index = 0; for (const auto &TS : Inputs) - ValuesBuffer.push_back(std::make_unique<char[]>(TS.getElementCount() * - TS.getElementByteSize())); + setUpBufferForTensor(Index++, TS, nullptr); } - -void *NoInferenceModelRunner::getTensorUntyped(size_t Index) { - return ValuesBuffer[Index].get(); -} -#endif // defined(LLVM_HAVE_TF_API)
diff --git a/src/llvm-project/llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp index 0826b30..6fe056d 100644 --- a/src/llvm-project/llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ObjCARCAliasAnalysis.cpp
@@ -26,8 +26,6 @@ #include "llvm/Analysis/ObjCARCAnalysisUtils.h" #include "llvm/Analysis/Passes.h" #include "llvm/IR/Function.h" -#include "llvm/IR/Instruction.h" -#include "llvm/IR/Value.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp b/src/llvm-project/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp index 6f3d4d5..17b40f0 100644 --- a/src/llvm-project/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp
@@ -47,7 +47,7 @@ bool OptimizationRemarkEmitter::invalidate( Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv) { - if (OwnedBFI.get()) { + if (OwnedBFI) { OwnedBFI.reset(); BFI = nullptr; } @@ -80,7 +80,7 @@ computeHotness(OptDiag); // Only emit it if its hotness meets the threshold. - if (OptDiag.getHotness().getValueOr(0) < + if (OptDiag.getHotness().value_or(0) < F->getContext().getDiagnosticsHotnessThreshold()) { return; }
diff --git a/src/llvm-project/llvm/lib/Analysis/OverflowInstAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/OverflowInstAnalysis.cpp index 87a85e6..8bfd664 100644 --- a/src/llvm-project/llvm/lib/Analysis/OverflowInstAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/OverflowInstAnalysis.cpp
@@ -12,7 +12,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/OverflowInstAnalysis.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/PatternMatch.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/PHITransAddr.cpp b/src/llvm-project/llvm/lib/Analysis/PHITransAddr.cpp index 02d0849..5b0fbca 100644 --- a/src/llvm-project/llvm/lib/Analysis/PHITransAddr.cpp +++ b/src/llvm-project/llvm/lib/Analysis/PHITransAddr.cpp
@@ -17,11 +17,14 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; +static cl::opt<bool> EnableAddPhiTranslation( + "gvn-add-phi-translation", cl::init(false), cl::Hidden, + cl::desc("Enable phi-translation of add instructions")); + static bool CanPHITrans(Instruction *Inst) { if (isa<PHINode>(Inst) || isa<GetElementPtrInst>(Inst)) @@ -35,9 +38,6 @@ isa<ConstantInt>(Inst->getOperand(1))) return true; - // cerr << "MEMDEP: Could not PHI translate: " << *Pointer; - // if (isa<BitCastInst>(PtrInst) || isa<GetElementPtrInst>(PtrInst)) - // cerr << "OP:\t\t\t\t" << *PtrInst->getOperand(0); return false; } @@ -226,7 +226,7 @@ return GEP; // Simplify the GEP to handle 'gep x, 0' -> x etc. - if (Value *V = SimplifyGEPInst(GEP->getSourceElementType(), GEPOps[0], + if (Value *V = simplifyGEPInst(GEP->getSourceElementType(), GEPOps[0], ArrayRef<Value *>(GEPOps).slice(1), GEP->isInBounds(), {DL, TLI, DT, AC})) { for (unsigned i = 0, e = GEPOps.size(); i != e; ++i) @@ -240,6 +240,7 @@ for (User *U : APHIOp->users()) { if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(U)) if (GEPI->getType() == GEP->getType() && + GEPI->getSourceElementType() == GEP->getSourceElementType() && GEPI->getNumOperands() == GEPOps.size() && GEPI->getParent()->getParent() == CurBB->getParent() && (!DT || DT->dominates(GEPI->getParent(), PredBB))) { @@ -277,7 +278,7 @@ } // See if the add simplifies away. - if (Value *Res = SimplifyAddInst(LHS, RHS, isNSW, isNUW, {DL, TLI, DT, AC})) { + if (Value *Res = simplifyAddInst(LHS, RHS, isNSW, isNUW, {DL, TLI, DT, AC})) { // If we simplified the operands, the LHS is no longer an input, but Res // is. RemoveInstInputs(LHS, InstInputs); @@ -413,14 +414,14 @@ return Result; } -#if 0 - // FIXME: This code works, but it is unclear that we actually want to insert - // a big chain of computation in order to make a value available in a block. - // This needs to be evaluated carefully to consider its cost trade offs. - // Handle add with a constant RHS. - if (Inst->getOpcode() == Instruction::Add && + if (EnableAddPhiTranslation && Inst->getOpcode() == Instruction::Add && isa<ConstantInt>(Inst->getOperand(1))) { + + // FIXME: This code works, but it is unclear that we actually want to insert + // a big chain of computation in order to make a value available in a block. + // This needs to be evaluated carefully to consider its cost trade offs. + // PHI translate the LHS. Value *OpVal = InsertPHITranslatedSubExpr(Inst->getOperand(0), CurBB, PredBB, DT, NewInsts); @@ -434,7 +435,6 @@ NewInsts.push_back(Res); return Res; } -#endif return nullptr; }
diff --git a/src/llvm-project/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/src/llvm-project/llvm/lib/Analysis/ProfileSummaryInfo.cpp index 268ed9d..64844f5 100644 --- a/src/llvm-project/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ProfileSummaryInfo.cpp
@@ -15,7 +15,6 @@ #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/InitializePasses.h" @@ -125,7 +124,7 @@ for (const auto &I : BB) if (isa<CallInst>(I) || isa<InvokeInst>(I)) if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr)) - TotalCallCount += CallCount.getValue(); + TotalCallCount += *CallCount; if (isHotCount(TotalCallCount)) return true; } @@ -154,7 +153,7 @@ for (const auto &I : BB) if (isa<CallInst>(I) || isa<InvokeInst>(I)) if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr)) - TotalCallCount += CallCount.getValue(); + TotalCallCount += *CallCount; if (!isColdCount(TotalCallCount)) return false; } @@ -166,7 +165,7 @@ bool ProfileSummaryInfo::isFunctionHotnessUnknown(const Function &F) const { assert(hasPartialSampleProfile() && "Expect partial sample profile"); - return !F.getEntryCount().hasValue(); + return !F.getEntryCount(); } template <bool isHot> @@ -188,7 +187,7 @@ for (const auto &I : BB) if (isa<CallInst>(I) || isa<InvokeInst>(I)) if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr)) - TotalCallCount += CallCount.getValue(); + TotalCallCount += *CallCount; if (isHot && isHotCountNthPercentile(PercentileCutoff, TotalCallCount)) return true; if (!isHot && !isColdCountNthPercentile(PercentileCutoff, TotalCallCount)) @@ -280,19 +279,19 @@ } bool ProfileSummaryInfo::hasHugeWorkingSetSize() const { - return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue(); + return HasHugeWorkingSetSize && HasHugeWorkingSetSize.value(); } bool ProfileSummaryInfo::hasLargeWorkingSetSize() const { - return HasLargeWorkingSetSize && HasLargeWorkingSetSize.getValue(); + return HasLargeWorkingSetSize && HasLargeWorkingSetSize.value(); } bool ProfileSummaryInfo::isHotCount(uint64_t C) const { - return HotCountThreshold && C >= HotCountThreshold.getValue(); + return HotCountThreshold && C >= HotCountThreshold.value(); } bool ProfileSummaryInfo::isColdCount(uint64_t C) const { - return ColdCountThreshold && C <= ColdCountThreshold.getValue(); + return ColdCountThreshold && C <= ColdCountThreshold.value(); } template <bool isHot> @@ -300,9 +299,9 @@ uint64_t C) const { auto CountThreshold = computeThreshold(PercentileCutoff); if (isHot) - return CountThreshold && C >= CountThreshold.getValue(); + return CountThreshold && C >= CountThreshold.value(); else - return CountThreshold && C <= CountThreshold.getValue(); + return CountThreshold && C <= CountThreshold.value(); } bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, @@ -316,11 +315,11 @@ } uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() const { - return HotCountThreshold.getValueOr(UINT64_MAX); + return HotCountThreshold.value_or(UINT64_MAX); } uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() const { - return ColdCountThreshold.getValueOr(0); + return ColdCountThreshold.value_or(0); } bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB,
diff --git a/src/llvm-project/llvm/lib/Analysis/PtrUseVisitor.cpp b/src/llvm-project/llvm/lib/Analysis/PtrUseVisitor.cpp index 9a834ba..4930481 100644 --- a/src/llvm-project/llvm/lib/Analysis/PtrUseVisitor.cpp +++ b/src/llvm-project/llvm/lib/Analysis/PtrUseVisitor.cpp
@@ -14,7 +14,6 @@ #include "llvm/Analysis/PtrUseVisitor.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" -#include <algorithm> using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/RegionInfo.cpp b/src/llvm-project/llvm/lib/Analysis/RegionInfo.cpp index 3ba0bb9..9be23a3 100644 --- a/src/llvm-project/llvm/lib/Analysis/RegionInfo.cpp +++ b/src/llvm-project/llvm/lib/Analysis/RegionInfo.cpp
@@ -10,6 +10,7 @@ #include "llvm/Analysis/RegionInfo.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/DominanceFrontier.h" #include "llvm/InitializePasses.h" #ifndef NDEBUG #include "llvm/Analysis/RegionPrinter.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/RegionPass.cpp b/src/llvm-project/llvm/lib/Analysis/RegionPass.cpp index 10c8569..ddef3be 100644 --- a/src/llvm-project/llvm/lib/Analysis/RegionPass.cpp +++ b/src/llvm-project/llvm/lib/Analysis/RegionPass.cpp
@@ -12,14 +12,16 @@ // Most of this code has been COPIED from LoopPass.cpp // //===----------------------------------------------------------------------===// + #include "llvm/Analysis/RegionPass.h" +#include "llvm/Analysis/RegionInfo.h" #include "llvm/IR/OptBisect.h" #include "llvm/IR/PassTimingInfo.h" #include "llvm/IR/PrintPasses.h" -#include "llvm/IR/StructuralHash.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" + using namespace llvm; #define DEBUG_TYPE "regionpassmgr" @@ -93,12 +95,12 @@ TimeRegion PassTimer(getPassTimer(P)); #ifdef EXPENSIVE_CHECKS - uint64_t RefHash = StructuralHash(F); + uint64_t RefHash = P->structuralHash(F); #endif LocalChanged = P->runOnRegion(CurrentRegion, *this); #ifdef EXPENSIVE_CHECKS - if (!LocalChanged && (RefHash != StructuralHash(F))) { + if (!LocalChanged && (RefHash != P->structuralHash(F))) { llvm::errs() << "Pass modifies its input and doesn't report it: " << P->getPassName() << "\n"; llvm_unreachable("Pass modifies its input and doesn't report it");
diff --git a/src/llvm-project/llvm/lib/Analysis/RegionPrinter.cpp b/src/llvm-project/llvm/lib/Analysis/RegionPrinter.cpp index 1fb5faa..fbd3d17 100644 --- a/src/llvm-project/llvm/lib/Analysis/RegionPrinter.cpp +++ b/src/llvm-project/llvm/lib/Analysis/RegionPrinter.cpp
@@ -10,15 +10,11 @@ #include "llvm/Analysis/RegionPrinter.h" #include "llvm/ADT/DepthFirstIterator.h" -#include "llvm/ADT/PostOrderIterator.h" -#include "llvm/ADT/Statistic.h" #include "llvm/Analysis/DOTGraphTraitsPass.h" -#include "llvm/Analysis/Passes.h" #include "llvm/Analysis/RegionInfo.h" #include "llvm/Analysis/RegionIterator.h" #include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #ifndef NDEBUG #include "llvm/IR/LegacyPassManager.h" @@ -35,28 +31,20 @@ cl::init(false)); namespace llvm { -template<> -struct DOTGraphTraits<RegionNode*> : public DefaultDOTGraphTraits { - DOTGraphTraits (bool isSimple=false) - : DefaultDOTGraphTraits(isSimple) {} +std::string DOTGraphTraits<RegionNode *>::getNodeLabel(RegionNode *Node, + RegionNode *Graph) { + if (!Node->isSubRegion()) { + BasicBlock *BB = Node->getNodeAs<BasicBlock>(); - std::string getNodeLabel(RegionNode *Node, RegionNode *Graph) { - - if (!Node->isSubRegion()) { - BasicBlock *BB = Node->getNodeAs<BasicBlock>(); - - if (isSimple()) - return DOTGraphTraits<DOTFuncInfo *> - ::getSimpleNodeLabel(BB, nullptr); - else - return DOTGraphTraits<DOTFuncInfo *> - ::getCompleteNodeLabel(BB, nullptr); - } - - return "Not implemented"; + if (isSimple()) + return DOTGraphTraits<DOTFuncInfo *>::getSimpleNodeLabel(BB, nullptr); + else + return DOTGraphTraits<DOTFuncInfo *>::getCompleteNodeLabel(BB, nullptr); } -}; + + return "Not implemented"; +} template <> struct DOTGraphTraits<RegionInfo *> : public DOTGraphTraits<RegionNode *> { @@ -138,7 +126,7 @@ printRegionCluster(*G->getTopLevelRegion(), GW, 4); } }; -} //end namespace llvm +} // end namespace llvm namespace { @@ -149,48 +137,49 @@ }; struct RegionPrinter - : public DOTGraphTraitsPrinter<RegionInfoPass, false, RegionInfo *, - RegionInfoPassGraphTraits> { + : public DOTGraphTraitsPrinterWrapperPass< + RegionInfoPass, false, RegionInfo *, RegionInfoPassGraphTraits> { static char ID; RegionPrinter() - : DOTGraphTraitsPrinter<RegionInfoPass, false, RegionInfo *, - RegionInfoPassGraphTraits>("reg", ID) { + : DOTGraphTraitsPrinterWrapperPass<RegionInfoPass, false, RegionInfo *, + RegionInfoPassGraphTraits>("reg", ID) { initializeRegionPrinterPass(*PassRegistry::getPassRegistry()); } }; char RegionPrinter::ID = 0; struct RegionOnlyPrinter - : public DOTGraphTraitsPrinter<RegionInfoPass, true, RegionInfo *, - RegionInfoPassGraphTraits> { + : public DOTGraphTraitsPrinterWrapperPass< + RegionInfoPass, true, RegionInfo *, RegionInfoPassGraphTraits> { static char ID; RegionOnlyPrinter() - : DOTGraphTraitsPrinter<RegionInfoPass, true, RegionInfo *, - RegionInfoPassGraphTraits>("reg", ID) { + : DOTGraphTraitsPrinterWrapperPass<RegionInfoPass, true, RegionInfo *, + RegionInfoPassGraphTraits>("reg", ID) { initializeRegionOnlyPrinterPass(*PassRegistry::getPassRegistry()); } }; char RegionOnlyPrinter::ID = 0; struct RegionViewer - : public DOTGraphTraitsViewer<RegionInfoPass, false, RegionInfo *, - RegionInfoPassGraphTraits> { + : public DOTGraphTraitsViewerWrapperPass< + RegionInfoPass, false, RegionInfo *, RegionInfoPassGraphTraits> { static char ID; RegionViewer() - : DOTGraphTraitsViewer<RegionInfoPass, false, RegionInfo *, - RegionInfoPassGraphTraits>("reg", ID) { + : DOTGraphTraitsViewerWrapperPass<RegionInfoPass, false, RegionInfo *, + RegionInfoPassGraphTraits>("reg", ID) { initializeRegionViewerPass(*PassRegistry::getPassRegistry()); } }; char RegionViewer::ID = 0; struct RegionOnlyViewer - : public DOTGraphTraitsViewer<RegionInfoPass, true, RegionInfo *, - RegionInfoPassGraphTraits> { + : public DOTGraphTraitsViewerWrapperPass<RegionInfoPass, true, RegionInfo *, + RegionInfoPassGraphTraits> { static char ID; RegionOnlyViewer() - : DOTGraphTraitsViewer<RegionInfoPass, true, RegionInfo *, - RegionInfoPassGraphTraits>("regonly", ID) { + : DOTGraphTraitsViewerWrapperPass<RegionInfoPass, true, RegionInfo *, + RegionInfoPassGraphTraits>("regonly", + ID) { initializeRegionOnlyViewerPass(*PassRegistry::getPassRegistry()); } };
diff --git a/src/llvm-project/llvm/lib/Analysis/ReplayInlineAdvisor.cpp b/src/llvm-project/llvm/lib/Analysis/ReplayInlineAdvisor.cpp index 294bc38..afc3d7f 100644 --- a/src/llvm-project/llvm/lib/Analysis/ReplayInlineAdvisor.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ReplayInlineAdvisor.cpp
@@ -14,9 +14,9 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/ReplayInlineAdvisor.h" -#include "llvm/IR/DebugInfoMetadata.h" -#include "llvm/IR/Instructions.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" #include <memory> using namespace llvm; @@ -26,8 +26,9 @@ ReplayInlineAdvisor::ReplayInlineAdvisor( Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr<InlineAdvisor> OriginalAdvisor, - const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks) - : InlineAdvisor(M, FAM), OriginalAdvisor(std::move(OriginalAdvisor)), + const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, + InlineContext IC) + : InlineAdvisor(M, FAM, IC), OriginalAdvisor(std::move(OriginalAdvisor)), ReplaySettings(ReplaySettings), EmitRemarks(EmitRemarks) { auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(ReplaySettings.ReplayFile); @@ -75,12 +76,15 @@ HasReplayRemarks = true; } -std::unique_ptr<InlineAdvisor> llvm::getReplayInlineAdvisor( - Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, - std::unique_ptr<InlineAdvisor> OriginalAdvisor, - const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks) { +std::unique_ptr<InlineAdvisor> +llvm::getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + LLVMContext &Context, + std::unique_ptr<InlineAdvisor> OriginalAdvisor, + const ReplayInlinerSettings &ReplaySettings, + bool EmitRemarks, InlineContext IC) { auto Advisor = std::make_unique<ReplayInlineAdvisor>( - M, FAM, Context, std::move(OriginalAdvisor), ReplaySettings, EmitRemarks); + M, FAM, Context, std::move(OriginalAdvisor), ReplaySettings, EmitRemarks, + IC); if (!Advisor->areReplayRemarksLoaded()) Advisor.reset(); return Advisor;
diff --git a/src/llvm-project/llvm/lib/Analysis/ScalarEvolution.cpp b/src/llvm-project/llvm/lib/Analysis/ScalarEvolution.cpp index 20661ef..2958a50 100644 --- a/src/llvm-project/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -79,7 +79,6 @@ #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/ScalarEvolutionDivision.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" @@ -96,7 +95,6 @@ #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalValue.h" -#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" @@ -104,7 +102,6 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Metadata.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PatternMatch.h" #include "llvm/IR/Type.h" @@ -125,7 +122,6 @@ #include <algorithm> #include <cassert> #include <climits> -#include <cstddef> #include <cstdint> #include <cstdlib> #include <map> @@ -146,17 +142,21 @@ STATISTIC(NumBruteForceTripCountsComputed, "Number of loops with trip counts computed by force"); -static cl::opt<unsigned> -MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, - cl::ZeroOrMore, - cl::desc("Maximum number of iterations SCEV will " - "symbolically execute a constant " - "derived loop"), - cl::init(100)); +#ifdef EXPENSIVE_CHECKS +bool llvm::VerifySCEV = true; +#else +bool llvm::VerifySCEV = false; +#endif -// FIXME: Enable this with EXPENSIVE_CHECKS when the test suite is clean. -static cl::opt<bool> VerifySCEV( - "verify-scev", cl::Hidden, +static cl::opt<unsigned> + MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, + cl::desc("Maximum number of iterations SCEV will " + "symbolically execute a constant " + "derived loop"), + cl::init(100)); + +static cl::opt<bool, true> VerifySCEVOpt( + "verify-scev", cl::Hidden, cl::location(VerifySCEV), cl::desc("Verify ScalarEvolution's backedge taken counts (slow)")); static cl::opt<bool> VerifySCEVStrict( "verify-scev-strict", cl::Hidden, @@ -231,6 +231,17 @@ cl::desc("Use more powerful methods of sharpening expression ranges. May " "be costly in terms of compile time")); +static cl::opt<unsigned> MaxPhiSCCAnalysisSize( + "scalar-evolution-max-scc-analysis-depth", cl::Hidden, + cl::desc("Maximum amount of nodes to process while searching SCEVUnknown " + "Phi strongly connected components"), + cl::init(8)); + +static cl::opt<bool> + EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden, + cl::desc("Handle <= and >= in finite loops"), + cl::init(true)); + //===----------------------------------------------------------------------===// // SCEV class definitions //===----------------------------------------------------------------------===// @@ -519,12 +530,13 @@ } void SCEVUnknown::allUsesReplacedWith(Value *New) { + // Clear this SCEVUnknown from various maps. + SE->forgetMemoizedResults(this); + // Remove this SCEVUnknown from the uniquing map. SE->UniqueSCEVs.RemoveNode(this); - // Update this SCEVUnknown to point to the new value. This is needed - // because there may still be outstanding SCEVs which still point to - // this SCEVUnknown. + // Replace the value pointer in case someone is still using this SCEVUnknown. setValPtr(New); } @@ -1146,7 +1158,7 @@ const SCEV *visitAddExpr(const SCEVAddExpr *Expr) { SmallVector<const SCEV *, 2> Operands; bool Changed = false; - for (auto *Op : Expr->operands()) { + for (const auto *Op : Expr->operands()) { Operands.push_back(visit(Op)); Changed |= Op != Operands.back(); } @@ -1156,7 +1168,7 @@ const SCEV *visitMulExpr(const SCEVMulExpr *Expr) { SmallVector<const SCEV *, 2> Operands; bool Changed = false; - for (auto *Op : Expr->operands()) { + for (const auto *Op : Expr->operands()) { Operands.push_back(visit(Op)); Changed |= Op != Operands.back(); } @@ -1643,10 +1655,12 @@ // If we have special knowledge that this addrec won't overflow, // we don't need to do any further analysis. - if (AR->hasNoUnsignedWrap()) - return getAddRecExpr( - getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + if (AR->hasNoUnsignedWrap()) { + Start = + getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); + } // Check whether the backedge-taken count is SCEVCouldNotCompute. // Note that this serves two purposes: It filters out loops that are @@ -1688,11 +1702,10 @@ // Cache knowledge of AR NUW, which is propagated to this AddRec. setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW); // Return the expression with the addrec on the outside. - return getAddRecExpr( - getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, - Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, + Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } // Similar to above, only this time treat the step value as signed. // This covers loops that count down. @@ -1707,11 +1720,10 @@ // Negative step causes unsigned wrap, but it still can't self-wrap. setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW); // Return the expression with the addrec on the outside. - return getAddRecExpr( - getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, - Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, + Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } } @@ -1733,11 +1745,10 @@ // issue. It's not clear that the order of checks does matter, but // it's one of two issue possible causes for a change which was // reverted. Be conservative for the moment. - return getAddRecExpr( - getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, - Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = + getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } // For a negative step, we can extend the operands iff doing so only @@ -1752,11 +1763,10 @@ // still can't self-wrap. setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW); // Return the expression with the addrec on the outside. - return getAddRecExpr( - getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, - Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, + Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } } @@ -1780,9 +1790,10 @@ if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) { setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW); - return getAddRecExpr( - getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + Start = + getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } @@ -1984,10 +1995,12 @@ // If we have special knowledge that this addrec won't overflow, // we don't need to do any further analysis. - if (AR->hasNoSignedWrap()) - return getAddRecExpr( - getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, SCEV::FlagNSW); + if (AR->hasNoSignedWrap()) { + Start = + getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, SCEV::FlagNSW); + } // Check whether the backedge-taken count is SCEVCouldNotCompute. // Note that this serves two purposes: It filters out loops that are @@ -2030,11 +2043,10 @@ // Cache knowledge of AR NSW, which is propagated to this AddRec. setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW); // Return the expression with the addrec on the outside. - return getAddRecExpr( - getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, - Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, + Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } // Similar to above, only this time treat the step value as unsigned. // This covers loops that count up with an unsigned step. @@ -2056,11 +2068,10 @@ setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW); // Return the expression with the addrec on the outside. - return getAddRecExpr( - getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, - Depth + 1), - getZeroExtendExpr(Step, Ty, Depth + 1), L, - AR->getNoWrapFlags()); + Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, + Depth + 1); + Step = getZeroExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } } @@ -2072,9 +2083,10 @@ // issue. It's not clear that the order of checks does matter, but // it's one of two issue possible causes for a change which was // reverted. Be conservative for the moment. - return getAddRecExpr( - getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + Start = + getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw> @@ -2096,9 +2108,10 @@ if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) { setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW); - return getAddRecExpr( - getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1), - getSignExtendExpr(Step, Ty, Depth + 1), L, AR->getNoWrapFlags()); + Start = + getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1); + Step = getSignExtendExpr(Step, Ty, Depth + 1); + return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags()); } } @@ -2300,15 +2313,19 @@ const SCEV *A = (this->*Extension)( (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0); - const SCEV *B = (this->*Operation)((this->*Extension)(LHS, WideTy, 0), - (this->*Extension)(RHS, WideTy, 0), - SCEV::FlagAnyWrap, 0); + const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0); + const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0); + const SCEV *B = (this->*Operation)(LHSB, RHSB, SCEV::FlagAnyWrap, 0); return A == B; } -std::pair<SCEV::NoWrapFlags, bool /*Deduced*/> +Optional<SCEV::NoWrapFlags> ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp( const OverflowingBinaryOperator *OBO) { + // It cannot be done any better. + if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap()) + return None; + SCEV::NoWrapFlags Flags = SCEV::NoWrapFlags::FlagAnyWrap; if (OBO->hasNoUnsignedWrap()) @@ -2318,13 +2335,10 @@ bool Deduced = false; - if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap()) - return {Flags, Deduced}; - if (OBO->getOpcode() != Instruction::Add && OBO->getOpcode() != Instruction::Sub && OBO->getOpcode() != Instruction::Mul) - return {Flags, Deduced}; + return None; const SCEV *LHS = getSCEV(OBO->getOperand(0)); const SCEV *RHS = getSCEV(OBO->getOperand(1)); @@ -2343,7 +2357,9 @@ Deduced = true; } - return {Flags, Deduced}; + if (Deduced) + return Flags; + return None; } // We're trying to construct a SCEV of type `Type' with `Ops' as operands and @@ -3106,12 +3122,13 @@ // TODO: There are some cases where this transformation is not // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of // this transformation should be narrowed down. - if (Add->getNumOperands() == 2 && containsConstantInAddMulChain(Add)) - return getAddExpr(getMulExpr(LHSC, Add->getOperand(0), - SCEV::FlagAnyWrap, Depth + 1), - getMulExpr(LHSC, Add->getOperand(1), - SCEV::FlagAnyWrap, Depth + 1), - SCEV::FlagAnyWrap, Depth + 1); + if (Add->getNumOperands() == 2 && containsConstantInAddMulChain(Add)) { + const SCEV *LHS = getMulExpr(LHSC, Add->getOperand(0), + SCEV::FlagAnyWrap, Depth + 1); + const SCEV *RHS = getMulExpr(LHSC, Add->getOperand(1), + SCEV::FlagAnyWrap, Depth + 1); + return getAddExpr(LHS, RHS, SCEV::FlagAnyWrap, Depth + 1); + } if (Ops[0]->isAllOnesValue()) { // If we have a mul by -1 of an add, try distributing the -1 among the @@ -3466,12 +3483,8 @@ } // Fold if both operands are constant. - if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { - Constant *LHSCV = LHSC->getValue(); - Constant *RHSCV = RHSC->getValue(); - return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV, - RHSCV))); - } + if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) + return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt())); } } @@ -4002,6 +4015,59 @@ } // namespace +/// Return true if V is poison given that AssumedPoison is already poison. +static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) { + // The only way poison may be introduced in a SCEV expression is from a + // poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown, + // not SCEVConstant). Notably, nowrap flags in SCEV nodes can *not* + // introduce poison -- they encode guaranteed, non-speculated knowledge. + // + // Additionally, all SCEV nodes propagate poison from inputs to outputs, + // with the notable exception of umin_seq, where only poison from the first + // operand is (unconditionally) propagated. + struct SCEVPoisonCollector { + bool LookThroughSeq; + SmallPtrSet<const SCEV *, 4> MaybePoison; + SCEVPoisonCollector(bool LookThroughSeq) : LookThroughSeq(LookThroughSeq) {} + + bool follow(const SCEV *S) { + // TODO: We can always follow the first operand, but the SCEVTraversal + // API doesn't support this. + if (!LookThroughSeq && isa<SCEVSequentialMinMaxExpr>(S)) + return false; + + if (auto *SU = dyn_cast<SCEVUnknown>(S)) { + if (!isGuaranteedNotToBePoison(SU->getValue())) + MaybePoison.insert(S); + } + return true; + } + bool isDone() const { return false; } + }; + + // First collect all SCEVs that might result in AssumedPoison to be poison. + // We need to look through umin_seq here, because we want to find all SCEVs + // that *might* result in poison, not only those that are *required* to. + SCEVPoisonCollector PC1(/* LookThroughSeq */ true); + visitAll(AssumedPoison, PC1); + + // AssumedPoison is never poison. As the assumption is false, the implication + // is true. Don't bother walking the other SCEV in this case. + if (PC1.MaybePoison.empty()) + return true; + + // Collect all SCEVs in S that, if poison, *will* result in S being poison + // as well. We cannot look through umin_seq here, as its argument only *may* + // make the result poison. + SCEVPoisonCollector PC2(/* LookThroughSeq */ false); + visitAll(S, PC2); + + // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison, + // it will also make S poison by being part of PC2.MaybePoison. + return all_of(PC1.MaybePoison, + [&](const SCEV *S) { return PC2.MaybePoison.contains(S); }); +} + const SCEV * ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind, SmallVectorImpl<const SCEV *> &Ops) { @@ -4010,11 +4076,6 @@ assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!"); if (Ops.size() == 1) return Ops[0]; - if (Ops.size() == 2 && - any_of(Ops, [](const SCEV *Op) { return isa<SCEVConstant>(Op); })) - return getMinMaxExpr( - SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Kind), - Ops); #ifndef NDEBUG Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); for (unsigned i = 1, e = Ops.size(); i != e; ++i) { @@ -4063,6 +4124,39 @@ return getSequentialMinMaxExpr(Kind, Ops); } + const SCEV *SaturationPoint; + ICmpInst::Predicate Pred; + switch (Kind) { + case scSequentialUMinExpr: + SaturationPoint = getZero(Ops[0]->getType()); + Pred = ICmpInst::ICMP_ULE; + break; + default: + llvm_unreachable("Not a sequential min/max type."); + } + + for (unsigned i = 1, e = Ops.size(); i != e; ++i) { + // We can replace %x umin_seq %y with %x umin %y if either: + // * %y being poison implies %x is also poison. + // * %x cannot be the saturating value (e.g. zero for umin). + if (::impliesPoison(Ops[i], Ops[i - 1]) || + isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, Ops[i - 1], + SaturationPoint)) { + SmallVector<const SCEV *> SeqOps = {Ops[i - 1], Ops[i]}; + Ops[i - 1] = getMinMaxExpr( + SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Kind), + SeqOps); + Ops.erase(Ops.begin() + i); + return getSequentialMinMaxExpr(Kind, Ops); + } + // Fold %x umin_seq %y to %x if %x ule %y. + // TODO: We might be able to prove the predicate for a later operand. + if (isKnownViaNonRecursiveReasoning(Pred, Ops[i - 1], Ops[i])) { + Ops.erase(Ops.begin() + i); + return getSequentialMinMaxExpr(Kind, Ops); + } + } + // Okay, it looks like we really DO need an expr. Check to see if we // already have one, otherwise create a new one. FoldingSetNodeID ID; @@ -4265,39 +4359,20 @@ return FoundAddRec; } -/// Try to split a SCEVAddExpr into a pair of {SCEV, ConstantInt}. -/// If \p S is a SCEVAddExpr and is composed of a sub SCEV S' and an -/// offset I, then return {S', I}, else return {\p S, nullptr}. -static std::pair<const SCEV *, ConstantInt *> splitAddExpr(const SCEV *S) { - const auto *Add = dyn_cast<SCEVAddExpr>(S); - if (!Add) - return {S, nullptr}; - - if (Add->getNumOperands() != 2) - return {S, nullptr}; - - auto *ConstOp = dyn_cast<SCEVConstant>(Add->getOperand(0)); - if (!ConstOp) - return {S, nullptr}; - - return {Add->getOperand(1), ConstOp->getValue()}; -} - /// Return the ValueOffsetPair set for \p S. \p S can be represented /// by the value and offset from any ValueOffsetPair in the set. -ScalarEvolution::ValueOffsetPairSetVector * -ScalarEvolution::getSCEVValues(const SCEV *S) { +ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) { ExprValueMapType::iterator SI = ExprValueMap.find_as(S); if (SI == ExprValueMap.end()) - return nullptr; + return None; #ifndef NDEBUG if (VerifySCEVMap) { // Check there is no dangling Value in the set returned. - for (const auto &VE : SI->second) - assert(ValueExprMap.count(VE.first)); + for (Value *V : SI->second) + assert(ValueExprMap.count(V)); } #endif - return &SI->second; + return SI->second.getArrayRef(); } /// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V) @@ -4306,20 +4381,11 @@ void ScalarEvolution::eraseValueFromMap(Value *V) { ValueExprMapType::iterator I = ValueExprMap.find_as(V); if (I != ValueExprMap.end()) { - const SCEV *S = I->second; - // Remove {V, 0} from the set of ExprValueMap[S] - if (auto *SV = getSCEVValues(S)) - SV->remove({V, nullptr}); - - // Remove {V, Offset} from the set of ExprValueMap[Stripped] - const SCEV *Stripped; - ConstantInt *Offset; - std::tie(Stripped, Offset) = splitAddExpr(S); - if (Offset != nullptr) { - if (auto *SV = getSCEVValues(Stripped)) - SV->remove({V, Offset}); - } - ValueExprMap.erase(V); + auto EVIt = ExprValueMap.find(I->second); + bool Removed = EVIt->second.remove(V); + (void) Removed; + assert(Removed && "Value not in ExprValueMap?"); + ValueExprMap.erase(I); } } @@ -4330,7 +4396,7 @@ auto It = ValueExprMap.find_as(V); if (It == ValueExprMap.end()) { ValueExprMap.insert({SCEVCallbackVH(V, this), S}); - ExprValueMap[S].insert({V, nullptr}); + ExprValueMap[S].insert(V); } } @@ -4339,33 +4405,9 @@ const SCEV *ScalarEvolution::getSCEV(Value *V) { assert(isSCEVable(V->getType()) && "Value is not SCEVable!"); - const SCEV *S = getExistingSCEV(V); - if (S == nullptr) { - S = createSCEV(V); - // During PHI resolution, it is possible to create two SCEVs for the same - // V, so it is needed to double check whether V->S is inserted into - // ValueExprMap before insert S->{V, 0} into ExprValueMap. - std::pair<ValueExprMapType::iterator, bool> Pair = - ValueExprMap.insert({SCEVCallbackVH(V, this), S}); - if (Pair.second) { - ExprValueMap[S].insert({V, nullptr}); - - // If S == Stripped + Offset, add Stripped -> {V, Offset} into - // ExprValueMap. - const SCEV *Stripped = S; - ConstantInt *Offset = nullptr; - std::tie(Stripped, Offset) = splitAddExpr(S); - // If stripped is SCEVUnknown, don't bother to save - // Stripped -> {V, offset}. It doesn't simplify and sometimes even - // increase the complexity of the expansion code. - // If V is GetElementPtrInst, don't save Stripped -> {V, offset} - // because it may generate add/sub instead of GEP in SCEV expansion. - if (Offset != nullptr && !isa<SCEVUnknown>(Stripped) && - !isa<GetElementPtrInst>(V)) - ExprValueMap[Stripped].insert({V, Offset}); - } - } - return S; + if (const SCEV *S = getExistingSCEV(V)) + return S; + return createSCEVIter(V); } const SCEV *ScalarEvolution::getExistingSCEV(Value *V) { @@ -4620,7 +4662,7 @@ // Find the max type first. Type *MaxType = nullptr; - for (auto *S : Ops) + for (const auto *S : Ops) if (MaxType) MaxType = getWiderType(MaxType, S->getType()); else @@ -4629,7 +4671,7 @@ // Extend all ops to max type. SmallVector<const SCEV *, 2> PromotedOps; - for (auto *S : Ops) + for (const auto *S : Ops) PromotedOps.push_back(getNoopOrZeroExtend(S, MaxType)); // Generate umin. @@ -4795,16 +4837,16 @@ SelectInst *SI = cast<SelectInst>(I); Optional<const SCEV *> Res = compareWithBackedgeCondition(SI->getCondition()); - if (Res.hasValue()) { - bool IsOne = cast<SCEVConstant>(Res.getValue())->getValue()->isOne(); + if (Res) { + bool IsOne = cast<SCEVConstant>(Res.value())->getValue()->isOne(); Result = SE.getSCEV(IsOne ? SI->getTrueValue() : SI->getFalseValue()); } break; } default: { Optional<const SCEV *> Res = compareWithBackedgeCondition(I); - if (Res.hasValue()) - Result = Res.getValue(); + if (Res) + Result = Res.value(); break; } } @@ -5067,6 +5109,9 @@ // Instcombine turns add of signmask into xor as a strength reduction step. if (RHSC->getValue().isSignMask()) return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1)); + // Binary `xor` is a bit-wise `add`. + if (V->getType()->isIntegerTy(1)) + return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1)); return BinaryOp(Op); case Instruction::LShr: @@ -5489,8 +5534,8 @@ return true; auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool { - if (Expr1 != Expr2 && !Preds.implies(SE.getEqualPredicate(Expr1, Expr2)) && - !Preds.implies(SE.getEqualPredicate(Expr2, Expr1))) + if (Expr1 != Expr2 && !Preds->implies(SE.getEqualPredicate(Expr1, Expr2)) && + !Preds->implies(SE.getEqualPredicate(Expr2, Expr1))) return false; return true; }; @@ -5872,31 +5917,53 @@ if (const SCEV *S = createNodeFromSelectLikePHI(PN)) return S; - // If the PHI has a single incoming value, follow that value, unless the - // PHI's incoming blocks are in a different loop, in which case doing so - // risks breaking LCSSA form. Instcombine would normally zap these, but - // it doesn't have DominatorTree information, so it may miss cases. - if (Value *V = SimplifyInstruction(PN, {getDataLayout(), &TLI, &DT, &AC})) - if (LI.replacementPreservesLCSSAForm(PN, V)) - return getSCEV(V); + if (Value *V = simplifyInstruction(PN, {getDataLayout(), &TLI, &DT, &AC})) + return getSCEV(V); // If it's not a loop phi, we can't handle it yet. return getUnknown(PN); } -const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Instruction *I, - Value *Cond, - Value *TrueVal, - Value *FalseVal) { - // Handle "constant" branch or select. This can occur for instance when a - // loop pass transforms an inner loop and moves on to process the outer loop. - if (auto *CI = dyn_cast<ConstantInt>(Cond)) - return getSCEV(CI->isOne() ? TrueVal : FalseVal); +bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind, + SCEVTypes RootKind) { + struct FindClosure { + const SCEV *OperandToFind; + const SCEVTypes RootKind; // Must be a sequential min/max expression. + const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind. + bool Found = false; + + bool canRecurseInto(SCEVTypes Kind) const { + // We can only recurse into the SCEV expression of the same effective type + // as the type of our root SCEV expression, and into zero-extensions. + return RootKind == Kind || NonSequentialRootKind == Kind || + scZeroExtend == Kind; + }; + + FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind) + : OperandToFind(OperandToFind), RootKind(RootKind), + NonSequentialRootKind( + SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType( + RootKind)) {} + + bool follow(const SCEV *S) { + Found = S == OperandToFind; + + return !isDone() && canRecurseInto(S->getSCEVType()); + } + + bool isDone() const { return Found; } + }; + + FindClosure FC(OperandToFind, RootKind); + visitAll(Root, FC); + return FC.Found; +} + +const SCEV *ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond( + Instruction *I, ICmpInst *Cond, Value *TrueVal, Value *FalseVal) { // Try to match some simple smax or umax patterns. - auto *ICI = dyn_cast<ICmpInst>(Cond); - if (!ICI) - return getUnknown(I); + auto *ICI = Cond; Value *LHS = ICI->getOperand(0); Value *RHS = ICI->getOperand(1); @@ -5958,31 +6025,36 @@ } break; case ICmpInst::ICMP_NE: - // n != 0 ? n+x : 1+x -> umax(n, 1)+x - if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType()) && - isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) { - const SCEV *One = getOne(I->getType()); - const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); - const SCEV *LA = getSCEV(TrueVal); - const SCEV *RA = getSCEV(FalseVal); - const SCEV *LDiff = getMinusSCEV(LA, LS); - const SCEV *RDiff = getMinusSCEV(RA, One); - if (LDiff == RDiff) - return getAddExpr(getUMaxExpr(One, LS), LDiff); - } - break; + // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y + std::swap(TrueVal, FalseVal); + LLVM_FALLTHROUGH; case ICmpInst::ICMP_EQ: - // n == 0 ? 1+x : n+x -> umax(n, 1)+x + // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType()) && isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) { - const SCEV *One = getOne(I->getType()); - const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); - const SCEV *LA = getSCEV(TrueVal); - const SCEV *RA = getSCEV(FalseVal); - const SCEV *LDiff = getMinusSCEV(LA, One); - const SCEV *RDiff = getMinusSCEV(RA, LS); - if (LDiff == RDiff) - return getAddExpr(getUMaxExpr(One, LS), LDiff); + const SCEV *X = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); + const SCEV *TrueValExpr = getSCEV(TrueVal); // C+y + const SCEV *FalseValExpr = getSCEV(FalseVal); // x+y + const SCEV *Y = getMinusSCEV(FalseValExpr, X); // y = (x+y)-x + const SCEV *C = getMinusSCEV(TrueValExpr, Y); // C = (C+y)-y + if (isa<SCEVConstant>(C) && cast<SCEVConstant>(C)->getAPInt().ule(1)) + return getAddExpr(getUMaxExpr(X, C), Y); + } + // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...)) + // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...)) + // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...) + // -> umin_seq(x, umin (..., umin_seq(...), ...)) + if (isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero() && + isa<ConstantInt>(TrueVal) && cast<ConstantInt>(TrueVal)->isZero()) { + const SCEV *X = getSCEV(LHS); + while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(X)) + X = ZExt->getOperand(); + if (getTypeSizeInBits(X->getType()) <= getTypeSizeInBits(I->getType())) { + const SCEV *FalseValExpr = getSCEV(FalseVal); + if (SCEVMinMaxExprContains(FalseValExpr, X, scSequentialUMinExpr)) + return getUMinExpr(getNoopOrZeroExtend(X, I->getType()), FalseValExpr, + /*Sequential=*/true); + } } break; default: @@ -5992,12 +6064,95 @@ return getUnknown(I); } +static Optional<const SCEV *> +createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr, + const SCEV *TrueExpr, const SCEV *FalseExpr) { + assert(CondExpr->getType()->isIntegerTy(1) && + TrueExpr->getType() == FalseExpr->getType() && + TrueExpr->getType()->isIntegerTy(1) && + "Unexpected operands of a select."); + + // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0) + // --> C + (umin_seq cond, x - C) + // + // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C)) + // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0) + // --> C + (umin_seq ~cond, x - C) + + // FIXME: while we can't legally model the case where both of the hands + // are fully variable, we only require that the *difference* is constant. + if (!isa<SCEVConstant>(TrueExpr) && !isa<SCEVConstant>(FalseExpr)) + return None; + + const SCEV *X, *C; + if (isa<SCEVConstant>(TrueExpr)) { + CondExpr = SE->getNotSCEV(CondExpr); + X = FalseExpr; + C = TrueExpr; + } else { + X = TrueExpr; + C = FalseExpr; + } + return SE->getAddExpr(C, SE->getUMinExpr(CondExpr, SE->getMinusSCEV(X, C), + /*Sequential=*/true)); +} + +static Optional<const SCEV *> createNodeForSelectViaUMinSeq(ScalarEvolution *SE, + Value *Cond, + Value *TrueVal, + Value *FalseVal) { + if (!isa<ConstantInt>(TrueVal) && !isa<ConstantInt>(FalseVal)) + return None; + + const auto *SECond = SE->getSCEV(Cond); + const auto *SETrue = SE->getSCEV(TrueVal); + const auto *SEFalse = SE->getSCEV(FalseVal); + return createNodeForSelectViaUMinSeq(SE, SECond, SETrue, SEFalse); +} + +const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq( + Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) { + assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?"); + assert(TrueVal->getType() == FalseVal->getType() && + V->getType() == TrueVal->getType() && + "Types of select hands and of the result must match."); + + // For now, only deal with i1-typed `select`s. + if (!V->getType()->isIntegerTy(1)) + return getUnknown(V); + + if (Optional<const SCEV *> S = + createNodeForSelectViaUMinSeq(this, Cond, TrueVal, FalseVal)) + return *S; + + return getUnknown(V); +} + +const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond, + Value *TrueVal, + Value *FalseVal) { + // Handle "constant" branch or select. This can occur for instance when a + // loop pass transforms an inner loop and moves on to process the outer loop. + if (auto *CI = dyn_cast<ConstantInt>(Cond)) + return getSCEV(CI->isOne() ? TrueVal : FalseVal); + + if (auto *I = dyn_cast<Instruction>(V)) { + if (auto *ICI = dyn_cast<ICmpInst>(Cond)) { + const SCEV *S = createNodeForSelectOrPHIInstWithICmpInstCond( + I, ICI, TrueVal, FalseVal); + if (!isa<SCEVUnknown>(S)) + return S; + } + } + + return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal); +} + /// Expand GEP instructions into add and multiply operations. This allows them /// to be analyzed by regular SCEV code. const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { - // Don't attempt to analyze GEPs over unsized objects. - if (!GEP->getSourceElementType()->isSized()) - return getUnknown(GEP); + assert(GEP->getSourceElementType()->isSized() && + "GEP source element type must be sized"); SmallVector<const SCEV *, 4> IndexExprs; for (Value *Index : GEP->indices()) @@ -6430,9 +6585,9 @@ // Check if the IR explicitly contains !range metadata. Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue()); - if (MDRange.hasValue()) - ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue(), - RangeType); + if (MDRange) + ConservativeResult = + ConservativeResult.intersectWith(MDRange.value(), RangeType); // Use facts about recurrences in the underlying IR. Note that add // recurrences are AddRecExprs and thus don't hit this path. This @@ -6481,7 +6636,7 @@ // Make sure that we do not run over cycled Phis. if (PendingPhiRanges.insert(Phi).second) { ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false); - for (auto &Op : Phi->operands()) { + for (const auto &Op : Phi->operands()) { auto OpRange = getRangeRef(getSCEV(Op), SignHint); RangeFromOps = RangeFromOps.unionWith(OpRange); // No point to continue if we already have a full set. @@ -6496,6 +6651,13 @@ } } + // vscale can't be equal to zero + if (const auto *II = dyn_cast<IntrinsicInst>(U->getValue())) + if (II->getIntrinsicID() == Intrinsic::vscale) { + ConstantRange Disallowed = APInt::getZero(BitWidth); + ConservativeResult = ConservativeResult.difference(Disallowed); + } + return setRange(U, SignHint, std::move(ConservativeResult)); } @@ -6719,7 +6881,7 @@ FalseValue = *FalseVal; // Re-apply the cast we peeled off earlier - if (CastOp.hasValue()) + if (CastOp) switch (*CastOp) { default: llvm_unreachable("Unknown SCEV cast type!"); @@ -6818,13 +6980,13 @@ Ops.push_back(S); }; if (auto *S2 = dyn_cast<SCEVCastExpr>(S)) - for (auto *Op : S2->operands()) + for (const auto *Op : S2->operands()) InsertUnique(Op); else if (auto *S2 = dyn_cast<SCEVNAryExpr>(S)) - for (auto *Op : S2->operands()) + for (const auto *Op : S2->operands()) InsertUnique(Op); else if (auto *S2 = dyn_cast<SCEVUDivExpr>(S)) - for (auto *Op : S2->operands()) + for (const auto *Op : S2->operands()) InsertUnique(Op); } @@ -6846,7 +7008,7 @@ Worklist.push_back(S); }; - for (auto *S : Ops) + for (const auto *S : Ops) pushOp(S); const Instruction *Bound = nullptr; @@ -6858,7 +7020,7 @@ } else { SmallVector<const SCEV *, 4> Ops; collectUniqueOps(S, Ops); - for (auto *Op : Ops) + for (const auto *Op : Ops) pushOp(Op); } } @@ -6962,7 +7124,7 @@ while (!PoisonStack.empty() && !LatchControlDependentOnPoison) { const Instruction *Poison = PoisonStack.pop_back_val(); - for (auto *PoisonUser : Poison->users()) { + for (const auto *PoisonUser : Poison->users()) { if (propagatesPoison(cast<Operator>(PoisonUser))) { if (Pushed.insert(cast<Instruction>(PoisonUser)).second) PoisonStack.push_back(cast<Instruction>(PoisonUser)); @@ -7020,6 +7182,241 @@ return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L)); } +const SCEV *ScalarEvolution::createSCEVIter(Value *V) { + // Worklist item with a Value and a bool indicating whether all operands have + // been visited already. + using PointerTy = PointerIntPair<Value *, 1, bool>; + SmallVector<PointerTy> Stack; + + Stack.emplace_back(V, true); + Stack.emplace_back(V, false); + while (!Stack.empty()) { + auto E = Stack.pop_back_val(); + Value *CurV = E.getPointer(); + + if (getExistingSCEV(CurV)) + continue; + + SmallVector<Value *> Ops; + const SCEV *CreatedSCEV = nullptr; + // If all operands have been visited already, create the SCEV. + if (E.getInt()) { + CreatedSCEV = createSCEV(CurV); + } else { + // Otherwise get the operands we need to create SCEV's for before creating + // the SCEV for CurV. If the SCEV for CurV can be constructed trivially, + // just use it. + CreatedSCEV = getOperandsToCreate(CurV, Ops); + } + + if (CreatedSCEV) { + insertValueToMap(CurV, CreatedSCEV); + } else { + // Queue CurV for SCEV creation, followed by its's operands which need to + // be constructed first. + Stack.emplace_back(CurV, true); + for (Value *Op : Ops) + Stack.emplace_back(Op, false); + } + } + + return getExistingSCEV(V); +} + +const SCEV * +ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) { + if (!isSCEVable(V->getType())) + return getUnknown(V); + + if (Instruction *I = dyn_cast<Instruction>(V)) { + // Don't attempt to analyze instructions in blocks that aren't + // reachable. Such instructions don't matter, and they aren't required + // to obey basic rules for definitions dominating uses which this + // analysis depends on. + if (!DT.isReachableFromEntry(I->getParent())) + return getUnknown(PoisonValue::get(V->getType())); + } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) + return getConstant(CI); + else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) { + if (!GA->isInterposable()) { + Ops.push_back(GA->getAliasee()); + return nullptr; + } + return getUnknown(V); + } else if (!isa<ConstantExpr>(V)) + return getUnknown(V); + + Operator *U = cast<Operator>(V); + if (auto BO = MatchBinaryOp(U, DT)) { + bool IsConstArg = isa<ConstantInt>(BO->RHS); + switch (BO->Opcode) { + case Instruction::Add: { + // For additions and multiplications, traverse add/mul chains for which we + // can potentially create a single SCEV, to reduce the number of + // get{Add,Mul}Expr calls. + do { + if (BO->Op) { + if (BO->Op != V && getExistingSCEV(BO->Op)) { + Ops.push_back(BO->Op); + break; + } + } + Ops.push_back(BO->RHS); + auto NewBO = MatchBinaryOp(BO->LHS, DT); + if (!NewBO || (NewBO->Opcode != Instruction::Add && + NewBO->Opcode != Instruction::Sub)) { + Ops.push_back(BO->LHS); + break; + } + BO = NewBO; + } while (true); + return nullptr; + } + + case Instruction::Mul: { + do { + if (BO->Op) { + if (BO->Op != V && getExistingSCEV(BO->Op)) { + Ops.push_back(BO->Op); + break; + } + } + Ops.push_back(BO->RHS); + auto NewBO = MatchBinaryOp(BO->LHS, DT); + if (!NewBO || NewBO->Opcode != Instruction::Mul) { + Ops.push_back(BO->LHS); + break; + } + BO = NewBO; + } while (true); + return nullptr; + } + case Instruction::Sub: + case Instruction::UDiv: + case Instruction::URem: + break; + case Instruction::AShr: + case Instruction::Shl: + case Instruction::Xor: + if (!IsConstArg) + return nullptr; + break; + case Instruction::And: + case Instruction::Or: + if (!IsConstArg && BO->LHS->getType()->isIntegerTy(1)) + return nullptr; + break; + case Instruction::LShr: + return getUnknown(V); + default: + llvm_unreachable("Unhandled binop"); + break; + } + + Ops.push_back(BO->LHS); + Ops.push_back(BO->RHS); + return nullptr; + } + + switch (U->getOpcode()) { + case Instruction::Trunc: + case Instruction::ZExt: + case Instruction::SExt: + case Instruction::PtrToInt: + Ops.push_back(U->getOperand(0)); + return nullptr; + + case Instruction::BitCast: + if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) { + Ops.push_back(U->getOperand(0)); + return nullptr; + } + return getUnknown(V); + + case Instruction::SDiv: + case Instruction::SRem: + Ops.push_back(U->getOperand(0)); + Ops.push_back(U->getOperand(1)); + return nullptr; + + case Instruction::GetElementPtr: + assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() && + "GEP source element type must be sized"); + for (Value *Index : U->operands()) + Ops.push_back(Index); + return nullptr; + + case Instruction::IntToPtr: + return getUnknown(V); + + case Instruction::PHI: + // Keep constructing SCEVs' for phis recursively for now. + return nullptr; + + case Instruction::Select: { + // Check if U is a select that can be simplified to a SCEVUnknown. + auto CanSimplifyToUnknown = [this, U]() { + if (U->getType()->isIntegerTy(1) || isa<ConstantInt>(U->getOperand(0))) + return false; + + auto *ICI = dyn_cast<ICmpInst>(U->getOperand(0)); + if (!ICI) + return false; + Value *LHS = ICI->getOperand(0); + Value *RHS = ICI->getOperand(1); + if (ICI->getPredicate() == CmpInst::ICMP_EQ || + ICI->getPredicate() == CmpInst::ICMP_NE) { + if (!(isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero())) + return true; + } else if (getTypeSizeInBits(LHS->getType()) > + getTypeSizeInBits(U->getType())) + return true; + return false; + }; + if (CanSimplifyToUnknown()) + return getUnknown(U); + + for (Value *Inc : U->operands()) + Ops.push_back(Inc); + return nullptr; + break; + } + case Instruction::Call: + case Instruction::Invoke: + if (Value *RV = cast<CallBase>(U)->getReturnedArgOperand()) { + Ops.push_back(RV); + return nullptr; + } + + if (auto *II = dyn_cast<IntrinsicInst>(U)) { + switch (II->getIntrinsicID()) { + case Intrinsic::abs: + Ops.push_back(II->getArgOperand(0)); + return nullptr; + case Intrinsic::umax: + case Intrinsic::umin: + case Intrinsic::smax: + case Intrinsic::smin: + case Intrinsic::usub_sat: + case Intrinsic::uadd_sat: + Ops.push_back(II->getArgOperand(0)); + Ops.push_back(II->getArgOperand(1)); + return nullptr; + case Intrinsic::start_loop_iterations: + case Intrinsic::annotation: + case Intrinsic::ptr_annotation: + Ops.push_back(II->getArgOperand(0)); + return nullptr; + default: + break; + } + } + break; + } + + return nullptr; +} + const SCEV *ScalarEvolution::createSCEV(Value *V) { if (!isSCEVable(V->getType())) return getUnknown(V); @@ -7030,7 +7427,7 @@ // to obey basic rules for definitions dominating uses which this // analysis depends on. if (!DT.isReachableFromEntry(I->getParent())) - return getUnknown(UndefValue::get(V->getType())); + return getUnknown(PoisonValue::get(V->getType())); } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) return getConstant(CI); else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) @@ -7038,6 +7435,9 @@ else if (!isa<ConstantExpr>(V)) return getUnknown(V); + const SCEV *LHS; + const SCEV *RHS; + Operator *U = cast<Operator>(V); if (auto BO = MatchBinaryOp(U, DT)) { switch (BO->Opcode) { @@ -7103,8 +7503,9 @@ SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op); if (Flags != SCEV::FlagAnyWrap) { - MulOps.push_back( - getMulExpr(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags)); + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + MulOps.push_back(getMulExpr(LHS, RHS, Flags)); break; } } @@ -7121,14 +7522,20 @@ return getMulExpr(MulOps); } case Instruction::UDiv: - return getUDivExpr(getSCEV(BO->LHS), getSCEV(BO->RHS)); + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getUDivExpr(LHS, RHS); case Instruction::URem: - return getURemExpr(getSCEV(BO->LHS), getSCEV(BO->RHS)); + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getURemExpr(LHS, RHS); case Instruction::Sub: { SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; if (BO->Op) Flags = getNoWrapFlagsFromUB(BO->Op); - return getMinusSCEV(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags); + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getMinusSCEV(LHS, RHS, Flags); } case Instruction::And: // For an expression like x&255 that merely masks off the high bits, @@ -7180,6 +7587,12 @@ MulCount); } } + // Binary `and` is a bit-wise `umin`. + if (BO->LHS->getType()->isIntegerTy(1)) { + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getUMinExpr(LHS, RHS); + } break; case Instruction::Or: @@ -7199,6 +7612,12 @@ (SCEV::NoWrapFlags)(SCEV::FlagNUW | SCEV::FlagNSW)); } } + // Binary `or` is a bit-wise `umax`. + if (BO->LHS->getType()->isIntegerTy(1)) { + LHS = getSCEV(BO->LHS); + RHS = getSCEV(BO->RHS); + return getUMaxExpr(LHS, RHS); + } break; case Instruction::Xor: @@ -7266,9 +7685,9 @@ Flags = (SCEV::NoWrapFlags)(Flags | SCEV::FlagNUW); } - Constant *X = ConstantInt::get( + ConstantInt *X = ConstantInt::get( getContext(), APInt::getOneBitSet(BitWidth, SA->getZExtValue())); - return getMulExpr(getSCEV(BO->LHS), getSCEV(X), Flags); + return getMulExpr(getSCEV(BO->LHS), getConstant(X), Flags); } break; @@ -7394,14 +7813,8 @@ return createNodeForPHI(cast<PHINode>(U)); case Instruction::Select: - // U can also be a select constant expr, which let fall through. Since - // createNodeForSelect only works for a condition that is an `ICmpInst`, and - // constant expressions cannot have instructions as operands, we'd have - // returned getUnknown for a select constant expressions anyway. - if (isa<Instruction>(U)) - return createNodeForSelectOrPHI(cast<Instruction>(U), U->getOperand(0), - U->getOperand(1), U->getOperand(2)); - break; + return createNodeForSelectOrPHI(U, U->getOperand(0), U->getOperand(1), + U->getOperand(2)); case Instruction::Call: case Instruction::Invoke: @@ -7415,17 +7828,21 @@ getSCEV(II->getArgOperand(0)), /*IsNSW=*/cast<ConstantInt>(II->getArgOperand(1))->isOne()); case Intrinsic::umax: - return getUMaxExpr(getSCEV(II->getArgOperand(0)), - getSCEV(II->getArgOperand(1))); + LHS = getSCEV(II->getArgOperand(0)); + RHS = getSCEV(II->getArgOperand(1)); + return getUMaxExpr(LHS, RHS); case Intrinsic::umin: - return getUMinExpr(getSCEV(II->getArgOperand(0)), - getSCEV(II->getArgOperand(1))); + LHS = getSCEV(II->getArgOperand(0)); + RHS = getSCEV(II->getArgOperand(1)); + return getUMinExpr(LHS, RHS); case Intrinsic::smax: - return getSMaxExpr(getSCEV(II->getArgOperand(0)), - getSCEV(II->getArgOperand(1))); + LHS = getSCEV(II->getArgOperand(0)); + RHS = getSCEV(II->getArgOperand(1)); + return getSMaxExpr(LHS, RHS); case Intrinsic::smin: - return getSMinExpr(getSCEV(II->getArgOperand(0)), - getSCEV(II->getArgOperand(1))); + LHS = getSCEV(II->getArgOperand(0)); + RHS = getSCEV(II->getArgOperand(1)); + return getSMinExpr(LHS, RHS); case Intrinsic::usub_sat: { const SCEV *X = getSCEV(II->getArgOperand(0)); const SCEV *Y = getSCEV(II->getArgOperand(1)); @@ -7439,8 +7856,10 @@ return getAddExpr(ClampedX, Y, SCEV::FlagNUW); } case Intrinsic::start_loop_iterations: - // A start_loop_iterations is just equivalent to the first operand for - // SCEV purposes. + case Intrinsic::annotation: + case Intrinsic::ptr_annotation: + // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is + // just eqivalent to the first operand for SCEV purposes. return getSCEV(II->getArgOperand(0)); default: break; @@ -7640,7 +8059,7 @@ Res = Multiple; Res = (unsigned)GreatestCommonDivisor64(*Res, Multiple); } - return Res.getValueOr(1); + return Res.value_or(1); } unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L, @@ -7708,7 +8127,7 @@ const SCEV * ScalarEvolution::getPredicatedBackedgeTakenCount(const Loop *L, - SCEVUnionPredicate &Preds) { + SmallVector<const SCEVPredicate *, 4> &Preds) { return getPredicatedBackedgeTakenInfo(L).getExact(L, this, &Preds); } @@ -7870,7 +8289,6 @@ if (LoopUsersItr != LoopUsers.end()) { ToForget.insert(ToForget.end(), LoopUsersItr->second.begin(), LoopUsersItr->second.end()); - LoopUsers.erase(LoopUsersItr); } // Drop information about expressions based on loop-header PHIs. @@ -7900,9 +8318,7 @@ } void ScalarEvolution::forgetTopmostLoop(const Loop *L) { - while (Loop *Parent = L->getParentLoop()) - L = Parent; - forgetLoop(L); + forgetLoop(L->getOutermostLoop()); } void ScalarEvolution::forgetValue(Value *V) { @@ -7944,7 +8360,7 @@ /// the relevant loop exiting block using getExact(ExitingBlock, SE). const SCEV * ScalarEvolution::BackedgeTakenInfo::getExact(const Loop *L, ScalarEvolution *SE, - SCEVUnionPredicate *Preds) const { + SmallVector<const SCEVPredicate *, 4> *Preds) const { // If any exits were not computable, the loop is not computable. if (!isComplete() || ExitNotTaken.empty()) return SE->getCouldNotCompute(); @@ -7957,7 +8373,7 @@ // All exiting blocks we have gathered dominate loop's latch, so exact trip // count is simply a minimum out of all these calculated exit counts. SmallVector<const SCEV *, 2> Ops; - for (auto &ENT : ExitNotTaken) { + for (const auto &ENT : ExitNotTaken) { const SCEV *BECount = ENT.ExactNotTaken; assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!"); assert(SE->DT.dominates(ENT.ExitingBlock, Latch) && @@ -7966,21 +8382,25 @@ Ops.push_back(BECount); - if (Preds && !ENT.hasAlwaysTruePredicate()) - Preds->add(ENT.Predicate.get()); + if (Preds) + for (const auto *P : ENT.Predicates) + Preds->push_back(P); assert((Preds || ENT.hasAlwaysTruePredicate()) && "Predicate should be always true!"); } - return SE->getUMinFromMismatchedTypes(Ops); + // If an earlier exit exits on the first iteration (exit count zero), then + // a later poison exit count should not propagate into the result. This are + // exactly the semantics provided by umin_seq. + return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true); } /// Get the exact not taken count for this loop exit. const SCEV * ScalarEvolution::BackedgeTakenInfo::getExact(const BasicBlock *ExitingBlock, ScalarEvolution *SE) const { - for (auto &ENT : ExitNotTaken) + for (const auto &ENT : ExitNotTaken) if (ENT.ExitingBlock == ExitingBlock && ENT.hasAlwaysTruePredicate()) return ENT.ExactNotTaken; @@ -7989,7 +8409,7 @@ const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax( const BasicBlock *ExitingBlock, ScalarEvolution *SE) const { - for (auto &ENT : ExitNotTaken) + for (const auto &ENT : ExitNotTaken) if (ENT.ExitingBlock == ExitingBlock && ENT.hasAlwaysTruePredicate()) return ENT.MaxNotTaken; @@ -8048,8 +8468,8 @@ assert((isa<SCEVCouldNotCompute>(MaxNotTaken) || isa<SCEVConstant>(MaxNotTaken)) && "No point in having a non-constant max backedge taken count!"); - for (auto *PredSet : PredSetList) - for (auto *P : *PredSet) + for (const auto *PredSet : PredSetList) + for (const auto *P : *PredSet) addPredicate(P); assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) && "Backedge count should be int"); @@ -8082,16 +8502,8 @@ [&](const EdgeExitInfo &EEI) { BasicBlock *ExitBB = EEI.first; const ExitLimit &EL = EEI.second; - if (EL.Predicates.empty()) - return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken, EL.MaxNotTaken, - nullptr); - - std::unique_ptr<SCEVUnionPredicate> Predicate(new SCEVUnionPredicate); - for (auto *Pred : EL.Predicates) - Predicate->add(Pred); - return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken, EL.MaxNotTaken, - std::move(Predicate)); + EL.Predicates); }); assert((isa<SCEVCouldNotCompute>(ConstantMax) || isa<SCEVConstant>(ConstantMax)) && @@ -8385,11 +8797,6 @@ BECount = getUMinFromMismatchedTypes( EL0.ExactNotTaken, EL1.ExactNotTaken, /*Sequential=*/!isa<BinaryOperator>(ExitCond)); - - // If EL0.ExactNotTaken was zero and ExitCond was a short-circuit form, - // it should have been simplified to zero (see the condition (3) above) - assert(!isa<BinaryOperator>(ExitCond) || !EL0.ExactNotTaken->isZero() || - BECount->isZero()); } if (EL0.MaxNotTaken == getCouldNotCompute()) MaxBECount = EL1.MaxNotTaken; @@ -8470,7 +8877,8 @@ ControlsExit && loopHasNoAbnormalExits(L) && loopIsFiniteByAssumption(L); // Simplify the operands before analyzing them. (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0, - ControllingFiniteLoop); + (EnableFiniteLoopControl ? ControllingFiniteLoop + : false)); // If we have a comparison of a chrec against a constant, try to use value // ranges to answer this query. @@ -8683,7 +9091,7 @@ // and the kind of shift should be match the kind of shift we peeled // off, if any. - (!PostShiftOpCode.hasValue() || *PostShiftOpCode == OpCodeOut); + (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut); }; PHINode *PN; @@ -8871,13 +9279,6 @@ Operands[i] = C; } - if (CmpInst *CI = dyn_cast<CmpInst>(I)) - return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], - Operands[1], DL, TLI); - if (LoadInst *LI = dyn_cast<LoadInst>(I)) { - if (!LI->isVolatile()) - return ConstantFoldLoadFromConstPtr(Operands[0], LI->getType(), DL); - } return ConstantFoldInstOperands(I, Operands, DL, TLI); } @@ -9121,67 +9522,44 @@ } case scAddExpr: { const SCEVAddExpr *SA = cast<SCEVAddExpr>(V); - if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) { - if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) { - unsigned AS = PTy->getAddressSpace(); - Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS); - C = ConstantExpr::getBitCast(C, DestPtrTy); + Constant *C = nullptr; + for (const SCEV *Op : SA->operands()) { + Constant *OpC = BuildConstantFromSCEV(Op); + if (!OpC) + return nullptr; + if (!C) { + C = OpC; + continue; } - for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) { - Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i)); - if (!C2) - return nullptr; - - // First pointer! - if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) { - unsigned AS = C2->getType()->getPointerAddressSpace(); - std::swap(C, C2); - Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS); - // The offsets have been converted to bytes. We can add bytes to an - // i8* by GEP with the byte count in the first index. - C = ConstantExpr::getBitCast(C, DestPtrTy); - } - - // Don't bother trying to sum two pointers. We probably can't - // statically compute a load that results from it anyway. - if (C2->getType()->isPointerTy()) - return nullptr; - - if (C->getType()->isPointerTy()) { - C = ConstantExpr::getGetElementPtr(Type::getInt8Ty(C->getContext()), - C, C2); - } else { - C = ConstantExpr::getAdd(C, C2); - } + assert(!C->getType()->isPointerTy() && + "Can only have one pointer, and it must be last"); + if (auto *PT = dyn_cast<PointerType>(OpC->getType())) { + // The offsets have been converted to bytes. We can add bytes to an + // i8* by GEP with the byte count in the first index. + Type *DestPtrTy = + Type::getInt8PtrTy(PT->getContext(), PT->getAddressSpace()); + OpC = ConstantExpr::getBitCast(OpC, DestPtrTy); + C = ConstantExpr::getGetElementPtr(Type::getInt8Ty(C->getContext()), + OpC, C); + } else { + C = ConstantExpr::getAdd(C, OpC); } - return C; } - return nullptr; + return C; } case scMulExpr: { const SCEVMulExpr *SM = cast<SCEVMulExpr>(V); - if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) { - // Don't bother with pointers at all. - if (C->getType()->isPointerTy()) + Constant *C = nullptr; + for (const SCEV *Op : SM->operands()) { + assert(!Op->getType()->isPointerTy() && "Can't multiply pointers"); + Constant *OpC = BuildConstantFromSCEV(Op); + if (!OpC) return nullptr; - for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) { - Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i)); - if (!C2 || C2->getType()->isPointerTy()) - return nullptr; - C = ConstantExpr::getMul(C, C2); - } - return C; + C = C ? ConstantExpr::getMul(C, OpC) : OpC; } - return nullptr; + return C; } - case scUDivExpr: { - const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V); - if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS())) - if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS())) - if (LHS->getType() == RHS->getType()) - return ConstantExpr::getUDiv(LHS, RHS); - return nullptr; - } + case scUDivExpr: case scSMaxExpr: case scUMaxExpr: case scSMinExpr: @@ -9297,15 +9675,7 @@ if (MadeImprovement) { Constant *C = nullptr; const DataLayout &DL = getDataLayout(); - if (const CmpInst *CI = dyn_cast<CmpInst>(I)) - C = ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], - Operands[1], DL, &TLI); - else if (const LoadInst *Load = dyn_cast<LoadInst>(I)) { - if (!Load->isVolatile()) - C = ConstantFoldLoadFromConstPtr(Operands[0], Load->getType(), - DL); - } else - C = ConstantFoldInstOperands(I, Operands, DL, &TLI); + C = ConstantFoldInstOperands(I, Operands, DL, &TLI); if (!C) return V; return getSCEV(C); } @@ -9535,15 +9905,15 @@ /// (b) if neither X nor Y exist, return None, /// (c) if exactly one of X and Y exists, return that value. static Optional<APInt> MinOptional(Optional<APInt> X, Optional<APInt> Y) { - if (X.hasValue() && Y.hasValue()) { + if (X && Y) { unsigned W = std::max(X->getBitWidth(), Y->getBitWidth()); - APInt XW = X->sextOrSelf(W); - APInt YW = Y->sextOrSelf(W); + APInt XW = X->sext(W); + APInt YW = Y->sext(W); return XW.slt(YW) ? *X : *Y; } - if (!X.hasValue() && !Y.hasValue()) + if (!X && !Y) return None; - return X.hasValue() ? *X : *Y; + return X ? *X : *Y; } /// Helper function to truncate an optional APInt to a given BitWidth. @@ -9558,7 +9928,7 @@ /// equation are BW+1 bits wide (to avoid truncation when converting from /// the addrec to the equation). static Optional<APInt> TruncIfPossible(Optional<APInt> X, unsigned BitWidth) { - if (!X.hasValue()) + if (!X) return None; unsigned W = X->getBitWidth(); if (BitWidth > 1 && BitWidth < W && X->isIntN(BitWidth)) @@ -9585,13 +9955,13 @@ APInt A, B, C, M; unsigned BitWidth; auto T = GetQuadraticEquation(AddRec); - if (!T.hasValue()) + if (!T) return None; std::tie(A, B, C, M, BitWidth) = *T; LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n"); Optional<APInt> X = APIntOps::SolveQuadraticEquationWrap(A, B, C, BitWidth+1); - if (!X.hasValue()) + if (!X) return None; ConstantInt *CX = ConstantInt::get(SE.getContext(), *X); @@ -9627,7 +9997,7 @@ APInt A, B, C, M; unsigned BitWidth; auto T = GetQuadraticEquation(AddRec); - if (!T.hasValue()) + if (!T) return None; // Be careful about the return value: there can be two reasons for not @@ -9672,7 +10042,7 @@ // If SolveQuadraticEquationWrap returns None, it means that there can // be a solution, but the function failed to find it. We cannot treat it // as "no solution". - if (!SO.hasValue() || !UO.hasValue()) + if (!SO || !UO) return { None, false }; // Check the smaller value first to see if it leaves the range. @@ -9690,8 +10060,8 @@ std::tie(A, B, C, M, BitWidth) = *T; // Lower bound is inclusive, subtract 1 to represent the exiting value. - APInt Lower = Range.getLower().sextOrSelf(A.getBitWidth()) - 1; - APInt Upper = Range.getUpper().sextOrSelf(A.getBitWidth()); + APInt Lower = Range.getLower().sext(A.getBitWidth()) - 1; + APInt Upper = Range.getUpper().sext(A.getBitWidth()); auto SL = SolveForBoundary(Lower); auto SU = SolveForBoundary(Upper); // If any of the solutions was unknown, no meaninigful conclusions can @@ -9776,7 +10146,7 @@ // value at this index. When solving for "X*X != 5", for example, we // should not accept a root of 2. if (auto S = SolveQuadraticAddRecExact(AddRec, *this)) { - const auto *R = cast<SCEVConstant>(getConstant(S.getValue())); + const auto *R = cast<SCEVConstant>(getConstant(*S)); return ExitLimit(R, R, false, Predicates); } return getCouldNotCompute(); @@ -10187,8 +10557,8 @@ // Domination relationship must be a linear order on collected loops. #ifndef NDEBUG - for (auto *L1 : LoopsUsed) - for (auto *L2 : LoopsUsed) + for (const auto *L1 : LoopsUsed) + for (const auto *L2 : LoopsUsed) assert((DT.dominates(L1->getHeader(), L2->getHeader()) || DT.dominates(L2->getHeader(), L1->getHeader())) && "Domination relationship is not a linear order"); @@ -10296,8 +10666,8 @@ auto ResultSwapped = getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred)); - assert(ResultSwapped.hasValue() && "should be able to analyze both!"); - assert(ResultSwapped.getValue() != Result.getValue() && + assert(ResultSwapped && "should be able to analyze both!"); + assert(ResultSwapped.value() != Result.value() && "monotonicity should flip as we flip the predicate"); } #endif @@ -10479,17 +10849,27 @@ return false; if (Pred == CmpInst::ICMP_NE) { - if (CheckRanges(getSignedRange(LHS), getSignedRange(RHS)) || - CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS))) + auto SL = getSignedRange(LHS); + auto SR = getSignedRange(RHS); + if (CheckRanges(SL, SR)) + return true; + auto UL = getUnsignedRange(LHS); + auto UR = getUnsignedRange(RHS); + if (CheckRanges(UL, UR)) return true; auto *Diff = getMinusSCEV(LHS, RHS); return !isa<SCEVCouldNotCompute>(Diff) && isKnownNonZero(Diff); } - if (CmpInst::isSigned(Pred)) - return CheckRanges(getSignedRange(LHS), getSignedRange(RHS)); + if (CmpInst::isSigned(Pred)) { + auto SL = getSignedRange(LHS); + auto SR = getSignedRange(RHS); + return CheckRanges(SL, SR); + } - return CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS)); + auto UL = getUnsignedRange(LHS); + auto UR = getUnsignedRange(RHS); + return CheckRanges(UL, UR); } bool ScalarEvolution::isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred, @@ -10632,8 +11012,10 @@ ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { // Interpret a null as meaning no loop, where there is obviously no guard - // (interprocedural conditions notwithstanding). - if (!L) return true; + // (interprocedural conditions notwithstanding). Do not bother about + // unreachable loops. + if (!L || !DT.isReachableFromEntry(L->getHeader())) + return true; if (VerifyIR) assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) && @@ -10690,12 +11072,6 @@ return true; } - // If the loop is not reachable from the entry block, we risk running into an - // infinite loop as we walk up into the dom tree. These loops do not matter - // anyway, so we just return a conservative answer when we see them. - if (!DT.isReachableFromEntry(L->getHeader())) - return false; - if (isImpliedViaGuard(Latch, Pred, LHS, RHS)) return true; @@ -10741,6 +11117,9 @@ ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { + // Do not bother proving facts for unreachable code. + if (!DT.isReachableFromEntry(BB)) + return true; if (VerifyIR) assert(!verifyFunction(*BB->getParent(), &dbgs()) && "This cannot be done on broken IR!"); @@ -10774,20 +11153,6 @@ return true; } - // Try to prove (Pred, LHS, RHS) using isImpliedViaGuard. - auto ProveViaGuard = [&](const BasicBlock *Block) { - if (isImpliedViaGuard(Block, Pred, LHS, RHS)) - return true; - if (ProvingStrictComparison) { - auto ProofFn = [&](ICmpInst::Predicate P) { - return isImpliedViaGuard(Block, P, LHS, RHS); - }; - if (SplitAndProve(ProofFn)) - return true; - } - return false; - }; - // Try to prove (Pred, LHS, RHS) using isImpliedCond. auto ProveViaCond = [&](const Value *Condition, bool Inverse) { const Instruction *CtxI = &BB->front(); @@ -10814,17 +11179,13 @@ PredBB = BB->getSinglePredecessor(); for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB); Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) { - if (ProveViaGuard(Pair.first)) - return true; - - const BranchInst *LoopEntryPredicate = + const BranchInst *BlockEntryPredicate = dyn_cast<BranchInst>(Pair.first->getTerminator()); - if (!LoopEntryPredicate || - LoopEntryPredicate->isUnconditional()) + if (!BlockEntryPredicate || BlockEntryPredicate->isUnconditional()) continue; - if (ProveViaCond(LoopEntryPredicate->getCondition(), - LoopEntryPredicate->getSuccessor(0) != Pair.second)) + if (ProveViaCond(BlockEntryPredicate->getCondition(), + BlockEntryPredicate->getSuccessor(0) != Pair.second)) return true; } @@ -10840,6 +11201,15 @@ return true; } + // Check conditions due to any @llvm.experimental.guard intrinsics. + auto *GuardDecl = F.getParent()->getFunction( + Intrinsic::getName(Intrinsic::experimental_guard)); + if (GuardDecl) + for (const auto *GU : GuardDecl->users()) + if (const auto *Guard = dyn_cast<IntrinsicInst>(GU)) + if (Guard->getFunction() == BB->getParent() && DT.dominates(Guard, BB)) + if (ProveViaCond(Guard->getArgOperand(0), false)) + return true; return false; } @@ -12596,7 +12966,7 @@ if (isQuadratic()) { if (auto S = SolveQuadraticAddRecRange(this, Range, SE)) - return SE.getConstant(S.getValue()); + return SE.getConstant(*S); } return SE.getCouldNotCompute(); @@ -12636,6 +13006,15 @@ }); } +// Return true when S contains a value that is a nullptr. +bool ScalarEvolution::containsErasedValue(const SCEV *S) const { + return SCEVExprContains(S, [](const SCEV *S) { + if (const auto *SU = dyn_cast<SCEVUnknown>(S)) + return SU->getValue() == nullptr; + return false; + }); +} + /// Return the size of an element read or written by Inst. const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) { Type *Ty; @@ -12820,12 +13199,13 @@ L->getHeader()->printAsOperand(OS, /*PrintType=*/false); OS << ": "; - SCEVUnionPredicate Pred; - auto PBT = SE->getPredicatedBackedgeTakenCount(L, Pred); + SmallVector<const SCEVPredicate *, 4> Preds; + auto PBT = SE->getPredicatedBackedgeTakenCount(L, Preds); if (!isa<SCEVCouldNotCompute>(PBT)) { OS << "Predicated backedge-taken count is " << *PBT << "\n"; OS << " Predicates:\n"; - Pred.print(OS, 4); + for (const auto *P : Preds) + P->print(OS, 4); } else { OS << "Unpredictable predicated backedge-taken count. "; } @@ -12901,7 +13281,7 @@ } bool First = true; - for (auto *Iter = L; Iter; Iter = Iter->getParentLoop()) { + for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) { if (First) { OS << "\t\t" "LoopDispositions: { "; First = false; @@ -12913,7 +13293,7 @@ OS << ": " << loopDispositionToStr(SE.getLoopDisposition(SV, Iter)); } - for (auto *InnerL : depth_first(L)) { + for (const auto *InnerL : depth_first(L)) { if (InnerL == L) continue; if (First) { @@ -12993,7 +13373,7 @@ // This recurrence is variant w.r.t. L if any of its operands // are variant. - for (auto *Op : AR->operands()) + for (const auto *Op : AR->operands()) if (!isLoopInvariant(Op, L)) return LoopVariant; @@ -13008,7 +13388,7 @@ case scSMinExpr: case scSequentialUMinExpr: { bool HasVarying = false; - for (auto *Op : cast<SCEVNAryExpr>(S)->operands()) { + for (const auto *Op : cast<SCEVNAryExpr>(S)->operands()) { LoopDisposition D = getLoopDisposition(Op, L); if (D == LoopVariant) return LoopVariant; @@ -13174,12 +13554,12 @@ const SCEV *Curr = Worklist.pop_back_val(); auto Users = SCEVUsers.find(Curr); if (Users != SCEVUsers.end()) - for (auto *User : Users->second) + for (const auto *User : Users->second) if (ToForget.insert(User).second) Worklist.push_back(User); } - for (auto *S : ToForget) + for (const auto *S : ToForget) forgetMemoizedResultsImpl(S); for (auto I = PredicatedSCEVRewrites.begin(); @@ -13202,12 +13582,10 @@ auto ExprIt = ExprValueMap.find(S); if (ExprIt != ExprValueMap.end()) { - for (auto &ValueAndOffset : ExprIt->second) { - if (ValueAndOffset.second == nullptr) { - auto ValueIt = ValueExprMap.find_as(ValueAndOffset.first); - if (ValueIt != ValueExprMap.end()) - ValueExprMap.erase(ValueIt); - } + for (Value *V : ExprIt->second) { + auto ValueIt = ValueExprMap.find_as(V); + if (ValueIt != ValueExprMap.end()) + ValueExprMap.erase(ValueIt); } ExprValueMap.erase(ExprIt); } @@ -13258,6 +13636,43 @@ SCEVTraversal<FindUsedLoops>(F).visitAll(S); } +void ScalarEvolution::getReachableBlocks( + SmallPtrSetImpl<BasicBlock *> &Reachable, Function &F) { + SmallVector<BasicBlock *> Worklist; + Worklist.push_back(&F.getEntryBlock()); + while (!Worklist.empty()) { + BasicBlock *BB = Worklist.pop_back_val(); + if (!Reachable.insert(BB).second) + continue; + + Value *Cond; + BasicBlock *TrueBB, *FalseBB; + if (match(BB->getTerminator(), m_Br(m_Value(Cond), m_BasicBlock(TrueBB), + m_BasicBlock(FalseBB)))) { + if (auto *C = dyn_cast<ConstantInt>(Cond)) { + Worklist.push_back(C->isOne() ? TrueBB : FalseBB); + continue; + } + + if (auto *Cmp = dyn_cast<ICmpInst>(Cond)) { + const SCEV *L = getSCEV(Cmp->getOperand(0)); + const SCEV *R = getSCEV(Cmp->getOperand(1)); + if (isKnownPredicateViaConstantRanges(Cmp->getPredicate(), L, R)) { + Worklist.push_back(TrueBB); + continue; + } + if (isKnownPredicateViaConstantRanges(Cmp->getInversePredicate(), L, + R)) { + Worklist.push_back(FalseBB); + continue; + } + } + } + + append_range(Worklist, successors(BB)); + } +} + void ScalarEvolution::verify() const { ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); ScalarEvolution SE2(F, TLI, AC, DT, LI); @@ -13282,13 +13697,44 @@ }; SCEVMapper SCM(SE2); + SmallPtrSet<BasicBlock *, 16> ReachableBlocks; + SE2.getReachableBlocks(ReachableBlocks, F); + + auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * { + if (containsUndefs(Old) || containsUndefs(New)) { + // SCEV treats "undef" as an unknown but consistent value (i.e. it does + // not propagate undef aggressively). This means we can (and do) fail + // verification in cases where a transform makes a value go from "undef" + // to "undef+1" (say). The transform is fine, since in both cases the + // result is "undef", but SCEV thinks the value increased by 1. + return nullptr; + } + + // Unless VerifySCEVStrict is set, we only compare constant deltas. + const SCEV *Delta = SE2.getMinusSCEV(Old, New); + if (!VerifySCEVStrict && !isa<SCEVConstant>(Delta)) + return nullptr; + + return Delta; + }; while (!LoopStack.empty()) { auto *L = LoopStack.pop_back_val(); llvm::append_range(LoopStack, *L); - auto *CurBECount = SCM.visit( - const_cast<ScalarEvolution *>(this)->getBackedgeTakenCount(L)); + // Only verify BECounts in reachable loops. For an unreachable loop, + // any BECount is legal. + if (!ReachableBlocks.contains(L->getHeader())) + continue; + + // Only verify cached BECounts. Computing new BECounts may change the + // results of subsequent SCEV uses. + auto It = BackedgeTakenCounts.find(L); + if (It == BackedgeTakenCounts.end()) + continue; + + auto *CurBECount = + SCM.visit(It->second.getExact(L, const_cast<ScalarEvolution *>(this))); auto *NewBECount = SE2.getBackedgeTakenCount(L); if (CurBECount == SE2.getCouldNotCompute() || @@ -13301,16 +13747,6 @@ continue; } - if (containsUndefs(CurBECount) || containsUndefs(NewBECount)) { - // SCEV treats "undef" as an unknown but consistent value (i.e. it does - // not propagate undef aggressively). This means we can (and do) fail - // verification in cases where a transform makes the trip count of a loop - // go from "undef" to "undef+1" (say). The transform is fine, since in - // both cases the loop iterates "undef" times, but SCEV thinks we - // increased the trip count of the loop by 1 incorrectly. - continue; - } - if (SE.getTypeSizeInBits(CurBECount->getType()) > SE.getTypeSizeInBits(NewBECount->getType())) NewBECount = SE2.getZeroExtendExpr(NewBECount, CurBECount->getType()); @@ -13318,10 +13754,8 @@ SE.getTypeSizeInBits(NewBECount->getType())) CurBECount = SE2.getZeroExtendExpr(CurBECount, NewBECount->getType()); - const SCEV *Delta = SE2.getMinusSCEV(CurBECount, NewBECount); - - // Unless VerifySCEVStrict is set, we only compare constant deltas. - if ((VerifySCEVStrict || isa<SCEVConstant>(Delta)) && !Delta->isZero()) { + const SCEV *Delta = GetDelta(CurBECount, NewBECount); + if (Delta && !Delta->isZero()) { dbgs() << "Trip Count for " << *L << " Changed!\n"; dbgs() << "Old: " << *CurBECount << "\n"; dbgs() << "New: " << *NewBECount << "\n"; @@ -13335,12 +13769,10 @@ SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end()); while (!Worklist.empty()) { Loop *L = Worklist.pop_back_val(); - if (ValidLoops.contains(L)) - continue; - ValidLoops.insert(L); - Worklist.append(L->begin(), L->end()); + if (ValidLoops.insert(L).second) + Worklist.append(L->begin(), L->end()); } - for (auto &KV : ValueExprMap) { + for (const auto &KV : ValueExprMap) { #ifndef NDEBUG // Check for SCEV expressions referencing invalid/deleted loops. if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) { @@ -13351,27 +13783,38 @@ // Check that the value is also part of the reverse map. auto It = ExprValueMap.find(KV.second); - if (It == ExprValueMap.end() || !It->second.contains({KV.first, nullptr})) { + if (It == ExprValueMap.end() || !It->second.contains(KV.first)) { dbgs() << "Value " << *KV.first << " is in ValueExprMap but not in ExprValueMap\n"; std::abort(); } + + if (auto *I = dyn_cast<Instruction>(&*KV.first)) { + if (!ReachableBlocks.contains(I->getParent())) + continue; + const SCEV *OldSCEV = SCM.visit(KV.second); + const SCEV *NewSCEV = SE2.getSCEV(I); + const SCEV *Delta = GetDelta(OldSCEV, NewSCEV); + if (Delta && !Delta->isZero()) { + dbgs() << "SCEV for value " << *I << " changed!\n" + << "Old: " << *OldSCEV << "\n" + << "New: " << *NewSCEV << "\n" + << "Delta: " << *Delta << "\n"; + std::abort(); + } + } } for (const auto &KV : ExprValueMap) { - for (const auto &ValueAndOffset : KV.second) { - if (ValueAndOffset.second != nullptr) - continue; - - auto It = ValueExprMap.find_as(ValueAndOffset.first); + for (Value *V : KV.second) { + auto It = ValueExprMap.find_as(V); if (It == ValueExprMap.end()) { - dbgs() << "Value " << *ValueAndOffset.first + dbgs() << "Value " << *V << " is in ExprValueMap but not in ValueExprMap\n"; std::abort(); } if (It->second != KV.first) { - dbgs() << "Value " << *ValueAndOffset.first - << " mapped to " << *It->second + dbgs() << "Value " << *V << " mapped to " << *It->second << " rather than " << *KV.first << "\n"; std::abort(); } @@ -13537,18 +13980,25 @@ const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS, const SCEV *RHS) { + return getComparePredicate(ICmpInst::ICMP_EQ, LHS, RHS); +} + +const SCEVPredicate * +ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS) { FoldingSetNodeID ID; assert(LHS->getType() == RHS->getType() && "Type mismatch between LHS and RHS"); // Unique this node based on the arguments - ID.AddInteger(SCEVPredicate::P_Equal); + ID.AddInteger(SCEVPredicate::P_Compare); + ID.AddInteger(Pred); ID.AddPointer(LHS); ID.AddPointer(RHS); void *IP = nullptr; if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP)) return S; - SCEVEqualPredicate *Eq = new (SCEVAllocator) - SCEVEqualPredicate(ID.Intern(SCEVAllocator), LHS, RHS); + SCEVComparePredicate *Eq = new (SCEVAllocator) + SCEVComparePredicate(ID.Intern(SCEVAllocator), Pred, LHS, RHS); UniquePreds.InsertNode(Eq, IP); return Eq; } @@ -13585,18 +14035,24 @@ /// \p NewPreds such that the result will be an AddRecExpr. static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE, SmallPtrSetImpl<const SCEVPredicate *> *NewPreds, - SCEVUnionPredicate *Pred) { + const SCEVPredicate *Pred) { SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred); return Rewriter.visit(S); } const SCEV *visitUnknown(const SCEVUnknown *Expr) { if (Pred) { - auto ExprPreds = Pred->getPredicatesForExpr(Expr); - for (auto *Pred : ExprPreds) - if (const auto *IPred = dyn_cast<SCEVEqualPredicate>(Pred)) - if (IPred->getLHS() == Expr) - return IPred->getRHS(); + if (auto *U = dyn_cast<SCEVUnionPredicate>(Pred)) { + for (const auto *Pred : U->getPredicates()) + if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Pred)) + if (IPred->getLHS() == Expr && + IPred->getPredicate() == ICmpInst::ICMP_EQ) + return IPred->getRHS(); + } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Pred)) { + if (IPred->getLHS() == Expr && + IPred->getPredicate() == ICmpInst::ICMP_EQ) + return IPred->getRHS(); + } } return convertToAddRecWithPreds(Expr); } @@ -13636,7 +14092,7 @@ private: explicit SCEVPredicateRewriter(const Loop *L, ScalarEvolution &SE, SmallPtrSetImpl<const SCEVPredicate *> *NewPreds, - SCEVUnionPredicate *Pred) + const SCEVPredicate *Pred) : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {} bool addOverflowAssumption(const SCEVPredicate *P) { @@ -13667,11 +14123,10 @@ PredicatedRewrite = SE.createAddRecFromPHIWithCasts(Expr); if (!PredicatedRewrite) return Expr; - for (auto *P : PredicatedRewrite->second){ + for (const auto *P : PredicatedRewrite->second){ // Wrap predicates from outer loops are not supported. if (auto *WP = dyn_cast<const SCEVWrapPredicate>(P)) { - auto *AR = cast<const SCEVAddRecExpr>(WP->getExpr()); - if (L != AR->getLoop()) + if (L != WP->getExpr()->getLoop()) return Expr; } if (!addOverflowAssumption(P)) @@ -13681,14 +14136,15 @@ } SmallPtrSetImpl<const SCEVPredicate *> *NewPreds; - SCEVUnionPredicate *Pred; + const SCEVPredicate *Pred; const Loop *L; }; } // end anonymous namespace -const SCEV *ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L, - SCEVUnionPredicate &Preds) { +const SCEV * +ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L, + const SCEVPredicate &Preds) { return SCEVPredicateRewriter::rewrite(S, L, *this, nullptr, &Preds); } @@ -13704,7 +14160,7 @@ // Since the transformation was successful, we can now transfer the SCEV // predicates. - for (auto *P : TransformPreds) + for (const auto *P : TransformPreds) Preds.insert(P); return AddRec; @@ -13715,28 +14171,36 @@ SCEVPredicateKind Kind) : FastID(ID), Kind(Kind) {} -SCEVEqualPredicate::SCEVEqualPredicate(const FoldingSetNodeIDRef ID, - const SCEV *LHS, const SCEV *RHS) - : SCEVPredicate(ID, P_Equal), LHS(LHS), RHS(RHS) { +SCEVComparePredicate::SCEVComparePredicate(const FoldingSetNodeIDRef ID, + const ICmpInst::Predicate Pred, + const SCEV *LHS, const SCEV *RHS) + : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) { assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match"); assert(LHS != RHS && "LHS and RHS are the same SCEV"); } -bool SCEVEqualPredicate::implies(const SCEVPredicate *N) const { - const auto *Op = dyn_cast<SCEVEqualPredicate>(N); +bool SCEVComparePredicate::implies(const SCEVPredicate *N) const { + const auto *Op = dyn_cast<SCEVComparePredicate>(N); if (!Op) return false; + if (Pred != ICmpInst::ICMP_EQ) + return false; + return Op->LHS == LHS && Op->RHS == RHS; } -bool SCEVEqualPredicate::isAlwaysTrue() const { return false; } +bool SCEVComparePredicate::isAlwaysTrue() const { return false; } -const SCEV *SCEVEqualPredicate::getExpr() const { return LHS; } +void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const { + if (Pred == ICmpInst::ICMP_EQ) + OS.indent(Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n"; + else + OS.indent(Depth) << "Compare predicate: " << *LHS + << " " << CmpInst::getPredicateName(Pred) << ") " + << *RHS << "\n"; -void SCEVEqualPredicate::print(raw_ostream &OS, unsigned Depth) const { - OS.indent(Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n"; } SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID, @@ -13744,7 +14208,7 @@ IncrementWrapFlags Flags) : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {} -const SCEV *SCEVWrapPredicate::getExpr() const { return AR; } +const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; } bool SCEVWrapPredicate::implies(const SCEVPredicate *N) const { const auto *Op = dyn_cast<SCEVWrapPredicate>(N); @@ -13793,68 +14257,51 @@ } /// Union predicates don't get cached so create a dummy set ID for it. -SCEVUnionPredicate::SCEVUnionPredicate() - : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {} +SCEVUnionPredicate::SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds) + : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) { + for (const auto *P : Preds) + add(P); +} bool SCEVUnionPredicate::isAlwaysTrue() const { return all_of(Preds, [](const SCEVPredicate *I) { return I->isAlwaysTrue(); }); } -ArrayRef<const SCEVPredicate *> -SCEVUnionPredicate::getPredicatesForExpr(const SCEV *Expr) { - auto I = SCEVToPreds.find(Expr); - if (I == SCEVToPreds.end()) - return ArrayRef<const SCEVPredicate *>(); - return I->second; -} - bool SCEVUnionPredicate::implies(const SCEVPredicate *N) const { if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N)) return all_of(Set->Preds, [this](const SCEVPredicate *I) { return this->implies(I); }); - auto ScevPredsIt = SCEVToPreds.find(N->getExpr()); - if (ScevPredsIt == SCEVToPreds.end()) - return false; - auto &SCEVPreds = ScevPredsIt->second; - - return any_of(SCEVPreds, + return any_of(Preds, [N](const SCEVPredicate *I) { return I->implies(N); }); } -const SCEV *SCEVUnionPredicate::getExpr() const { return nullptr; } - void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const { - for (auto Pred : Preds) + for (const auto *Pred : Preds) Pred->print(OS, Depth); } void SCEVUnionPredicate::add(const SCEVPredicate *N) { if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N)) { - for (auto Pred : Set->Preds) + for (const auto *Pred : Set->Preds) add(Pred); return; } - if (implies(N)) - return; - - const SCEV *Key = N->getExpr(); - assert(Key && "Only SCEVUnionPredicate doesn't have an " - " associated expression!"); - - SCEVToPreds[Key].push_back(N); Preds.push_back(N); } PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L) - : SE(SE), L(L) {} + : SE(SE), L(L) { + SmallVector<const SCEVPredicate*, 4> Empty; + Preds = std::make_unique<SCEVUnionPredicate>(Empty); +} void ScalarEvolution::registerUser(const SCEV *User, ArrayRef<const SCEV *> Ops) { - for (auto *Op : Ops) + for (const auto *Op : Ops) // We do not expect that forgetting cached data for SCEVConstants will ever // open any prospects for sharpening or introduce any correctness issues, // so we don't bother storing their dependencies. @@ -13875,7 +14322,7 @@ if (Entry.second) Expr = Entry.second; - const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, Preds); + const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, *Preds); Entry = {Generation, NewSCEV}; return NewSCEV; @@ -13883,22 +14330,27 @@ const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() { if (!BackedgeCount) { - SCEVUnionPredicate BackedgePred; - BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, BackedgePred); - addPredicate(BackedgePred); + SmallVector<const SCEVPredicate *, 4> Preds; + BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, Preds); + for (const auto *P : Preds) + addPredicate(*P); } return BackedgeCount; } void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) { - if (Preds.implies(&Pred)) + if (Preds->implies(&Pred)) return; - Preds.add(&Pred); + + auto &OldPreds = Preds->getPredicates(); + SmallVector<const SCEVPredicate*, 4> NewPreds(OldPreds.begin(), OldPreds.end()); + NewPreds.push_back(&Pred); + Preds = std::make_unique<SCEVUnionPredicate>(NewPreds); updateGeneration(); } -const SCEVUnionPredicate &PredicatedScalarEvolution::getUnionPredicate() const { - return Preds; +const SCEVPredicate &PredicatedScalarEvolution::getPredicate() const { + return *Preds; } void PredicatedScalarEvolution::updateGeneration() { @@ -13906,7 +14358,7 @@ if (++Generation == 0) { for (auto &II : RewriteMap) { const SCEV *Rewritten = II.second.second; - II.second = {Generation, SE.rewriteUsingPredicate(Rewritten, &L, Preds)}; + II.second = {Generation, SE.rewriteUsingPredicate(Rewritten, &L, *Preds)}; } } } @@ -13951,18 +14403,18 @@ if (!New) return nullptr; - for (auto *P : NewPreds) - Preds.add(P); + for (const auto *P : NewPreds) + addPredicate(*P); - updateGeneration(); RewriteMap[SE.getSCEV(V)] = {Generation, New}; return New; } PredicatedScalarEvolution::PredicatedScalarEvolution( const PredicatedScalarEvolution &Init) - : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L), Preds(Init.Preds), - Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) { + : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L), + Preds(std::make_unique<SCEVUnionPredicate>(Init.Preds->getPredicates())), + Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) { for (auto I : Init.FlagsMap) FlagsMap.insert(I); } @@ -14243,12 +14695,23 @@ ExprsToRewrite.push_back(LHS); } }; - // First, collect conditions from dominating branches. Starting at the loop + + SmallVector<std::pair<Value *, bool>> Terms; + // First, collect information from assumptions dominating the loop. + for (auto &AssumeVH : AC.assumptions()) { + if (!AssumeVH) + continue; + auto *AssumeI = cast<CallInst>(AssumeVH); + if (!DT.dominates(AssumeI, L->getHeader())) + continue; + Terms.emplace_back(AssumeI->getOperand(0), true); + } + + // Second, collect conditions from dominating branches. Starting at the loop // predecessor, climb up the predecessor chain, as long as there are // predecessors that can be found that have unique successors leading to the // original header. // TODO: share this logic with isLoopEntryGuardedByCond. - SmallVector<std::pair<Value *, bool>> Terms; for (std::pair<const BasicBlock *, const BasicBlock *> Pair( L->getLoopPredecessor(), L->getHeader()); Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) { @@ -14280,8 +14743,9 @@ if (auto *Cmp = dyn_cast<ICmpInst>(Cond)) { auto Predicate = EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate(); - CollectCondition(Predicate, getSCEV(Cmp->getOperand(0)), - getSCEV(Cmp->getOperand(1)), RewriteMap); + const auto *LHS = getSCEV(Cmp->getOperand(0)); + const auto *RHS = getSCEV(Cmp->getOperand(1)); + CollectCondition(Predicate, LHS, RHS, RewriteMap); continue; } @@ -14294,18 +14758,6 @@ } } - // Also collect information from assumptions dominating the loop. - for (auto &AssumeVH : AC.assumptions()) { - if (!AssumeVH) - continue; - auto *AssumeI = cast<CallInst>(AssumeVH); - auto *Cmp = dyn_cast<ICmpInst>(AssumeI->getOperand(0)); - if (!Cmp || !DT.dominates(AssumeI, L->getHeader())) - continue; - CollectCondition(Cmp->getPredicate(), getSCEV(Cmp->getOperand(0)), - getSCEV(Cmp->getOperand(1)), RewriteMap); - } - if (RewriteMap.empty()) return Expr;
diff --git a/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp index f4fa159..3d47dc6 100644 --- a/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
@@ -20,6 +20,7 @@ #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" #include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/InitializePasses.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionDivision.cpp b/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionDivision.cpp index 64e908b..0619569 100644 --- a/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionDivision.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
@@ -15,9 +15,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/IR/Constants.h" #include "llvm/Support/Casting.h" -#include "llvm/Support/ErrorHandling.h" #include <cassert> #include <cstdint>
diff --git a/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp b/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp index 209ae66..22dff5e 100644 --- a/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
@@ -13,6 +13,7 @@ #include "llvm/Analysis/ScalarEvolutionNormalization.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/ScopedNoAliasAA.cpp b/src/llvm-project/llvm/lib/Analysis/ScopedNoAliasAA.cpp index e847bf8..f510991 100644 --- a/src/llvm-project/llvm/lib/Analysis/ScopedNoAliasAA.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ScopedNoAliasAA.cpp
@@ -36,7 +36,6 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/IR/InstrTypes.h" -#include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/InitializePasses.h"
diff --git a/src/llvm-project/llvm/lib/Analysis/StackLifetime.cpp b/src/llvm-project/llvm/lib/Analysis/StackLifetime.cpp index 9056cc0..162fd75 100644 --- a/src/llvm-project/llvm/lib/Analysis/StackLifetime.cpp +++ b/src/llvm-project/llvm/lib/Analysis/StackLifetime.cpp
@@ -19,17 +19,12 @@ #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Intrinsics.h" -#include "llvm/IR/User.h" #include "llvm/IR/Value.h" -#include "llvm/Pass.h" #include "llvm/Support/Casting.h" -#include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FormattedStream.h" #include <algorithm> -#include <memory> #include <tuple> using namespace llvm; @@ -75,7 +70,7 @@ auto AllocaSizeInBits = AI->getAllocationSizeInBits(DL); if (!AllocaSizeInBits) return nullptr; - int64_t AllocaSize = AllocaSizeInBits.getValue() / 8; + int64_t AllocaSize = *AllocaSizeInBits / 8; auto *Size = dyn_cast<ConstantInt>(II.getArgOperand(0)); if (!Size) @@ -187,7 +182,7 @@ // Compute LiveIn by unioning together the LiveOut sets of all preds. BitVector LocalLiveIn; - for (auto *PredBB : predecessors(BB)) { + for (const auto *PredBB : predecessors(BB)) { LivenessMap::const_iterator I = BlockLiveness.find(PredBB); // If a predecessor is unreachable, ignore it. if (I == BlockLiveness.end())
diff --git a/src/llvm-project/llvm/lib/Analysis/StackSafetyAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/StackSafetyAnalysis.cpp index 54f3605..9698af3 100644 --- a/src/llvm-project/llvm/lib/Analysis/StackSafetyAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/StackSafetyAnalysis.cpp
@@ -15,7 +15,6 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/StackLifetime.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/DerivedTypes.h" @@ -207,7 +206,7 @@ O << " allocas uses:\n"; if (F) { - for (auto &I : instructions(F)) { + for (const auto &I : instructions(F)) { if (const AllocaInst *AI = dyn_cast<AllocaInst>(&I)) { auto &AS = Allocas.find(AI)->second; O << " " << AI->getName() << "[" @@ -384,9 +383,9 @@ const SCEV *Max = SE.getMinusSCEV(ToDiffTy(SE.getConstant(Size.getUpper())), ToDiffTy(AccessSize)); return SE.evaluatePredicateAt(ICmpInst::Predicate::ICMP_SGE, Diff, Min, I) - .getValueOr(false) && + .value_or(false) && SE.evaluatePredicateAt(ICmpInst::Predicate::ICMP_SLE, Diff, Max, I) - .getValueOr(false); + .value_or(false); } /// The function analyzes all local uses of Ptr (alloca or argument) and @@ -764,7 +763,7 @@ uint32_t ParamNo) { assert(FS.isLive()); assert(FS.isDSOLocal()); - for (auto &PS : FS.paramAccesses()) + for (const auto &PS : FS.paramAccesses()) if (ParamNo == PS.ParamNo) return &PS.Use; return nullptr; @@ -824,7 +823,7 @@ Copy.begin()->first->getParent()->getDataLayout().getPointerSizeInBits(); StackSafetyDataFlowAnalysis<GlobalValue> SSDFA(PointerSize, std::move(Copy)); - for (auto &F : SSDFA.run()) { + for (const auto &F : SSDFA.run()) { auto FI = F.second; auto &SrcF = Functions[F.first]; for (auto &KV : FI.Allocas) { @@ -923,7 +922,7 @@ FunctionSummary::ParamAccess &Param = ParamAccesses.back(); Param.Calls.reserve(PS.Calls.size()); - for (auto &C : PS.Calls) { + for (const auto &C : PS.Calls) { // Parameter forwarded into another function by any or unknown offset // will make ParamAccess::Range as FullSet anyway. So we can drop the // entire parameter like we did above. @@ -979,7 +978,7 @@ if (SSI.empty()) return; const Module &M = *SSI.begin()->first->getParent(); - for (auto &F : M.functions()) { + for (const auto &F : M.functions()) { if (!F.isDeclaration()) { SSI.find(&F)->second.print(O, F.getName(), &F); O << " safe accesses:" @@ -1095,7 +1094,7 @@ bool llvm::needsParamAccessSummary(const Module &M) { if (StackSafetyRun) return true; - for (auto &F : M.functions()) + for (const auto &F : M.functions()) if (F.hasFnAttribute(Attribute::SanitizeMemTag)) return true; return false; @@ -1127,13 +1126,13 @@ continue; if (FS->isLive() && FS->isDSOLocal()) { FunctionInfo<FunctionSummary> FI; - for (auto &PS : FS->paramAccesses()) { + for (const auto &PS : FS->paramAccesses()) { auto &US = FI.Params .emplace(PS.ParamNo, FunctionSummary::ParamAccess::RangeWidth) .first->second; US.Range = PS.Use; - for (auto &Call : PS.Calls) { + for (const auto &Call : PS.Calls) { assert(!Call.Offsets.isFullSet()); FunctionSummary *S = findCalleeFunctionSummary(Call.Callee, FS->modulePath()); @@ -1159,10 +1158,10 @@ NumCombinedDataFlowNodes += Functions.size(); StackSafetyDataFlowAnalysis<FunctionSummary> SSDFA( FunctionSummary::ParamAccess::RangeWidth, std::move(Functions)); - for (auto &KV : SSDFA.run()) { + for (const auto &KV : SSDFA.run()) { std::vector<FunctionSummary::ParamAccess> NewParams; NewParams.reserve(KV.second.Params.size()); - for (auto &Param : KV.second.Params) { + for (const auto &Param : KV.second.Params) { // It's not needed as FullSet is processed the same as a missing value. if (Param.second.Range.isFullSet()) continue;
diff --git a/src/llvm-project/llvm/lib/Analysis/StratifiedSets.h b/src/llvm-project/llvm/lib/Analysis/StratifiedSets.h index 60ea245..883ebd2 100644 --- a/src/llvm-project/llvm/lib/Analysis/StratifiedSets.h +++ b/src/llvm-project/llvm/lib/Analysis/StratifiedSets.h
@@ -340,10 +340,10 @@ return StratifiedSets<T>(std::move(Values), std::move(StratLinks)); } - bool has(const T &Elem) const { return get(Elem).hasValue(); } + bool has(const T &Elem) const { return get(Elem).has_value(); } bool add(const T &Main) { - if (get(Main).hasValue()) + if (get(Main)) return false; auto NewIndex = getNewUnlinkedIndex(); @@ -560,7 +560,7 @@ Optional<StratifiedIndex> indexOf(const T &Val) { auto MaybeVal = get(Val); - if (!MaybeVal.hasValue()) + if (!MaybeVal) return None; auto *Info = *MaybeVal; auto &Link = linksAt(Info->Index);
diff --git a/src/llvm-project/llvm/lib/Analysis/SyncDependenceAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/SyncDependenceAnalysis.cpp index ff833b5..3446e50 100644 --- a/src/llvm-project/llvm/lib/Analysis/SyncDependenceAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/SyncDependenceAnalysis.cpp
@@ -116,18 +116,16 @@ // around from the latch. // //===----------------------------------------------------------------------===// + #include "llvm/Analysis/SyncDependenceAnalysis.h" -#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/Analysis/PostDominators.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include <functional> -#include <stack> -#include <unordered_set> #define DEBUG_TYPE "sync-dependence" @@ -257,7 +255,7 @@ [&](const BasicBlock &BB) { LoopPO.appendBlock(BB); }); } -SyncDependenceAnalysis::~SyncDependenceAnalysis() {} +SyncDependenceAnalysis::~SyncDependenceAnalysis() = default; // divergence propagator for reducible CFGs struct DivergencePropagator {
diff --git a/src/llvm-project/llvm/lib/Analysis/SyntheticCountsUtils.cpp b/src/llvm-project/llvm/lib/Analysis/SyntheticCountsUtils.cpp index a3edce7..29c41fd 100644 --- a/src/llvm-project/llvm/lib/Analysis/SyntheticCountsUtils.cpp +++ b/src/llvm-project/llvm/lib/Analysis/SyntheticCountsUtils.cpp
@@ -14,9 +14,6 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SCCIterator.h" #include "llvm/Analysis/CallGraph.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/InstIterator.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/ModuleSummaryIndex.h" using namespace llvm; @@ -57,7 +54,7 @@ if (!OptProfCount) continue; auto Callee = CGT::edge_dest(E.second); - AdditionalCounts[Callee] += OptProfCount.getValue(); + AdditionalCounts[Callee] += *OptProfCount; } // Update the counts for the nodes in the SCC. @@ -70,7 +67,7 @@ if (!OptProfCount) continue; auto Callee = CGT::edge_dest(E.second); - AddCount(Callee, OptProfCount.getValue()); + AddCount(Callee, *OptProfCount); } }
diff --git a/src/llvm-project/llvm/lib/Analysis/TFUtils.cpp b/src/llvm-project/llvm/lib/Analysis/TFUtils.cpp index 26bc639..682fc09 100644 --- a/src/llvm-project/llvm/lib/Analysis/TFUtils.cpp +++ b/src/llvm-project/llvm/lib/Analysis/TFUtils.cpp
@@ -18,7 +18,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/JSON.h" -#include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" @@ -49,19 +48,17 @@ struct TFInitializer { TFInitializer() { - assert(!IsInitialized && "TFInitialized should be called only once"); int Argc = 1; const char *Name = ""; const char **NamePtr = &Name; TF_InitMain(Name, &Argc, const_cast<char ***>(&NamePtr)); - IsInitialized = true; } - bool IsInitialized = false; }; -llvm::ManagedStatic<TFInitializer> TFLibInitializer; - -bool ensureInitTF() { return TFLibInitializer->IsInitialized; } +bool ensureInitTF() { + static TFInitializer TFLibInitializer; + return true; +} TFGraphPtr createTFGraph() { return TFGraphPtr(TF_NewGraph(), &TF_DeleteGraph); @@ -82,6 +79,33 @@ *OutStr = SE.SerializeAsString(); } } + +int getTFTypeIndex(TensorType TType) { + switch (TType) { + case TensorType::Double: + return TF_DOUBLE; + case TensorType::Float: + return TF_FLOAT; + case TensorType::Int8: + return TF_INT8; + case TensorType::UInt8: + return TF_UINT8; + case TensorType::Int16: + return TF_INT16; + case TensorType::UInt16: + return TF_UINT16; + case TensorType::Int32: + return TF_INT32; + case TensorType::UInt32: + return TF_UINT32; + case TensorType::Int64: + return TF_INT64; + case TensorType::UInt64: + return TF_UINT64; + case TensorType::Invalid: + llvm_unreachable("Unknown tensor type"); + } +} } // namespace namespace llvm { @@ -105,116 +129,6 @@ std::vector<TF_Tensor *> Output; }; -size_t TensorSpec::getElementByteSize() const { - return TF_DataTypeSize(static_cast<TF_DataType>(TypeIndex)); -} - -TensorSpec::TensorSpec(const std::string &Name, int Port, int TypeIndex, - const std::vector<int64_t> &Shape) - : Name(Name), Port(Port), TypeIndex(TypeIndex), Shape(Shape), - ElementCount(std::accumulate(Shape.begin(), Shape.end(), 1, - std::multiplies<int64_t>())) {} - -Optional<TensorSpec> getTensorSpecFromJSON(LLVMContext &Ctx, - const json::Value &Value) { - auto EmitError = [&](const llvm::Twine &Message) -> Optional<TensorSpec> { - std::string S; - llvm::raw_string_ostream OS(S); - OS << Value; - Ctx.emitError("Unable to parse JSON Value as spec (" + Message + "): " + S); - return None; - }; - // FIXME: accept a Path as a parameter, and use it for error reporting. - json::Path::Root Root("tensor_spec"); - json::ObjectMapper Mapper(Value, Root); - if (!Mapper) - return EmitError("Value is not a dict"); - - std::string TensorName; - int TensorPort = -1; - std::string TensorType; - std::vector<int64_t> TensorShape; - - if (!Mapper.map<std::string>("name", TensorName)) - return EmitError("'name' property not present or not a string"); - if (!Mapper.map<std::string>("type", TensorType)) - return EmitError("'type' property not present or not a string"); - if (!Mapper.map<int>("port", TensorPort)) - return EmitError("'port' property not present or not an int"); - if (!Mapper.map<std::vector<int64_t>>("shape", TensorShape)) - return EmitError("'shape' property not present or not an int array"); - -#define PARSE_TYPE(T, E) \ - if (TensorType == #T) \ - return TensorSpec::createSpec<T>(TensorName, TensorShape, TensorPort); - TFUTILS_SUPPORTED_TYPES(PARSE_TYPE) -#undef PARSE_TYPE - return None; -} - -Optional<std::vector<LoggedFeatureSpec>> -loadOutputSpecs(LLVMContext &Ctx, StringRef ExpectedDecisionName, - StringRef ModelPath, StringRef SpecFileOverride) { - SmallVector<char, 128> OutputSpecsPath; - StringRef FileName = SpecFileOverride; - if (FileName.empty()) { - llvm::sys::path::append(OutputSpecsPath, ModelPath, "output_spec.json"); - FileName = {OutputSpecsPath.data(), OutputSpecsPath.size()}; - } - - auto BufferOrError = MemoryBuffer::getFileOrSTDIN(FileName); - if (!BufferOrError) { - Ctx.emitError("Error opening output specs file: " + FileName + " : " + - BufferOrError.getError().message()); - return None; - } - auto ParsedJSONValues = json::parse(BufferOrError.get()->getBuffer()); - if (!ParsedJSONValues) { - Ctx.emitError("Could not parse specs file: " + FileName); - return None; - } - auto ValuesArray = ParsedJSONValues->getAsArray(); - if (!ValuesArray) { - Ctx.emitError("Expected an array of {tensor_spec:<TensorSpec>, " - "logging_name:<name>} dictionaries"); - return None; - } - std::vector<LoggedFeatureSpec> Ret; - for (const auto &Value : *ValuesArray) - if (const auto *Obj = Value.getAsObject()) - if (const auto *SpecPart = Obj->get("tensor_spec")) - if (auto TensorSpec = getTensorSpecFromJSON(Ctx, *SpecPart)) - if (auto LoggingName = Obj->getString("logging_name")) { - if (!TensorSpec->isElementType<int64_t>() && - !TensorSpec->isElementType<int32_t>() && - !TensorSpec->isElementType<float>()) { - Ctx.emitError( - "Only int64, int32, and float tensors are supported. " - "Found unsupported type for tensor named " + - TensorSpec->name()); - return None; - } - Ret.push_back({*TensorSpec, LoggingName->str()}); - } - - if (ValuesArray->size() != Ret.size()) { - Ctx.emitError( - "Unable to parse output spec. It should be a json file containing an " - "array of dictionaries. Each dictionary must have a 'tensor_spec' key, " - "with a json object describing a TensorSpec; and a 'logging_name' key, " - "which is a string to use as name when logging this tensor in the " - "training log."); - return None; - } - if (Ret.empty() || *Ret[0].LoggingName != ExpectedDecisionName) { - Ctx.emitError("The first output spec must describe the decision tensor, " - "and must have the logging_name " + - StringRef(ExpectedDecisionName)); - return None; - } - return Ret; -} - class TFModelEvaluatorImpl { public: TFModelEvaluatorImpl(StringRef SavedModelPath, @@ -383,16 +297,29 @@ errs() << TF_Message(Status.get()); invalidate(); } + size_t NrSupported = 0; for (size_t I = 0; I < InputSpecs.size(); ++I) { auto &InputSpec = InputSpecs[I]; InputFeed[I] = { TF_GraphOperationByName(Graph.get(), (InputSpec.name()).c_str()), InputSpec.port()}; + if (!InputFeed[I].oper) { + continue; + } + if (NrSupported++ != I) { + errs() + << "Unsupported features must be placed at the end of the InputSpecs"; + invalidate(); + return; + } if (!checkReportAndInvalidate(InputFeed[I], InputSpec)) return; - initInput(I, static_cast<TF_DataType>(InputSpec.typeIndex()), + initInput(I, static_cast<TF_DataType>(getTFTypeIndex(InputSpec.type())), InputSpec.shape()); } + InputFeed.resize(NrSupported); + Input.resize(NrSupported); + for (size_t I = 0; I < OutputSpecsSize; ++I) { auto OutputSpec = GetOutputSpecs(I); OutputFeed[I] = { @@ -470,7 +397,9 @@ } void *TFModelEvaluator::getUntypedInput(size_t Index) { - return TF_TensorData(Impl->getInput()[Index]); + if (Index < Impl->getInput().size()) + return TF_TensorData(Impl->getInput()[Index]); + return nullptr; } TFModelEvaluator::EvaluationResult::EvaluationResult( @@ -495,13 +424,6 @@ return TF_TensorData(Impl->getOutput()[Index]); } -#define TFUTILS_GETDATATYPE_IMPL(T, E) \ - template <> int TensorSpec::getDataType<T>() { return E; } - -TFUTILS_SUPPORTED_TYPES(TFUTILS_GETDATATYPE_IMPL) - -#undef TFUTILS_GETDATATYPE_IMPL - TFModelEvaluator::EvaluationResult::~EvaluationResult() {} TFModelEvaluator::~TFModelEvaluator() {}
diff --git a/src/llvm-project/llvm/lib/Analysis/TargetLibraryInfo.cpp b/src/llvm-project/llvm/lib/Analysis/TargetLibraryInfo.cpp index 22d6a5f..8ebdb65 100644 --- a/src/llvm-project/llvm/lib/Analysis/TargetLibraryInfo.cpp +++ b/src/llvm-project/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -659,12 +659,12 @@ TLI.setUnavailable(LibFunc_stpncpy); } - if (T.isPS4()) { - // PS4 does have memalign. + if (T.isPS()) { + // PS4/PS5 do have memalign. TLI.setAvailable(LibFunc_memalign); - // PS4 does not have new/delete with "unsigned int" size parameter; - // it only has the "unsigned long" versions. + // PS4/PS5 do not have new/delete with "unsigned int" size parameter; + // they only have the "unsigned long" versions. TLI.setUnavailable(LibFunc_ZdaPvj); TLI.setUnavailable(LibFunc_ZdaPvjSt11align_val_t); TLI.setUnavailable(LibFunc_ZdlPvj); @@ -1110,9 +1110,11 @@ case LibFunc_system: return (NumParams == 1 && FTy.getParamType(0)->isPointerTy()); case LibFunc___kmpc_alloc_shared: + return NumParams == 1 && FTy.getReturnType()->isPointerTy(); case LibFunc_malloc: case LibFunc_vec_malloc: - return (NumParams == 1 && FTy.getReturnType()->isPointerTy()); + return NumParams == 1 && FTy.getParamType(0)->isIntegerTy(SizeTBits) && + FTy.getReturnType()->isPointerTy(); case LibFunc_memcmp: return NumParams == 3 && FTy.getReturnType()->isIntegerTy(32) && FTy.getParamType(0)->isPointerTy() && @@ -1793,26 +1795,6 @@ else return false; } - - case LibFunc_rust_alloc: - case LibFunc_rust_alloc_zeroed: - return (NumParams == 2 && FTy.getReturnType()->isPointerTy() && - FTy.getParamType(0)->isIntegerTy() && - FTy.getParamType(1)->isIntegerTy()); - - case LibFunc_rust_dealloc: - return (NumParams == 3 && FTy.getReturnType()->isVoidTy() && - FTy.getParamType(0)->isPointerTy() && - FTy.getParamType(1)->isIntegerTy() && - FTy.getParamType(2)->isIntegerTy()); - - case LibFunc_rust_realloc: - return (NumParams == 4 && FTy.getReturnType()->isPointerTy() && - FTy.getParamType(0)->isPointerTy() && - FTy.getParamType(1)->isIntegerTy() && - FTy.getParamType(2)->isIntegerTy() && - FTy.getParamType(3)->isIntegerTy()); - case LibFunc::NumLibFuncs: case LibFunc::NotLibFunc: break;
diff --git a/src/llvm-project/llvm/lib/Analysis/TargetTransformInfo.cpp b/src/llvm-project/llvm/lib/Analysis/TargetTransformInfo.cpp index 25e9dee..143f03c 100644 --- a/src/llvm-project/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/src/llvm-project/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -11,7 +11,6 @@ #include "llvm/Analysis/LoopIterator.h" #include "llvm/Analysis/TargetTransformInfoImpl.h" #include "llvm/IR/CFG.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" @@ -21,7 +20,6 @@ #include "llvm/IR/PatternMatch.h" #include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" #include <utility> using namespace llvm; @@ -33,6 +31,11 @@ cl::Hidden, cl::desc("Recognize reduction patterns.")); +static cl::opt<unsigned> CacheLineSize( + "cache-line-size", cl::init(0), cl::Hidden, + cl::desc("Use this to override the target cache line size when " + "specified by the user.")); + namespace { /// No-op implementation of the TTI interface using the utility base /// classes. @@ -55,14 +58,16 @@ } IntrinsicCostAttributes::IntrinsicCostAttributes( - Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost) + Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost, + bool TypeBasedOnly) : II(dyn_cast<IntrinsicInst>(&CI)), RetTy(CI.getType()), IID(Id), ScalarizationCost(ScalarizationCost) { if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI)) FMF = FPMO->getFastMathFlags(); - Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end()); + if (!TypeBasedOnly) + Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end()); FunctionType *FTy = CI.getCalledFunction()->getFunctionType(); ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end()); } @@ -179,7 +184,7 @@ TargetTransformInfo::TargetTransformInfo(const DataLayout &DL) : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {} -TargetTransformInfo::~TargetTransformInfo() {} +TargetTransformInfo::~TargetTransformInfo() = default; TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg) : TTIImpl(std::move(Arg.TTIImpl)) {} @@ -291,11 +296,11 @@ bool TargetTransformInfo::preferPredicateOverEpilogue( Loop *L, LoopInfo *LI, ScalarEvolution &SE, AssumptionCache &AC, TargetLibraryInfo *TLI, DominatorTree *DT, - const LoopAccessInfo *LAI) const { - return TTIImpl->preferPredicateOverEpilogue(L, LI, SE, AC, TLI, DT, LAI); + LoopVectorizationLegality *LVL) const { + return TTIImpl->preferPredicateOverEpilogue(L, LI, SE, AC, TLI, DT, LVL); } -bool TargetTransformInfo::emitGetActiveLaneMask() const { +PredicationStyle TargetTransformInfo::emitGetActiveLaneMask() const { return TTIImpl->emitGetActiveLaneMask(); } @@ -350,7 +355,8 @@ Scale, AddrSpace, I); } -bool TargetTransformInfo::isLSRCostLess(LSRCost &C1, LSRCost &C2) const { +bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1, + const LSRCost &C2) const { return TTIImpl->isLSRCostLess(C1, C2); } @@ -398,11 +404,22 @@ return TTIImpl->isLegalNTLoad(DataType, Alignment); } +bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy, + ElementCount NumElements) const { + return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements); +} + bool TargetTransformInfo::isLegalMaskedGather(Type *DataType, Align Alignment) const { return TTIImpl->isLegalMaskedGather(DataType, Alignment); } +bool TargetTransformInfo::isLegalAltInstr( + VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, + const SmallBitVector &OpcodeMask) const { + return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask); +} + bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType, Align Alignment) const { return TTIImpl->isLegalMaskedScatter(DataType, Alignment); @@ -470,7 +487,7 @@ return TTIImpl->isTypeLegal(Ty); } -InstructionCost TargetTransformInfo::getRegUsageForType(Type *Ty) const { +unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const { return TTIImpl->getRegUsageForType(Ty); } @@ -507,6 +524,10 @@ return TTIImpl->supportsEfficientVectorElementLoadStore(); } +bool TargetTransformInfo::supportsTailCalls() const { + return TTIImpl->supportsTailCalls(); +} + bool TargetTransformInfo::enableAggressiveInterleaving( bool LoopHasReductions) const { return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); @@ -623,8 +644,9 @@ return TTIImpl->getVScaleForTuning(); } -bool TargetTransformInfo::shouldMaximizeVectorBandwidth() const { - return TTIImpl->shouldMaximizeVectorBandwidth(); +bool TargetTransformInfo::shouldMaximizeVectorBandwidth( + TargetTransformInfo::RegisterKind K) const { + return TTIImpl->shouldMaximizeVectorBandwidth(K); } ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth, @@ -637,6 +659,11 @@ return TTIImpl->getMaximumVF(ElemWidth, Opcode); } +unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, + Type *ScalarValTy) const { + return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy); +} + bool TargetTransformInfo::shouldConsiderAddressTypePromotion( const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const { return TTIImpl->shouldConsiderAddressTypePromotion( @@ -644,7 +671,8 @@ } unsigned TargetTransformInfo::getCacheLineSize() const { - return TTIImpl->getCacheLineSize(); + return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize + : TTIImpl->getCacheLineSize(); } llvm::Optional<unsigned> @@ -742,12 +770,11 @@ return Cost; } -InstructionCost TargetTransformInfo::getShuffleCost(ShuffleKind Kind, - VectorType *Ty, - ArrayRef<int> Mask, - int Index, - VectorType *SubTp) const { - InstructionCost Cost = TTIImpl->getShuffleCost(Kind, Ty, Mask, Index, SubTp); +InstructionCost TargetTransformInfo::getShuffleCost( + ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask, int Index, + VectorType *SubTp, ArrayRef<const Value *> Args) const { + InstructionCost Cost = + TTIImpl->getShuffleCost(Kind, Ty, Mask, Index, SubTp, Args); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } @@ -973,18 +1000,21 @@ Type *TargetTransformInfo::getMemcpyLoopLoweringType( LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, - unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const { + unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicElementSize) const { return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace, - DestAddrSpace, SrcAlign, DestAlign); + DestAddrSpace, SrcAlign, DestAlign, + AtomicElementSize); } void TargetTransformInfo::getMemcpyLoopResidualLoweringType( SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, - unsigned SrcAlign, unsigned DestAlign) const { - TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, - SrcAddrSpace, DestAddrSpace, - SrcAlign, DestAlign); + unsigned SrcAlign, unsigned DestAlign, + Optional<uint32_t> AtomicCpySize) const { + TTIImpl->getMemcpyLoopResidualLoweringType( + OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign, + DestAlign, AtomicCpySize); } bool TargetTransformInfo::areInlineCompatible(const Function *Caller, @@ -1078,6 +1108,10 @@ return TTIImpl->getGISelRematGlobalCost(); } +unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const { + return TTIImpl->getMinTripCountTailFoldingThreshold(); +} + bool TargetTransformInfo::supportsScalableVectors() const { return TTIImpl->supportsScalableVectors(); } @@ -1155,7 +1189,7 @@ } } -TargetTransformInfo::Concept::~Concept() {} +TargetTransformInfo::Concept::~Concept() = default; TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
diff --git a/src/llvm-project/llvm/lib/Analysis/TensorSpec.cpp b/src/llvm-project/llvm/lib/Analysis/TensorSpec.cpp new file mode 100644 index 0000000..f6a5882 --- /dev/null +++ b/src/llvm-project/llvm/lib/Analysis/TensorSpec.cpp
@@ -0,0 +1,144 @@ +//===- TensorSpec.cpp - tensor type abstraction ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implementation file for the abstraction of a tensor type, and JSON loading +// utils. +// +//===----------------------------------------------------------------------===// +#include "llvm/Config/config.h" + +#include "llvm/ADT/Twine.h" +#include "llvm/Analysis/TensorSpec.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" +#include <cassert> +#include <numeric> + +using namespace llvm; + +namespace llvm { + +#define TFUTILS_GETDATATYPE_IMPL(T, E) \ + template <> TensorType TensorSpec::getDataType<T>() { return TensorType::E; } + +SUPPORTED_TENSOR_TYPES(TFUTILS_GETDATATYPE_IMPL) + +#undef TFUTILS_GETDATATYPE_IMPL + +TensorSpec::TensorSpec(const std::string &Name, int Port, TensorType Type, + size_t ElementSize, const std::vector<int64_t> &Shape) + : Name(Name), Port(Port), Type(Type), Shape(Shape), + ElementCount(std::accumulate(Shape.begin(), Shape.end(), 1, + std::multiplies<int64_t>())), + ElementSize(ElementSize) {} + +Optional<TensorSpec> getTensorSpecFromJSON(LLVMContext &Ctx, + const json::Value &Value) { + auto EmitError = [&](const llvm::Twine &Message) -> Optional<TensorSpec> { + std::string S; + llvm::raw_string_ostream OS(S); + OS << Value; + Ctx.emitError("Unable to parse JSON Value as spec (" + Message + "): " + S); + return None; + }; + // FIXME: accept a Path as a parameter, and use it for error reporting. + json::Path::Root Root("tensor_spec"); + json::ObjectMapper Mapper(Value, Root); + if (!Mapper) + return EmitError("Value is not a dict"); + + std::string TensorName; + int TensorPort = -1; + std::string TensorType; + std::vector<int64_t> TensorShape; + + if (!Mapper.map<std::string>("name", TensorName)) + return EmitError("'name' property not present or not a string"); + if (!Mapper.map<std::string>("type", TensorType)) + return EmitError("'type' property not present or not a string"); + if (!Mapper.map<int>("port", TensorPort)) + return EmitError("'port' property not present or not an int"); + if (!Mapper.map<std::vector<int64_t>>("shape", TensorShape)) + return EmitError("'shape' property not present or not an int array"); + +#define PARSE_TYPE(T, E) \ + if (TensorType == #T) \ + return TensorSpec::createSpec<T>(TensorName, TensorShape, TensorPort); + SUPPORTED_TENSOR_TYPES(PARSE_TYPE) +#undef PARSE_TYPE + return None; +} + +Optional<std::vector<LoggedFeatureSpec>> +loadOutputSpecs(LLVMContext &Ctx, StringRef ExpectedDecisionName, + StringRef ModelPath, StringRef SpecFileOverride) { + SmallVector<char, 128> OutputSpecsPath; + StringRef FileName = SpecFileOverride; + if (FileName.empty()) { + llvm::sys::path::append(OutputSpecsPath, ModelPath, "output_spec.json"); + FileName = {OutputSpecsPath.data(), OutputSpecsPath.size()}; + } + + auto BufferOrError = MemoryBuffer::getFileOrSTDIN(FileName); + if (!BufferOrError) { + Ctx.emitError("Error opening output specs file: " + FileName + " : " + + BufferOrError.getError().message()); + return None; + } + auto ParsedJSONValues = json::parse(BufferOrError.get()->getBuffer()); + if (!ParsedJSONValues) { + Ctx.emitError("Could not parse specs file: " + FileName); + return None; + } + auto ValuesArray = ParsedJSONValues->getAsArray(); + if (!ValuesArray) { + Ctx.emitError("Expected an array of {tensor_spec:<TensorSpec>, " + "logging_name:<name>} dictionaries"); + return None; + } + std::vector<LoggedFeatureSpec> Ret; + for (const auto &Value : *ValuesArray) + if (const auto *Obj = Value.getAsObject()) + if (const auto *SpecPart = Obj->get("tensor_spec")) + if (auto TensorSpec = getTensorSpecFromJSON(Ctx, *SpecPart)) + if (auto LoggingName = Obj->getString("logging_name")) { + if (!TensorSpec->isElementType<int64_t>() && + !TensorSpec->isElementType<int32_t>() && + !TensorSpec->isElementType<float>()) { + Ctx.emitError( + "Only int64, int32, and float tensors are supported. " + "Found unsupported type for tensor named " + + TensorSpec->name()); + return None; + } + Ret.push_back({*TensorSpec, LoggingName->str()}); + } + + if (ValuesArray->size() != Ret.size()) { + Ctx.emitError( + "Unable to parse output spec. It should be a json file containing an " + "array of dictionaries. Each dictionary must have a 'tensor_spec' key, " + "with a json object describing a TensorSpec; and a 'logging_name' key, " + "which is a string to use as name when logging this tensor in the " + "training log."); + return None; + } + if (Ret.empty() || *Ret[0].LoggingName != ExpectedDecisionName) { + Ctx.emitError("The first output spec must describe the decision tensor, " + "and must have the logging_name " + + StringRef(ExpectedDecisionName)); + return None; + } + return Ret; +} +} // namespace llvm
diff --git a/src/llvm-project/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp b/src/llvm-project/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp index 627a78a..2160277 100644 --- a/src/llvm-project/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp +++ b/src/llvm-project/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
@@ -112,7 +112,6 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InstrTypes.h" -#include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/InitializePasses.h" @@ -304,24 +303,27 @@ /// given offset. Update the offset to be relative to the field type. TBAAStructTypeNode getField(uint64_t &Offset) const { bool NewFormat = isNewFormat(); + const ArrayRef<MDOperand> Operands = Node->operands(); + const unsigned NumOperands = Operands.size(); + if (NewFormat) { // New-format root and scalar type nodes have no fields. - if (Node->getNumOperands() < 6) + if (NumOperands < 6) return TBAAStructTypeNode(); } else { // Parent can be omitted for the root node. - if (Node->getNumOperands() < 2) + if (NumOperands < 2) return TBAAStructTypeNode(); // Fast path for a scalar type node and a struct type node with a single // field. - if (Node->getNumOperands() <= 3) { - uint64_t Cur = Node->getNumOperands() == 2 - ? 0 - : mdconst::extract<ConstantInt>(Node->getOperand(2)) - ->getZExtValue(); + if (NumOperands <= 3) { + uint64_t Cur = + NumOperands == 2 + ? 0 + : mdconst::extract<ConstantInt>(Operands[2])->getZExtValue(); Offset -= Cur; - MDNode *P = dyn_cast_or_null<MDNode>(Node->getOperand(1)); + MDNode *P = dyn_cast_or_null<MDNode>(Operands[1]); if (!P) return TBAAStructTypeNode(); return TBAAStructTypeNode(P); @@ -333,10 +335,11 @@ unsigned FirstFieldOpNo = NewFormat ? 3 : 1; unsigned NumOpsPerField = NewFormat ? 3 : 2; unsigned TheIdx = 0; - for (unsigned Idx = FirstFieldOpNo; Idx < Node->getNumOperands(); + + for (unsigned Idx = FirstFieldOpNo; Idx < NumOperands; Idx += NumOpsPerField) { - uint64_t Cur = mdconst::extract<ConstantInt>(Node->getOperand(Idx + 1)) - ->getZExtValue(); + uint64_t Cur = + mdconst::extract<ConstantInt>(Operands[Idx + 1])->getZExtValue(); if (Cur > Offset) { assert(Idx >= FirstFieldOpNo + NumOpsPerField && "TBAAStructTypeNode::getField should have an offset match!"); @@ -346,11 +349,11 @@ } // Move along the last field. if (TheIdx == 0) - TheIdx = Node->getNumOperands() - NumOpsPerField; - uint64_t Cur = mdconst::extract<ConstantInt>(Node->getOperand(TheIdx + 1)) - ->getZExtValue(); + TheIdx = NumOperands - NumOpsPerField; + uint64_t Cur = + mdconst::extract<ConstantInt>(Operands[TheIdx + 1])->getZExtValue(); Offset -= Cur; - MDNode *P = dyn_cast_or_null<MDNode>(Node->getOperand(TheIdx)); + MDNode *P = dyn_cast_or_null<MDNode>(Operands[TheIdx]); if (!P) return TBAAStructTypeNode(); return TBAAStructTypeNode(P); @@ -808,7 +811,8 @@ return nullptr; // Otherwise, create TBAA with the new Len - SmallVector<Metadata *, 4> NextNodes(MD->operands()); + ArrayRef<MDOperand> MDOperands = MD->operands(); + SmallVector<Metadata *, 4> NextNodes(MDOperands.begin(), MDOperands.end()); ConstantInt *PreviousSize = mdconst::extract<ConstantInt>(NextNodes[3]); // Don't create a new MDNode if it is the same length.
diff --git a/src/llvm-project/llvm/lib/Analysis/TypeMetadataUtils.cpp b/src/llvm-project/llvm/lib/Analysis/TypeMetadataUtils.cpp index 80051fd..e128187 100644 --- a/src/llvm-project/llvm/lib/Analysis/TypeMetadataUtils.cpp +++ b/src/llvm-project/llvm/lib/Analysis/TypeMetadataUtils.cpp
@@ -16,7 +16,6 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Module.h" using namespace llvm; @@ -76,7 +75,9 @@ SmallVectorImpl<DevirtCallSite> &DevirtCalls, SmallVectorImpl<CallInst *> &Assumes, const CallInst *CI, DominatorTree &DT) { - assert(CI->getCalledFunction()->getIntrinsicID() == Intrinsic::type_test); + assert(CI->getCalledFunction()->getIntrinsicID() == Intrinsic::type_test || + CI->getCalledFunction()->getIntrinsicID() == + Intrinsic::public_type_test); const Module *M = CI->getParent()->getParent()->getParent();
diff --git a/src/llvm-project/llvm/lib/Analysis/VFABIDemangling.cpp b/src/llvm-project/llvm/lib/Analysis/VFABIDemangling.cpp index 7573975..e6d2978 100644 --- a/src/llvm-project/llvm/lib/Analysis/VFABIDemangling.cpp +++ b/src/llvm-project/llvm/lib/Analysis/VFABIDemangling.cpp
@@ -6,8 +6,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallString.h" #include "llvm/Analysis/VectorUtils.h" using namespace llvm;
diff --git a/src/llvm-project/llvm/lib/Analysis/ValueLatticeUtils.cpp b/src/llvm-project/llvm/lib/Analysis/ValueLatticeUtils.cpp index 53638c3..2bcb4d5 100644 --- a/src/llvm-project/llvm/lib/Analysis/ValueLatticeUtils.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ValueLatticeUtils.cpp
@@ -29,12 +29,13 @@ !GV->hasDefinitiveInitializer()) return false; return all_of(GV->users(), [&](User *U) { - // Currently all users of a global variable have to be none-volatile loads - // or stores and the global cannot be stored itself. + // Currently all users of a global variable have to be non-volatile loads + // or stores of the global type, and the global cannot be stored itself. if (auto *Store = dyn_cast<StoreInst>(U)) - return Store->getValueOperand() != GV && !Store->isVolatile(); + return Store->getValueOperand() != GV && !Store->isVolatile() && + Store->getValueOperand()->getType() == GV->getValueType(); if (auto *Load = dyn_cast<LoadInst>(U)) - return !Load->isVolatile(); + return !Load->isVolatile() && Load->getType() == GV->getValueType(); return false; });
diff --git a/src/llvm-project/llvm/lib/Analysis/ValueTracking.cpp b/src/llvm-project/llvm/lib/Analysis/ValueTracking.cpp index c14bdb8..2dd671b 100644 --- a/src/llvm-project/llvm/lib/Analysis/ValueTracking.cpp +++ b/src/llvm-project/llvm/lib/Analysis/ValueTracking.cpp
@@ -26,6 +26,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AssumeBundleQueries.h" #include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/GuardUtils.h" #include "llvm/Analysis/InstructionSimplify.h" @@ -70,10 +71,8 @@ #include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" #include <algorithm> -#include <array> #include <cassert> #include <cstdint> -#include <iterator> #include <utility> using namespace llvm; @@ -86,13 +85,12 @@ // According to the LangRef, branching on a poison condition is absolutely // immediate full UB. However, historically we haven't implemented that -// consistently as we have an important transformation (non-trivial unswitch) -// which introduces instances of branch on poison/undef to otherwise well -// defined programs. This flag exists to let us test optimization benefit -// of exploiting the specified behavior (in combination with enabling the -// unswitch fix.) +// consistently as we had an important transformation (non-trivial unswitch) +// which introduced instances of branch on poison/undef to otherwise well +// defined programs. This issue has since been fixed, but the flag is +// temporarily retained to easily diagnose potential regressions. static cl::opt<bool> BranchOnPoisonAsUB("branch-on-poison-as-ub", - cl::Hidden, cl::init(false)); + cl::Hidden, cl::init(true)); /// Returns the bitwidth of the given scalar or pointer type. For vector types, @@ -275,13 +273,39 @@ assert(LHS->getType()->isIntOrIntVectorTy() && "LHS and RHS should be integers"); // Look for an inverted mask: (X & ~M) op (Y & M). - Value *M; - if (match(LHS, m_c_And(m_Not(m_Value(M)), m_Value())) && - match(RHS, m_c_And(m_Specific(M), m_Value()))) + { + Value *M; + if (match(LHS, m_c_And(m_Not(m_Value(M)), m_Value())) && + match(RHS, m_c_And(m_Specific(M), m_Value()))) + return true; + if (match(RHS, m_c_And(m_Not(m_Value(M)), m_Value())) && + match(LHS, m_c_And(m_Specific(M), m_Value()))) + return true; + } + + // X op (Y & ~X) + if (match(RHS, m_c_And(m_Not(m_Specific(LHS)), m_Value())) || + match(LHS, m_c_And(m_Not(m_Specific(RHS)), m_Value()))) return true; - if (match(RHS, m_c_And(m_Not(m_Value(M)), m_Value())) && - match(LHS, m_c_And(m_Specific(M), m_Value()))) + + // X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern + // for constant Y. + Value *Y; + if (match(RHS, + m_c_Xor(m_c_And(m_Specific(LHS), m_Value(Y)), m_Deferred(Y))) || + match(LHS, m_c_Xor(m_c_And(m_Specific(RHS), m_Value(Y)), m_Deferred(Y)))) return true; + + // Look for: (A & B) op ~(A | B) + { + Value *A, *B; + if (match(LHS, m_And(m_Value(A), m_Value(B))) && + match(RHS, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))) + return true; + if (match(RHS, m_And(m_Value(A), m_Value(B))) && + match(LHS, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))) + return true; + } IntegerType *IT = cast<IntegerType>(LHS->getType()->getScalarType()); KnownBits LHSKnown(IT->getBitWidth()); KnownBits RHSKnown(IT->getBitWidth()); @@ -451,7 +475,12 @@ } } - Known = KnownBits::mul(Known, Known2); + bool SelfMultiply = Op0 == Op1; + // TODO: SelfMultiply can be poison, but not undef. + if (SelfMultiply) + SelfMultiply &= + isGuaranteedNotToBeUndefOrPoison(Op0, Q.AC, Q.CxtI, Q.DT, Depth + 1); + Known = KnownBits::mul(Known, Known2, SelfMultiply); // Only make use of no-wrap flags if we failed to compute the sign bit // directly. This matters if the multiplication always overflows, in @@ -656,7 +685,8 @@ if (V->getType()->isPointerTy()) { if (RetainedKnowledge RK = getKnowledgeValidInContext( V, {Attribute::Alignment}, Q.CxtI, Q.DT, Q.AC)) { - Known.Zero.setLowBits(Log2_64(RK.ArgValue)); + if (isPowerOf2_64(RK.ArgValue)) + Known.Zero.setLowBits(Log2_64(RK.ArgValue)); } } @@ -1041,7 +1071,7 @@ // bits. This check is sunk down as far as possible to avoid the expensive // call to isKnownNonZero if the cheaper checks above fail. if (ShiftAmt == 0) { - if (!ShifterOperandIsNonZero.hasValue()) + if (!ShifterOperandIsNonZero) ShifterOperandIsNonZero = isKnownNonZero(I->getOperand(1), DemandedElts, Depth + 1, Q); if (*ShifterOperandIsNonZero) @@ -1726,8 +1756,7 @@ break; } - unsigned FirstZeroHighBit = - 32 - countLeadingZeros(VScaleMax.getValue()); + unsigned FirstZeroHighBit = 32 - countLeadingZeros(*VScaleMax); if (FirstZeroHighBit < BitWidth) Known.Zero.setBitsFrom(FirstZeroHighBit); @@ -2007,6 +2036,63 @@ assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?"); } +/// Try to detect a recurrence that the value of the induction variable is +/// always a power of two (or zero). +static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero, + unsigned Depth, Query &Q) { + BinaryOperator *BO = nullptr; + Value *Start = nullptr, *Step = nullptr; + if (!matchSimpleRecurrence(PN, BO, Start, Step)) + return false; + + // Initial value must be a power of two. + for (const Use &U : PN->operands()) { + if (U.get() == Start) { + // Initial value comes from a different BB, need to adjust context + // instruction for analysis. + Q.CxtI = PN->getIncomingBlock(U)->getTerminator(); + if (!isKnownToBeAPowerOfTwo(Start, OrZero, Depth, Q)) + return false; + } + } + + // Except for Mul, the induction variable must be on the left side of the + // increment expression, otherwise its value can be arbitrary. + if (BO->getOpcode() != Instruction::Mul && BO->getOperand(1) != Step) + return false; + + Q.CxtI = BO->getParent()->getTerminator(); + switch (BO->getOpcode()) { + case Instruction::Mul: + // Power of two is closed under multiplication. + return (OrZero || Q.IIQ.hasNoUnsignedWrap(BO) || + Q.IIQ.hasNoSignedWrap(BO)) && + isKnownToBeAPowerOfTwo(Step, OrZero, Depth, Q); + case Instruction::SDiv: + // Start value must not be signmask for signed division, so simply being a + // power of two is not sufficient, and it has to be a constant. + if (!match(Start, m_Power2()) || match(Start, m_SignMask())) + return false; + LLVM_FALLTHROUGH; + case Instruction::UDiv: + // Divisor must be a power of two. + // If OrZero is false, cannot guarantee induction variable is non-zero after + // division, same for Shr, unless it is exact division. + return (OrZero || Q.IIQ.isExact(BO)) && + isKnownToBeAPowerOfTwo(Step, false, Depth, Q); + case Instruction::Shl: + return OrZero || Q.IIQ.hasNoUnsignedWrap(BO) || Q.IIQ.hasNoSignedWrap(BO); + case Instruction::AShr: + if (!match(Start, m_Power2()) || match(Start, m_SignMask())) + return false; + LLVM_FALLTHROUGH; + case Instruction::LShr: + return OrZero || Q.IIQ.isExact(BO); + default: + return false; + } +} + /// Return true if the given value is known to have exactly one /// bit set when defined. For vectors return true if every element is known to /// be a power of two when defined. Supports values with integer or pointer @@ -2098,6 +2184,30 @@ } } + // A PHI node is power of two if all incoming values are power of two, or if + // it is an induction variable where in each step its value is a power of two. + if (const PHINode *PN = dyn_cast<PHINode>(V)) { + Query RecQ = Q; + + // Check if it is an induction variable and always power of two. + if (isPowerOfTwoRecurrence(PN, OrZero, Depth, RecQ)) + return true; + + // Recursively check all incoming values. Limit recursion to 2 levels, so + // that search complexity is limited to number of operands^2. + unsigned NewDepth = std::max(Depth, MaxAnalysisRecursionDepth - 1); + return llvm::all_of(PN->operands(), [&](const Use &U) { + // Value is power of 2 if it is coming from PHI node itself by induction. + if (U.get() == PN) + return true; + + // Change the context instruction to the incoming block where it is + // evaluated. + RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator(); + return isKnownToBeAPowerOfTwo(U.get(), OrZero, NewDepth, RecQ); + }); + } + // An exact divide or right shift can only shift off zero bits, so the result // is a power of two only if the first operand is a power of two and not // copying a sign bit (sdiv int_min, 2). @@ -2187,7 +2297,7 @@ return false; unsigned NumUsesExplored = 0; - for (auto *U : V->users()) { + for (const auto *U : V->users()) { // Avoid massive lists if (NumUsesExplored >= DomConditionsMaxUses) break; @@ -2228,7 +2338,7 @@ SmallVector<const User *, 4> WorkList; SmallPtrSet<const User *, 4> Visited; - for (auto *CmpU : U->users()) { + for (const auto *CmpU : U->users()) { assert(WorkList.empty() && "Should be!"); if (Visited.insert(CmpU).second) WorkList.push_back(CmpU); @@ -2242,7 +2352,7 @@ // TODO: Support similar logic of OR and EQ predicate? if (NonNullIfTrue) if (match(Curr, m_LogicalAnd(m_Value(), m_Value()))) { - for (auto *CurrU : Curr->users()) + for (const auto *CurrU : Curr->users()) if (Visited.insert(CurrU).second) WorkList.push_back(CurrU); continue; @@ -2588,6 +2698,9 @@ if (isKnownNonZero(Op, Depth, Q) && isGuaranteedNotToBePoison(Op, Q.AC, Q.CxtI, Q.DT, Depth)) return true; + } else if (const auto *II = dyn_cast<IntrinsicInst>(V)) { + if (II->getIntrinsicID() == Intrinsic::vscale) + return true; } KnownBits Known(BitWidth); @@ -2885,6 +2998,24 @@ return CLow->sle(*CHigh); } +static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II, + const APInt *&CLow, + const APInt *&CHigh) { + assert((II->getIntrinsicID() == Intrinsic::smin || + II->getIntrinsicID() == Intrinsic::smax) && "Must be smin/smax"); + + Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(II->getIntrinsicID()); + auto *InnerII = dyn_cast<IntrinsicInst>(II->getArgOperand(0)); + if (!InnerII || InnerII->getIntrinsicID() != InverseID || + !match(II->getArgOperand(1), m_APInt(CLow)) || + !match(InnerII->getArgOperand(1), m_APInt(CHigh))) + return false; + + if (II->getIntrinsicID() == Intrinsic::smin) + std::swap(CLow, CHigh); + return CLow->sle(*CHigh); +} + /// For vector constants, loop over the elements and find the constant with the /// minimum number of sign bits. Return 0 if the value is not a vector constant /// or if any element was not analyzed; otherwise, return the count for the @@ -3225,6 +3356,12 @@ // Absolute value reduces number of sign bits by at most 1. return Tmp - 1; + case Intrinsic::smin: + case Intrinsic::smax: { + const APInt *CLow, *CHigh; + if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh)) + return std::min(CLow->getNumSignBits(), CHigh->getNumSignBits()); + } } } } @@ -3358,9 +3495,6 @@ /// NOTE: Do not check 'nsz' here because that fast-math-flag does not guarantee /// that a value is not -0.0. It only guarantees that -0.0 may be treated /// the same as +0.0 in floating-point ops. -/// -/// NOTE: this function will need to be revisited when we support non-default -/// rounding modes! bool llvm::CannotBeNegativeZero(const Value *V, const TargetLibraryInfo *TLI, unsigned Depth) { if (auto *CFP = dyn_cast<ConstantFP>(V)) @@ -3390,9 +3524,21 @@ case Intrinsic::sqrt: case Intrinsic::canonicalize: return CannotBeNegativeZero(Call->getArgOperand(0), TLI, Depth + 1); + case Intrinsic::experimental_constrained_sqrt: { + // NOTE: This rounding mode restriction may be too strict. + const auto *CI = cast<ConstrainedFPIntrinsic>(Call); + if (CI->getRoundingMode() == RoundingMode::NearestTiesToEven) + return CannotBeNegativeZero(Call->getArgOperand(0), TLI, Depth + 1); + else + return false; + } // fabs(x) != -0.0 case Intrinsic::fabs: return true; + // sitofp and uitofp turn into +0.0 for zero. + case Intrinsic::experimental_constrained_sitofp: + case Intrinsic::experimental_constrained_uitofp: + return true; } } @@ -4032,69 +4178,83 @@ return true; } +// If V refers to an initialized global constant, set Slice either to +// its initializer if the size of its elements equals ElementSize, or, +// for ElementSize == 8, to its representation as an array of unsiged +// char. Return true on success. bool llvm::getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset) { assert(V); - // Look through bitcast instructions and geps. - V = V->stripPointerCasts(); - - // If the value is a GEP instruction or constant expression, treat it as an - // offset. - if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { - // The GEP operator should be based on a pointer to string constant, and is - // indexing into the string constant. - if (!isGEPBasedOnPointerToString(GEP, ElementSize)) - return false; - - // If the second index isn't a ConstantInt, then this is a variable index - // into the array. If this occurs, we can't say anything meaningful about - // the string. - uint64_t StartIdx = 0; - if (const ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(2))) - StartIdx = CI->getZExtValue(); - else - return false; - return getConstantDataArrayInfo(GEP->getOperand(0), Slice, ElementSize, - StartIdx + Offset); - } - - // The GEP instruction, constant or instruction, must reference a global - // variable that is a constant and is initialized. The referenced constant - // initializer is the array that we'll use for optimization. - const GlobalVariable *GV = dyn_cast<GlobalVariable>(V); + // Drill down into the pointer expression V, ignoring any intervening + // casts, and determine the identity of the object it references along + // with the cumulative byte offset into it. + const GlobalVariable *GV = + dyn_cast<GlobalVariable>(getUnderlyingObject(V)); if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer()) + // Fail if V is not based on constant global object. return false; - const ConstantDataArray *Array; - ArrayType *ArrayTy; + const DataLayout &DL = GV->getParent()->getDataLayout(); + APInt Off(DL.getIndexTypeSizeInBits(V->getType()), 0); + + if (GV != V->stripAndAccumulateConstantOffsets(DL, Off, + /*AllowNonInbounds*/ true)) + // Fail if a constant offset could not be determined. + return false; + + uint64_t StartIdx = Off.getLimitedValue(); + if (StartIdx == UINT64_MAX) + // Fail if the constant offset is excessive. + return false; + + Offset += StartIdx; + + ConstantDataArray *Array = nullptr; + ArrayType *ArrayTy = nullptr; + if (GV->getInitializer()->isNullValue()) { Type *GVTy = GV->getValueType(); - if ( (ArrayTy = dyn_cast<ArrayType>(GVTy)) ) { - // A zeroinitializer for the array; there is no ConstantDataArray. - Array = nullptr; - } else { - const DataLayout &DL = GV->getParent()->getDataLayout(); - uint64_t SizeInBytes = DL.getTypeStoreSize(GVTy).getFixedSize(); - uint64_t Length = SizeInBytes / (ElementSize / 8); - if (Length <= Offset) - return false; + uint64_t SizeInBytes = DL.getTypeStoreSize(GVTy).getFixedSize(); + uint64_t Length = SizeInBytes / (ElementSize / 8); - Slice.Array = nullptr; - Slice.Offset = 0; - Slice.Length = Length - Offset; - return true; - } - } else { - // This must be a ConstantDataArray. - Array = dyn_cast<ConstantDataArray>(GV->getInitializer()); - if (!Array) - return false; - ArrayTy = Array->getType(); + Slice.Array = nullptr; + Slice.Offset = 0; + // Return an empty Slice for undersized constants to let callers + // transform even undefined library calls into simpler, well-defined + // expressions. This is preferable to making the calls although it + // prevents sanitizers from detecting such calls. + Slice.Length = Length < Offset ? 0 : Length - Offset; + return true; } - if (!ArrayTy->getElementType()->isIntegerTy(ElementSize)) - return false; + + auto *Init = const_cast<Constant *>(GV->getInitializer()); + if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Init)) { + Type *InitElTy = ArrayInit->getElementType(); + if (InitElTy->isIntegerTy(ElementSize)) { + // If Init is an initializer for an array of the expected type + // and size, use it as is. + Array = ArrayInit; + ArrayTy = ArrayInit->getType(); + } + } + + if (!Array) { + if (ElementSize != 8) + // TODO: Handle conversions to larger integral types. + return false; + + // Otherwise extract the portion of the initializer starting + // at Offset as an array of bytes, and reset Offset. + Init = ReadByteArrayFromGlobal(GV, Offset); + if (!Init) + return false; + + Offset = 0; + Array = dyn_cast<ConstantDataArray>(Init); + ArrayTy = dyn_cast<ArrayType>(Init->getType()); + } uint64_t NumElts = ArrayTy->getArrayNumElements(); if (Offset > NumElts) @@ -4106,9 +4266,10 @@ return true; } -/// This function computes the length of a null-terminated C string pointed to -/// by V. If successful, it returns true and returns the string in Str. -/// If unsuccessful, it returns false. +/// Extract bytes from the initializer of the constant array V, which need +/// not be a nul-terminated string. On success, store the bytes in Str and +/// return true. When TrimAtNul is set, Str will contain only the bytes up +/// to but not including the first nul. Return false on failure. bool llvm::getConstantStringInfo(const Value *V, StringRef &Str, uint64_t Offset, bool TrimAtNul) { ConstantDataArraySlice Slice; @@ -4117,6 +4278,12 @@ if (Slice.Array == nullptr) { if (TrimAtNul) { + // Return a nul-terminated string even for an empty Slice. This is + // safe because all existing SimplifyLibcalls callers require string + // arguments and the behavior of the functions they fold is undefined + // otherwise. Folding the calls this way is preferable to making + // the undefined library calls, even though it prevents sanitizers + // from reporting such calls. Str = StringRef(); return true; } @@ -4196,9 +4363,13 @@ return 0; if (Slice.Array == nullptr) + // Zeroinitializer (including an empty one). return 1; - // Search for nul characters + // Search for the first nul character. Return a conservative result even + // when there is no nul. This is safe since otherwise the string function + // being folded such as strlen is undefined, and can be preferable to + // making the undefined library call. unsigned NullIndex = 0; for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) { if (Slice.Array->getElementAsInteger(Slice.Offset + NullIndex) == 0) @@ -4509,21 +4680,38 @@ F.hasFnAttribute(Attribute::SanitizeHWAddress); } - -bool llvm::isSafeToSpeculativelyExecute(const Value *V, +bool llvm::isSafeToSpeculativelyExecute(const Instruction *Inst, const Instruction *CtxI, const DominatorTree *DT, const TargetLibraryInfo *TLI) { - const Operator *Inst = dyn_cast<Operator>(V); - if (!Inst) - return false; + return isSafeToSpeculativelyExecuteWithOpcode(Inst->getOpcode(), Inst, CtxI, + DT, TLI); +} - for (unsigned i = 0, e = Inst->getNumOperands(); i != e; ++i) - if (Constant *C = dyn_cast<Constant>(Inst->getOperand(i))) - if (C->canTrap()) - return false; +bool llvm::isSafeToSpeculativelyExecuteWithOpcode( + unsigned Opcode, const Instruction *Inst, const Instruction *CtxI, + const DominatorTree *DT, const TargetLibraryInfo *TLI) { +#ifndef NDEBUG + if (Inst->getOpcode() != Opcode) { + // Check that the operands are actually compatible with the Opcode override. + auto hasEqualReturnAndLeadingOperandTypes = + [](const Instruction *Inst, unsigned NumLeadingOperands) { + if (Inst->getNumOperands() < NumLeadingOperands) + return false; + const Type *ExpectedType = Inst->getType(); + for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp) + if (Inst->getOperand(ItOp)->getType() != ExpectedType) + return false; + return true; + }; + assert(!Instruction::isBinaryOp(Opcode) || + hasEqualReturnAndLeadingOperandTypes(Inst, 2)); + assert(!Instruction::isUnaryOp(Opcode) || + hasEqualReturnAndLeadingOperandTypes(Inst, 1)); + } +#endif - switch (Inst->getOpcode()) { + switch (Opcode) { default: return true; case Instruction::UDiv: @@ -4554,7 +4742,9 @@ return false; } case Instruction::Load: { - const LoadInst *LI = cast<LoadInst>(Inst); + const LoadInst *LI = dyn_cast<LoadInst>(Inst); + if (!LI) + return false; if (mustSuppressSpeculation(*LI)) return false; const DataLayout &DL = LI->getModule()->getDataLayout(); @@ -4563,7 +4753,9 @@ TLI); } case Instruction::Call: { - auto *CI = cast<const CallInst>(Inst); + auto *CI = dyn_cast<const CallInst>(Inst); + if (!CI) + return false; const Function *Callee = CI->getCalledFunction(); // The called function could have undefined behavior or side-effects, even @@ -4595,8 +4787,20 @@ } } -bool llvm::mayBeMemoryDependent(const Instruction &I) { - return I.mayReadOrWriteMemory() || !isSafeToSpeculativelyExecute(&I); +bool llvm::mayHaveNonDefUseDependency(const Instruction &I) { + if (I.mayReadOrWriteMemory()) + // Memory dependency possible + return true; + if (!isSafeToSpeculativelyExecute(&I)) + // Can't move above a maythrow call or infinite loop. Or if an + // inalloca alloca, above a stacksave call. + return true; + if (!isGuaranteedToTransferExecutionToSuccessor(&I)) + // 1) Can't reorder two inf-loop calls, even if readonly + // 2) Also can't reorder an inf-loop call below a instruction which isn't + // safe to speculative execute. (Inverse of above) + return true; + return false; } /// Convert ConstantRange OverflowResult into ValueTracking OverflowResult. @@ -4766,6 +4970,22 @@ AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { + // X - (X % ?) + // The remainder of a value can't have greater magnitude than itself, + // so the subtraction can't overflow. + + // X - (X -nuw ?) + // In the minimal case, this would simplify to "?", so there's no subtract + // at all. But if this analysis is used to peek through casts, for example, + // then determining no-overflow may allow other transforms. + + // TODO: There are other patterns like this. + // See simplifyICmpWithBinOpOnLHS() for candidates. + if (match(RHS, m_URem(m_Specific(LHS), m_Value())) || + match(RHS, m_NUWSub(m_Specific(LHS), m_Value()))) + if (isGuaranteedNotToBeUndefOrPoison(LHS, AC, CxtI, DT)) + return OverflowResult::NeverOverflows; + // Checking for conditions implied by dominating conditions may be expensive. // Limit it to usub_with_overflow calls for now. if (match(CxtI, @@ -4789,6 +5009,19 @@ AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { + // X - (X % ?) + // The remainder of a value can't have greater magnitude than itself, + // so the subtraction can't overflow. + + // X - (X -nsw ?) + // In the minimal case, this would simplify to "?", so there's no subtract + // at all. But if this analysis is used to peek through casts, for example, + // then determining no-overflow may allow other transforms. + if (match(RHS, m_SRem(m_Specific(LHS), m_Value())) || + match(RHS, m_NSWSub(m_Specific(LHS), m_Value()))) + if (isGuaranteedNotToBeUndefOrPoison(LHS, AC, CxtI, DT)) + return OverflowResult::NeverOverflows; + // If LHS and RHS each have at least two sign bits, the subtraction // cannot overflow. if (ComputeNumSignBits(LHS, DL, 0, AC, CxtI, DT) > 1 && @@ -4841,7 +5074,7 @@ if (DT.dominates(NoWrapEdge, Result->getParent())) continue; - for (auto &RU : Result->uses()) + for (const auto &RU : Result->uses()) if (!DT.dominates(NoWrapEdge, RU)) return false; } @@ -5100,7 +5333,9 @@ } if (auto *I = dyn_cast<LoadInst>(V)) - if (I->getMetadata(LLVMContext::MD_noundef)) + if (I->hasMetadata(LLVMContext::MD_noundef) || + I->hasMetadata(LLVMContext::MD_dereferenceable) || + I->hasMetadata(LLVMContext::MD_dereferenceable_or_null)) return true; if (programUndefinedIfUndefOrPoison(V, PoisonOnly)) @@ -5125,10 +5360,10 @@ auto *TI = Dominator->getBlock()->getTerminator(); Value *Cond = nullptr; - if (auto BI = dyn_cast<BranchInst>(TI)) { + if (auto BI = dyn_cast_or_null<BranchInst>(TI)) { if (BI->isConditional()) Cond = BI->getCondition(); - } else if (auto SI = dyn_cast<SwitchInst>(TI)) { + } else if (auto SI = dyn_cast_or_null<SwitchInst>(TI)) { Cond = SI->getCondition(); } @@ -5411,7 +5646,7 @@ // whether a value is directly passed to an instruction that must take // well-defined operands. - for (auto &I : make_range(Begin, End)) { + for (const auto &I : make_range(Begin, End)) { if (isa<DbgInfoIntrinsic>(I)) continue; if (--ScanLimit == 0) @@ -5442,7 +5677,7 @@ Visited.insert(BB); while (true) { - for (auto &I : make_range(Begin, End)) { + for (const auto &I : make_range(Begin, End)) { if (isa<DbgInfoIntrinsic>(I)) continue; if (--ScanLimit == 0) @@ -5763,20 +5998,6 @@ if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT) return {SPF_UNKNOWN, SPNB_NA, false}; - // Z = X -nsw Y - // (X >s Y) ? 0 : Z ==> (Z >s 0) ? 0 : Z ==> SMIN(Z, 0) - // (X <s Y) ? 0 : Z ==> (Z <s 0) ? 0 : Z ==> SMAX(Z, 0) - if (match(TrueVal, m_Zero()) && - match(FalseVal, m_NSWSub(m_Specific(CmpLHS), m_Specific(CmpRHS)))) - return {Pred == CmpInst::ICMP_SGT ? SPF_SMIN : SPF_SMAX, SPNB_NA, false}; - - // Z = X -nsw Y - // (X >s Y) ? Z : 0 ==> (Z >s 0) ? Z : 0 ==> SMAX(Z, 0) - // (X <s Y) ? Z : 0 ==> (Z <s 0) ? Z : 0 ==> SMIN(Z, 0) - if (match(FalseVal, m_Zero()) && - match(TrueVal, m_NSWSub(m_Specific(CmpLHS), m_Specific(CmpRHS)))) - return {Pred == CmpInst::ICMP_SGT ? SPF_SMAX : SPF_SMIN, SPNB_NA, false}; - const APInt *C1; if (!match(CmpRHS, m_APInt(C1))) return {SPF_UNKNOWN, SPNB_NA, false}; @@ -6323,7 +6544,6 @@ static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS, const Value *RHS, const DataLayout &DL, unsigned Depth) { - assert(!LHS->getType()->isVectorTy() && "TODO: extend to handle vectors!"); if (ICmpInst::isTrueWhenEqual(Pred) && LHS == RHS) return true; @@ -6436,14 +6656,12 @@ /// Return true if "icmp APred X, C1" implies "icmp BPred X, C2" is true. /// Return false if "icmp APred X, C1" implies "icmp BPred X, C2" is false. /// Otherwise, return None if we can't infer anything. -static Optional<bool> -isImpliedCondMatchingImmOperands(CmpInst::Predicate APred, - const ConstantInt *C1, - CmpInst::Predicate BPred, - const ConstantInt *C2) { - ConstantRange DomCR = - ConstantRange::makeExactICmpRegion(APred, C1->getValue()); - ConstantRange CR = ConstantRange::makeExactICmpRegion(BPred, C2->getValue()); +static Optional<bool> isImpliedCondMatchingImmOperands(CmpInst::Predicate APred, + const APInt &C1, + CmpInst::Predicate BPred, + const APInt &C2) { + ConstantRange DomCR = ConstantRange::makeExactICmpRegion(APred, C1); + ConstantRange CR = ConstantRange::makeExactICmpRegion(BPred, C2); ConstantRange Intersection = DomCR.intersectWith(CR); ConstantRange Difference = DomCR.difference(CR); if (Intersection.isEmptySet()) @@ -6481,14 +6699,9 @@ // Can we infer anything when the LHS operands match and the RHS operands are // constants (not necessarily matching)? - if (ALHS == BLHS && isa<ConstantInt>(ARHS) && isa<ConstantInt>(BRHS)) { - if (Optional<bool> Implication = isImpliedCondMatchingImmOperands( - APred, cast<ConstantInt>(ARHS), BPred, cast<ConstantInt>(BRHS))) - return Implication; - // No amount of additional analysis will infer the second condition, so - // early exit. - return None; - } + const APInt *AC, *BC; + if (ALHS == BLHS && match(ARHS, m_APInt(AC)) && match(BRHS, m_APInt(BC))) + return isImpliedCondMatchingImmOperands(APred, *AC, BPred, *BC); if (APred == BPred) return isImpliedCondOperands(APred, ALHS, ARHS, BLHS, BRHS, DL, Depth); @@ -6541,14 +6754,8 @@ if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy()) return None; - Type *OpTy = LHS->getType(); - assert(OpTy->isIntOrIntVectorTy(1) && "Expected integer type only!"); - - // FIXME: Extending the code below to handle vectors. - if (OpTy->isVectorTy()) - return None; - - assert(OpTy->isIntegerTy(1) && "implied by above"); + assert(LHS->getType()->isIntOrIntVectorTy(1) && + "Expected integer type only!"); // Both LHS and RHS are icmps. const ICmpInst *LHSCmp = dyn_cast<ICmpInst>(LHS); @@ -6576,11 +6783,38 @@ if (LHS == RHS) return LHSIsTrue; - const ICmpInst *RHSCmp = dyn_cast<ICmpInst>(RHS); - if (RHSCmp) + if (const ICmpInst *RHSCmp = dyn_cast<ICmpInst>(RHS)) return isImpliedCondition(LHS, RHSCmp->getPredicate(), RHSCmp->getOperand(0), RHSCmp->getOperand(1), DL, LHSIsTrue, Depth); + + if (Depth == MaxAnalysisRecursionDepth) + return None; + + // LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2 + // LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2 + const Value *RHS1, *RHS2; + if (match(RHS, m_LogicalOr(m_Value(RHS1), m_Value(RHS2)))) { + if (Optional<bool> Imp = + isImpliedCondition(LHS, RHS1, DL, LHSIsTrue, Depth + 1)) + if (*Imp == true) + return true; + if (Optional<bool> Imp = + isImpliedCondition(LHS, RHS2, DL, LHSIsTrue, Depth + 1)) + if (*Imp == true) + return true; + } + if (match(RHS, m_LogicalAnd(m_Value(RHS1), m_Value(RHS2)))) { + if (Optional<bool> Imp = + isImpliedCondition(LHS, RHS1, DL, LHSIsTrue, Depth + 1)) + if (*Imp == false) + return false; + if (Optional<bool> Imp = + isImpliedCondition(LHS, RHS2, DL, LHSIsTrue, Depth + 1)) + if (*Imp == false) + return false; + } + return None; } @@ -7072,66 +7306,25 @@ Optional<int64_t> llvm::isPointerOffset(const Value *Ptr1, const Value *Ptr2, const DataLayout &DL) { - Ptr1 = Ptr1->stripPointerCasts(); - Ptr2 = Ptr2->stripPointerCasts(); + APInt Offset1(DL.getIndexTypeSizeInBits(Ptr1->getType()), 0); + APInt Offset2(DL.getIndexTypeSizeInBits(Ptr2->getType()), 0); + Ptr1 = Ptr1->stripAndAccumulateConstantOffsets(DL, Offset1, true); + Ptr2 = Ptr2->stripAndAccumulateConstantOffsets(DL, Offset2, true); // Handle the trivial case first. - if (Ptr1 == Ptr2) { - return 0; - } + if (Ptr1 == Ptr2) + return Offset2.getSExtValue() - Offset1.getSExtValue(); const GEPOperator *GEP1 = dyn_cast<GEPOperator>(Ptr1); const GEPOperator *GEP2 = dyn_cast<GEPOperator>(Ptr2); - // If one pointer is a GEP see if the GEP is a constant offset from the base, - // as in "P" and "gep P, 1". - // Also do this iteratively to handle the the following case: - // Ptr_t1 = GEP Ptr1, c1 - // Ptr_t2 = GEP Ptr_t1, c2 - // Ptr2 = GEP Ptr_t2, c3 - // where we will return c1+c2+c3. - // TODO: Handle the case when both Ptr1 and Ptr2 are GEPs of some common base - // -- replace getOffsetFromBase with getOffsetAndBase, check that the bases - // are the same, and return the difference between offsets. - auto getOffsetFromBase = [&DL](const GEPOperator *GEP, - const Value *Ptr) -> Optional<int64_t> { - const GEPOperator *GEP_T = GEP; - int64_t OffsetVal = 0; - bool HasSameBase = false; - while (GEP_T) { - auto Offset = getOffsetFromIndex(GEP_T, 1, DL); - if (!Offset) - return None; - OffsetVal += *Offset; - auto Op0 = GEP_T->getOperand(0)->stripPointerCasts(); - if (Op0 == Ptr) { - HasSameBase = true; - break; - } - GEP_T = dyn_cast<GEPOperator>(Op0); - } - if (!HasSameBase) - return None; - return OffsetVal; - }; - - if (GEP1) { - auto Offset = getOffsetFromBase(GEP1, Ptr2); - if (Offset) - return -*Offset; - } - if (GEP2) { - auto Offset = getOffsetFromBase(GEP2, Ptr1); - if (Offset) - return Offset; - } - // Right now we handle the case when Ptr1/Ptr2 are both GEPs with an identical // base. After that base, they may have some number of common (and // potentially variable) indices. After that they handle some constant // offset, which determines their offset from each other. At this point, we // handle no other case. - if (!GEP1 || !GEP2 || GEP1->getOperand(0) != GEP2->getOperand(0)) + if (!GEP1 || !GEP2 || GEP1->getOperand(0) != GEP2->getOperand(0) || + GEP1->getSourceElementType() != GEP2->getSourceElementType()) return None; // Skip any common indices and track the GEP types. @@ -7140,9 +7333,10 @@ if (GEP1->getOperand(Idx) != GEP2->getOperand(Idx)) break; - auto Offset1 = getOffsetFromIndex(GEP1, Idx, DL); - auto Offset2 = getOffsetFromIndex(GEP2, Idx, DL); - if (!Offset1 || !Offset2) + auto IOffset1 = getOffsetFromIndex(GEP1, Idx, DL); + auto IOffset2 = getOffsetFromIndex(GEP2, Idx, DL); + if (!IOffset1 || !IOffset2) return None; - return *Offset2 - *Offset1; + return *IOffset2 - *IOffset1 + Offset2.getSExtValue() - + Offset1.getSExtValue(); }
diff --git a/src/llvm-project/llvm/lib/Analysis/VectorUtils.cpp b/src/llvm-project/llvm/lib/Analysis/VectorUtils.cpp index 655c248..c4795a8 100644 --- a/src/llvm-project/llvm/lib/Analysis/VectorUtils.cpp +++ b/src/llvm-project/llvm/lib/Analysis/VectorUtils.cpp
@@ -40,7 +40,7 @@ /// Return true if all of the intrinsic's arguments and return type are scalars /// for the scalar form of the intrinsic, and vectors for the vector form of the /// intrinsic (except operands that are marked as always being scalar by -/// hasVectorInstrinsicScalarOpd). +/// isVectorIntrinsicWithScalarOpAtArg). bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { switch (ID) { case Intrinsic::abs: // Begin integer bit-manipulation. @@ -89,6 +89,8 @@ case Intrinsic::fmuladd: case Intrinsic::powi: case Intrinsic::canonicalize: + case Intrinsic::fptosi_sat: + case Intrinsic::fptoui_sat: return true; default: return false; @@ -96,8 +98,8 @@ } /// Identifies if the vector form of the intrinsic has a scalar operand. -bool llvm::hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, - unsigned ScalarOpdIdx) { +bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, + unsigned ScalarOpdIdx) { switch (ID) { case Intrinsic::abs: case Intrinsic::ctlz: @@ -114,11 +116,14 @@ } } -bool llvm::hasVectorInstrinsicOverloadedScalarOpd(Intrinsic::ID ID, - unsigned ScalarOpdIdx) { +bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, + unsigned OpdIdx) { switch (ID) { + case Intrinsic::fptosi_sat: + case Intrinsic::fptoui_sat: + return OpdIdx == 0; case Intrinsic::powi: - return (ScalarOpdIdx == 1); + return OpdIdx == 1; default: return false; } @@ -496,6 +501,116 @@ return true; } +void llvm::processShuffleMasks( + ArrayRef<int> Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, + unsigned NumOfUsedRegs, function_ref<void()> NoInputAction, + function_ref<void(ArrayRef<int>, unsigned, unsigned)> SingleInputAction, + function_ref<void(ArrayRef<int>, unsigned, unsigned)> ManyInputsAction) { + SmallVector<SmallVector<SmallVector<int>>> Res(NumOfDestRegs); + // Try to perform better estimation of the permutation. + // 1. Split the source/destination vectors into real registers. + // 2. Do the mask analysis to identify which real registers are + // permuted. + int Sz = Mask.size(); + unsigned SzDest = Sz / NumOfDestRegs; + unsigned SzSrc = Sz / NumOfSrcRegs; + for (unsigned I = 0; I < NumOfDestRegs; ++I) { + auto &RegMasks = Res[I]; + RegMasks.assign(NumOfSrcRegs, {}); + // Check that the values in dest registers are in the one src + // register. + for (unsigned K = 0; K < SzDest; ++K) { + int Idx = I * SzDest + K; + if (Idx == Sz) + break; + if (Mask[Idx] >= Sz || Mask[Idx] == UndefMaskElem) + continue; + int SrcRegIdx = Mask[Idx] / SzSrc; + // Add a cost of PermuteTwoSrc for each new source register permute, + // if we have more than one source registers. + if (RegMasks[SrcRegIdx].empty()) + RegMasks[SrcRegIdx].assign(SzDest, UndefMaskElem); + RegMasks[SrcRegIdx][K] = Mask[Idx] % SzSrc; + } + } + // Process split mask. + for (unsigned I = 0; I < NumOfUsedRegs; ++I) { + auto &Dest = Res[I]; + int NumSrcRegs = + count_if(Dest, [](ArrayRef<int> Mask) { return !Mask.empty(); }); + switch (NumSrcRegs) { + case 0: + // No input vectors were used! + NoInputAction(); + break; + case 1: { + // Find the only mask with at least single undef mask elem. + auto *It = + find_if(Dest, [](ArrayRef<int> Mask) { return !Mask.empty(); }); + unsigned SrcReg = std::distance(Dest.begin(), It); + SingleInputAction(*It, SrcReg, I); + break; + } + default: { + // The first mask is a permutation of a single register. Since we have >2 + // input registers to shuffle, we merge the masks for 2 first registers + // and generate a shuffle of 2 registers rather than the reordering of the + // first register and then shuffle with the second register. Next, + // generate the shuffles of the resulting register + the remaining + // registers from the list. + auto &&CombineMasks = [](MutableArrayRef<int> FirstMask, + ArrayRef<int> SecondMask) { + for (int Idx = 0, VF = FirstMask.size(); Idx < VF; ++Idx) { + if (SecondMask[Idx] != UndefMaskElem) { + assert(FirstMask[Idx] == UndefMaskElem && + "Expected undefined mask element."); + FirstMask[Idx] = SecondMask[Idx] + VF; + } + } + }; + auto &&NormalizeMask = [](MutableArrayRef<int> Mask) { + for (int Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) { + if (Mask[Idx] != UndefMaskElem) + Mask[Idx] = Idx; + } + }; + int SecondIdx; + do { + int FirstIdx = -1; + SecondIdx = -1; + MutableArrayRef<int> FirstMask, SecondMask; + for (unsigned I = 0; I < NumOfDestRegs; ++I) { + SmallVectorImpl<int> &RegMask = Dest[I]; + if (RegMask.empty()) + continue; + + if (FirstIdx == SecondIdx) { + FirstIdx = I; + FirstMask = RegMask; + continue; + } + SecondIdx = I; + SecondMask = RegMask; + CombineMasks(FirstMask, SecondMask); + ManyInputsAction(FirstMask, FirstIdx, SecondIdx); + NormalizeMask(FirstMask); + RegMask.clear(); + SecondMask = FirstMask; + SecondIdx = FirstIdx; + } + if (FirstIdx != SecondIdx && SecondIdx >= 0) { + CombineMasks(SecondMask, FirstMask); + ManyInputsAction(SecondMask, SecondIdx, FirstIdx); + Dest[FirstIdx].clear(); + NormalizeMask(SecondMask); + } + } while (SecondIdx >= 0); + break; + } + } + } +} + MapVector<Instruction *, uint64_t> llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB, const TargetTransformInfo *TTI) { @@ -543,9 +658,8 @@ Value *Val = Worklist.pop_back_val(); Value *Leader = ECs.getOrInsertLeaderValue(Val); - if (Visited.count(Val)) + if (!Visited.insert(Val).second) continue; - Visited.insert(Val); // Non-instructions terminate a chain successfully. if (!isa<Instruction>(Val)) @@ -648,7 +762,7 @@ return; } - for (auto &AccGroupListOp : AccGroups->operands()) { + for (const auto &AccGroupListOp : AccGroups->operands()) { auto *Item = cast<MDNode>(AccGroupListOp.get()); assert(isValidAsAccessGroup(Item) && "List item must be an access group"); List.insert(Item); @@ -1383,12 +1497,12 @@ SmallVector<StringRef, 8> ListAttr; S.split(ListAttr, ","); - for (auto &S : SetVector<StringRef>(ListAttr.begin(), ListAttr.end())) { + for (const auto &S : SetVector<StringRef>(ListAttr.begin(), ListAttr.end())) { #ifndef NDEBUG LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << S << "'\n"); Optional<VFInfo> Info = VFABI::tryDemangleForVFABI(S, *(CI.getModule())); - assert(Info.hasValue() && "Invalid name for a VFABI variant."); - assert(CI.getModule()->getFunction(Info.getValue().VectorName) && + assert(Info && "Invalid name for a VFABI variant."); + assert(CI.getModule()->getFunction(Info.value().VectorName) && "Vector function is missing."); #endif VariantMappings.push_back(std::string(S));
diff --git a/src/llvm-project/llvm/lib/Analysis/models/gen-regalloc-eviction-test-model.py b/src/llvm-project/llvm/lib/Analysis/models/gen-regalloc-eviction-test-model.py index 1cb2492..476163d 100644 --- a/src/llvm-project/llvm/lib/Analysis/models/gen-regalloc-eviction-test-model.py +++ b/src/llvm-project/llvm/lib/Analysis/models/gen-regalloc-eviction-test-model.py
@@ -22,19 +22,7 @@ } ] """ -PER_REGISTER_INT64_FEATURE_LIST = [ - 'mask', 'is_hint', 'is_local', 'is_free', 'max_stage', 'min_stage' -] -PER_REGISTER_FLOAT32_FEATURE_LIST = ['nr_urgent', - 'weighed_reads_by_max', 'weighed_writes_by_max', - 'weighed_read_writes_by_max', 'weighed_indvars_by_max', - 'hint_weights_by_max', 'start_bb_freq_by_max', 'end_bb_freq_by_max', - 'hottest_bb_freq_by_max', 'liverange_size', 'use_def_density', - 'nr_defs_and_uses', 'nr_broken_hints', 'nr_rematerializable' -] -PER_REGISTER_FEATURE_LIST = PER_REGISTER_FLOAT32_FEATURE_LIST + \ - PER_REGISTER_INT64_FEATURE_LIST -CONTEXT_FEATURE_LIST = ('progress', 'discount', 'reward', 'step_type') +PER_REGISTER_FEATURE_LIST = ['mask'] NUM_REGISTERS = 33 @@ -42,19 +30,7 @@ """Returns (time_step_spec, action_spec) for LLVM register allocation.""" inputs = dict( (key, tf.TensorSpec(dtype=tf.int64, shape=(NUM_REGISTERS), name=key)) - for key in PER_REGISTER_INT64_FEATURE_LIST) - inputs.update( - dict((key, - tf.TensorSpec(dtype=tf.float32, shape=(NUM_REGISTERS), name=key)) - for key in PER_REGISTER_FLOAT32_FEATURE_LIST)) - inputs['progress'] = tf.TensorSpec( - dtype=tf.float32, shape=(), name='progress') - inputs.update( - dict((key, tf.TensorSpec(dtype=tf.float32, shape=(), name=key)) - for key in ['discount', 'reward'])) - inputs.update( - dict((key, tf.TensorSpec(dtype=tf.int32, shape=(), name=key)) - for key in ['step_type'])) + for key in PER_REGISTER_FEATURE_LIST) return inputs @@ -70,17 +46,7 @@ module.var = tf.Variable(0, dtype=tf.int64) def action(*inputs): - s1 = tf.reduce_sum([ - tf.cast(inputs[0][key], tf.float32) for key in PER_REGISTER_FEATURE_LIST - ], - axis=0) - s2 = tf.reduce_sum( - [tf.cast(inputs[0][key], tf.float32) for key in CONTEXT_FEATURE_LIST]) - # Add a large number so s won't be 0. - s = s1 + s2 + 123456789.123456789 - # Equals to mask feature. - mask_alias = tf.not_equal(s * tf.cast(inputs[0]['mask'], tf.float32), 0) - result = tf.math.argmax(mask_alias, axis=-1) + module.var + result = tf.math.argmax(inputs[0]['mask'], axis=-1) + module.var return {POLICY_DECISION_LABEL: result} module.action = tf.function()(action) action = {