llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp

//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/// \file
///
/// This file implements the OpenMPIRBuilder class, which is used as a
/// convenient way to create LLVM instructions for OpenMP directives.
///
//===----------------------------------------------------------------------===//

#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/Frontend/Offloading/Utility.h"
#include "llvm/Frontend/OpenMP/OMPGridValues.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/PassInstrumentation.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ReplaceConstant.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/CodeExtractor.h"
#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/Transforms/Utils/UnrollLoop.h"

#include <cstdint>
#include <optional>
#include <stack>

#define DEBUG_TYPE "openmp-ir-builder"

using namespace llvm;
using namespace omp;

static cl::opt<bool>
    OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
                         cl::desc("Use optimistic attributes describing "
                                  "'as-if' properties of runtime calls."),
                         cl::init(false));

static cl::opt<double> UnrollThresholdFactor(
    "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
    cl::desc("Factor for the unroll threshold to account for code "
             "simplifications still taking place"),
    cl::init(1.5));

#ifndef NDEBUG
/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
/// an InsertPoint stores the instruction before something is inserted. For
/// instance, if both point to the same instruction, two IRBuilders alternating
/// creating instruction will cause the instructions to be interleaved.
static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
                         IRBuilder<>::InsertPoint IP2) {
  if (!IP1.isSet() || !IP2.isSet())
    return false;
  return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
}

static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
  // Valid ordered/unordered and base algorithm combinations.
  switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
  case OMPScheduleType::UnorderedStaticChunked:
  case OMPScheduleType::UnorderedStatic:
  case OMPScheduleType::UnorderedDynamicChunked:
  case OMPScheduleType::UnorderedGuidedChunked:
  case OMPScheduleType::UnorderedRuntime:
  case OMPScheduleType::UnorderedAuto:
  case OMPScheduleType::UnorderedTrapezoidal:
  case OMPScheduleType::UnorderedGreedy:
  case OMPScheduleType::UnorderedBalanced:
  case OMPScheduleType::UnorderedGuidedIterativeChunked:
  case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
  case OMPScheduleType::UnorderedSteal:
  case OMPScheduleType::UnorderedStaticBalancedChunked:
  case OMPScheduleType::UnorderedGuidedSimd:
  case OMPScheduleType::UnorderedRuntimeSimd:
  case OMPScheduleType::OrderedStaticChunked:
  case OMPScheduleType::OrderedStatic:
  case OMPScheduleType::OrderedDynamicChunked:
  case OMPScheduleType::OrderedGuidedChunked:
  case OMPScheduleType::OrderedRuntime:
  case OMPScheduleType::OrderedAuto:
  case OMPScheduleType::OrderdTrapezoidal:
  case OMPScheduleType::NomergeUnorderedStaticChunked:
  case OMPScheduleType::NomergeUnorderedStatic:
  case OMPScheduleType::NomergeUnorderedDynamicChunked:
  case OMPScheduleType::NomergeUnorderedGuidedChunked:
  case OMPScheduleType::NomergeUnorderedRuntime:
  case OMPScheduleType::NomergeUnorderedAuto:
  case OMPScheduleType::NomergeUnorderedTrapezoidal:
  case OMPScheduleType::NomergeUnorderedGreedy:
  case OMPScheduleType::NomergeUnorderedBalanced:
  case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
  case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
  case OMPScheduleType::NomergeUnorderedSteal:
  case OMPScheduleType::NomergeOrderedStaticChunked:
  case OMPScheduleType::NomergeOrderedStatic:
  case OMPScheduleType::NomergeOrderedDynamicChunked:
  case OMPScheduleType::NomergeOrderedGuidedChunked:
  case OMPScheduleType::NomergeOrderedRuntime:
  case OMPScheduleType::NomergeOrderedAuto:
  case OMPScheduleType::NomergeOrderedTrapezoidal:
    break;
  default:
    return false;
  }

  // Must not set both monotonicity modifiers at the same time.
  OMPScheduleType MonotonicityFlags =
      SchedType & OMPScheduleType::MonotonicityMask;
  if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
    return false;

  return true;
}
#endif

static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
  if (T.isAMDGPU()) {
    StringRef Features =
        Kernel->getFnAttribute("target-features").getValueAsString();
    if (Features.count("+wavefrontsize64"))
      return omp::getAMDGPUGridValues<64>();
    return omp::getAMDGPUGridValues<32>();
  }
  if (T.isNVPTX())
    return omp::NVPTXGridValues;
  llvm_unreachable("No grid value available for this architecture!");
}

/// Determine which scheduling algorithm to use, determined from schedule clause
/// arguments.
static OMPScheduleType
getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
                          bool HasSimdModifier) {
  // Currently, the default schedule it static.
  switch (ClauseKind) {
  case OMP_SCHEDULE_Default:
  case OMP_SCHEDULE_Static:
    return HasChunks ? OMPScheduleType::BaseStaticChunked
                     : OMPScheduleType::BaseStatic;
  case OMP_SCHEDULE_Dynamic:
    return OMPScheduleType::BaseDynamicChunked;
  case OMP_SCHEDULE_Guided:
    return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
                           : OMPScheduleType::BaseGuidedChunked;
  case OMP_SCHEDULE_Auto:
    return llvm::omp::OMPScheduleType::BaseAuto;
  case OMP_SCHEDULE_Runtime:
    return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
                           : OMPScheduleType::BaseRuntime;
  }
  llvm_unreachable("unhandled schedule clause argument");
}

/// Adds ordering modifier flags to schedule type.
static OMPScheduleType
getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
                              bool HasOrderedClause) {
  assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
             OMPScheduleType::None &&
         "Must not have ordering nor monotonicity flags already set");

  OMPScheduleType OrderingModifier = HasOrderedClause
                                         ? OMPScheduleType::ModifierOrdered
                                         : OMPScheduleType::ModifierUnordered;
  OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;

  // Unsupported combinations
  if (OrderingScheduleType ==
      (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
    return OMPScheduleType::OrderedGuidedChunked;
  else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
                                    OMPScheduleType::ModifierOrdered))
    return OMPScheduleType::OrderedRuntime;

  return OrderingScheduleType;
}

/// Adds monotonicity modifier flags to schedule type.
static OMPScheduleType
getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
                                  bool HasSimdModifier, bool HasMonotonic,
                                  bool HasNonmonotonic, bool HasOrderedClause) {
  assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
             OMPScheduleType::None &&
         "Must not have monotonicity flags already set");
  assert((!HasMonotonic || !HasNonmonotonic) &&
         "Monotonic and Nonmonotonic are contradicting each other");

  if (HasMonotonic) {
    return ScheduleType | OMPScheduleType::ModifierMonotonic;
  } else if (HasNonmonotonic) {
    return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
  } else {
    // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
    // If the static schedule kind is specified or if the ordered clause is
    // specified, and if the nonmonotonic modifier is not specified, the
    // effect is as if the monotonic modifier is specified. Otherwise, unless
    // the monotonic modifier is specified, the effect is as if the
    // nonmonotonic modifier is specified.
    OMPScheduleType BaseScheduleType =
        ScheduleType & ~OMPScheduleType::ModifierMask;
    if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
        (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
        HasOrderedClause) {
      // The monotonic is used by default in openmp runtime library, so no need
      // to set it.
      return ScheduleType;
    } else {
      return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
    }
  }
}

/// Determine the schedule type using schedule and ordering clause arguments.
static OMPScheduleType
computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
                          bool HasSimdModifier, bool HasMonotonicModifier,
                          bool HasNonmonotonicModifier, bool HasOrderedClause) {
  OMPScheduleType BaseSchedule =
      getOpenMPBaseScheduleType(ClauseKind, HasChunks, HasSimdModifier);
  OMPScheduleType OrderedSchedule =
      getOpenMPOrderingScheduleType(BaseSchedule, HasOrderedClause);
  OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
      OrderedSchedule, HasSimdModifier, HasMonotonicModifier,
      HasNonmonotonicModifier, HasOrderedClause);

  assert(isValidWorkshareLoopScheduleType(Result));
  return Result;
}

/// Make \p Source branch to \p Target.
///
/// Handles two situations:
/// * \p Source already has an unconditional branch.
/// * \p Source is a degenerate block (no terminator because the BB is
///             the current head of the IR construction).
static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
  if (Instruction *Term = Source->getTerminator()) {
    auto *Br = cast<BranchInst>(Term);
    assert(!Br->isConditional() &&
           "BB's terminator must be an unconditional branch (or degenerate)");
    BasicBlock *Succ = Br->getSuccessor(0);
    Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true);
    Br->setSuccessor(0, Target);
    return;
  }

  auto *NewBr = BranchInst::Create(Target, Source);
  NewBr->setDebugLoc(DL);
}

void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
                    bool CreateBranch) {
  assert(New->getFirstInsertionPt() == New->begin() &&
         "Target BB must not have PHI nodes");

  // Move instructions to new block.
  BasicBlock *Old = IP.getBlock();
  New->splice(New->begin(), Old, IP.getPoint(), Old->end());

  if (CreateBranch)
    BranchInst::Create(New, Old);
}

void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
  DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
  BasicBlock *Old = Builder.GetInsertBlock();

  spliceBB(Builder.saveIP(), New, CreateBranch);
  if (CreateBranch)
    Builder.SetInsertPoint(Old->getTerminator());
  else
    Builder.SetInsertPoint(Old);

  // SetInsertPoint also updates the Builder's debug location, but we want to
  // keep the one the Builder was configured to use.
  Builder.SetCurrentDebugLocation(DebugLoc);
}

BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
                          llvm::Twine Name) {
  BasicBlock *Old = IP.getBlock();
  BasicBlock *New = BasicBlock::Create(
      Old->getContext(), Name.isTriviallyEmpty() ? Old->getName() : Name,
      Old->getParent(), Old->getNextNode());
  spliceBB(IP, New, CreateBranch);
  New->replaceSuccessorsPhiUsesWith(Old, New);
  return New;
}

BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
                          llvm::Twine Name) {
  DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
  BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
  if (CreateBranch)
    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
  else
    Builder.SetInsertPoint(Builder.GetInsertBlock());
  // SetInsertPoint also updates the Builder's debug location, but we want to
  // keep the one the Builder was configured to use.
  Builder.SetCurrentDebugLocation(DebugLoc);
  return New;
}

BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
                          llvm::Twine Name) {
  DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
  BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name);
  if (CreateBranch)
    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
  else
    Builder.SetInsertPoint(Builder.GetInsertBlock());
  // SetInsertPoint also updates the Builder's debug location, but we want to
  // keep the one the Builder was configured to use.
  Builder.SetCurrentDebugLocation(DebugLoc);
  return New;
}

BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
                                    llvm::Twine Suffix) {
  BasicBlock *Old = Builder.GetInsertBlock();
  return splitBB(Builder, CreateBranch, Old->getName() + Suffix);
}

// This function creates a fake integer value and a fake use for the integer
// value. It returns the fake value created. This is useful in modeling the
// extra arguments to the outlined functions.
Value *createFakeIntVal(IRBuilderBase &Builder,
                        OpenMPIRBuilder::InsertPointTy OuterAllocaIP,
                        llvm::SmallVectorImpl<Instruction *> &ToBeDeleted,
                        OpenMPIRBuilder::InsertPointTy InnerAllocaIP,
                        const Twine &Name = "", bool AsPtr = true) {
  Builder.restoreIP(OuterAllocaIP);
  Instruction *FakeVal;
  AllocaInst *FakeValAddr =
      Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, Name + ".addr");
  ToBeDeleted.push_back(FakeValAddr);

  if (AsPtr) {
    FakeVal = FakeValAddr;
  } else {
    FakeVal =
        Builder.CreateLoad(Builder.getInt32Ty(), FakeValAddr, Name + ".val");
    ToBeDeleted.push_back(FakeVal);
  }

  // Generate a fake use of this value
  Builder.restoreIP(InnerAllocaIP);
  Instruction *UseFakeVal;
  if (AsPtr) {
    UseFakeVal =
        Builder.CreateLoad(Builder.getInt32Ty(), FakeVal, Name + ".use");
  } else {
    UseFakeVal =
        cast<BinaryOperator>(Builder.CreateAdd(FakeVal, Builder.getInt32(10)));
  }
  ToBeDeleted.push_back(UseFakeVal);
  return FakeVal;
}

//===----------------------------------------------------------------------===//
// OpenMPIRBuilderConfig
//===----------------------------------------------------------------------===//

namespace {
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
/// Values for bit flags for marking which requires clauses have been used.
enum OpenMPOffloadingRequiresDirFlags {
  /// flag undefined.
  OMP_REQ_UNDEFINED = 0x000,
  /// no requires directive present.
  OMP_REQ_NONE = 0x001,
  /// reverse_offload clause.
  OMP_REQ_REVERSE_OFFLOAD = 0x002,
  /// unified_address clause.
  OMP_REQ_UNIFIED_ADDRESS = 0x004,
  /// unified_shared_memory clause.
  OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
  /// dynamic_allocators clause.
  OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
};

} // anonymous namespace

OpenMPIRBuilderConfig::OpenMPIRBuilderConfig()
    : RequiresFlags(OMP_REQ_UNDEFINED) {}

OpenMPIRBuilderConfig::OpenMPIRBuilderConfig(
    bool IsTargetDevice, bool IsGPU, bool OpenMPOffloadMandatory,
    bool HasRequiresReverseOffload, bool HasRequiresUnifiedAddress,
    bool HasRequiresUnifiedSharedMemory, bool HasRequiresDynamicAllocators)
    : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
      OpenMPOffloadMandatory(OpenMPOffloadMandatory),
      RequiresFlags(OMP_REQ_UNDEFINED) {
  if (HasRequiresReverseOffload)
    RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
  if (HasRequiresUnifiedAddress)
    RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
  if (HasRequiresUnifiedSharedMemory)
    RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
  if (HasRequiresDynamicAllocators)
    RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
}

bool OpenMPIRBuilderConfig::hasRequiresReverseOffload() const {
  return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
}

bool OpenMPIRBuilderConfig::hasRequiresUnifiedAddress() const {
  return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
}

bool OpenMPIRBuilderConfig::hasRequiresUnifiedSharedMemory() const {
  return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
}

bool OpenMPIRBuilderConfig::hasRequiresDynamicAllocators() const {
  return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
}

int64_t OpenMPIRBuilderConfig::getRequiresFlags() const {
  return hasRequiresFlags() ? RequiresFlags
                            : static_cast<int64_t>(OMP_REQ_NONE);
}

void OpenMPIRBuilderConfig::setHasRequiresReverseOffload(bool Value) {
  if (Value)
    RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
  else
    RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
}

void OpenMPIRBuilderConfig::setHasRequiresUnifiedAddress(bool Value) {
  if (Value)
    RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
  else
    RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
}

void OpenMPIRBuilderConfig::setHasRequiresUnifiedSharedMemory(bool Value) {
  if (Value)
    RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
  else
    RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
}

void OpenMPIRBuilderConfig::setHasRequiresDynamicAllocators(bool Value) {
  if (Value)
    RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
  else
    RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
}

//===----------------------------------------------------------------------===//
// OpenMPIRBuilder
//===----------------------------------------------------------------------===//

void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
                                          IRBuilderBase &Builder,
                                          SmallVector<Value *> &ArgsVector) {
  Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);
  Value *PointerNum = Builder.getInt32(KernelArgs.NumTargetItems);
  auto Int32Ty = Type::getInt32Ty(Builder.getContext());
  constexpr const size_t MaxDim = 3;
  Value *ZeroArray = Constant::getNullValue(ArrayType::get(Int32Ty, MaxDim));
  Value *Flags = Builder.getInt64(KernelArgs.HasNoWait);

  assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());

  Value *NumTeams3D =
      Builder.CreateInsertValue(ZeroArray, KernelArgs.NumTeams[0], {0});
  Value *NumThreads3D =
      Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads[0], {0});
  for (unsigned I :
       seq<unsigned>(1, std::min(KernelArgs.NumTeams.size(), MaxDim)))
    NumTeams3D =
        Builder.CreateInsertValue(NumTeams3D, KernelArgs.NumTeams[I], {I});
  for (unsigned I :
       seq<unsigned>(1, std::min(KernelArgs.NumThreads.size(), MaxDim)))
    NumThreads3D =
        Builder.CreateInsertValue(NumThreads3D, KernelArgs.NumThreads[I], {I});

  ArgsVector = {Version,
                PointerNum,
                KernelArgs.RTArgs.BasePointersArray,
                KernelArgs.RTArgs.PointersArray,
                KernelArgs.RTArgs.SizesArray,
                KernelArgs.RTArgs.MapTypesArray,
                KernelArgs.RTArgs.MapNamesArray,
                KernelArgs.RTArgs.MappersArray,
                KernelArgs.NumIterations,
                Flags,
                NumTeams3D,
                NumThreads3D,
                KernelArgs.DynCGGroupMem};
}

void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
  LLVMContext &Ctx = Fn.getContext();

  // Get the function's current attributes.
  auto Attrs = Fn.getAttributes();
  auto FnAttrs = Attrs.getFnAttrs();
  auto RetAttrs = Attrs.getRetAttrs();
  SmallVector<AttributeSet, 4> ArgAttrs;
  for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
    ArgAttrs.emplace_back(Attrs.getParamAttrs(ArgNo));

  // Add AS to FnAS while taking special care with integer extensions.
  auto addAttrSet = [&](AttributeSet &FnAS, const AttributeSet &AS,
                        bool Param = true) -> void {
    bool HasSignExt = AS.hasAttribute(Attribute::SExt);
    bool HasZeroExt = AS.hasAttribute(Attribute::ZExt);
    if (HasSignExt || HasZeroExt) {
      assert(AS.getNumAttributes() == 1 &&
             "Currently not handling extension attr combined with others.");
      if (Param) {
        if (auto AK = TargetLibraryInfo::getExtAttrForI32Param(T, HasSignExt))
          FnAS = FnAS.addAttribute(Ctx, AK);
      } else if (auto AK =
                     TargetLibraryInfo::getExtAttrForI32Return(T, HasSignExt))
        FnAS = FnAS.addAttribute(Ctx, AK);
    } else {
      FnAS = FnAS.addAttributes(Ctx, AS);
    }
  };

#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
#include "llvm/Frontend/OpenMP/OMPKinds.def"

  // Add attributes to the function declaration.
  switch (FnID) {
#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets)                \
  case Enum:                                                                   \
    FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet);                           \
    addAttrSet(RetAttrs, RetAttrSet, /*Param*/ false);                         \
    for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo)                \
      addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]);                         \
    Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs));    \
    break;
#include "llvm/Frontend/OpenMP/OMPKinds.def"
  default:
    // Attributes are optional.
    break;
  }
}

FunctionCallee
OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
  FunctionType *FnTy = nullptr;
  Function *Fn = nullptr;

  // Try to find the declation in the module first.
  switch (FnID) {
#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...)                          \
  case Enum:                                                                   \
    FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__},        \
                             IsVarArg);                                        \
    Fn = M.getFunction(Str);                                                   \
    break;
#include "llvm/Frontend/OpenMP/OMPKinds.def"
  }

  if (!Fn) {
    // Create a new declaration if we need one.
    switch (FnID) {
#define OMP_RTL(Enum, Str, ...)                                                \
  case Enum:                                                                   \
    Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M);         \
    break;
#include "llvm/Frontend/OpenMP/OMPKinds.def"
    }

    // Add information if the runtime function takes a callback function
    if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
      if (!Fn->hasMetadata(LLVMContext::MD_callback)) {
        LLVMContext &Ctx = Fn->getContext();
        MDBuilder MDB(Ctx);
        // Annotate the callback behavior of the runtime function:
        //  - The callback callee is argument number 2 (microtask).
        //  - The first two arguments of the callback callee are unknown (-1).
        //  - All variadic arguments to the runtime function are passed to the
        //    callback callee.
        Fn->addMetadata(
            LLVMContext::MD_callback,
            *MDNode::get(Ctx, {MDB.createCallbackEncoding(
                                  2, {-1, -1}, /* VarArgsArePassed */ true)}));
      }
    }

    LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()
                      << " with type " << *Fn->getFunctionType() << "\n");
    addAttributes(FnID, *Fn);

  } else {
    LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()
                      << " with type " << *Fn->getFunctionType() << "\n");
  }

  assert(Fn && "Failed to create OpenMP runtime function");

  return {FnTy, Fn};
}

Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
  FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
  auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee());
  assert(Fn && "Failed to create OpenMP runtime function pointer");
  return Fn;
}

void OpenMPIRBuilder::initialize() { initializeTypes(M); }

static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,
                                                     Function *Function) {
  BasicBlock &EntryBlock = Function->getEntryBlock();
  Instruction *MoveLocInst = EntryBlock.getFirstNonPHI();

  // Loop over blocks looking for constant allocas, skipping the entry block
  // as any allocas there are already in the desired location.
  for (auto Block = std::next(Function->begin(), 1); Block != Function->end();
       Block++) {
    for (auto Inst = Block->getReverseIterator()->begin();
         Inst != Block->getReverseIterator()->end();) {
      if (auto *AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Inst)) {
        Inst++;
        if (!isa<ConstantData>(AllocaInst->getArraySize()))
          continue;
        AllocaInst->moveBeforePreserving(MoveLocInst);
      } else {
        Inst++;
      }
    }
  }
}

void OpenMPIRBuilder::finalize(Function *Fn) {
  SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
  SmallVector<BasicBlock *, 32> Blocks;
  SmallVector<OutlineInfo, 16> DeferredOutlines;
  for (OutlineInfo &OI : OutlineInfos) {
    // Skip functions that have not finalized yet; may happen with nested
    // function generation.
    if (Fn && OI.getFunction() != Fn) {
      DeferredOutlines.push_back(OI);
      continue;
    }

    ParallelRegionBlockSet.clear();
    Blocks.clear();
    OI.collectBlocks(ParallelRegionBlockSet, Blocks);

    Function *OuterFn = OI.getFunction();
    CodeExtractorAnalysisCache CEAC(*OuterFn);
    // If we generate code for the target device, we need to allocate
    // struct for aggregate params in the device default alloca address space.
    // OpenMP runtime requires that the params of the extracted functions are
    // passed as zero address space pointers. This flag ensures that
    // CodeExtractor generates correct code for extracted functions
    // which are used by OpenMP runtime.
    bool ArgsInZeroAddressSpace = Config.isTargetDevice();
    CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
                            /* AggregateArgs */ true,
                            /* BlockFrequencyInfo */ nullptr,
                            /* BranchProbabilityInfo */ nullptr,
                            /* AssumptionCache */ nullptr,
                            /* AllowVarArgs */ true,
                            /* AllowAlloca */ true,
                            /* AllocaBlock*/ OI.OuterAllocaBB,
                            /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);

    LLVM_DEBUG(dbgs() << "Before     outlining: " << *OuterFn << "\n");
    LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()
                      << " Exit: " << OI.ExitBB->getName() << "\n");
    assert(Extractor.isEligible() &&
           "Expected OpenMP outlining to be possible!");

    for (auto *V : OI.ExcludeArgsFromAggregate)
      Extractor.excludeArgFromAggregate(V);

    Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);

    // Forward target-cpu, target-features attributes to the outlined function.
    auto TargetCpuAttr = OuterFn->getFnAttribute("target-cpu");
    if (TargetCpuAttr.isStringAttribute())
      OutlinedFn->addFnAttr(TargetCpuAttr);

    auto TargetFeaturesAttr = OuterFn->getFnAttribute("target-features");
    if (TargetFeaturesAttr.isStringAttribute())
      OutlinedFn->addFnAttr(TargetFeaturesAttr);

    LLVM_DEBUG(dbgs() << "After      outlining: " << *OuterFn << "\n");
    LLVM_DEBUG(dbgs() << "   Outlined function: " << *OutlinedFn << "\n");
    assert(OutlinedFn->getReturnType()->isVoidTy() &&
           "OpenMP outlined functions should not return a value!");

    // For compability with the clang CG we move the outlined function after the
    // one with the parallel region.
    OutlinedFn->removeFromParent();
    M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn);

    // Remove the artificial entry introduced by the extractor right away, we
    // made our own entry block after all.
    {
      BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
      assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB);
      assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry);
      // Move instructions from the to-be-deleted ArtificialEntry to the entry
      // basic block of the parallel region. CodeExtractor generates
      // instructions to unwrap the aggregate argument and may sink
      // allocas/bitcasts for values that are solely used in the outlined region
      // and do not escape.
      assert(!ArtificialEntry.empty() &&
             "Expected instructions to add in the outlined region entry");
      for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
                                        End = ArtificialEntry.rend();
           It != End;) {
        Instruction &I = *It;
        It++;

        if (I.isTerminator())
          continue;

        I.moveBeforePreserving(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
      }

      OI.EntryBB->moveBefore(&ArtificialEntry);
      ArtificialEntry.eraseFromParent();
    }
    assert(&OutlinedFn->getEntryBlock() == OI.EntryBB);
    assert(OutlinedFn && OutlinedFn->getNumUses() == 1);

    // Run a user callback, e.g. to add attributes.
    if (OI.PostOutlineCB)
      OI.PostOutlineCB(*OutlinedFn);
  }

  // Remove work items that have been completed.
  OutlineInfos = std::move(DeferredOutlines);

  // The createTarget functions embeds user written code into
  // the target region which may inject allocas which need to
  // be moved to the entry block of our target or risk malformed
  // optimisations by later passes, this is only relevant for
  // the device pass which appears to be a little more delicate
  // when it comes to optimisations (however, we do not block on
  // that here, it's up to the inserter to the list to do so).
  // This notbaly has to occur after the OutlinedInfo candidates
  // have been extracted so we have an end product that will not
  // be implicitly adversely affected by any raises unless
  // intentionally appended to the list.
  // NOTE: This only does so for ConstantData, it could be extended
  // to ConstantExpr's with further effort, however, they should
  // largely be folded when they get here. Extending it to runtime
  // defined/read+writeable allocation sizes would be non-trivial
  // (need to factor in movement of any stores to variables the
  // allocation size depends on, as well as the usual loads,
  // otherwise it'll yield the wrong result after movement) and
  // likely be more suitable as an LLVM optimisation pass.
  for (Function *F : ConstantAllocaRaiseCandidates)
    raiseUserConstantDataAllocasToEntryBlock(Builder, F);

  EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
      [](EmitMetadataErrorKind Kind,
         const TargetRegionEntryInfo &EntryInfo) -> void {
    errs() << "Error of kind: " << Kind
           << " when emitting offload entries and metadata during "
              "OMPIRBuilder finalization \n";
  };

  if (!OffloadInfoManager.empty())
    createOffloadEntriesAndInfoMetadata(ErrorReportFn);

  if (Config.EmitLLVMUsedMetaInfo.value_or(false)) {
    std::vector<WeakTrackingVH> LLVMCompilerUsed = {
        M.getGlobalVariable("__openmp_nvptx_data_transfer_temporary_storage")};
    emitUsed("llvm.compiler.used", LLVMCompilerUsed);
  }
}

OpenMPIRBuilder::~OpenMPIRBuilder() {
  assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
}

GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
  IntegerType *I32Ty = Type::getInt32Ty(M.getContext());
  auto *GV =
      new GlobalVariable(M, I32Ty,
                         /* isConstant = */ true, GlobalValue::WeakODRLinkage,
                         ConstantInt::get(I32Ty, Value), Name);
  GV->setVisibility(GlobalValue::HiddenVisibility);

  return GV;
}

Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
                                            uint32_t SrcLocStrSize,
                                            IdentFlag LocFlags,
                                            unsigned Reserve2Flags) {
  // Enable "C-mode".
  LocFlags |= OMP_IDENT_FLAG_KMPC;

  Constant *&Ident =
      IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
  if (!Ident) {
    Constant *I32Null = ConstantInt::getNullValue(Int32);
    Constant *IdentData[] = {I32Null,
                             ConstantInt::get(Int32, uint32_t(LocFlags)),
                             ConstantInt::get(Int32, Reserve2Flags),
                             ConstantInt::get(Int32, SrcLocStrSize), SrcLocStr};
    Constant *Initializer =
        ConstantStruct::get(OpenMPIRBuilder::Ident, IdentData);

    // Look for existing encoding of the location + flags, not needed but
    // minimizes the difference to the existing solution while we transition.
    for (GlobalVariable &GV : M.globals())
      if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
        if (GV.getInitializer() == Initializer)
          Ident = &GV;

    if (!Ident) {
      auto *GV = new GlobalVariable(
          M, OpenMPIRBuilder::Ident,
          /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
          nullptr, GlobalValue::NotThreadLocal,
          M.getDataLayout().getDefaultGlobalsAddressSpace());
      GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
      GV->setAlignment(Align(8));
      Ident = GV;
    }
  }

  return ConstantExpr::getPointerBitCastOrAddrSpaceCast(Ident, IdentPtr);
}

Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
                                                uint32_t &SrcLocStrSize) {
  SrcLocStrSize = LocStr.size();
  Constant *&SrcLocStr = SrcLocStrMap[LocStr];
  if (!SrcLocStr) {
    Constant *Initializer =
        ConstantDataArray::getString(M.getContext(), LocStr);

    // Look for existing encoding of the location, not needed but minimizes the
    // difference to the existing solution while we transition.
    for (GlobalVariable &GV : M.globals())
      if (GV.isConstant() && GV.hasInitializer() &&
          GV.getInitializer() == Initializer)
        return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr);

    SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "",
                                              /* AddressSpace */ 0, &M);
  }
  return SrcLocStr;
}

Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
                                                StringRef FileName,
                                                unsigned Line, unsigned Column,
                                                uint32_t &SrcLocStrSize) {
  SmallString<128> Buffer;
  Buffer.push_back(';');
  Buffer.append(FileName);
  Buffer.push_back(';');
  Buffer.append(FunctionName);
  Buffer.push_back(';');
  Buffer.append(std::to_string(Line));
  Buffer.push_back(';');
  Buffer.append(std::to_string(Column));
  Buffer.push_back(';');
  Buffer.push_back(';');
  return getOrCreateSrcLocStr(Buffer.str(), SrcLocStrSize);
}

Constant *
OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
  StringRef UnknownLoc = ";unknown;unknown;0;0;;";
  return getOrCreateSrcLocStr(UnknownLoc, SrcLocStrSize);
}

Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
                                                uint32_t &SrcLocStrSize,
                                                Function *F) {
  DILocation *DIL = DL.get();
  if (!DIL)
    return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
  StringRef FileName = M.getName();
  if (DIFile *DIF = DIL->getFile())
    if (std::optional<StringRef> Source = DIF->getSource())
      FileName = *Source;
  StringRef Function = DIL->getScope()->getSubprogram()->getName();
  if (Function.empty() && F)
    Function = F->getName();
  return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(),
                              DIL->getColumn(), SrcLocStrSize);
}

Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
                                                uint32_t &SrcLocStrSize) {
  return getOrCreateSrcLocStr(Loc.DL, SrcLocStrSize,
                              Loc.IP.getBlock()->getParent());
}

Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
  return Builder.CreateCall(
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,
      "omp_global_thread_num");
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
                               bool ForceSimpleCall, bool CheckCancelFlag) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  // Build call __kmpc_cancel_barrier(loc, thread_id) or
  //            __kmpc_barrier(loc, thread_id);

  IdentFlag BarrierLocFlags;
  switch (Kind) {
  case OMPD_for:
    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
    break;
  case OMPD_sections:
    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
    break;
  case OMPD_single:
    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
    break;
  case OMPD_barrier:
    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
    break;
  default:
    BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
    break;
  }

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Args[] = {
      getOrCreateIdent(SrcLocStr, SrcLocStrSize, BarrierLocFlags),
      getOrCreateThreadID(getOrCreateIdent(SrcLocStr, SrcLocStrSize))};

  // If we are in a cancellable parallel region, barriers are cancellation
  // points.
  // TODO: Check why we would force simple calls or to ignore the cancel flag.
  bool UseCancelBarrier =
      !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);

  Value *Result =
      Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
                             UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
                                              : OMPRTL___kmpc_barrier),
                         Args);

  if (UseCancelBarrier && CheckCancelFlag)
    emitCancelationCheckImpl(Result, OMPD_parallel);

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
                              Value *IfCondition,
                              omp::Directive CanceledDirective) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  // LLVM utilities like blocks with terminators.
  auto *UI = Builder.CreateUnreachable();

  Instruction *ThenTI = UI, *ElseTI = nullptr;
  if (IfCondition)
    SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
  Builder.SetInsertPoint(ThenTI);

  Value *CancelKind = nullptr;
  switch (CanceledDirective) {
#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value)                       \
  case DirectiveEnum:                                                          \
    CancelKind = Builder.getInt32(Value);                                      \
    break;
#include "llvm/Frontend/OpenMP/OMPKinds.def"
  default:
    llvm_unreachable("Unknown cancel kind!");
  }

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
  Value *Result = Builder.CreateCall(
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);
  auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) {
    if (CanceledDirective == OMPD_parallel) {
      IRBuilder<>::InsertPointGuard IPG(Builder);
      Builder.restoreIP(IP);
      createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
                    omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
                    /* CheckCancelFlag */ false);
    }
  };

  // The actual cancel logic is shared with others, e.g., cancel_barriers.
  emitCancelationCheckImpl(Result, CanceledDirective, ExitCB);

  // Update the insertion point and remove the terminator we introduced.
  Builder.SetInsertPoint(UI->getParent());
  UI->eraseFromParent();

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
    const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
    Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
    Value *HostPtr, ArrayRef<Value *> KernelArgs) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  Builder.restoreIP(AllocaIP);
  auto *KernelArgsPtr =
      Builder.CreateAlloca(OpenMPIRBuilder::KernelArgs, nullptr, "kernel_args");
  Builder.restoreIP(Loc.IP);

  for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
    llvm::Value *Arg =
        Builder.CreateStructGEP(OpenMPIRBuilder::KernelArgs, KernelArgsPtr, I);
    Builder.CreateAlignedStore(
        KernelArgs[I], Arg,
        M.getDataLayout().getPrefTypeAlign(KernelArgs[I]->getType()));
  }

  SmallVector<Value *> OffloadingArgs{Ident,      DeviceID, NumTeams,
                                      NumThreads, HostPtr,  KernelArgsPtr};

  Return = Builder.CreateCall(
      getOrCreateRuntimeFunction(M, OMPRTL___tgt_target_kernel),
      OffloadingArgs);

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitKernelLaunch(
    const LocationDescription &Loc, Function *OutlinedFn, Value *OutlinedFnID,
    EmitFallbackCallbackTy emitTargetCallFallbackCB, TargetKernelArgs &Args,
    Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {

  if (!updateToLocation(Loc))
    return Loc.IP;

  Builder.restoreIP(Loc.IP);
  // On top of the arrays that were filled up, the target offloading call
  // takes as arguments the device id as well as the host pointer. The host
  // pointer is used by the runtime library to identify the current target
  // region, so it only has to be unique and not necessarily point to
  // anything. It could be the pointer to the outlined function that
  // implements the target region, but we aren't using that so that the
  // compiler doesn't need to keep that, and could therefore inline the host
  // function if proven worthwhile during optimization.

  // From this point on, we need to have an ID of the target region defined.
  assert(OutlinedFnID && "Invalid outlined function ID!");
  (void)OutlinedFnID;

  // Return value of the runtime offloading call.
  Value *Return = nullptr;

  // Arguments for the target kernel.
  SmallVector<Value *> ArgsVector;
  getKernelArgsVector(Args, Builder, ArgsVector);

  // The target region is an outlined function launched by the runtime
  // via calls to __tgt_target_kernel().
  //
  // Note that on the host and CPU targets, the runtime implementation of
  // these calls simply call the outlined function without forking threads.
  // The outlined functions themselves have runtime calls to
  // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
  // the compiler in emitTeamsCall() and emitParallelCall().
  //
  // In contrast, on the NVPTX target, the implementation of
  // __tgt_target_teams() launches a GPU kernel with the requested number
  // of teams and threads so no additional calls to the runtime are required.
  // Check the error code and execute the host version if required.
  Builder.restoreIP(emitTargetKernel(
      Builder, AllocaIP, Return, RTLoc, DeviceID, Args.NumTeams.front(),
      Args.NumThreads.front(), OutlinedFnID, ArgsVector));

  BasicBlock *OffloadFailedBlock =
      BasicBlock::Create(Builder.getContext(), "omp_offload.failed");
  BasicBlock *OffloadContBlock =
      BasicBlock::Create(Builder.getContext(), "omp_offload.cont");
  Value *Failed = Builder.CreateIsNotNull(Return);
  Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);

  auto CurFn = Builder.GetInsertBlock()->getParent();
  emitBlock(OffloadFailedBlock, CurFn);
  Builder.restoreIP(emitTargetCallFallbackCB(Builder.saveIP()));
  emitBranch(OffloadContBlock);
  emitBlock(OffloadContBlock, CurFn, /*IsFinished=*/true);
  return Builder.saveIP();
}

void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag,
                                               omp::Directive CanceledDirective,
                                               FinalizeCallbackTy ExitCB) {
  assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
         "Unexpected cancellation!");

  // For a cancel barrier we create two new blocks.
  BasicBlock *BB = Builder.GetInsertBlock();
  BasicBlock *NonCancellationBlock;
  if (Builder.GetInsertPoint() == BB->end()) {
    // TODO: This branch will not be needed once we moved to the
    // OpenMPIRBuilder codegen completely.
    NonCancellationBlock = BasicBlock::Create(
        BB->getContext(), BB->getName() + ".cont", BB->getParent());
  } else {
    NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());
    BB->getTerminator()->eraseFromParent();
    Builder.SetInsertPoint(BB);
  }
  BasicBlock *CancellationBlock = BasicBlock::Create(
      BB->getContext(), BB->getName() + ".cncl", BB->getParent());

  // Jump to them based on the return value.
  Value *Cmp = Builder.CreateIsNull(CancelFlag);
  Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
                       /* TODO weight */ nullptr, nullptr);

  // From the cancellation block we finalize all variables and go to the
  // post finalization block that is known to the FiniCB callback.
  Builder.SetInsertPoint(CancellationBlock);
  if (ExitCB)
    ExitCB(Builder.saveIP());
  auto &FI = FinalizationStack.back();
  FI.FiniCB(Builder.saveIP());

  // The continuation block is where code generation continues.
  Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
}

// Callback used to create OpenMP runtime calls to support
// omp parallel clause for the device.
// We need to use this callback to replace call to the OutlinedFn in OuterFn
// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_51)
static void targetParallelCallback(
    OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn,
    BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition,
    Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr,
    Value *ThreadID, const SmallVector<Instruction *, 4> &ToBeDeleted) {
  // Add some known attributes.
  IRBuilder<> &Builder = OMPIRBuilder->Builder;
  OutlinedFn.addParamAttr(0, Attribute::NoAlias);
  OutlinedFn.addParamAttr(1, Attribute::NoAlias);
  OutlinedFn.addParamAttr(0, Attribute::NoUndef);
  OutlinedFn.addParamAttr(1, Attribute::NoUndef);
  OutlinedFn.addFnAttr(Attribute::NoUnwind);

  assert(OutlinedFn.arg_size() >= 2 &&
         "Expected at least tid and bounded tid as arguments");
  unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;

  CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
  assert(CI && "Expected call instruction to outlined function");
  CI->getParent()->setName("omp_parallel");

  Builder.SetInsertPoint(CI);
  Type *PtrTy = OMPIRBuilder->VoidPtr;
  Value *NullPtrValue = Constant::getNullValue(PtrTy);

  // Add alloca for kernel args
  OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();
  Builder.SetInsertPoint(OuterAllocaBB, OuterAllocaBB->getFirstInsertionPt());
  AllocaInst *ArgsAlloca =
      Builder.CreateAlloca(ArrayType::get(PtrTy, NumCapturedVars));
  Value *Args = ArgsAlloca;
  // Add address space cast if array for storing arguments is not allocated
  // in address space 0
  if (ArgsAlloca->getAddressSpace())
    Args = Builder.CreatePointerCast(ArgsAlloca, PtrTy);
  Builder.restoreIP(CurrentIP);

  // Store captured vars which are used by kmpc_parallel_51
  for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) {
    Value *V = *(CI->arg_begin() + 2 + Idx);
    Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(
        ArrayType::get(PtrTy, NumCapturedVars), Args, 0, Idx);
    Builder.CreateStore(V, StoreAddress);
  }

  Value *Cond =
      IfCondition ? Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32)
                  : Builder.getInt32(1);

  // Build kmpc_parallel_51 call
  Value *Parallel51CallArgs[] = {
      /* identifier*/ Ident,
      /* global thread num*/ ThreadID,
      /* if expression */ Cond,
      /* number of threads */ NumThreads ? NumThreads : Builder.getInt32(-1),
      /* Proc bind */ Builder.getInt32(-1),
      /* outlined function */
      Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr),
      /* wrapper function */ NullPtrValue,
      /* arguments of the outlined funciton*/ Args,
      /* number of arguments */ Builder.getInt64(NumCapturedVars)};

  FunctionCallee RTLFn =
      OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_parallel_51);

  Builder.CreateCall(RTLFn, Parallel51CallArgs);

  LLVM_DEBUG(dbgs() << "With kmpc_parallel_51 placed: "
                    << *Builder.GetInsertBlock()->getParent() << "\n");

  // Initialize the local TID stack location with the argument value.
  Builder.SetInsertPoint(PrivTID);
  Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
  Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI),
                      PrivTIDAddr);

  // Remove redundant call to the outlined function.
  CI->eraseFromParent();

  for (Instruction *I : ToBeDeleted) {
    I->eraseFromParent();
  }
}

// Callback used to create OpenMP runtime calls to support
// omp parallel clause for the host.
// We need to use this callback to replace call to the OutlinedFn in OuterFn
// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if])
static void
hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
                     Function *OuterFn, Value *Ident, Value *IfCondition,
                     Instruction *PrivTID, AllocaInst *PrivTIDAddr,
                     const SmallVector<Instruction *, 4> &ToBeDeleted) {
  IRBuilder<> &Builder = OMPIRBuilder->Builder;
  FunctionCallee RTLFn;
  if (IfCondition) {
    RTLFn =
        OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call_if);
  } else {
    RTLFn =
        OMPIRBuilder->getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);
  }
  if (auto *F = dyn_cast<Function>(RTLFn.getCallee())) {
    if (!F->hasMetadata(LLVMContext::MD_callback)) {
      LLVMContext &Ctx = F->getContext();
      MDBuilder MDB(Ctx);
      // Annotate the callback behavior of the __kmpc_fork_call:
      //  - The callback callee is argument number 2 (microtask).
      //  - The first two arguments of the callback callee are unknown (-1).
      //  - All variadic arguments to the __kmpc_fork_call are passed to the
      //    callback callee.
      F->addMetadata(LLVMContext::MD_callback,
                     *MDNode::get(Ctx, {MDB.createCallbackEncoding(
                                           2, {-1, -1},
                                           /* VarArgsArePassed */ true)}));
    }
  }
  // Add some known attributes.
  OutlinedFn.addParamAttr(0, Attribute::NoAlias);
  OutlinedFn.addParamAttr(1, Attribute::NoAlias);
  OutlinedFn.addFnAttr(Attribute::NoUnwind);

  assert(OutlinedFn.arg_size() >= 2 &&
         "Expected at least tid and bounded tid as arguments");
  unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;

  CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
  CI->getParent()->setName("omp_parallel");
  Builder.SetInsertPoint(CI);

  // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn);
  Value *ForkCallArgs[] = {
      Ident, Builder.getInt32(NumCapturedVars),
      Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr)};

  SmallVector<Value *, 16> RealArgs;
  RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
  if (IfCondition) {
    Value *Cond = Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32);
    RealArgs.push_back(Cond);
  }
  RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end());

  // __kmpc_fork_call_if always expects a void ptr as the last argument
  // If there are no arguments, pass a null pointer.
  auto PtrTy = OMPIRBuilder->VoidPtr;
  if (IfCondition && NumCapturedVars == 0) {
    Value *NullPtrValue = Constant::getNullValue(PtrTy);
    RealArgs.push_back(NullPtrValue);
  }
  if (IfCondition && RealArgs.back()->getType() != PtrTy)
    RealArgs.back() = Builder.CreateBitCast(RealArgs.back(), PtrTy);

  Builder.CreateCall(RTLFn, RealArgs);

  LLVM_DEBUG(dbgs() << "With fork_call placed: "
                    << *Builder.GetInsertBlock()->getParent() << "\n");

  // Initialize the local TID stack location with the argument value.
  Builder.SetInsertPoint(PrivTID);
  Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
  Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI),
                      PrivTIDAddr);

  // Remove redundant call to the outlined function.
  CI->eraseFromParent();

  for (Instruction *I : ToBeDeleted) {
    I->eraseFromParent();
  }
}

IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
    const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
    BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
    FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
    omp::ProcBindKind ProcBind, bool IsCancellable) {
  assert(!isConflictIP(Loc.IP, OuterAllocaIP) && "IPs must not be ambiguous");

  if (!updateToLocation(Loc))
    return Loc.IP;

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadID = getOrCreateThreadID(Ident);
  // If we generate code for the target device, we need to allocate
  // struct for aggregate params in the device default alloca address space.
  // OpenMP runtime requires that the params of the extracted functions are
  // passed as zero address space pointers. This flag ensures that extracted
  // function arguments are declared in zero address space
  bool ArgsInZeroAddressSpace = Config.isTargetDevice();

  // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
  // only if we compile for host side.
  if (NumThreads && !Config.isTargetDevice()) {
    Value *Args[] = {
        Ident, ThreadID,
        Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};
    Builder.CreateCall(
        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args);
  }

  if (ProcBind != OMP_PROC_BIND_default) {
    // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
    Value *Args[] = {
        Ident, ThreadID,
        ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)};
    Builder.CreateCall(
        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args);
  }

  BasicBlock *InsertBB = Builder.GetInsertBlock();
  Function *OuterFn = InsertBB->getParent();

  // Save the outer alloca block because the insertion iterator may get
  // invalidated and we still need this later.
  BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();

  // Vector to remember instructions we used only during the modeling but which
  // we want to delete at the end.
  SmallVector<Instruction *, 4> ToBeDeleted;

  // Change the location to the outer alloca insertion point to create and
  // initialize the allocas we pass into the parallel region.
  InsertPointTy NewOuter(OuterAllocaBlock, OuterAllocaBlock->begin());
  Builder.restoreIP(NewOuter);
  AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
  AllocaInst *ZeroAddrAlloca =
      Builder.CreateAlloca(Int32, nullptr, "zero.addr");
  Instruction *TIDAddr = TIDAddrAlloca;
  Instruction *ZeroAddr = ZeroAddrAlloca;
  if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) {
    // Add additional casts to enforce pointers in zero address space
    TIDAddr = new AddrSpaceCastInst(
        TIDAddrAlloca, PointerType ::get(M.getContext(), 0), "tid.addr.ascast");
    TIDAddr->insertAfter(TIDAddrAlloca);
    ToBeDeleted.push_back(TIDAddr);
    ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca,
                                     PointerType ::get(M.getContext(), 0),
                                     "zero.addr.ascast");
    ZeroAddr->insertAfter(ZeroAddrAlloca);
    ToBeDeleted.push_back(ZeroAddr);
  }

  // We only need TIDAddr and ZeroAddr for modeling purposes to get the
  // associated arguments in the outlined function, so we delete them later.
  ToBeDeleted.push_back(TIDAddrAlloca);
  ToBeDeleted.push_back(ZeroAddrAlloca);

  // Create an artificial insertion point that will also ensure the blocks we
  // are about to split are not degenerated.
  auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);

  BasicBlock *EntryBB = UI->getParent();
  BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(UI, "omp.par.entry");
  BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(UI, "omp.par.region");
  BasicBlock *PRegPreFiniBB =
      PRegBodyBB->splitBasicBlock(UI, "omp.par.pre_finalize");
  BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(UI, "omp.par.exit");

  auto FiniCBWrapper = [&](InsertPointTy IP) {
    // Hide "open-ended" blocks from the given FiniCB by setting the right jump
    // target to the region exit block.
    if (IP.getBlock()->end() == IP.getPoint()) {
      IRBuilder<>::InsertPointGuard IPG(Builder);
      Builder.restoreIP(IP);
      Instruction *I = Builder.CreateBr(PRegExitBB);
      IP = InsertPointTy(I->getParent(), I->getIterator());
    }
    assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
           IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
           "Unexpected insertion point for finalization call!");
    return FiniCB(IP);
  };

  FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});

  // Generate the privatization allocas in the block that will become the entry
  // of the outlined function.
  Builder.SetInsertPoint(PRegEntryBB->getTerminator());
  InsertPointTy InnerAllocaIP = Builder.saveIP();

  AllocaInst *PrivTIDAddr =
      Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
  Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");

  // Add some fake uses for OpenMP provided arguments.
  ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));
  Instruction *ZeroAddrUse =
      Builder.CreateLoad(Int32, ZeroAddr, "zero.addr.use");
  ToBeDeleted.push_back(ZeroAddrUse);

  // EntryBB
  //   |
  //   V
  // PRegionEntryBB         <- Privatization allocas are placed here.
  //   |
  //   V
  // PRegionBodyBB          <- BodeGen is invoked here.
  //   |
  //   V
  // PRegPreFiniBB          <- The block we will start finalization from.
  //   |
  //   V
  // PRegionExitBB          <- A common exit to simplify block collection.
  //

  LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");

  // Let the caller create the body.
  assert(BodyGenCB && "Expected body generation callback!");
  InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
  BodyGenCB(InnerAllocaIP, CodeGenIP);

  LLVM_DEBUG(dbgs() << "After  body codegen: " << *OuterFn << "\n");

  OutlineInfo OI;
  if (Config.isTargetDevice()) {
    // Generate OpenMP target specific runtime call
    OI.PostOutlineCB = [=, ToBeDeletedVec =
                               std::move(ToBeDeleted)](Function &OutlinedFn) {
      targetParallelCallback(this, OutlinedFn, OuterFn, OuterAllocaBlock, Ident,
                             IfCondition, NumThreads, PrivTID, PrivTIDAddr,
                             ThreadID, ToBeDeletedVec);
    };
  } else {
    // Generate OpenMP host runtime call
    OI.PostOutlineCB = [=, ToBeDeletedVec =
                               std::move(ToBeDeleted)](Function &OutlinedFn) {
      hostParallelCallback(this, OutlinedFn, OuterFn, Ident, IfCondition,
                           PrivTID, PrivTIDAddr, ToBeDeletedVec);
    };
  }

  OI.OuterAllocaBB = OuterAllocaBlock;
  OI.EntryBB = PRegEntryBB;
  OI.ExitBB = PRegExitBB;

  SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
  SmallVector<BasicBlock *, 32> Blocks;
  OI.collectBlocks(ParallelRegionBlockSet, Blocks);

  // Ensure a single exit node for the outlined region by creating one.
  // We might have multiple incoming edges to the exit now due to finalizations,
  // e.g., cancel calls that cause the control flow to leave the region.
  BasicBlock *PRegOutlinedExitBB = PRegExitBB;
  PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt());
  PRegOutlinedExitBB->setName("omp.par.outlined.exit");
  Blocks.push_back(PRegOutlinedExitBB);

  CodeExtractorAnalysisCache CEAC(*OuterFn);
  CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
                          /* AggregateArgs */ false,
                          /* BlockFrequencyInfo */ nullptr,
                          /* BranchProbabilityInfo */ nullptr,
                          /* AssumptionCache */ nullptr,
                          /* AllowVarArgs */ true,
                          /* AllowAlloca */ true,
                          /* AllocationBlock */ OuterAllocaBlock,
                          /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);

  // Find inputs to, outputs from the code region.
  BasicBlock *CommonExit = nullptr;
  SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
  Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
  Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands);

  LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");

  FunctionCallee TIDRTLFn =
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);

  auto PrivHelper = [&](Value &V) {
    if (&V == TIDAddr || &V == ZeroAddr) {
      OI.ExcludeArgsFromAggregate.push_back(&V);
      return;
    }

    SetVector<Use *> Uses;
    for (Use &U : V.uses())
      if (auto *UserI = dyn_cast<Instruction>(U.getUser()))
        if (ParallelRegionBlockSet.count(UserI->getParent()))
          Uses.insert(&U);

    // __kmpc_fork_call expects extra arguments as pointers. If the input
    // already has a pointer type, everything is fine. Otherwise, store the
    // value onto stack and load it back inside the to-be-outlined region. This
    // will ensure only the pointer will be passed to the function.
    // FIXME: if there are more than 15 trailing arguments, they must be
    // additionally packed in a struct.
    Value *Inner = &V;
    if (!V.getType()->isPointerTy()) {
      IRBuilder<>::InsertPointGuard Guard(Builder);
      LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");

      Builder.restoreIP(OuterAllocaIP);
      Value *Ptr =
          Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");

      // Store to stack at end of the block that currently branches to the entry
      // block of the to-be-outlined region.
      Builder.SetInsertPoint(InsertBB,
                             InsertBB->getTerminator()->getIterator());
      Builder.CreateStore(&V, Ptr);

      // Load back next to allocations in the to-be-outlined region.
      Builder.restoreIP(InnerAllocaIP);
      Inner = Builder.CreateLoad(V.getType(), Ptr);
    }

    Value *ReplacementValue = nullptr;
    CallInst *CI = dyn_cast<CallInst>(&V);
    if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
      ReplacementValue = PrivTID;
    } else {
      Builder.restoreIP(
          PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
      InnerAllocaIP = {
          InnerAllocaIP.getBlock(),
          InnerAllocaIP.getBlock()->getTerminator()->getIterator()};

      assert(ReplacementValue &&
             "Expected copy/create callback to set replacement value!");
      if (ReplacementValue == &V)
        return;
    }

    for (Use *UPtr : Uses)
      UPtr->set(ReplacementValue);
  };

  // Reset the inner alloca insertion as it will be used for loading the values
  // wrapped into pointers before passing them into the to-be-outlined region.
  // Configure it to insert immediately after the fake use of zero address so
  // that they are available in the generated body and so that the
  // OpenMP-related values (thread ID and zero address pointers) remain leading
  // in the argument list.
  InnerAllocaIP = IRBuilder<>::InsertPoint(
      ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());

  // Reset the outer alloca insertion point to the entry of the relevant block
  // in case it was invalidated.
  OuterAllocaIP = IRBuilder<>::InsertPoint(
      OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());

  for (Value *Input : Inputs) {
    LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
    PrivHelper(*Input);
  }
  LLVM_DEBUG({
    for (Value *Output : Outputs)
      LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
  });
  assert(Outputs.empty() &&
         "OpenMP outlining should not produce live-out values!");

  LLVM_DEBUG(dbgs() << "After  privatization: " << *OuterFn << "\n");
  LLVM_DEBUG({
    for (auto *BB : Blocks)
      dbgs() << " PBR: " << BB->getName() << "\n";
  });

  // Adjust the finalization stack, verify the adjustment, and call the
  // finalize function a last time to finalize values between the pre-fini
  // block and the exit block if we left the parallel "the normal way".
  auto FiniInfo = FinalizationStack.pop_back_val();
  (void)FiniInfo;
  assert(FiniInfo.DK == OMPD_parallel &&
         "Unexpected finalization stack state!");

  Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();

  InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
  FiniCB(PreFiniIP);

  // Register the outlined info.
  addOutlineInfo(std::move(OI));

  InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
  UI->eraseFromParent();

  return AfterIP;
}

void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
  // Build call void __kmpc_flush(ident_t *loc)
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};

  Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args);
}

void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
  if (!updateToLocation(Loc))
    return;
  emitFlush(Loc);
}

void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
  // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
  // global_tid);
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *Args[] = {Ident, getOrCreateThreadID(Ident)};

  // Ignore return result until untied tasks are supported.
  Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
                     Args);
}

void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc) {
  if (!updateToLocation(Loc))
    return;
  emitTaskwaitImpl(Loc);
}

void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
  // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Constant *I32Null = ConstantInt::getNullValue(Int32);
  Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};

  Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
                     Args);
}

void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
  if (!updateToLocation(Loc))
    return;
  emitTaskyieldImpl(Loc);
}

// Processes the dependencies in Dependencies and does the following
// - Allocates space on the stack of an array of DependInfo objects
// - Populates each DependInfo object with relevant information of
//   the corresponding dependence.
// - All code is inserted in the entry block of the current function.
static Value *emitTaskDependencies(
    OpenMPIRBuilder &OMPBuilder,
    SmallVectorImpl<OpenMPIRBuilder::DependData> &Dependencies) {
  // Early return if we have no dependencies to process
  if (Dependencies.empty())
    return nullptr;

  // Given a vector of DependData objects, in this function we create an
  // array on the stack that holds kmp_dep_info objects corresponding
  // to each dependency. This is then passed to the OpenMP runtime.
  // For example, if there are 'n' dependencies then the following psedo
  // code is generated. Assume the first dependence is on a variable 'a'
  //
  // \code{c}
  // DepArray = alloc(n x sizeof(kmp_depend_info);
  // idx = 0;
  // DepArray[idx].base_addr = ptrtoint(&a);
  // DepArray[idx].len = 8;
  // DepArray[idx].flags = Dep.DepKind; /*(See OMPContants.h for DepKind)*/
  // ++idx;
  // DepArray[idx].base_addr = ...;
  // \endcode

  IRBuilderBase &Builder = OMPBuilder.Builder;
  Type *DependInfo = OMPBuilder.DependInfo;
  Module &M = OMPBuilder.M;

  Value *DepArray = nullptr;
  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
  Builder.SetInsertPoint(
      OldIP.getBlock()->getParent()->getEntryBlock().getTerminator());

  Type *DepArrayTy = ArrayType::get(DependInfo, Dependencies.size());
  DepArray = Builder.CreateAlloca(DepArrayTy, nullptr, ".dep.arr.addr");

  for (const auto &[DepIdx, Dep] : enumerate(Dependencies)) {
    Value *Base =
        Builder.CreateConstInBoundsGEP2_64(DepArrayTy, DepArray, 0, DepIdx);
    // Store the pointer to the variable
    Value *Addr = Builder.CreateStructGEP(
        DependInfo, Base,
        static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
    Value *DepValPtr = Builder.CreatePtrToInt(Dep.DepVal, Builder.getInt64Ty());
    Builder.CreateStore(DepValPtr, Addr);
    // Store the size of the variable
    Value *Size = Builder.CreateStructGEP(
        DependInfo, Base, static_cast<unsigned int>(RTLDependInfoFields::Len));
    Builder.CreateStore(
        Builder.getInt64(M.getDataLayout().getTypeStoreSize(Dep.DepValueType)),
        Size);
    // Store the dependency kind
    Value *Flags = Builder.CreateStructGEP(
        DependInfo, Base,
        static_cast<unsigned int>(RTLDependInfoFields::Flags));
    Builder.CreateStore(
        ConstantInt::get(Builder.getInt8Ty(),
                         static_cast<unsigned int>(Dep.DepKind)),
        Flags);
  }
  Builder.restoreIP(OldIP);
  return DepArray;
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createTask(const LocationDescription &Loc,
                            InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB,
                            bool Tied, Value *Final, Value *IfCondition,
                            SmallVector<DependData> Dependencies) {

  if (!updateToLocation(Loc))
    return InsertPointTy();

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  // The current basic block is split into four basic blocks. After outlining,
  // they will be mapped as follows:
  // ```
  // def current_fn() {
  //   current_basic_block:
  //     br label %task.exit
  //   task.exit:
  //     ; instructions after task
  // }
  // def outlined_fn() {
  //   task.alloca:
  //     br label %task.body
  //   task.body:
  //     ret void
  // }
  // ```
  BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, "task.exit");
  BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, "task.body");
  BasicBlock *TaskAllocaBB =
      splitBB(Builder, /*CreateBranch=*/true, "task.alloca");

  InsertPointTy TaskAllocaIP =
      InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
  InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
  BodyGenCB(TaskAllocaIP, TaskBodyIP);

  OutlineInfo OI;
  OI.EntryBB = TaskAllocaBB;
  OI.OuterAllocaBB = AllocaIP.getBlock();
  OI.ExitBB = TaskExitBB;

  // Add the thread ID argument.
  SmallVector<Instruction *, 4> ToBeDeleted;
  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(
      Builder, AllocaIP, ToBeDeleted, TaskAllocaIP, "global.tid", false));

  OI.PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,
                      TaskAllocaBB, ToBeDeleted](Function &OutlinedFn) mutable {
    // Replace the Stale CI by appropriate RTL function call.
    assert(OutlinedFn.getNumUses() == 1 &&
           "there must be a single user for the outlined function");
    CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());

    // HasShareds is true if any variables are captured in the outlined region,
    // false otherwise.
    bool HasShareds = StaleCI->arg_size() > 1;
    Builder.SetInsertPoint(StaleCI);

    // Gather the arguments for emitting the runtime call for
    // @__kmpc_omp_task_alloc
    Function *TaskAllocFn =
        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);

    // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
    // call.
    Value *ThreadID = getOrCreateThreadID(Ident);

    // Argument - `flags`
    // Task is tied iff (Flags & 1) == 1.
    // Task is untied iff (Flags & 1) == 0.
    // Task is final iff (Flags & 2) == 2.
    // Task is not final iff (Flags & 2) == 0.
    // TODO: Handle the other flags.
    Value *Flags = Builder.getInt32(Tied);
    if (Final) {
      Value *FinalFlag =
          Builder.CreateSelect(Final, Builder.getInt32(2), Builder.getInt32(0));
      Flags = Builder.CreateOr(FinalFlag, Flags);
    }

    // Argument - `sizeof_kmp_task_t` (TaskSize)
    // Tasksize refers to the size in bytes of kmp_task_t data structure
    // including private vars accessed in task.
    // TODO: add kmp_task_t_with_privates (privates)
    Value *TaskSize = Builder.getInt64(
        divideCeil(M.getDataLayout().getTypeSizeInBits(Task), 8));

    // Argument - `sizeof_shareds` (SharedsSize)
    // SharedsSize refers to the shareds array size in the kmp_task_t data
    // structure.
    Value *SharedsSize = Builder.getInt64(0);
    if (HasShareds) {
      AllocaInst *ArgStructAlloca =
          dyn_cast<AllocaInst>(StaleCI->getArgOperand(1));
      assert(ArgStructAlloca &&
             "Unable to find the alloca instruction corresponding to arguments "
             "for extracted function");
      StructType *ArgStructType =
          dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
      assert(ArgStructType && "Unable to find struct type corresponding to "
                              "arguments for extracted function");
      SharedsSize =
          Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
    }
    // Emit the @__kmpc_omp_task_alloc runtime call
    // The runtime call returns a pointer to an area where the task captured
    // variables must be copied before the task is run (TaskData)
    CallInst *TaskData = Builder.CreateCall(
        TaskAllocFn, {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
                      /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
                      /*task_func=*/&OutlinedFn});

    // Copy the arguments for outlined function
    if (HasShareds) {
      Value *Shareds = StaleCI->getArgOperand(1);
      Align Alignment = TaskData->getPointerAlignment(M.getDataLayout());
      Value *TaskShareds = Builder.CreateLoad(VoidPtr, TaskData);
      Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment,
                           SharedsSize);
    }

    Value *DepArray = nullptr;
    if (Dependencies.size()) {
      InsertPointTy OldIP = Builder.saveIP();
      Builder.SetInsertPoint(
          &OldIP.getBlock()->getParent()->getEntryBlock().back());

      Type *DepArrayTy = ArrayType::get(DependInfo, Dependencies.size());
      DepArray = Builder.CreateAlloca(DepArrayTy, nullptr, ".dep.arr.addr");

      unsigned P = 0;
      for (const DependData &Dep : Dependencies) {
        Value *Base =
            Builder.CreateConstInBoundsGEP2_64(DepArrayTy, DepArray, 0, P);
        // Store the pointer to the variable
        Value *Addr = Builder.CreateStructGEP(
            DependInfo, Base,
            static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
        Value *DepValPtr =
            Builder.CreatePtrToInt(Dep.DepVal, Builder.getInt64Ty());
        Builder.CreateStore(DepValPtr, Addr);
        // Store the size of the variable
        Value *Size = Builder.CreateStructGEP(
            DependInfo, Base,
            static_cast<unsigned int>(RTLDependInfoFields::Len));
        Builder.CreateStore(Builder.getInt64(M.getDataLayout().getTypeStoreSize(
                                Dep.DepValueType)),
                            Size);
        // Store the dependency kind
        Value *Flags = Builder.CreateStructGEP(
            DependInfo, Base,
            static_cast<unsigned int>(RTLDependInfoFields::Flags));
        Builder.CreateStore(
            ConstantInt::get(Builder.getInt8Ty(),
                             static_cast<unsigned int>(Dep.DepKind)),
            Flags);
        ++P;
      }

      Builder.restoreIP(OldIP);
    }

    // In the presence of the `if` clause, the following IR is generated:
    //    ...
    //    %data = call @__kmpc_omp_task_alloc(...)
    //    br i1 %if_condition, label %then, label %else
    //  then:
    //    call @__kmpc_omp_task(...)
    //    br label %exit
    //  else:
    //    ;; Wait for resolution of dependencies, if any, before
    //    ;; beginning the task
    //    call @__kmpc_omp_wait_deps(...)
    //    call @__kmpc_omp_task_begin_if0(...)
    //    call @outlined_fn(...)
    //    call @__kmpc_omp_task_complete_if0(...)
    //    br label %exit
    //  exit:
    //    ...
    if (IfCondition) {
      // `SplitBlockAndInsertIfThenElse` requires the block to have a
      // terminator.
      splitBB(Builder, /*CreateBranch=*/true, "if.end");
      Instruction *IfTerminator =
          Builder.GetInsertPoint()->getParent()->getTerminator();
      Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;
      Builder.SetInsertPoint(IfTerminator);
      SplitBlockAndInsertIfThenElse(IfCondition, IfTerminator, &ThenTI,
                                    &ElseTI);
      Builder.SetInsertPoint(ElseTI);

      if (Dependencies.size()) {
        Function *TaskWaitFn =
            getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_wait_deps);
        Builder.CreateCall(
            TaskWaitFn,
            {Ident, ThreadID, Builder.getInt32(Dependencies.size()), DepArray,
             ConstantInt::get(Builder.getInt32Ty(), 0),
             ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});
      }
      Function *TaskBeginFn =
          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0);
      Function *TaskCompleteFn =
          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0);
      Builder.CreateCall(TaskBeginFn, {Ident, ThreadID, TaskData});
      CallInst *CI = nullptr;
      if (HasShareds)
        CI = Builder.CreateCall(&OutlinedFn, {ThreadID, TaskData});
      else
        CI = Builder.CreateCall(&OutlinedFn, {ThreadID});
      CI->setDebugLoc(StaleCI->getDebugLoc());
      Builder.CreateCall(TaskCompleteFn, {Ident, ThreadID, TaskData});
      Builder.SetInsertPoint(ThenTI);
    }

    if (Dependencies.size()) {
      Function *TaskFn =
          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps);
      Builder.CreateCall(
          TaskFn,
          {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()),
           DepArray, ConstantInt::get(Builder.getInt32Ty(), 0),
           ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});

    } else {
      // Emit the @__kmpc_omp_task runtime call to spawn the task
      Function *TaskFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task);
      Builder.CreateCall(TaskFn, {Ident, ThreadID, TaskData});
    }

    StaleCI->eraseFromParent();

    Builder.SetInsertPoint(TaskAllocaBB, TaskAllocaBB->begin());
    if (HasShareds) {
      LoadInst *Shareds = Builder.CreateLoad(VoidPtr, OutlinedFn.getArg(1));
      OutlinedFn.getArg(1)->replaceUsesWithIf(
          Shareds, [Shareds](Use &U) { return U.getUser() != Shareds; });
    }

    for (Instruction *I : llvm::reverse(ToBeDeleted))
      I->eraseFromParent();
  };

  addOutlineInfo(std::move(OI));
  Builder.SetInsertPoint(TaskExitBB, TaskExitBB->begin());

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
                                 InsertPointTy AllocaIP,
                                 BodyGenCallbackTy BodyGenCB) {
  if (!updateToLocation(Loc))
    return InsertPointTy();

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadID = getOrCreateThreadID(Ident);

  // Emit the @__kmpc_taskgroup runtime call to start the taskgroup
  Function *TaskgroupFn =
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup);
  Builder.CreateCall(TaskgroupFn, {Ident, ThreadID});

  BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit");
  BodyGenCB(AllocaIP, Builder.saveIP());

  Builder.SetInsertPoint(TaskgroupExitBB);
  // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
  Function *EndTaskgroupFn =
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup);
  Builder.CreateCall(EndTaskgroupFn, {Ident, ThreadID});

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
    const LocationDescription &Loc, InsertPointTy AllocaIP,
    ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
    FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
  assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required");

  if (!updateToLocation(Loc))
    return Loc.IP;

  auto FiniCBWrapper = [&](InsertPointTy IP) {
    if (IP.getBlock()->end() != IP.getPoint())
      return FiniCB(IP);
    // This must be done otherwise any nested constructs using FinalizeOMPRegion
    // will fail because that function requires the Finalization Basic Block to
    // have a terminator, which is already removed by EmitOMPRegionBody.
    // IP is currently at cancelation block.
    // We need to backtrack to the condition block to fetch
    // the exit block and create a branch from cancelation
    // to exit block.
    IRBuilder<>::InsertPointGuard IPG(Builder);
    Builder.restoreIP(IP);
    auto *CaseBB = IP.getBlock()->getSinglePredecessor();
    auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
    auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
    Instruction *I = Builder.CreateBr(ExitBB);
    IP = InsertPointTy(I->getParent(), I->getIterator());
    return FiniCB(IP);
  };

  FinalizationStack.push_back({FiniCBWrapper, OMPD_sections, IsCancellable});

  // Each section is emitted as a switch case
  // Each finalization callback is handled from clang.EmitOMPSectionDirective()
  // -> OMP.createSection() which generates the IR for each section
  // Iterate through all sections and emit a switch construct:
  // switch (IV) {
  //   case 0:
  //     <SectionStmt[0]>;
  //     break;
  // ...
  //   case <NumSection> - 1:
  //     <SectionStmt[<NumSection> - 1]>;
  //     break;
  // }
  // ...
  // section_loop.after:
  // <FiniCB>;
  auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) {
    Builder.restoreIP(CodeGenIP);
    BasicBlock *Continue =
        splitBBWithSuffix(Builder, /*CreateBranch=*/false, ".sections.after");
    Function *CurFn = Continue->getParent();
    SwitchInst *SwitchStmt = Builder.CreateSwitch(IndVar, Continue);

    unsigned CaseNumber = 0;
    for (auto SectionCB : SectionCBs) {
      BasicBlock *CaseBB = BasicBlock::Create(
          M.getContext(), "omp_section_loop.body.case", CurFn, Continue);
      SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB);
      Builder.SetInsertPoint(CaseBB);
      BranchInst *CaseEndBr = Builder.CreateBr(Continue);
      SectionCB(InsertPointTy(),
                {CaseEndBr->getParent(), CaseEndBr->getIterator()});
      CaseNumber++;
    }
    // remove the existing terminator from body BB since there can be no
    // terminators after switch/case
  };
  // Loop body ends here
  // LowerBound, UpperBound, and STride for createCanonicalLoop
  Type *I32Ty = Type::getInt32Ty(M.getContext());
  Value *LB = ConstantInt::get(I32Ty, 0);
  Value *UB = ConstantInt::get(I32Ty, SectionCBs.size());
  Value *ST = ConstantInt::get(I32Ty, 1);
  llvm::CanonicalLoopInfo *LoopInfo = createCanonicalLoop(
      Loc, LoopBodyGenCB, LB, UB, ST, true, false, AllocaIP, "section_loop");
  InsertPointTy AfterIP =
      applyStaticWorkshareLoop(Loc.DL, LoopInfo, AllocaIP, !IsNowait);

  // Apply the finalization callback in LoopAfterBB
  auto FiniInfo = FinalizationStack.pop_back_val();
  assert(FiniInfo.DK == OMPD_sections &&
         "Unexpected finalization stack state!");
  if (FinalizeCallbackTy &CB = FiniInfo.FiniCB) {
    Builder.restoreIP(AfterIP);
    BasicBlock *FiniBB =
        splitBBWithSuffix(Builder, /*CreateBranch=*/true, "sections.fini");
    CB(Builder.saveIP());
    AfterIP = {FiniBB, FiniBB->begin()};
  }

  return AfterIP;
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createSection(const LocationDescription &Loc,
                               BodyGenCallbackTy BodyGenCB,
                               FinalizeCallbackTy FiniCB) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  auto FiniCBWrapper = [&](InsertPointTy IP) {
    if (IP.getBlock()->end() != IP.getPoint())
      return FiniCB(IP);
    // This must be done otherwise any nested constructs using FinalizeOMPRegion
    // will fail because that function requires the Finalization Basic Block to
    // have a terminator, which is already removed by EmitOMPRegionBody.
    // IP is currently at cancelation block.
    // We need to backtrack to the condition block to fetch
    // the exit block and create a branch from cancelation
    // to exit block.
    IRBuilder<>::InsertPointGuard IPG(Builder);
    Builder.restoreIP(IP);
    auto *CaseBB = Loc.IP.getBlock();
    auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
    auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
    Instruction *I = Builder.CreateBr(ExitBB);
    IP = InsertPointTy(I->getParent(), I->getIterator());
    return FiniCB(IP);
  };

  Directive OMPD = Directive::OMPD_sections;
  // Since we are using Finalization Callback here, HasFinalize
  // and IsCancellable have to be true
  return EmitOMPInlinedRegion(OMPD, nullptr, nullptr, BodyGenCB, FiniCBWrapper,
                              /*Conditional*/ false, /*hasFinalize*/ true,
                              /*IsCancellable*/ true);
}

static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I) {
  BasicBlock::iterator IT(I);
  IT++;
  return OpenMPIRBuilder::InsertPointTy(I->getParent(), IT);
}

void OpenMPIRBuilder::emitUsed(StringRef Name,
                               std::vector<WeakTrackingVH> &List) {
  if (List.empty())
    return;

  // Convert List to what ConstantArray needs.
  SmallVector<Constant *, 8> UsedArray;
  UsedArray.resize(List.size());
  for (unsigned I = 0, E = List.size(); I != E; ++I)
    UsedArray[I] = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
        cast<Constant>(&*List[I]), Builder.getPtrTy());

  if (UsedArray.empty())
    return;
  ArrayType *ATy = ArrayType::get(Builder.getPtrTy(), UsedArray.size());

  auto *GV = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
                                ConstantArray::get(ATy, UsedArray), Name);

  GV->setSection("llvm.metadata");
}

Value *OpenMPIRBuilder::getGPUThreadID() {
  return Builder.CreateCall(
      getOrCreateRuntimeFunction(M,
                                 OMPRTL___kmpc_get_hardware_thread_id_in_block),
      {});
}

Value *OpenMPIRBuilder::getGPUWarpSize() {
  return Builder.CreateCall(
      getOrCreateRuntimeFunction(M, OMPRTL___kmpc_get_warp_size), {});
}

Value *OpenMPIRBuilder::getNVPTXWarpID() {
  unsigned LaneIDBits = Log2_32(Config.getGridValue().GV_Warp_Size);
  return Builder.CreateAShr(getGPUThreadID(), LaneIDBits, "nvptx_warp_id");
}

Value *OpenMPIRBuilder::getNVPTXLaneID() {
  unsigned LaneIDBits = Log2_32(Config.getGridValue().GV_Warp_Size);
  assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
  unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
  return Builder.CreateAnd(getGPUThreadID(), Builder.getInt32(LaneIDMask),
                           "nvptx_lane_id");
}

Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP, Value *From,
                                        Type *ToType) {
  Type *FromType = From->getType();
  uint64_t FromSize = M.getDataLayout().getTypeStoreSize(FromType);
  uint64_t ToSize = M.getDataLayout().getTypeStoreSize(ToType);
  assert(FromSize > 0 && "From size must be greater than zero");
  assert(ToSize > 0 && "To size must be greater than zero");
  if (FromType == ToType)
    return From;
  if (FromSize == ToSize)
    return Builder.CreateBitCast(From, ToType);
  if (ToType->isIntegerTy() && FromType->isIntegerTy())
    return Builder.CreateIntCast(From, ToType, /*isSigned*/ true);
  InsertPointTy SaveIP = Builder.saveIP();
  Builder.restoreIP(AllocaIP);
  Value *CastItem = Builder.CreateAlloca(ToType);
  Builder.restoreIP(SaveIP);

  Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
      CastItem, FromType->getPointerTo());
  Builder.CreateStore(From, ValCastItem);
  return Builder.CreateLoad(ToType, CastItem);
}

Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
                                                     Value *Element,
                                                     Type *ElementType,
                                                     Value *Offset) {
  uint64_t Size = M.getDataLayout().getTypeStoreSize(ElementType);
  assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");

  // Cast all types to 32- or 64-bit values before calling shuffle routines.
  Type *CastTy = Builder.getIntNTy(Size <= 4 ? 32 : 64);
  Value *ElemCast = castValueToType(AllocaIP, Element, CastTy);
  Value *WarpSize =
      Builder.CreateIntCast(getGPUWarpSize(), Builder.getInt16Ty(), true);
  Function *ShuffleFunc = getOrCreateRuntimeFunctionPtr(
      Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
                : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
  Value *WarpSizeCast =
      Builder.CreateIntCast(WarpSize, Builder.getInt16Ty(), /*isSigned=*/true);
  Value *ShuffleCall =
      Builder.CreateCall(ShuffleFunc, {ElemCast, Offset, WarpSizeCast});
  return castValueToType(AllocaIP, ShuffleCall, CastTy);
}

void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr,
                                      Value *DstAddr, Type *ElemType,
                                      Value *Offset, Type *ReductionArrayTy) {
  uint64_t Size = M.getDataLayout().getTypeStoreSize(ElemType);
  // Create the loop over the big sized data.
  // ptr = (void*)Elem;
  // ptrEnd = (void*) Elem + 1;
  // Step = 8;
  // while (ptr + Step < ptrEnd)
  //   shuffle((int64_t)*ptr);
  // Step = 4;
  // while (ptr + Step < ptrEnd)
  //   shuffle((int32_t)*ptr);
  // ...
  Type *IndexTy = Builder.getIndexTy(
      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
  Value *ElemPtr = DstAddr;
  Value *Ptr = SrcAddr;
  for (unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
    if (Size < IntSize)
      continue;
    Type *IntType = Builder.getIntNTy(IntSize * 8);
    Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
        Ptr, IntType->getPointerTo(), Ptr->getName() + ".ascast");
    Value *SrcAddrGEP =
        Builder.CreateGEP(ElemType, SrcAddr, {ConstantInt::get(IndexTy, 1)});
    ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
        ElemPtr, IntType->getPointerTo(), ElemPtr->getName() + ".ascast");

    Function *CurFunc = Builder.GetInsertBlock()->getParent();
    if ((Size / IntSize) > 1) {
      Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
          SrcAddrGEP, Builder.getPtrTy());
      BasicBlock *PreCondBB =
          BasicBlock::Create(M.getContext(), ".shuffle.pre_cond");
      BasicBlock *ThenBB = BasicBlock::Create(M.getContext(), ".shuffle.then");
      BasicBlock *ExitBB = BasicBlock::Create(M.getContext(), ".shuffle.exit");
      BasicBlock *CurrentBB = Builder.GetInsertBlock();
      emitBlock(PreCondBB, CurFunc);
      PHINode *PhiSrc =
          Builder.CreatePHI(Ptr->getType(), /*NumReservedValues=*/2);
      PhiSrc->addIncoming(Ptr, CurrentBB);
      PHINode *PhiDest =
          Builder.CreatePHI(ElemPtr->getType(), /*NumReservedValues=*/2);
      PhiDest->addIncoming(ElemPtr, CurrentBB);
      Ptr = PhiSrc;
      ElemPtr = PhiDest;
      Value *PtrDiff = Builder.CreatePtrDiff(
          Builder.getInt8Ty(), PtrEnd,
          Builder.CreatePointerBitCastOrAddrSpaceCast(Ptr, Builder.getPtrTy()));
      Builder.CreateCondBr(
          Builder.CreateICmpSGT(PtrDiff, Builder.getInt64(IntSize - 1)), ThenBB,
          ExitBB);
      emitBlock(ThenBB, CurFunc);
      Value *Res = createRuntimeShuffleFunction(
          AllocaIP,
          Builder.CreateAlignedLoad(
              IntType, Ptr, M.getDataLayout().getPrefTypeAlign(ElemType)),
          IntType, Offset);
      Builder.CreateAlignedStore(Res, ElemPtr,
                                 M.getDataLayout().getPrefTypeAlign(ElemType));
      Value *LocalPtr =
          Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)});
      Value *LocalElemPtr =
          Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)});
      PhiSrc->addIncoming(LocalPtr, ThenBB);
      PhiDest->addIncoming(LocalElemPtr, ThenBB);
      emitBranch(PreCondBB);
      emitBlock(ExitBB, CurFunc);
    } else {
      Value *Res = createRuntimeShuffleFunction(
          AllocaIP, Builder.CreateLoad(IntType, Ptr), IntType, Offset);
      if (ElemType->isIntegerTy() && ElemType->getScalarSizeInBits() <
                                         Res->getType()->getScalarSizeInBits())
        Res = Builder.CreateTrunc(Res, ElemType);
      Builder.CreateStore(Res, ElemPtr);
      Ptr = Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)});
      ElemPtr =
          Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)});
    }
    Size = Size % IntSize;
  }
}

void OpenMPIRBuilder::emitReductionListCopy(
    InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,
    ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,
    CopyOptionsTy CopyOptions) {
  Type *IndexTy = Builder.getIndexTy(
      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
  Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;

  // Iterates, element-by-element, through the source Reduce list and
  // make a copy.
  for (auto En : enumerate(ReductionInfos)) {
    const ReductionInfo &RI = En.value();
    Value *SrcElementAddr = nullptr;
    Value *DestElementAddr = nullptr;
    Value *DestElementPtrAddr = nullptr;
    // Should we shuffle in an element from a remote lane?
    bool ShuffleInElement = false;
    // Set to true to update the pointer in the dest Reduce list to a
    // newly created element.
    bool UpdateDestListPtr = false;

    // Step 1.1: Get the address for the src element in the Reduce list.
    Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(
        ReductionArrayTy, SrcBase,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    SrcElementAddr = Builder.CreateLoad(Builder.getPtrTy(), SrcElementPtrAddr);

    // Step 1.2: Create a temporary to store the element in the destination
    // Reduce list.
    DestElementPtrAddr = Builder.CreateInBoundsGEP(
        ReductionArrayTy, DestBase,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    switch (Action) {
    case CopyAction::RemoteLaneToThread: {
      InsertPointTy CurIP = Builder.saveIP();
      Builder.restoreIP(AllocaIP);
      AllocaInst *DestAlloca = Builder.CreateAlloca(RI.ElementType, nullptr,
                                                    ".omp.reduction.element");
      DestAlloca->setAlignment(
          M.getDataLayout().getPrefTypeAlign(RI.ElementType));
      DestElementAddr = DestAlloca;
      DestElementAddr =
          Builder.CreateAddrSpaceCast(DestElementAddr, Builder.getPtrTy(),
                                      DestElementAddr->getName() + ".ascast");
      Builder.restoreIP(CurIP);
      ShuffleInElement = true;
      UpdateDestListPtr = true;
      break;
    }
    case CopyAction::ThreadCopy: {
      DestElementAddr =
          Builder.CreateLoad(Builder.getPtrTy(), DestElementPtrAddr);
      break;
    }
    }

    // Now that all active lanes have read the element in the
    // Reduce list, shuffle over the value from the remote lane.
    if (ShuffleInElement) {
      shuffleAndStore(AllocaIP, SrcElementAddr, DestElementAddr, RI.ElementType,
                      RemoteLaneOffset, ReductionArrayTy);
    } else {
      switch (RI.EvaluationKind) {
      case EvalKind::Scalar: {
        Value *Elem = Builder.CreateLoad(RI.ElementType, SrcElementAddr);
        // Store the source element value to the dest element address.
        Builder.CreateStore(Elem, DestElementAddr);
        break;
      }
      case EvalKind::Complex: {
        Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
            RI.ElementType, SrcElementAddr, 0, 0, ".realp");
        Value *SrcReal = Builder.CreateLoad(
            RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");
        Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
            RI.ElementType, SrcElementAddr, 0, 1, ".imagp");
        Value *SrcImg = Builder.CreateLoad(
            RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");

        Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
            RI.ElementType, DestElementAddr, 0, 0, ".realp");
        Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
            RI.ElementType, DestElementAddr, 0, 1, ".imagp");
        Builder.CreateStore(SrcReal, DestRealPtr);
        Builder.CreateStore(SrcImg, DestImgPtr);
        break;
      }
      case EvalKind::Aggregate: {
        Value *SizeVal = Builder.getInt64(
            M.getDataLayout().getTypeStoreSize(RI.ElementType));
        Builder.CreateMemCpy(
            DestElementAddr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),
            SrcElementAddr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),
            SizeVal, false);
        break;
      }
      };
    }

    // Step 3.1: Modify reference in dest Reduce list as needed.
    // Modifying the reference in Reduce list to point to the newly
    // created element.  The element is live in the current function
    // scope and that of functions it invokes (i.e., reduce_function).
    // RemoteReduceData[i] = (void*)&RemoteElem
    if (UpdateDestListPtr) {
      Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
          DestElementAddr, Builder.getPtrTy(),
          DestElementAddr->getName() + ".ascast");
      Builder.CreateStore(CastDestAddr, DestElementPtrAddr);
    }
  }
}

Function *OpenMPIRBuilder::emitInterWarpCopyFunction(
    const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,
    AttributeList FuncAttrs) {
  InsertPointTy SavedIP = Builder.saveIP();
  LLVMContext &Ctx = M.getContext();
  FunctionType *FuncTy = FunctionType::get(
      Builder.getVoidTy(), {Builder.getPtrTy(), Builder.getInt32Ty()},
      /* IsVarArg */ false);
  Function *WcFunc =
      Function::Create(FuncTy, GlobalVariable::InternalLinkage,
                       "_omp_reduction_inter_warp_copy_func", &M);
  WcFunc->setAttributes(FuncAttrs);
  WcFunc->addParamAttr(0, Attribute::NoUndef);
  WcFunc->addParamAttr(1, Attribute::NoUndef);
  BasicBlock *EntryBB = BasicBlock::Create(M.getContext(), "entry", WcFunc);
  Builder.SetInsertPoint(EntryBB);

  // ReduceList: thread local Reduce list.
  // At the stage of the computation when this function is called, partially
  // aggregated values reside in the first lane of every active warp.
  Argument *ReduceListArg = WcFunc->getArg(0);
  // NumWarps: number of warps active in the parallel region.  This could
  // be smaller than 32 (max warps in a CTA) for partial block reduction.
  Argument *NumWarpsArg = WcFunc->getArg(1);

  // This array is used as a medium to transfer, one reduce element at a time,
  // the data from the first lane of every warp to lanes in the first warp
  // in order to perform the final step of a reduction in a parallel region
  // (reduction across warps).  The array is placed in NVPTX __shared__ memory
  // for reduced latency, as well as to have a distinct copy for concurrently
  // executing target regions.  The array is declared with common linkage so
  // as to be shared across compilation units.
  StringRef TransferMediumName =
      "__openmp_nvptx_data_transfer_temporary_storage";
  GlobalVariable *TransferMedium = M.getGlobalVariable(TransferMediumName);
  unsigned WarpSize = Config.getGridValue().GV_Warp_Size;
  ArrayType *ArrayTy = ArrayType::get(Builder.getInt32Ty(), WarpSize);
  if (!TransferMedium) {
    TransferMedium = new GlobalVariable(
        M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,
        UndefValue::get(ArrayTy), TransferMediumName,
        /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
        /*AddressSpace=*/3);
  }

  // Get the CUDA thread id of the current OpenMP thread on the GPU.
  Value *GPUThreadID = getGPUThreadID();
  // nvptx_lane_id = nvptx_id % warpsize
  Value *LaneID = getNVPTXLaneID();
  // nvptx_warp_id = nvptx_id / warpsize
  Value *WarpID = getNVPTXWarpID();

  InsertPointTy AllocaIP =
      InsertPointTy(Builder.GetInsertBlock(),
                    Builder.GetInsertBlock()->getFirstInsertionPt());
  Type *Arg0Type = ReduceListArg->getType();
  Type *Arg1Type = NumWarpsArg->getType();
  Builder.restoreIP(AllocaIP);
  AllocaInst *ReduceListAlloca = Builder.CreateAlloca(
      Arg0Type, nullptr, ReduceListArg->getName() + ".addr");
  AllocaInst *NumWarpsAlloca =
      Builder.CreateAlloca(Arg1Type, nullptr, NumWarpsArg->getName() + ".addr");
  Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      ReduceListAlloca, Arg0Type, ReduceListAlloca->getName() + ".ascast");
  Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      NumWarpsAlloca, Arg1Type->getPointerTo(),
      NumWarpsAlloca->getName() + ".ascast");
  Builder.CreateStore(ReduceListArg, ReduceListAddrCast);
  Builder.CreateStore(NumWarpsArg, NumWarpsAddrCast);
  AllocaIP = getInsertPointAfterInstr(NumWarpsAlloca);
  InsertPointTy CodeGenIP =
      getInsertPointAfterInstr(&Builder.GetInsertBlock()->back());
  Builder.restoreIP(CodeGenIP);

  Value *ReduceList =
      Builder.CreateLoad(Builder.getPtrTy(), ReduceListAddrCast);

  for (auto En : enumerate(ReductionInfos)) {
    //
    // Warp master copies reduce element to transfer medium in __shared__
    // memory.
    //
    const ReductionInfo &RI = En.value();
    unsigned RealTySize = M.getDataLayout().getTypeAllocSize(RI.ElementType);
    for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
      Type *CType = Builder.getIntNTy(TySize * 8);

      unsigned NumIters = RealTySize / TySize;
      if (NumIters == 0)
        continue;
      Value *Cnt = nullptr;
      Value *CntAddr = nullptr;
      BasicBlock *PrecondBB = nullptr;
      BasicBlock *ExitBB = nullptr;
      if (NumIters > 1) {
        CodeGenIP = Builder.saveIP();
        Builder.restoreIP(AllocaIP);
        CntAddr =
            Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, ".cnt.addr");

        CntAddr = Builder.CreateAddrSpaceCast(CntAddr, Builder.getPtrTy(),
                                              CntAddr->getName() + ".ascast");
        Builder.restoreIP(CodeGenIP);
        Builder.CreateStore(Constant::getNullValue(Builder.getInt32Ty()),
                            CntAddr,
                            /*Volatile=*/false);
        PrecondBB = BasicBlock::Create(Ctx, "precond");
        ExitBB = BasicBlock::Create(Ctx, "exit");
        BasicBlock *BodyBB = BasicBlock::Create(Ctx, "body");
        emitBlock(PrecondBB, Builder.GetInsertBlock()->getParent());
        Cnt = Builder.CreateLoad(Builder.getInt32Ty(), CntAddr,
                                 /*Volatile=*/false);
        Value *Cmp = Builder.CreateICmpULT(
            Cnt, ConstantInt::get(Builder.getInt32Ty(), NumIters));
        Builder.CreateCondBr(Cmp, BodyBB, ExitBB);
        emitBlock(BodyBB, Builder.GetInsertBlock()->getParent());
      }

      // kmpc_barrier.
      createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
                    omp::Directive::OMPD_unknown,
                    /* ForceSimpleCall */ false,
                    /* CheckCancelFlag */ true);
      BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then");
      BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else");
      BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont");

      // if (lane_id  == 0)
      Value *IsWarpMaster = Builder.CreateIsNull(LaneID, "warp_master");
      Builder.CreateCondBr(IsWarpMaster, ThenBB, ElseBB);
      emitBlock(ThenBB, Builder.GetInsertBlock()->getParent());

      // Reduce element = LocalReduceList[i]
      auto *RedListArrayTy =
          ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
      Type *IndexTy = Builder.getIndexTy(
          M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
      Value *ElemPtrPtr =
          Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList,
                                    {ConstantInt::get(IndexTy, 0),
                                     ConstantInt::get(IndexTy, En.index())});
      // elemptr = ((CopyType*)(elemptrptr)) + I
      Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);
      if (NumIters > 1)
        ElemPtr = Builder.CreateGEP(Builder.getInt32Ty(), ElemPtr, Cnt);

      // Get pointer to location in transfer medium.
      // MediumPtr = &medium[warp_id]
      Value *MediumPtr = Builder.CreateInBoundsGEP(
          ArrayTy, TransferMedium, {Builder.getInt64(0), WarpID});
      // elem = *elemptr
      //*MediumPtr = elem
      Value *Elem = Builder.CreateLoad(CType, ElemPtr);
      // Store the source element value to the dest element address.
      Builder.CreateStore(Elem, MediumPtr,
                          /*IsVolatile*/ true);
      Builder.CreateBr(MergeBB);

      // else
      emitBlock(ElseBB, Builder.GetInsertBlock()->getParent());
      Builder.CreateBr(MergeBB);

      // endif
      emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());
      createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
                    omp::Directive::OMPD_unknown,
                    /* ForceSimpleCall */ false,
                    /* CheckCancelFlag */ true);

      // Warp 0 copies reduce element from transfer medium
      BasicBlock *W0ThenBB = BasicBlock::Create(Ctx, "then");
      BasicBlock *W0ElseBB = BasicBlock::Create(Ctx, "else");
      BasicBlock *W0MergeBB = BasicBlock::Create(Ctx, "ifcont");

      Value *NumWarpsVal =
          Builder.CreateLoad(Builder.getInt32Ty(), NumWarpsAddrCast);
      // Up to 32 threads in warp 0 are active.
      Value *IsActiveThread =
          Builder.CreateICmpULT(GPUThreadID, NumWarpsVal, "is_active_thread");
      Builder.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB);

      emitBlock(W0ThenBB, Builder.GetInsertBlock()->getParent());

      // SecMediumPtr = &medium[tid]
      // SrcMediumVal = *SrcMediumPtr
      Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
          ArrayTy, TransferMedium, {Builder.getInt64(0), GPUThreadID});
      // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
      Value *TargetElemPtrPtr =
          Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList,
                                    {ConstantInt::get(IndexTy, 0),
                                     ConstantInt::get(IndexTy, En.index())});
      Value *TargetElemPtrVal =
          Builder.CreateLoad(Builder.getPtrTy(), TargetElemPtrPtr);
      Value *TargetElemPtr = TargetElemPtrVal;
      if (NumIters > 1)
        TargetElemPtr =
            Builder.CreateGEP(Builder.getInt32Ty(), TargetElemPtr, Cnt);

      // *TargetElemPtr = SrcMediumVal;
      Value *SrcMediumValue =
          Builder.CreateLoad(CType, SrcMediumPtrVal, /*IsVolatile*/ true);
      Builder.CreateStore(SrcMediumValue, TargetElemPtr);
      Builder.CreateBr(W0MergeBB);

      emitBlock(W0ElseBB, Builder.GetInsertBlock()->getParent());
      Builder.CreateBr(W0MergeBB);

      emitBlock(W0MergeBB, Builder.GetInsertBlock()->getParent());

      if (NumIters > 1) {
        Cnt = Builder.CreateNSWAdd(
            Cnt, ConstantInt::get(Builder.getInt32Ty(), /*V=*/1));
        Builder.CreateStore(Cnt, CntAddr, /*Volatile=*/false);

        auto *CurFn = Builder.GetInsertBlock()->getParent();
        emitBranch(PrecondBB);
        emitBlock(ExitBB, CurFn);
      }
      RealTySize %= TySize;
    }
  }

  Builder.CreateRetVoid();
  Builder.restoreIP(SavedIP);

  return WcFunc;
}

Function *OpenMPIRBuilder::emitShuffleAndReduceFunction(
    ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
    AttributeList FuncAttrs) {
  LLVMContext &Ctx = M.getContext();
  FunctionType *FuncTy =
      FunctionType::get(Builder.getVoidTy(),
                        {Builder.getPtrTy(), Builder.getInt16Ty(),
                         Builder.getInt16Ty(), Builder.getInt16Ty()},
                        /* IsVarArg */ false);
  Function *SarFunc =
      Function::Create(FuncTy, GlobalVariable::InternalLinkage,
                       "_omp_reduction_shuffle_and_reduce_func", &M);
  SarFunc->setAttributes(FuncAttrs);
  SarFunc->addParamAttr(0, Attribute::NoUndef);
  SarFunc->addParamAttr(1, Attribute::NoUndef);
  SarFunc->addParamAttr(2, Attribute::NoUndef);
  SarFunc->addParamAttr(3, Attribute::NoUndef);
  SarFunc->addParamAttr(1, Attribute::SExt);
  SarFunc->addParamAttr(2, Attribute::SExt);
  SarFunc->addParamAttr(3, Attribute::SExt);
  BasicBlock *EntryBB = BasicBlock::Create(M.getContext(), "entry", SarFunc);
  Builder.SetInsertPoint(EntryBB);

  // Thread local Reduce list used to host the values of data to be reduced.
  Argument *ReduceListArg = SarFunc->getArg(0);
  // Current lane id; could be logical.
  Argument *LaneIDArg = SarFunc->getArg(1);
  // Offset of the remote source lane relative to the current lane.
  Argument *RemoteLaneOffsetArg = SarFunc->getArg(2);
  // Algorithm version.  This is expected to be known at compile time.
  Argument *AlgoVerArg = SarFunc->getArg(3);

  Type *ReduceListArgType = ReduceListArg->getType();
  Type *LaneIDArgType = LaneIDArg->getType();
  Type *LaneIDArgPtrType = LaneIDArg->getType()->getPointerTo();
  Value *ReduceListAlloca = Builder.CreateAlloca(
      ReduceListArgType, nullptr, ReduceListArg->getName() + ".addr");
  Value *LaneIdAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr,
                                             LaneIDArg->getName() + ".addr");
  Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(
      LaneIDArgType, nullptr, RemoteLaneOffsetArg->getName() + ".addr");
  Value *AlgoVerAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr,
                                              AlgoVerArg->getName() + ".addr");
  ArrayType *RedListArrayTy =
      ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());

  // Create a local thread-private variable to host the Reduce list
  // from a remote lane.
  Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
      RedListArrayTy, nullptr, ".omp.reduction.remote_reduce_list");

  Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      ReduceListAlloca, ReduceListArgType,
      ReduceListAlloca->getName() + ".ascast");
  Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      LaneIdAlloca, LaneIDArgPtrType, LaneIdAlloca->getName() + ".ascast");
  Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      RemoteLaneOffsetAlloca, LaneIDArgPtrType,
      RemoteLaneOffsetAlloca->getName() + ".ascast");
  Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      AlgoVerAlloca, LaneIDArgPtrType, AlgoVerAlloca->getName() + ".ascast");
  Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      RemoteReductionListAlloca, Builder.getPtrTy(),
      RemoteReductionListAlloca->getName() + ".ascast");

  Builder.CreateStore(ReduceListArg, ReduceListAddrCast);
  Builder.CreateStore(LaneIDArg, LaneIdAddrCast);
  Builder.CreateStore(RemoteLaneOffsetArg, RemoteLaneOffsetAddrCast);
  Builder.CreateStore(AlgoVerArg, AlgoVerAddrCast);

  Value *ReduceList = Builder.CreateLoad(ReduceListArgType, ReduceListAddrCast);
  Value *LaneId = Builder.CreateLoad(LaneIDArgType, LaneIdAddrCast);
  Value *RemoteLaneOffset =
      Builder.CreateLoad(LaneIDArgType, RemoteLaneOffsetAddrCast);
  Value *AlgoVer = Builder.CreateLoad(LaneIDArgType, AlgoVerAddrCast);

  InsertPointTy AllocaIP = getInsertPointAfterInstr(RemoteReductionListAlloca);

  // This loop iterates through the list of reduce elements and copies,
  // element by element, from a remote lane in the warp to RemoteReduceList,
  // hosted on the thread's stack.
  emitReductionListCopy(
      AllocaIP, CopyAction::RemoteLaneToThread, RedListArrayTy, ReductionInfos,
      ReduceList, RemoteListAddrCast, {RemoteLaneOffset, nullptr, nullptr});

  // The actions to be performed on the Remote Reduce list is dependent
  // on the algorithm version.
  //
  //  if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
  //  LaneId % 2 == 0 && Offset > 0):
  //    do the reduction value aggregation
  //
  //  The thread local variable Reduce list is mutated in place to host the
  //  reduced data, which is the aggregated value produced from local and
  //  remote lanes.
  //
  //  Note that AlgoVer is expected to be a constant integer known at compile
  //  time.
  //  When AlgoVer==0, the first conjunction evaluates to true, making
  //    the entire predicate true during compile time.
  //  When AlgoVer==1, the second conjunction has only the second part to be
  //    evaluated during runtime.  Other conjunctions evaluates to false
  //    during compile time.
  //  When AlgoVer==2, the third conjunction has only the second part to be
  //    evaluated during runtime.  Other conjunctions evaluates to false
  //    during compile time.
  Value *CondAlgo0 = Builder.CreateIsNull(AlgoVer);
  Value *Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
  Value *LaneComp = Builder.CreateICmpULT(LaneId, RemoteLaneOffset);
  Value *CondAlgo1 = Builder.CreateAnd(Algo1, LaneComp);
  Value *Algo2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(2));
  Value *LaneIdAnd1 = Builder.CreateAnd(LaneId, Builder.getInt16(1));
  Value *LaneIdComp = Builder.CreateIsNull(LaneIdAnd1);
  Value *Algo2AndLaneIdComp = Builder.CreateAnd(Algo2, LaneIdComp);
  Value *RemoteOffsetComp =
      Builder.CreateICmpSGT(RemoteLaneOffset, Builder.getInt16(0));
  Value *CondAlgo2 = Builder.CreateAnd(Algo2AndLaneIdComp, RemoteOffsetComp);
  Value *CA0OrCA1 = Builder.CreateOr(CondAlgo0, CondAlgo1);
  Value *CondReduce = Builder.CreateOr(CA0OrCA1, CondAlgo2);

  BasicBlock *ThenBB = BasicBlock::Create(Ctx, "then");
  BasicBlock *ElseBB = BasicBlock::Create(Ctx, "else");
  BasicBlock *MergeBB = BasicBlock::Create(Ctx, "ifcont");

  Builder.CreateCondBr(CondReduce, ThenBB, ElseBB);
  emitBlock(ThenBB, Builder.GetInsertBlock()->getParent());
  Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
      ReduceList, Builder.getPtrTy());
  Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
      RemoteListAddrCast, Builder.getPtrTy());
  Builder.CreateCall(ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr})
      ->addFnAttr(Attribute::NoUnwind);
  Builder.CreateBr(MergeBB);

  emitBlock(ElseBB, Builder.GetInsertBlock()->getParent());
  Builder.CreateBr(MergeBB);

  emitBlock(MergeBB, Builder.GetInsertBlock()->getParent());

  // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local
  // Reduce list.
  Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1));
  Value *LaneIdGtOffset = Builder.CreateICmpUGE(LaneId, RemoteLaneOffset);
  Value *CondCopy = Builder.CreateAnd(Algo1, LaneIdGtOffset);

  BasicBlock *CpyThenBB = BasicBlock::Create(Ctx, "then");
  BasicBlock *CpyElseBB = BasicBlock::Create(Ctx, "else");
  BasicBlock *CpyMergeBB = BasicBlock::Create(Ctx, "ifcont");
  Builder.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);

  emitBlock(CpyThenBB, Builder.GetInsertBlock()->getParent());
  emitReductionListCopy(AllocaIP, CopyAction::ThreadCopy, RedListArrayTy,
                        ReductionInfos, RemoteListAddrCast, ReduceList);
  Builder.CreateBr(CpyMergeBB);

  emitBlock(CpyElseBB, Builder.GetInsertBlock()->getParent());
  Builder.CreateBr(CpyMergeBB);

  emitBlock(CpyMergeBB, Builder.GetInsertBlock()->getParent());

  Builder.CreateRetVoid();

  return SarFunc;
}

Function *OpenMPIRBuilder::emitListToGlobalCopyFunction(
    ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
    AttributeList FuncAttrs) {
  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
  LLVMContext &Ctx = M.getContext();
  FunctionType *FuncTy = FunctionType::get(
      Builder.getVoidTy(),
      {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
      /* IsVarArg */ false);
  Function *LtGCFunc =
      Function::Create(FuncTy, GlobalVariable::InternalLinkage,
                       "_omp_reduction_list_to_global_copy_func", &M);
  LtGCFunc->setAttributes(FuncAttrs);
  LtGCFunc->addParamAttr(0, Attribute::NoUndef);
  LtGCFunc->addParamAttr(1, Attribute::NoUndef);
  LtGCFunc->addParamAttr(2, Attribute::NoUndef);

  BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGCFunc);
  Builder.SetInsertPoint(EntryBlock);

  // Buffer: global reduction buffer.
  Argument *BufferArg = LtGCFunc->getArg(0);
  // Idx: index of the buffer.
  Argument *IdxArg = LtGCFunc->getArg(1);
  // ReduceList: thread local Reduce list.
  Argument *ReduceListArg = LtGCFunc->getArg(2);

  Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
                                                BufferArg->getName() + ".addr");
  Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
                                             IdxArg->getName() + ".addr");
  Value *ReduceListArgAlloca = Builder.CreateAlloca(
      Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
  Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      BufferArgAlloca, Builder.getPtrTy(),
      BufferArgAlloca->getName() + ".ascast");
  Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
  Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      ReduceListArgAlloca, Builder.getPtrTy(),
      ReduceListArgAlloca->getName() + ".ascast");

  Builder.CreateStore(BufferArg, BufferArgAddrCast);
  Builder.CreateStore(IdxArg, IdxArgAddrCast);
  Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);

  Value *LocalReduceList =
      Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
  Value *BufferArgVal =
      Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
  Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
  Type *IndexTy = Builder.getIndexTy(
      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
  for (auto En : enumerate(ReductionInfos)) {
    const ReductionInfo &RI = En.value();
    auto *RedListArrayTy =
        ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
    // Reduce element = LocalReduceList[i]
    Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
        RedListArrayTy, LocalReduceList,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    // elemptr = ((CopyType*)(elemptrptr)) + I
    Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);

    // Global = Buffer.VD[Idx];
    Value *BufferVD =
        Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArgVal, Idxs);
    Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(
        ReductionsBufferTy, BufferVD, 0, En.index());

    switch (RI.EvaluationKind) {
    case EvalKind::Scalar: {
      Value *TargetElement = Builder.CreateLoad(RI.ElementType, ElemPtr);
      Builder.CreateStore(TargetElement, GlobVal);
      break;
    }
    case EvalKind::Complex: {
      Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
          RI.ElementType, ElemPtr, 0, 0, ".realp");
      Value *SrcReal = Builder.CreateLoad(
          RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");
      Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
          RI.ElementType, ElemPtr, 0, 1, ".imagp");
      Value *SrcImg = Builder.CreateLoad(
          RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");

      Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
          RI.ElementType, GlobVal, 0, 0, ".realp");
      Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
          RI.ElementType, GlobVal, 0, 1, ".imagp");
      Builder.CreateStore(SrcReal, DestRealPtr);
      Builder.CreateStore(SrcImg, DestImgPtr);
      break;
    }
    case EvalKind::Aggregate: {
      Value *SizeVal =
          Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));
      Builder.CreateMemCpy(
          GlobVal, M.getDataLayout().getPrefTypeAlign(RI.ElementType), ElemPtr,
          M.getDataLayout().getPrefTypeAlign(RI.ElementType), SizeVal, false);
      break;
    }
    }
  }

  Builder.CreateRetVoid();
  Builder.restoreIP(OldIP);
  return LtGCFunc;
}

Function *OpenMPIRBuilder::emitListToGlobalReduceFunction(
    ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
    Type *ReductionsBufferTy, AttributeList FuncAttrs) {
  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
  LLVMContext &Ctx = M.getContext();
  FunctionType *FuncTy = FunctionType::get(
      Builder.getVoidTy(),
      {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
      /* IsVarArg */ false);
  Function *LtGRFunc =
      Function::Create(FuncTy, GlobalVariable::InternalLinkage,
                       "_omp_reduction_list_to_global_reduce_func", &M);
  LtGRFunc->setAttributes(FuncAttrs);
  LtGRFunc->addParamAttr(0, Attribute::NoUndef);
  LtGRFunc->addParamAttr(1, Attribute::NoUndef);
  LtGRFunc->addParamAttr(2, Attribute::NoUndef);

  BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGRFunc);
  Builder.SetInsertPoint(EntryBlock);

  // Buffer: global reduction buffer.
  Argument *BufferArg = LtGRFunc->getArg(0);
  // Idx: index of the buffer.
  Argument *IdxArg = LtGRFunc->getArg(1);
  // ReduceList: thread local Reduce list.
  Argument *ReduceListArg = LtGRFunc->getArg(2);

  Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
                                                BufferArg->getName() + ".addr");
  Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
                                             IdxArg->getName() + ".addr");
  Value *ReduceListArgAlloca = Builder.CreateAlloca(
      Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
  auto *RedListArrayTy =
      ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());

  // 1. Build a list of reduction variables.
  // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
  Value *LocalReduceList =
      Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");

  Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      BufferArgAlloca, Builder.getPtrTy(),
      BufferArgAlloca->getName() + ".ascast");
  Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
  Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      ReduceListArgAlloca, Builder.getPtrTy(),
      ReduceListArgAlloca->getName() + ".ascast");
  Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      LocalReduceList, Builder.getPtrTy(),
      LocalReduceList->getName() + ".ascast");

  Builder.CreateStore(BufferArg, BufferArgAddrCast);
  Builder.CreateStore(IdxArg, IdxArgAddrCast);
  Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);

  Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
  Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
  Type *IndexTy = Builder.getIndexTy(
      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
  for (auto En : enumerate(ReductionInfos)) {
    Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
        RedListArrayTy, LocalReduceListAddrCast,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    Value *BufferVD =
        Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
    // Global = Buffer.VD[Idx];
    Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
        ReductionsBufferTy, BufferVD, 0, En.index());
    Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
  }

  // Call reduce_function(GlobalReduceList, ReduceList)
  Value *ReduceList =
      Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
  Builder.CreateCall(ReduceFn, {LocalReduceListAddrCast, ReduceList})
      ->addFnAttr(Attribute::NoUnwind);
  Builder.CreateRetVoid();
  Builder.restoreIP(OldIP);
  return LtGRFunc;
}

Function *OpenMPIRBuilder::emitGlobalToListCopyFunction(
    ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
    AttributeList FuncAttrs) {
  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
  LLVMContext &Ctx = M.getContext();
  FunctionType *FuncTy = FunctionType::get(
      Builder.getVoidTy(),
      {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
      /* IsVarArg */ false);
  Function *LtGCFunc =
      Function::Create(FuncTy, GlobalVariable::InternalLinkage,
                       "_omp_reduction_global_to_list_copy_func", &M);
  LtGCFunc->setAttributes(FuncAttrs);
  LtGCFunc->addParamAttr(0, Attribute::NoUndef);
  LtGCFunc->addParamAttr(1, Attribute::NoUndef);
  LtGCFunc->addParamAttr(2, Attribute::NoUndef);

  BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGCFunc);
  Builder.SetInsertPoint(EntryBlock);

  // Buffer: global reduction buffer.
  Argument *BufferArg = LtGCFunc->getArg(0);
  // Idx: index of the buffer.
  Argument *IdxArg = LtGCFunc->getArg(1);
  // ReduceList: thread local Reduce list.
  Argument *ReduceListArg = LtGCFunc->getArg(2);

  Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
                                                BufferArg->getName() + ".addr");
  Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
                                             IdxArg->getName() + ".addr");
  Value *ReduceListArgAlloca = Builder.CreateAlloca(
      Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
  Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      BufferArgAlloca, Builder.getPtrTy(),
      BufferArgAlloca->getName() + ".ascast");
  Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
  Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      ReduceListArgAlloca, Builder.getPtrTy(),
      ReduceListArgAlloca->getName() + ".ascast");
  Builder.CreateStore(BufferArg, BufferArgAddrCast);
  Builder.CreateStore(IdxArg, IdxArgAddrCast);
  Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);

  Value *LocalReduceList =
      Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
  Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
  Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
  Type *IndexTy = Builder.getIndexTy(
      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
  for (auto En : enumerate(ReductionInfos)) {
    const OpenMPIRBuilder::ReductionInfo &RI = En.value();
    auto *RedListArrayTy =
        ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
    // Reduce element = LocalReduceList[i]
    Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
        RedListArrayTy, LocalReduceList,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    // elemptr = ((CopyType*)(elemptrptr)) + I
    Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr);
    // Global = Buffer.VD[Idx];
    Value *BufferVD =
        Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
    Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
        ReductionsBufferTy, BufferVD, 0, En.index());

    switch (RI.EvaluationKind) {
    case EvalKind::Scalar: {
      Value *TargetElement = Builder.CreateLoad(RI.ElementType, GlobValPtr);
      Builder.CreateStore(TargetElement, ElemPtr);
      break;
    }
    case EvalKind::Complex: {
      Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
          RI.ElementType, GlobValPtr, 0, 0, ".realp");
      Value *SrcReal = Builder.CreateLoad(
          RI.ElementType->getStructElementType(0), SrcRealPtr, ".real");
      Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
          RI.ElementType, GlobValPtr, 0, 1, ".imagp");
      Value *SrcImg = Builder.CreateLoad(
          RI.ElementType->getStructElementType(1), SrcImgPtr, ".imag");

      Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
          RI.ElementType, ElemPtr, 0, 0, ".realp");
      Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
          RI.ElementType, ElemPtr, 0, 1, ".imagp");
      Builder.CreateStore(SrcReal, DestRealPtr);
      Builder.CreateStore(SrcImg, DestImgPtr);
      break;
    }
    case EvalKind::Aggregate: {
      Value *SizeVal =
          Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType));
      Builder.CreateMemCpy(
          ElemPtr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),
          GlobValPtr, M.getDataLayout().getPrefTypeAlign(RI.ElementType),
          SizeVal, false);
      break;
    }
    }
  }

  Builder.CreateRetVoid();
  Builder.restoreIP(OldIP);
  return LtGCFunc;
}

Function *OpenMPIRBuilder::emitGlobalToListReduceFunction(
    ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
    Type *ReductionsBufferTy, AttributeList FuncAttrs) {
  OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP();
  LLVMContext &Ctx = M.getContext();
  auto *FuncTy = FunctionType::get(
      Builder.getVoidTy(),
      {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
      /* IsVarArg */ false);
  Function *LtGRFunc =
      Function::Create(FuncTy, GlobalVariable::InternalLinkage,
                       "_omp_reduction_global_to_list_reduce_func", &M);
  LtGRFunc->setAttributes(FuncAttrs);
  LtGRFunc->addParamAttr(0, Attribute::NoUndef);
  LtGRFunc->addParamAttr(1, Attribute::NoUndef);
  LtGRFunc->addParamAttr(2, Attribute::NoUndef);

  BasicBlock *EntryBlock = BasicBlock::Create(Ctx, "entry", LtGRFunc);
  Builder.SetInsertPoint(EntryBlock);

  // Buffer: global reduction buffer.
  Argument *BufferArg = LtGRFunc->getArg(0);
  // Idx: index of the buffer.
  Argument *IdxArg = LtGRFunc->getArg(1);
  // ReduceList: thread local Reduce list.
  Argument *ReduceListArg = LtGRFunc->getArg(2);

  Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr,
                                                BufferArg->getName() + ".addr");
  Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr,
                                             IdxArg->getName() + ".addr");
  Value *ReduceListArgAlloca = Builder.CreateAlloca(
      Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr");
  ArrayType *RedListArrayTy =
      ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());

  // 1. Build a list of reduction variables.
  // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
  Value *LocalReduceList =
      Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list");

  Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      BufferArgAlloca, Builder.getPtrTy(),
      BufferArgAlloca->getName() + ".ascast");
  Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast");
  Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      ReduceListArgAlloca, Builder.getPtrTy(),
      ReduceListArgAlloca->getName() + ".ascast");
  Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
      LocalReduceList, Builder.getPtrTy(),
      LocalReduceList->getName() + ".ascast");

  Builder.CreateStore(BufferArg, BufferArgAddrCast);
  Builder.CreateStore(IdxArg, IdxArgAddrCast);
  Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast);

  Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast);
  Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)};
  Type *IndexTy = Builder.getIndexTy(
      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
  for (auto En : enumerate(ReductionInfos)) {
    Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
        RedListArrayTy, ReductionList,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    // Global = Buffer.VD[Idx];
    Value *BufferVD =
        Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs);
    Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
        ReductionsBufferTy, BufferVD, 0, En.index());
    Builder.CreateStore(GlobValPtr, TargetElementPtrPtr);
  }

  // Call reduce_function(ReduceList, GlobalReduceList)
  Value *ReduceList =
      Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast);
  Builder.CreateCall(ReduceFn, {ReduceList, ReductionList})
      ->addFnAttr(Attribute::NoUnwind);
  Builder.CreateRetVoid();
  Builder.restoreIP(OldIP);
  return LtGRFunc;
}

std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
  std::string Suffix =
      createPlatformSpecificName({"omp", "reduction", "reduction_func"});
  return (Name + Suffix).str();
}

Function *OpenMPIRBuilder::createReductionFunction(
    StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
    ReductionGenCBKind ReductionGenCBKind, AttributeList FuncAttrs) {
  auto *FuncTy = FunctionType::get(Builder.getVoidTy(),
                                   {Builder.getPtrTy(), Builder.getPtrTy()},
                                   /* IsVarArg */ false);
  std::string Name = getReductionFuncName(ReducerName);
  Function *ReductionFunc =
      Function::Create(FuncTy, GlobalVariable::InternalLinkage, Name, &M);
  ReductionFunc->setAttributes(FuncAttrs);
  ReductionFunc->addParamAttr(0, Attribute::NoUndef);
  ReductionFunc->addParamAttr(1, Attribute::NoUndef);
  BasicBlock *EntryBB =
      BasicBlock::Create(M.getContext(), "entry", ReductionFunc);
  Builder.SetInsertPoint(EntryBB);

  // Need to alloca memory here and deal with the pointers before getting
  // LHS/RHS pointers out
  Value *LHSArrayPtr = nullptr;
  Value *RHSArrayPtr = nullptr;
  Argument *Arg0 = ReductionFunc->getArg(0);
  Argument *Arg1 = ReductionFunc->getArg(1);
  Type *Arg0Type = Arg0->getType();
  Type *Arg1Type = Arg1->getType();

  Value *LHSAlloca =
      Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr");
  Value *RHSAlloca =
      Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr");
  Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      LHSAlloca, Arg0Type, LHSAlloca->getName() + ".ascast");
  Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
      RHSAlloca, Arg1Type, RHSAlloca->getName() + ".ascast");
  Builder.CreateStore(Arg0, LHSAddrCast);
  Builder.CreateStore(Arg1, RHSAddrCast);
  LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast);
  RHSArrayPtr = Builder.CreateLoad(Arg1Type, RHSAddrCast);

  Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), ReductionInfos.size());
  Type *IndexTy = Builder.getIndexTy(
      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
  SmallVector<Value *> LHSPtrs, RHSPtrs;
  for (auto En : enumerate(ReductionInfos)) {
    const ReductionInfo &RI = En.value();
    Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(
        RedArrayTy, RHSArrayPtr,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
    Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
        RHSI8Ptr, RI.PrivateVariable->getType(),
        RHSI8Ptr->getName() + ".ascast");

    Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(
        RedArrayTy, LHSArrayPtr,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
    Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
        LHSI8Ptr, RI.Variable->getType(), LHSI8Ptr->getName() + ".ascast");

    if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
      LHSPtrs.emplace_back(LHSPtr);
      RHSPtrs.emplace_back(RHSPtr);
    } else {
      Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
      Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
      Value *Reduced;
      RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
      if (!Builder.GetInsertBlock())
        return ReductionFunc;
      Builder.CreateStore(Reduced, LHSPtr);
    }
  }

  if (ReductionGenCBKind == ReductionGenCBKind::Clang)
    for (auto En : enumerate(ReductionInfos)) {
      unsigned Index = En.index();
      const ReductionInfo &RI = En.value();
      Value *LHSFixupPtr, *RHSFixupPtr;
      Builder.restoreIP(RI.ReductionGenClang(
          Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));

      // Fix the CallBack code genereated to use the correct Values for the LHS
      // and RHS
      LHSFixupPtr->replaceUsesWithIf(
          LHSPtrs[Index], [ReductionFunc](const Use &U) {
            return cast<Instruction>(U.getUser())->getParent()->getParent() ==
                   ReductionFunc;
          });
      RHSFixupPtr->replaceUsesWithIf(
          RHSPtrs[Index], [ReductionFunc](const Use &U) {
            return cast<Instruction>(U.getUser())->getParent()->getParent() ==
                   ReductionFunc;
          });
    }

  Builder.CreateRetVoid();
  return ReductionFunc;
}

static void
checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
                    bool IsGPU) {
  for (const OpenMPIRBuilder::ReductionInfo &RI : ReductionInfos) {
    (void)RI;
    assert(RI.Variable && "expected non-null variable");
    assert(RI.PrivateVariable && "expected non-null private variable");
    assert((RI.ReductionGen || RI.ReductionGenClang) &&
           "expected non-null reduction generator callback");
    if (!IsGPU) {
      assert(
          RI.Variable->getType() == RI.PrivateVariable->getType() &&
          "expected variables and their private equivalents to have the same "
          "type");
    }
    assert(RI.Variable->getType()->isPointerTy() &&
           "expected variables to be pointers");
  }
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductionsGPU(
    const LocationDescription &Loc, InsertPointTy AllocaIP,
    InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
    bool IsNoWait, bool IsTeamsReduction, bool HasDistribute,
    ReductionGenCBKind ReductionGenCBKind, std::optional<omp::GV> GridValue,
    unsigned ReductionBufNum, Value *SrcLocInfo) {
  if (!updateToLocation(Loc))
    return InsertPointTy();
  Builder.restoreIP(CodeGenIP);
  checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
  LLVMContext &Ctx = M.getContext();

  // Source location for the ident struct
  if (!SrcLocInfo) {
    uint32_t SrcLocStrSize;
    Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
    SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  }

  if (ReductionInfos.size() == 0)
    return Builder.saveIP();

  Function *CurFunc = Builder.GetInsertBlock()->getParent();
  AttributeList FuncAttrs;
  AttrBuilder AttrBldr(Ctx);
  for (auto Attr : CurFunc->getAttributes().getFnAttrs())
    AttrBldr.addAttribute(Attr);
  AttrBldr.removeAttribute(Attribute::OptimizeNone);
  FuncAttrs = FuncAttrs.addFnAttributes(Ctx, AttrBldr);

  Function *ReductionFunc = nullptr;
  CodeGenIP = Builder.saveIP();
  ReductionFunc =
      createReductionFunction(Builder.GetInsertBlock()->getParent()->getName(),
                              ReductionInfos, ReductionGenCBKind, FuncAttrs);
  Builder.restoreIP(CodeGenIP);

  // Set the grid value in the config needed for lowering later on
  if (GridValue.has_value())
    Config.setGridValue(GridValue.value());
  else
    Config.setGridValue(getGridValue(T, ReductionFunc));

  // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),
  // RedList, shuffle_reduce_func, interwarp_copy_func);
  // or
  // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);
  Value *Res;

  // 1. Build a list of reduction variables.
  // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
  auto Size = ReductionInfos.size();
  Type *PtrTy = PointerType::getUnqual(Ctx);
  Type *RedArrayTy = ArrayType::get(PtrTy, Size);
  CodeGenIP = Builder.saveIP();
  Builder.restoreIP(AllocaIP);
  Value *ReductionListAlloca =
      Builder.CreateAlloca(RedArrayTy, nullptr, ".omp.reduction.red_list");
  Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
      ReductionListAlloca, PtrTy, ReductionListAlloca->getName() + ".ascast");
  Builder.restoreIP(CodeGenIP);
  Type *IndexTy = Builder.getIndexTy(
      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
  for (auto En : enumerate(ReductionInfos)) {
    const ReductionInfo &RI = En.value();
    Value *ElemPtr = Builder.CreateInBoundsGEP(
        RedArrayTy, ReductionList,
        {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
    Value *CastElem =
        Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy);
    Builder.CreateStore(CastElem, ElemPtr);
  }
  CodeGenIP = Builder.saveIP();
  Function *SarFunc =
      emitShuffleAndReduceFunction(ReductionInfos, ReductionFunc, FuncAttrs);
  Function *WcFunc = emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs);
  Builder.restoreIP(CodeGenIP);

  Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy);

  unsigned MaxDataSize = 0;
  SmallVector<Type *> ReductionTypeArgs;
  for (auto En : enumerate(ReductionInfos)) {
    auto Size = M.getDataLayout().getTypeStoreSize(En.value().ElementType);
    if (Size > MaxDataSize)
      MaxDataSize = Size;
    ReductionTypeArgs.emplace_back(En.value().ElementType);
  }
  Value *ReductionDataSize =
      Builder.getInt64(MaxDataSize * ReductionInfos.size());
  if (!IsTeamsReduction) {
    Value *SarFuncCast =
        Builder.CreatePointerBitCastOrAddrSpaceCast(SarFunc, PtrTy);
    Value *WcFuncCast =
        Builder.CreatePointerBitCastOrAddrSpaceCast(WcFunc, PtrTy);
    Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,
                     WcFuncCast};
    Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(
        RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
    Res = Builder.CreateCall(Pv2Ptr, Args);
  } else {
    CodeGenIP = Builder.saveIP();
    StructType *ReductionsBufferTy = StructType::create(
        Ctx, ReductionTypeArgs, "struct._globalized_locals_ty");
    Function *RedFixedBuferFn = getOrCreateRuntimeFunctionPtr(
        RuntimeFunction::OMPRTL___kmpc_reduction_get_fixed_buffer);
    Function *LtGCFunc = emitListToGlobalCopyFunction(
        ReductionInfos, ReductionsBufferTy, FuncAttrs);
    Function *LtGRFunc = emitListToGlobalReduceFunction(
        ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);
    Function *GtLCFunc = emitGlobalToListCopyFunction(
        ReductionInfos, ReductionsBufferTy, FuncAttrs);
    Function *GtLRFunc = emitGlobalToListReduceFunction(
        ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);
    Builder.restoreIP(CodeGenIP);

    Value *KernelTeamsReductionPtr = Builder.CreateCall(
        RedFixedBuferFn, {}, "_openmp_teams_reductions_buffer_$_$ptr");

    Value *Args3[] = {SrcLocInfo,
                      KernelTeamsReductionPtr,
                      Builder.getInt32(ReductionBufNum),
                      ReductionDataSize,
                      RL,
                      SarFunc,
                      WcFunc,
                      LtGCFunc,
                      LtGRFunc,
                      GtLCFunc,
                      GtLRFunc};

    Function *TeamsReduceFn = getOrCreateRuntimeFunctionPtr(
        RuntimeFunction::OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2);
    Res = Builder.CreateCall(TeamsReduceFn, Args3);
  }

  // 5. Build if (res == 1)
  BasicBlock *ExitBB = BasicBlock::Create(Ctx, ".omp.reduction.done");
  BasicBlock *ThenBB = BasicBlock::Create(Ctx, ".omp.reduction.then");
  Value *Cond = Builder.CreateICmpEQ(Res, Builder.getInt32(1));
  Builder.CreateCondBr(Cond, ThenBB, ExitBB);

  // 6. Build then branch: where we have reduced values in the master
  //    thread in each team.
  //    __kmpc_end_reduce{_nowait}(<gtid>);
  //    break;
  emitBlock(ThenBB, CurFunc);

  // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
  for (auto En : enumerate(ReductionInfos)) {
    const ReductionInfo &RI = En.value();
    Value *LHS = RI.Variable;
    Value *RHS =
        Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy);

    if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
      Value *LHSPtr, *RHSPtr;
      Builder.restoreIP(RI.ReductionGenClang(Builder.saveIP(), En.index(),
                                             &LHSPtr, &RHSPtr, CurFunc));

      // Fix the CallBack code genereated to use the correct Values for the LHS
      // and RHS
      LHSPtr->replaceUsesWithIf(LHS, [ReductionFunc](const Use &U) {
        return cast<Instruction>(U.getUser())->getParent()->getParent() ==
               ReductionFunc;
      });
      RHSPtr->replaceUsesWithIf(RHS, [ReductionFunc](const Use &U) {
        return cast<Instruction>(U.getUser())->getParent()->getParent() ==
               ReductionFunc;
      });
    } else {
      assert(false && "Unhandled ReductionGenCBKind");
    }
  }
  emitBlock(ExitBB, CurFunc);

  Config.setEmitLLVMUsed();

  return Builder.saveIP();
}

static Function *getFreshReductionFunc(Module &M) {
  Type *VoidTy = Type::getVoidTy(M.getContext());
  Type *Int8PtrTy = PointerType::getUnqual(M.getContext());
  auto *FuncTy =
      FunctionType::get(VoidTy, {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ false);
  return Function::Create(FuncTy, GlobalVariable::InternalLinkage,
                          ".omp.reduction.func", &M);
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
                                  InsertPointTy AllocaIP,
                                  ArrayRef<ReductionInfo> ReductionInfos,
                                  ArrayRef<bool> IsByRef, bool IsNoWait) {
  assert(ReductionInfos.size() == IsByRef.size());
  for (const ReductionInfo &RI : ReductionInfos) {
    (void)RI;
    assert(RI.Variable && "expected non-null variable");
    assert(RI.PrivateVariable && "expected non-null private variable");
    assert(RI.ReductionGen && "expected non-null reduction generator callback");
    assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&
           "expected variables and their private equivalents to have the same "
           "type");
    assert(RI.Variable->getType()->isPointerTy() &&
           "expected variables to be pointers");
  }

  if (!updateToLocation(Loc))
    return InsertPointTy();

  BasicBlock *InsertBlock = Loc.IP.getBlock();
  BasicBlock *ContinuationBlock =
      InsertBlock->splitBasicBlock(Loc.IP.getPoint(), "reduce.finalize");
  InsertBlock->getTerminator()->eraseFromParent();

  // Create and populate array of type-erased pointers to private reduction
  // values.
  unsigned NumReductions = ReductionInfos.size();
  Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions);
  Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator());
  Value *RedArray = Builder.CreateAlloca(RedArrayTy, nullptr, "red.array");

  Builder.SetInsertPoint(InsertBlock, InsertBlock->end());

  for (auto En : enumerate(ReductionInfos)) {
    unsigned Index = En.index();
    const ReductionInfo &RI = En.value();
    Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
        RedArrayTy, RedArray, 0, Index, "red.array.elem." + Twine(Index));
    Builder.CreateStore(RI.PrivateVariable, RedArrayElemPtr);
  }

  // Emit a call to the runtime function that orchestrates the reduction.
  // Declare the reduction function in the process.
  Function *Func = Builder.GetInsertBlock()->getParent();
  Module *Module = Func->getParent();
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  bool CanGenerateAtomic = all_of(ReductionInfos, [](const ReductionInfo &RI) {
    return RI.AtomicReductionGen;
  });
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
                                  CanGenerateAtomic
                                      ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
                                      : IdentFlag(0));
  Value *ThreadId = getOrCreateThreadID(Ident);
  Constant *NumVariables = Builder.getInt32(NumReductions);
  const DataLayout &DL = Module->getDataLayout();
  unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);
  Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
  Function *ReductionFunc = getFreshReductionFunc(*Module);
  Value *Lock = getOMPCriticalRegionLock(".reduction");
  Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
      IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
               : RuntimeFunction::OMPRTL___kmpc_reduce);
  CallInst *ReduceCall =
      Builder.CreateCall(ReduceFunc,
                         {Ident, ThreadId, NumVariables, RedArraySize, RedArray,
                          ReductionFunc, Lock},
                         "reduce");

  // Create final reduction entry blocks for the atomic and non-atomic case.
  // Emit IR that dispatches control flow to one of the blocks based on the
  // reduction supporting the atomic mode.
  BasicBlock *NonAtomicRedBlock =
      BasicBlock::Create(Module->getContext(), "reduce.switch.nonatomic", Func);
  BasicBlock *AtomicRedBlock =
      BasicBlock::Create(Module->getContext(), "reduce.switch.atomic", Func);
  SwitchInst *Switch =
      Builder.CreateSwitch(ReduceCall, ContinuationBlock, /* NumCases */ 2);
  Switch->addCase(Builder.getInt32(1), NonAtomicRedBlock);
  Switch->addCase(Builder.getInt32(2), AtomicRedBlock);

  // Populate the non-atomic reduction using the elementwise reduction function.
  // This loads the elements from the global and private variables and reduces
  // them before storing back the result to the global variable.
  Builder.SetInsertPoint(NonAtomicRedBlock);
  for (auto En : enumerate(ReductionInfos)) {
    const ReductionInfo &RI = En.value();
    Type *ValueType = RI.ElementType;
    // We have one less load for by-ref case because that load is now inside of
    // the reduction region
    Value *RedValue = nullptr;
    if (!IsByRef[En.index()]) {
      RedValue = Builder.CreateLoad(ValueType, RI.Variable,
                                    "red.value." + Twine(En.index()));
    }
    Value *PrivateRedValue =
        Builder.CreateLoad(ValueType, RI.PrivateVariable,
                           "red.private.value." + Twine(En.index()));
    Value *Reduced;
    if (IsByRef[En.index()]) {
      Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RI.Variable,
                                        PrivateRedValue, Reduced));
    } else {
      Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RedValue,
                                        PrivateRedValue, Reduced));
    }
    if (!Builder.GetInsertBlock())
      return InsertPointTy();
    // for by-ref case, the load is inside of the reduction region
    if (!IsByRef[En.index()])
      Builder.CreateStore(Reduced, RI.Variable);
  }
  Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
      IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
               : RuntimeFunction::OMPRTL___kmpc_end_reduce);
  Builder.CreateCall(EndReduceFunc, {Ident, ThreadId, Lock});
  Builder.CreateBr(ContinuationBlock);

  // Populate the atomic reduction using the atomic elementwise reduction
  // function. There are no loads/stores here because they will be happening
  // inside the atomic elementwise reduction.
  Builder.SetInsertPoint(AtomicRedBlock);
  if (CanGenerateAtomic && llvm::none_of(IsByRef, [](bool P) { return P; })) {
    for (const ReductionInfo &RI : ReductionInfos) {
      Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
                                              RI.Variable, RI.PrivateVariable));
      if (!Builder.GetInsertBlock())
        return InsertPointTy();
    }
    Builder.CreateBr(ContinuationBlock);
  } else {
    Builder.CreateUnreachable();
  }

  // Populate the outlined reduction function using the elementwise reduction
  // function. Partial values are extracted from the type-erased array of
  // pointers to private variables.
  BasicBlock *ReductionFuncBlock =
      BasicBlock::Create(Module->getContext(), "", ReductionFunc);
  Builder.SetInsertPoint(ReductionFuncBlock);
  Value *LHSArrayPtr = ReductionFunc->getArg(0);
  Value *RHSArrayPtr = ReductionFunc->getArg(1);

  for (auto En : enumerate(ReductionInfos)) {
    const ReductionInfo &RI = En.value();
    Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
        RedArrayTy, LHSArrayPtr, 0, En.index());
    Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr);
    Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType());
    Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
    Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
        RedArrayTy, RHSArrayPtr, 0, En.index());
    Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr);
    Value *RHSPtr =
        Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
    Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
    Value *Reduced;
    Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
    if (!Builder.GetInsertBlock())
      return InsertPointTy();
    // store is inside of the reduction region when using by-ref
    if (!IsByRef[En.index()])
      Builder.CreateStore(Reduced, LHSPtr);
  }
  Builder.CreateRetVoid();

  Builder.SetInsertPoint(ContinuationBlock);
  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
                              BodyGenCallbackTy BodyGenCB,
                              FinalizeCallbackTy FiniCB) {

  if (!updateToLocation(Loc))
    return Loc.IP;

  Directive OMPD = Directive::OMPD_master;
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  Value *Args[] = {Ident, ThreadId};

  Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);
  Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);

  Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);
  Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);

  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
                              /*Conditional*/ true, /*hasFinalize*/ true);
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
                              BodyGenCallbackTy BodyGenCB,
                              FinalizeCallbackTy FiniCB, Value *Filter) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  Directive OMPD = Directive::OMPD_masked;
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  Value *Args[] = {Ident, ThreadId, Filter};
  Value *ArgsEnd[] = {Ident, ThreadId};

  Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_masked);
  Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);

  Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_masked);
  Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, ArgsEnd);

  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
                              /*Conditional*/ true, /*hasFinalize*/ true);
}

CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
    DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
    BasicBlock *PostInsertBefore, const Twine &Name) {
  Module *M = F->getParent();
  LLVMContext &Ctx = M->getContext();
  Type *IndVarTy = TripCount->getType();

  // Create the basic block structure.
  BasicBlock *Preheader =
      BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);
  BasicBlock *Header =
      BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);
  BasicBlock *Cond =
      BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);
  BasicBlock *Body =
      BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);
  BasicBlock *Latch =
      BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);
  BasicBlock *Exit =
      BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);
  BasicBlock *After =
      BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);

  // Use specified DebugLoc for new instructions.
  Builder.SetCurrentDebugLocation(DL);

  Builder.SetInsertPoint(Preheader);
  Builder.CreateBr(Header);

  Builder.SetInsertPoint(Header);
  PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
  IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
  Builder.CreateBr(Cond);

  Builder.SetInsertPoint(Cond);
  Value *Cmp =
      Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
  Builder.CreateCondBr(Cmp, Body, Exit);

  Builder.SetInsertPoint(Body);
  Builder.CreateBr(Latch);

  Builder.SetInsertPoint(Latch);
  Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
                                  "omp_" + Name + ".next", /*HasNUW=*/true);
  Builder.CreateBr(Header);
  IndVarPHI->addIncoming(Next, Latch);

  Builder.SetInsertPoint(Exit);
  Builder.CreateBr(After);

  // Remember and return the canonical control flow.
  LoopInfos.emplace_front();
  CanonicalLoopInfo *CL = &LoopInfos.front();

  CL->Header = Header;
  CL->Cond = Cond;
  CL->Latch = Latch;
  CL->Exit = Exit;

#ifndef NDEBUG
  CL->assertOK();
#endif
  return CL;
}

CanonicalLoopInfo *
OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
                                     LoopBodyGenCallbackTy BodyGenCB,
                                     Value *TripCount, const Twine &Name) {
  BasicBlock *BB = Loc.IP.getBlock();
  BasicBlock *NextBB = BB->getNextNode();

  CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),
                                             NextBB, NextBB, Name);
  BasicBlock *After = CL->getAfter();

  // If location is not set, don't connect the loop.
  if (updateToLocation(Loc)) {
    // Split the loop at the insertion point: Branch to the preheader and move
    // every following instruction to after the loop (the After BB). Also, the
    // new successor is the loop's after block.
    spliceBB(Builder, After, /*CreateBranch=*/false);
    Builder.CreateBr(CL->getPreheader());
  }

  // Emit the body content. We do it after connecting the loop to the CFG to
  // avoid that the callback encounters degenerate BBs.
  BodyGenCB(CL->getBodyIP(), CL->getIndVar());

#ifndef NDEBUG
  CL->assertOK();
#endif
  return CL;
}

CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
    const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
    Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
    InsertPointTy ComputeIP, const Twine &Name) {

  // Consider the following difficulties (assuming 8-bit signed integers):
  //  * Adding \p Step to the loop counter which passes \p Stop may overflow:
  //      DO I = 1, 100, 50
  ///  * A \p Step of INT_MIN cannot not be normalized to a positive direction:
  //      DO I = 100, 0, -128

  // Start, Stop and Step must be of the same integer type.
  auto *IndVarTy = cast<IntegerType>(Start->getType());
  assert(IndVarTy == Stop->getType() && "Stop type mismatch");
  assert(IndVarTy == Step->getType() && "Step type mismatch");

  LocationDescription ComputeLoc =
      ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
  updateToLocation(ComputeLoc);

  ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
  ConstantInt *One = ConstantInt::get(IndVarTy, 1);

  // Like Step, but always positive.
  Value *Incr = Step;

  // Distance between Start and Stop; always positive.
  Value *Span;

  // Condition whether there are no iterations are executed at all, e.g. because
  // UB < LB.
  Value *ZeroCmp;

  if (IsSigned) {
    // Ensure that increment is positive. If not, negate and invert LB and UB.
    Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);
    Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);
    Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);
    Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);
    Span = Builder.CreateSub(UB, LB, "", false, true);
    ZeroCmp = Builder.CreateICmp(
        InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, UB, LB);
  } else {
    Span = Builder.CreateSub(Stop, Start, "", true);
    ZeroCmp = Builder.CreateICmp(
        InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Stop, Start);
  }

  Value *CountIfLooping;
  if (InclusiveStop) {
    CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);
  } else {
    // Avoid incrementing past stop since it could overflow.
    Value *CountIfTwo = Builder.CreateAdd(
        Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);
    Value *OneCmp = Builder.CreateICmp(CmpInst::ICMP_ULE, Span, Incr);
    CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
  }
  Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
                                          "omp_" + Name + ".tripcount");

  auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
    Builder.restoreIP(CodeGenIP);
    Value *Span = Builder.CreateMul(IV, Step);
    Value *IndVar = Builder.CreateAdd(Span, Start);
    BodyGenCB(Builder.saveIP(), IndVar);
  };
  LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
  return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
}

// Returns an LLVM function to call for initializing loop bounds using OpenMP
// static scheduling depending on `type`. Only i32 and i64 are supported by the
// runtime. Always interpret integers as unsigned similarly to
// CanonicalLoopInfo.
static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
                                                  OpenMPIRBuilder &OMPBuilder) {
  unsigned Bitwidth = Ty->getIntegerBitWidth();
  if (Bitwidth == 32)
    return OMPBuilder.getOrCreateRuntimeFunction(
        M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
  if (Bitwidth == 64)
    return OMPBuilder.getOrCreateRuntimeFunction(
        M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
  llvm_unreachable("unknown OpenMP loop iterator bitwidth");
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
                                          InsertPointTy AllocaIP,
                                          bool NeedsBarrier) {
  assert(CLI->isValid() && "Requires a valid canonical loop");
  assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
         "Require dedicated allocate IP");

  // Set up the source location value for OpenMP runtime.
  Builder.restoreIP(CLI->getPreheaderIP());
  Builder.SetCurrentDebugLocation(DL);

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
  Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);

  // Declare useful OpenMP runtime functions.
  Value *IV = CLI->getIndVar();
  Type *IVTy = IV->getType();
  FunctionCallee StaticInit = getKmpcForStaticInitForType(IVTy, M, *this);
  FunctionCallee StaticFini =
      getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);

  // Allocate space for computed loop bounds as expected by the "init" function.
  Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());

  Type *I32Type = Type::getInt32Ty(M.getContext());
  Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
  Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
  Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
  Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");

  // At the end of the preheader, prepare for calling the "init" function by
  // storing the current loop bounds into the allocated space. A canonical loop
  // always iterates from 0 to trip-count with step 1. Note that "init" expects
  // and produces an inclusive upper bound.
  Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
  Constant *Zero = ConstantInt::get(IVTy, 0);
  Constant *One = ConstantInt::get(IVTy, 1);
  Builder.CreateStore(Zero, PLowerBound);
  Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
  Builder.CreateStore(UpperBound, PUpperBound);
  Builder.CreateStore(One, PStride);

  Value *ThreadNum = getOrCreateThreadID(SrcLoc);

  Constant *SchedulingType = ConstantInt::get(
      I32Type, static_cast<int>(OMPScheduleType::UnorderedStatic));

  // Call the "init" function and update the trip count of the loop with the
  // value it produced.
  Builder.CreateCall(StaticInit,
                     {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
                      PUpperBound, PStride, One, Zero});
  Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);
  Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);
  Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
  Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
  CLI->setTripCount(TripCount);

  // Update all uses of the induction variable except the one in the condition
  // block that compares it with the actual upper bound, and the increment in
  // the latch block.

  CLI->mapIndVar([&](Instruction *OldIV) -> Value * {
    Builder.SetInsertPoint(CLI->getBody(),
                           CLI->getBody()->getFirstInsertionPt());
    Builder.SetCurrentDebugLocation(DL);
    return Builder.CreateAdd(OldIV, LowerBound);
  });

  // In the "exit" block, call the "fini" function.
  Builder.SetInsertPoint(CLI->getExit(),
                         CLI->getExit()->getTerminator()->getIterator());
  Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});

  // Add the barrier if requested.
  if (NeedsBarrier)
    createBarrier(LocationDescription(Builder.saveIP(), DL),
                  omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
                  /* CheckCancelFlag */ false);

  InsertPointTy AfterIP = CLI->getAfterIP();
  CLI->invalidate();

  return AfterIP;
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
    bool NeedsBarrier, Value *ChunkSize) {
  assert(CLI->isValid() && "Requires a valid canonical loop");
  assert(ChunkSize && "Chunk size is required");

  LLVMContext &Ctx = CLI->getFunction()->getContext();
  Value *IV = CLI->getIndVar();
  Value *OrigTripCount = CLI->getTripCount();
  Type *IVTy = IV->getType();
  assert(IVTy->getIntegerBitWidth() <= 64 &&
         "Max supported tripcount bitwidth is 64 bits");
  Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(Ctx)
                                                        : Type::getInt64Ty(Ctx);
  Type *I32Type = Type::getInt32Ty(M.getContext());
  Constant *Zero = ConstantInt::get(InternalIVTy, 0);
  Constant *One = ConstantInt::get(InternalIVTy, 1);

  // Declare useful OpenMP runtime functions.
  FunctionCallee StaticInit =
      getKmpcForStaticInitForType(InternalIVTy, M, *this);
  FunctionCallee StaticFini =
      getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);

  // Allocate space for computed loop bounds as expected by the "init" function.
  Builder.restoreIP(AllocaIP);
  Builder.SetCurrentDebugLocation(DL);
  Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
  Value *PLowerBound =
      Builder.CreateAlloca(InternalIVTy, nullptr, "p.lowerbound");
  Value *PUpperBound =
      Builder.CreateAlloca(InternalIVTy, nullptr, "p.upperbound");
  Value *PStride = Builder.CreateAlloca(InternalIVTy, nullptr, "p.stride");

  // Set up the source location value for the OpenMP runtime.
  Builder.restoreIP(CLI->getPreheaderIP());
  Builder.SetCurrentDebugLocation(DL);

  // TODO: Detect overflow in ubsan or max-out with current tripcount.
  Value *CastedChunkSize =
      Builder.CreateZExtOrTrunc(ChunkSize, InternalIVTy, "chunksize");
  Value *CastedTripCount =
      Builder.CreateZExt(OrigTripCount, InternalIVTy, "tripcount");

  Constant *SchedulingType = ConstantInt::get(
      I32Type, static_cast<int>(OMPScheduleType::UnorderedStaticChunked));
  Builder.CreateStore(Zero, PLowerBound);
  Value *OrigUpperBound = Builder.CreateSub(CastedTripCount, One);
  Builder.CreateStore(OrigUpperBound, PUpperBound);
  Builder.CreateStore(One, PStride);

  // Call the "init" function and update the trip count of the loop with the
  // value it produced.
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
  Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadNum = getOrCreateThreadID(SrcLoc);
  Builder.CreateCall(StaticInit,
                     {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
                      /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
                      /*plower=*/PLowerBound, /*pupper=*/PUpperBound,
                      /*pstride=*/PStride, /*incr=*/One,
                      /*chunk=*/CastedChunkSize});

  // Load values written by the "init" function.
  Value *FirstChunkStart =
      Builder.CreateLoad(InternalIVTy, PLowerBound, "omp_firstchunk.lb");
  Value *FirstChunkStop =
      Builder.CreateLoad(InternalIVTy, PUpperBound, "omp_firstchunk.ub");
  Value *FirstChunkEnd = Builder.CreateAdd(FirstChunkStop, One);
  Value *ChunkRange =
      Builder.CreateSub(FirstChunkEnd, FirstChunkStart, "omp_chunk.range");
  Value *NextChunkStride =
      Builder.CreateLoad(InternalIVTy, PStride, "omp_dispatch.stride");

  // Create outer "dispatch" loop for enumerating the chunks.
  BasicBlock *DispatchEnter = splitBB(Builder, true);
  Value *DispatchCounter;
  CanonicalLoopInfo *DispatchCLI = createCanonicalLoop(
      {Builder.saveIP(), DL},
      [&](InsertPointTy BodyIP, Value *Counter) { DispatchCounter = Counter; },
      FirstChunkStart, CastedTripCount, NextChunkStride,
      /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
      "dispatch");

  // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
  // not have to preserve the canonical invariant.
  BasicBlock *DispatchBody = DispatchCLI->getBody();
  BasicBlock *DispatchLatch = DispatchCLI->getLatch();
  BasicBlock *DispatchExit = DispatchCLI->getExit();
  BasicBlock *DispatchAfter = DispatchCLI->getAfter();
  DispatchCLI->invalidate();

  // Rewire the original loop to become the chunk loop inside the dispatch loop.
  redirectTo(DispatchAfter, CLI->getAfter(), DL);
  redirectTo(CLI->getExit(), DispatchLatch, DL);
  redirectTo(DispatchBody, DispatchEnter, DL);

  // Prepare the prolog of the chunk loop.
  Builder.restoreIP(CLI->getPreheaderIP());
  Builder.SetCurrentDebugLocation(DL);

  // Compute the number of iterations of the chunk loop.
  Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
  Value *ChunkEnd = Builder.CreateAdd(DispatchCounter, ChunkRange);
  Value *IsLastChunk =
      Builder.CreateICmpUGE(ChunkEnd, CastedTripCount, "omp_chunk.is_last");
  Value *CountUntilOrigTripCount =
      Builder.CreateSub(CastedTripCount, DispatchCounter);
  Value *ChunkTripCount = Builder.CreateSelect(
      IsLastChunk, CountUntilOrigTripCount, ChunkRange, "omp_chunk.tripcount");
  Value *BackcastedChunkTC =
      Builder.CreateTrunc(ChunkTripCount, IVTy, "omp_chunk.tripcount.trunc");
  CLI->setTripCount(BackcastedChunkTC);

  // Update all uses of the induction variable except the one in the condition
  // block that compares it with the actual upper bound, and the increment in
  // the latch block.
  Value *BackcastedDispatchCounter =
      Builder.CreateTrunc(DispatchCounter, IVTy, "omp_dispatch.iv.trunc");
  CLI->mapIndVar([&](Instruction *) -> Value * {
    Builder.restoreIP(CLI->getBodyIP());
    return Builder.CreateAdd(IV, BackcastedDispatchCounter);
  });

  // In the "exit" block, call the "fini" function.
  Builder.SetInsertPoint(DispatchExit, DispatchExit->getFirstInsertionPt());
  Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});

  // Add the barrier if requested.
  if (NeedsBarrier)
    createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for,
                  /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);

#ifndef NDEBUG
  // Even though we currently do not support applying additional methods to it,
  // the chunk loop should remain a canonical loop.
  CLI->assertOK();
#endif

  return {DispatchAfter, DispatchAfter->getFirstInsertionPt()};
}

// Returns an LLVM function to call for executing an OpenMP static worksharing
// for loop depending on `type`. Only i32 and i64 are supported by the runtime.
// Always interpret integers as unsigned similarly to CanonicalLoopInfo.
static FunctionCallee
getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
                            WorksharingLoopType LoopType) {
  unsigned Bitwidth = Ty->getIntegerBitWidth();
  Module &M = OMPBuilder->M;
  switch (LoopType) {
  case WorksharingLoopType::ForStaticLoop:
    if (Bitwidth == 32)
      return OMPBuilder->getOrCreateRuntimeFunction(
          M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
    if (Bitwidth == 64)
      return OMPBuilder->getOrCreateRuntimeFunction(
          M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
    break;
  case WorksharingLoopType::DistributeStaticLoop:
    if (Bitwidth == 32)
      return OMPBuilder->getOrCreateRuntimeFunction(
          M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
    if (Bitwidth == 64)
      return OMPBuilder->getOrCreateRuntimeFunction(
          M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
    break;
  case WorksharingLoopType::DistributeForStaticLoop:
    if (Bitwidth == 32)
      return OMPBuilder->getOrCreateRuntimeFunction(
          M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
    if (Bitwidth == 64)
      return OMPBuilder->getOrCreateRuntimeFunction(
          M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
    break;
  }
  if (Bitwidth != 32 && Bitwidth != 64) {
    llvm_unreachable("Unknown OpenMP loop iterator bitwidth");
  }
  llvm_unreachable("Unknown type of OpenMP worksharing loop");
}

// Inserts a call to proper OpenMP Device RTL function which handles
// loop worksharing.
static void createTargetLoopWorkshareCall(
    OpenMPIRBuilder *OMPBuilder, WorksharingLoopType LoopType,
    BasicBlock *InsertBlock, Value *Ident, Value *LoopBodyArg,
    Type *ParallelTaskPtr, Value *TripCount, Function &LoopBodyFn) {
  Type *TripCountTy = TripCount->getType();
  Module &M = OMPBuilder->M;
  IRBuilder<> &Builder = OMPBuilder->Builder;
  FunctionCallee RTLFn =
      getKmpcForStaticLoopForType(TripCountTy, OMPBuilder, LoopType);
  SmallVector<Value *, 8> RealArgs;
  RealArgs.push_back(Ident);
  RealArgs.push_back(Builder.CreateBitCast(&LoopBodyFn, ParallelTaskPtr));
  RealArgs.push_back(LoopBodyArg);
  RealArgs.push_back(TripCount);
  if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
    RealArgs.push_back(ConstantInt::get(TripCountTy, 0));
    Builder.CreateCall(RTLFn, RealArgs);
    return;
  }
  FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction(
      M, omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
  Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});
  Value *NumThreads = Builder.CreateCall(RTLNumThreads, {});

  RealArgs.push_back(
      Builder.CreateZExtOrTrunc(NumThreads, TripCountTy, "num.threads.cast"));
  RealArgs.push_back(ConstantInt::get(TripCountTy, 0));
  if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
    RealArgs.push_back(ConstantInt::get(TripCountTy, 0));
  }

  Builder.CreateCall(RTLFn, RealArgs);
}

static void
workshareLoopTargetCallback(OpenMPIRBuilder *OMPIRBuilder,
                            CanonicalLoopInfo *CLI, Value *Ident,
                            Function &OutlinedFn, Type *ParallelTaskPtr,
                            const SmallVector<Instruction *, 4> &ToBeDeleted,
                            WorksharingLoopType LoopType) {
  IRBuilder<> &Builder = OMPIRBuilder->Builder;
  BasicBlock *Preheader = CLI->getPreheader();
  Value *TripCount = CLI->getTripCount();

  // After loop body outling, the loop body contains only set up
  // of loop body argument structure and the call to the outlined
  // loop body function. Firstly, we need to move setup of loop body args
  // into loop preheader.
  Preheader->splice(std::prev(Preheader->end()), CLI->getBody(),
                    CLI->getBody()->begin(), std::prev(CLI->getBody()->end()));

  // The next step is to remove the whole loop. We do not it need anymore.
  // That's why make an unconditional branch from loop preheader to loop
  // exit block
  Builder.restoreIP({Preheader, Preheader->end()});
  Preheader->getTerminator()->eraseFromParent();
  Builder.CreateBr(CLI->getExit());

  // Delete dead loop blocks
  OpenMPIRBuilder::OutlineInfo CleanUpInfo;
  SmallPtrSet<BasicBlock *, 32> RegionBlockSet;
  SmallVector<BasicBlock *, 32> BlocksToBeRemoved;
  CleanUpInfo.EntryBB = CLI->getHeader();
  CleanUpInfo.ExitBB = CLI->getExit();
  CleanUpInfo.collectBlocks(RegionBlockSet, BlocksToBeRemoved);
  DeleteDeadBlocks(BlocksToBeRemoved);

  // Find the instruction which corresponds to loop body argument structure
  // and remove the call to loop body function instruction.
  Value *LoopBodyArg;
  User *OutlinedFnUser = OutlinedFn.getUniqueUndroppableUser();
  assert(OutlinedFnUser &&
         "Expected unique undroppable user of outlined function");
  CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(OutlinedFnUser);
  assert(OutlinedFnCallInstruction && "Expected outlined function call");
  assert((OutlinedFnCallInstruction->getParent() == Preheader) &&
         "Expected outlined function call to be located in loop preheader");
  // Check in case no argument structure has been passed.
  if (OutlinedFnCallInstruction->arg_size() > 1)
    LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(1);
  else
    LoopBodyArg = Constant::getNullValue(Builder.getPtrTy());
  OutlinedFnCallInstruction->eraseFromParent();

  createTargetLoopWorkshareCall(OMPIRBuilder, LoopType, Preheader, Ident,
                                LoopBodyArg, ParallelTaskPtr, TripCount,
                                OutlinedFn);

  for (auto &ToBeDeletedItem : ToBeDeleted)
    ToBeDeletedItem->eraseFromParent();
  CLI->invalidate();
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::applyWorkshareLoopTarget(DebugLoc DL, CanonicalLoopInfo *CLI,
                                          InsertPointTy AllocaIP,
                                          WorksharingLoopType LoopType) {
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);

  OutlineInfo OI;
  OI.OuterAllocaBB = CLI->getPreheader();
  Function *OuterFn = CLI->getPreheader()->getParent();

  // Instructions which need to be deleted at the end of code generation
  SmallVector<Instruction *, 4> ToBeDeleted;

  OI.OuterAllocaBB = AllocaIP.getBlock();

  // Mark the body loop as region which needs to be extracted
  OI.EntryBB = CLI->getBody();
  OI.ExitBB = CLI->getLatch()->splitBasicBlock(CLI->getLatch()->begin(),
                                               "omp.prelatch", true);

  // Prepare loop body for extraction
  Builder.restoreIP({CLI->getPreheader(), CLI->getPreheader()->begin()});

  // Insert new loop counter variable which will be used only in loop
  // body.
  AllocaInst *NewLoopCnt = Builder.CreateAlloca(CLI->getIndVarType(), 0, "");
  Instruction *NewLoopCntLoad =
      Builder.CreateLoad(CLI->getIndVarType(), NewLoopCnt);
  // New loop counter instructions are redundant in the loop preheader when
  // code generation for workshare loop is finshed. That's why mark them as
  // ready for deletion.
  ToBeDeleted.push_back(NewLoopCntLoad);
  ToBeDeleted.push_back(NewLoopCnt);

  // Analyse loop body region. Find all input variables which are used inside
  // loop body region.
  SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
  SmallVector<BasicBlock *, 32> Blocks;
  OI.collectBlocks(ParallelRegionBlockSet, Blocks);
  SmallVector<BasicBlock *, 32> BlocksT(ParallelRegionBlockSet.begin(),
                                        ParallelRegionBlockSet.end());

  CodeExtractorAnalysisCache CEAC(*OuterFn);
  CodeExtractor Extractor(Blocks,
                          /* DominatorTree */ nullptr,
                          /* AggregateArgs */ true,
                          /* BlockFrequencyInfo */ nullptr,
                          /* BranchProbabilityInfo */ nullptr,
                          /* AssumptionCache */ nullptr,
                          /* AllowVarArgs */ true,
                          /* AllowAlloca */ true,
                          /* AllocationBlock */ CLI->getPreheader(),
                          /* Suffix */ ".omp_wsloop",
                          /* AggrArgsIn0AddrSpace */ true);

  BasicBlock *CommonExit = nullptr;
  SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;

  // Find allocas outside the loop body region which are used inside loop
  // body
  Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);

  // We need to model loop body region as the function f(cnt, loop_arg).
  // That's why we replace loop induction variable by the new counter
  // which will be one of loop body function argument
  SmallVector<User *> Users(CLI->getIndVar()->user_begin(),
                            CLI->getIndVar()->user_end());
  for (auto Use : Users) {
    if (Instruction *Inst = dyn_cast<Instruction>(Use)) {
      if (ParallelRegionBlockSet.count(Inst->getParent())) {
        Inst->replaceUsesOfWith(CLI->getIndVar(), NewLoopCntLoad);
      }
    }
  }
  // Make sure that loop counter variable is not merged into loop body
  // function argument structure and it is passed as separate variable
  OI.ExcludeArgsFromAggregate.push_back(NewLoopCntLoad);

  // PostOutline CB is invoked when loop body function is outlined and
  // loop body is replaced by call to outlined function. We need to add
  // call to OpenMP device rtl inside loop preheader. OpenMP device rtl
  // function will handle loop control logic.
  //
  OI.PostOutlineCB = [=, ToBeDeletedVec =
                             std::move(ToBeDeleted)](Function &OutlinedFn) {
    workshareLoopTargetCallback(this, CLI, Ident, OutlinedFn, ParallelTaskPtr,
                                ToBeDeletedVec, LoopType);
  };
  addOutlineInfo(std::move(OI));
  return CLI->getAfterIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoop(
    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
    bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,
    bool HasSimdModifier, bool HasMonotonicModifier,
    bool HasNonmonotonicModifier, bool HasOrderedClause,
    WorksharingLoopType LoopType) {
  if (Config.isTargetDevice())
    return applyWorkshareLoopTarget(DL, CLI, AllocaIP, LoopType);
  OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
      SchedKind, ChunkSize, HasSimdModifier, HasMonotonicModifier,
      HasNonmonotonicModifier, HasOrderedClause);

  bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
                   OMPScheduleType::ModifierOrdered;
  switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
  case OMPScheduleType::BaseStatic:
    assert(!ChunkSize && "No chunk size with static-chunked schedule");
    if (IsOrdered)
      return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
                                       NeedsBarrier, ChunkSize);
    // FIXME: Monotonicity ignored?
    return applyStaticWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier);

  case OMPScheduleType::BaseStaticChunked:
    if (IsOrdered)
      return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
                                       NeedsBarrier, ChunkSize);
    // FIXME: Monotonicity ignored?
    return applyStaticChunkedWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier,
                                           ChunkSize);

  case OMPScheduleType::BaseRuntime:
  case OMPScheduleType::BaseAuto:
  case OMPScheduleType::BaseGreedy:
  case OMPScheduleType::BaseBalanced:
  case OMPScheduleType::BaseSteal:
  case OMPScheduleType::BaseGuidedSimd:
  case OMPScheduleType::BaseRuntimeSimd:
    assert(!ChunkSize &&
           "schedule type does not support user-defined chunk sizes");
    [[fallthrough]];
  case OMPScheduleType::BaseDynamicChunked:
  case OMPScheduleType::BaseGuidedChunked:
  case OMPScheduleType::BaseGuidedIterativeChunked:
  case OMPScheduleType::BaseGuidedAnalyticalChunked:
  case OMPScheduleType::BaseStaticBalancedChunked:
    return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, EffectiveScheduleType,
                                     NeedsBarrier, ChunkSize);

  default:
    llvm_unreachable("Unknown/unimplemented schedule kind");
  }
}

/// Returns an LLVM function to call for initializing loop bounds using OpenMP
/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
/// the runtime. Always interpret integers as unsigned similarly to
/// CanonicalLoopInfo.
static FunctionCallee
getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
  unsigned Bitwidth = Ty->getIntegerBitWidth();
  if (Bitwidth == 32)
    return OMPBuilder.getOrCreateRuntimeFunction(
        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
  if (Bitwidth == 64)
    return OMPBuilder.getOrCreateRuntimeFunction(
        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
  llvm_unreachable("unknown OpenMP loop iterator bitwidth");
}

/// Returns an LLVM function to call for updating the next loop using OpenMP
/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
/// the runtime. Always interpret integers as unsigned similarly to
/// CanonicalLoopInfo.
static FunctionCallee
getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
  unsigned Bitwidth = Ty->getIntegerBitWidth();
  if (Bitwidth == 32)
    return OMPBuilder.getOrCreateRuntimeFunction(
        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
  if (Bitwidth == 64)
    return OMPBuilder.getOrCreateRuntimeFunction(
        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
  llvm_unreachable("unknown OpenMP loop iterator bitwidth");
}

/// Returns an LLVM function to call for finalizing the dynamic loop using
/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
/// interpret integers as unsigned similarly to CanonicalLoopInfo.
static FunctionCallee
getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
  unsigned Bitwidth = Ty->getIntegerBitWidth();
  if (Bitwidth == 32)
    return OMPBuilder.getOrCreateRuntimeFunction(
        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
  if (Bitwidth == 64)
    return OMPBuilder.getOrCreateRuntimeFunction(
        M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
  llvm_unreachable("unknown OpenMP loop iterator bitwidth");
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop(
    DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
    OMPScheduleType SchedType, bool NeedsBarrier, Value *Chunk) {
  assert(CLI->isValid() && "Requires a valid canonical loop");
  assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
         "Require dedicated allocate IP");
  assert(isValidWorkshareLoopScheduleType(SchedType) &&
         "Require valid schedule type");

  bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
                 OMPScheduleType::ModifierOrdered;

  // Set up the source location value for OpenMP runtime.
  Builder.SetCurrentDebugLocation(DL);

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
  Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);

  // Declare useful OpenMP runtime functions.
  Value *IV = CLI->getIndVar();
  Type *IVTy = IV->getType();
  FunctionCallee DynamicInit = getKmpcForDynamicInitForType(IVTy, M, *this);
  FunctionCallee DynamicNext = getKmpcForDynamicNextForType(IVTy, M, *this);

  // Allocate space for computed loop bounds as expected by the "init" function.
  Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca());
  Type *I32Type = Type::getInt32Ty(M.getContext());
  Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
  Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
  Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
  Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");

  // At the end of the preheader, prepare for calling the "init" function by
  // storing the current loop bounds into the allocated space. A canonical loop
  // always iterates from 0 to trip-count with step 1. Note that "init" expects
  // and produces an inclusive upper bound.
  BasicBlock *PreHeader = CLI->getPreheader();
  Builder.SetInsertPoint(PreHeader->getTerminator());
  Constant *One = ConstantInt::get(IVTy, 1);
  Builder.CreateStore(One, PLowerBound);
  Value *UpperBound = CLI->getTripCount();
  Builder.CreateStore(UpperBound, PUpperBound);
  Builder.CreateStore(One, PStride);

  BasicBlock *Header = CLI->getHeader();
  BasicBlock *Exit = CLI->getExit();
  BasicBlock *Cond = CLI->getCond();
  BasicBlock *Latch = CLI->getLatch();
  InsertPointTy AfterIP = CLI->getAfterIP();

  // The CLI will be "broken" in the code below, as the loop is no longer
  // a valid canonical loop.

  if (!Chunk)
    Chunk = One;

  Value *ThreadNum = getOrCreateThreadID(SrcLoc);

  Constant *SchedulingType =
      ConstantInt::get(I32Type, static_cast<int>(SchedType));

  // Call the "init" function.
  Builder.CreateCall(DynamicInit,
                     {SrcLoc, ThreadNum, SchedulingType, /* LowerBound */ One,
                      UpperBound, /* step */ One, Chunk});

  // An outer loop around the existing one.
  BasicBlock *OuterCond = BasicBlock::Create(
      PreHeader->getContext(), Twine(PreHeader->getName()) + ".outer.cond",
      PreHeader->getParent());
  // This needs to be 32-bit always, so can't use the IVTy Zero above.
  Builder.SetInsertPoint(OuterCond, OuterCond->getFirstInsertionPt());
  Value *Res =
      Builder.CreateCall(DynamicNext, {SrcLoc, ThreadNum, PLastIter,
                                       PLowerBound, PUpperBound, PStride});
  Constant *Zero32 = ConstantInt::get(I32Type, 0);
  Value *MoreWork = Builder.CreateCmp(CmpInst::ICMP_NE, Res, Zero32);
  Value *LowerBound =
      Builder.CreateSub(Builder.CreateLoad(IVTy, PLowerBound), One, "lb");
  Builder.CreateCondBr(MoreWork, Header, Exit);

  // Change PHI-node in loop header to use outer cond rather than preheader,
  // and set IV to the LowerBound.
  Instruction *Phi = &Header->front();
  auto *PI = cast<PHINode>(Phi);
  PI->setIncomingBlock(0, OuterCond);
  PI->setIncomingValue(0, LowerBound);

  // Then set the pre-header to jump to the OuterCond
  Instruction *Term = PreHeader->getTerminator();
  auto *Br = cast<BranchInst>(Term);
  Br->setSuccessor(0, OuterCond);

  // Modify the inner condition:
  // * Use the UpperBound returned from the DynamicNext call.
  // * jump to the loop outer loop when done with one of the inner loops.
  Builder.SetInsertPoint(Cond, Cond->getFirstInsertionPt());
  UpperBound = Builder.CreateLoad(IVTy, PUpperBound, "ub");
  Instruction *Comp = &*Builder.GetInsertPoint();
  auto *CI = cast<CmpInst>(Comp);
  CI->setOperand(1, UpperBound);
  // Redirect the inner exit to branch to outer condition.
  Instruction *Branch = &Cond->back();
  auto *BI = cast<BranchInst>(Branch);
  assert(BI->getSuccessor(1) == Exit);
  BI->setSuccessor(1, OuterCond);

  // Call the "fini" function if "ordered" is present in wsloop directive.
  if (Ordered) {
    Builder.SetInsertPoint(&Latch->back());
    FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(IVTy, M, *this);
    Builder.CreateCall(DynamicFini, {SrcLoc, ThreadNum});
  }

  // Add the barrier if requested.
  if (NeedsBarrier) {
    Builder.SetInsertPoint(&Exit->back());
    createBarrier(LocationDescription(Builder.saveIP(), DL),
                  omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
                  /* CheckCancelFlag */ false);
  }

  CLI->invalidate();
  return AfterIP;
}

/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
/// after this \p OldTarget will be orphaned.
static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
                                      BasicBlock *NewTarget, DebugLoc DL) {
  for (BasicBlock *Pred : make_early_inc_range(predecessors(OldTarget)))
    redirectTo(Pred, NewTarget, DL);
}

/// Determine which blocks in \p BBs are reachable from outside and remove the
/// ones that are not reachable from the function.
static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
  SmallPtrSet<BasicBlock *, 6> BBsToErase{BBs.begin(), BBs.end()};
  auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {
    for (Use &U : BB->uses()) {
      auto *UseInst = dyn_cast<Instruction>(U.getUser());
      if (!UseInst)
        continue;
      if (BBsToErase.count(UseInst->getParent()))
        continue;
      return true;
    }
    return false;
  };

  while (BBsToErase.remove_if(HasRemainingUses)) {
    // Try again if anything was removed.
  }

  SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());
  DeleteDeadBlocks(BBVec);
}

CanonicalLoopInfo *
OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
                               InsertPointTy ComputeIP) {
  assert(Loops.size() >= 1 && "At least one loop required");
  size_t NumLoops = Loops.size();

  // Nothing to do if there is already just one loop.
  if (NumLoops == 1)
    return Loops.front();

  CanonicalLoopInfo *Outermost = Loops.front();
  CanonicalLoopInfo *Innermost = Loops.back();
  BasicBlock *OrigPreheader = Outermost->getPreheader();
  BasicBlock *OrigAfter = Outermost->getAfter();
  Function *F = OrigPreheader->getParent();

  // Loop control blocks that may become orphaned later.
  SmallVector<BasicBlock *, 12> OldControlBBs;
  OldControlBBs.reserve(6 * Loops.size());
  for (CanonicalLoopInfo *Loop : Loops)
    Loop->collectControlBlocks(OldControlBBs);

  // Setup the IRBuilder for inserting the trip count computation.
  Builder.SetCurrentDebugLocation(DL);
  if (ComputeIP.isSet())
    Builder.restoreIP(ComputeIP);
  else
    Builder.restoreIP(Outermost->getPreheaderIP());

  // Derive the collapsed' loop trip count.
  // TODO: Find common/largest indvar type.
  Value *CollapsedTripCount = nullptr;
  for (CanonicalLoopInfo *L : Loops) {
    assert(L->isValid() &&
           "All loops to collapse must be valid canonical loops");
    Value *OrigTripCount = L->getTripCount();
    if (!CollapsedTripCount) {
      CollapsedTripCount = OrigTripCount;
      continue;
    }

    // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
    CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
                                           {}, /*HasNUW=*/true);
  }

  // Create the collapsed loop control flow.
  CanonicalLoopInfo *Result =
      createLoopSkeleton(DL, CollapsedTripCount, F,
                         OrigPreheader->getNextNode(), OrigAfter, "collapsed");

  // Build the collapsed loop body code.
  // Start with deriving the input loop induction variables from the collapsed
  // one, using a divmod scheme. To preserve the original loops' order, the
  // innermost loop use the least significant bits.
  Builder.restoreIP(Result->getBodyIP());

  Value *Leftover = Result->getIndVar();
  SmallVector<Value *> NewIndVars;
  NewIndVars.resize(NumLoops);
  for (int i = NumLoops - 1; i >= 1; --i) {
    Value *OrigTripCount = Loops[i]->getTripCount();

    Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);
    NewIndVars[i] = NewIndVar;

    Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);
  }
  // Outermost loop gets all the remaining bits.
  NewIndVars[0] = Leftover;

  // Construct the loop body control flow.
  // We progressively construct the branch structure following in direction of
  // the control flow, from the leading in-between code, the loop nest body, the
  // trailing in-between code, and rejoining the collapsed loop's latch.
  // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
  // the ContinueBlock is set, continue with that block. If ContinuePred, use
  // its predecessors as sources.
  BasicBlock *ContinueBlock = Result->getBody();
  BasicBlock *ContinuePred = nullptr;
  auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
                                                          BasicBlock *NextSrc) {
    if (ContinueBlock)
      redirectTo(ContinueBlock, Dest, DL);
    else
      redirectAllPredecessorsTo(ContinuePred, Dest, DL);

    ContinueBlock = nullptr;
    ContinuePred = NextSrc;
  };

  // The code before the nested loop of each level.
  // Because we are sinking it into the nest, it will be executed more often
  // that the original loop. More sophisticated schemes could keep track of what
  // the in-between code is and instantiate it only once per thread.
  for (size_t i = 0; i < NumLoops - 1; ++i)
    ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());

  // Connect the loop nest body.
  ContinueWith(Innermost->getBody(), Innermost->getLatch());

  // The code after the nested loop at each level.
  for (size_t i = NumLoops - 1; i > 0; --i)
    ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());

  // Connect the finished loop to the collapsed loop latch.
  ContinueWith(Result->getLatch(), nullptr);

  // Replace the input loops with the new collapsed loop.
  redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL);
  redirectTo(Result->getAfter(), Outermost->getAfter(), DL);

  // Replace the input loop indvars with the derived ones.
  for (size_t i = 0; i < NumLoops; ++i)
    Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);

  // Remove unused parts of the input loops.
  removeUnusedBlocksFromParent(OldControlBBs);

  for (CanonicalLoopInfo *L : Loops)
    L->invalidate();

#ifndef NDEBUG
  Result->assertOK();
#endif
  return Result;
}

std::vector<CanonicalLoopInfo *>
OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
                           ArrayRef<Value *> TileSizes) {
  assert(TileSizes.size() == Loops.size() &&
         "Must pass as many tile sizes as there are loops");
  int NumLoops = Loops.size();
  assert(NumLoops >= 1 && "At least one loop to tile required");

  CanonicalLoopInfo *OutermostLoop = Loops.front();
  CanonicalLoopInfo *InnermostLoop = Loops.back();
  Function *F = OutermostLoop->getBody()->getParent();
  BasicBlock *InnerEnter = InnermostLoop->getBody();
  BasicBlock *InnerLatch = InnermostLoop->getLatch();

  // Loop control blocks that may become orphaned later.
  SmallVector<BasicBlock *, 12> OldControlBBs;
  OldControlBBs.reserve(6 * Loops.size());
  for (CanonicalLoopInfo *Loop : Loops)
    Loop->collectControlBlocks(OldControlBBs);

  // Collect original trip counts and induction variable to be accessible by
  // index. Also, the structure of the original loops is not preserved during
  // the construction of the tiled loops, so do it before we scavenge the BBs of
  // any original CanonicalLoopInfo.
  SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
  for (CanonicalLoopInfo *L : Loops) {
    assert(L->isValid() && "All input loops must be valid canonical loops");
    OrigTripCounts.push_back(L->getTripCount());
    OrigIndVars.push_back(L->getIndVar());
  }

  // Collect the code between loop headers. These may contain SSA definitions
  // that are used in the loop nest body. To be usable with in the innermost
  // body, these BasicBlocks will be sunk into the loop nest body. That is,
  // these instructions may be executed more often than before the tiling.
  // TODO: It would be sufficient to only sink them into body of the
  // corresponding tile loop.
  SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
  for (int i = 0; i < NumLoops - 1; ++i) {
    CanonicalLoopInfo *Surrounding = Loops[i];
    CanonicalLoopInfo *Nested = Loops[i + 1];

    BasicBlock *EnterBB = Surrounding->getBody();
    BasicBlock *ExitBB = Nested->getHeader();
    InbetweenCode.emplace_back(EnterBB, ExitBB);
  }

  // Compute the trip counts of the floor loops.
  Builder.SetCurrentDebugLocation(DL);
  Builder.restoreIP(OutermostLoop->getPreheaderIP());
  SmallVector<Value *, 4> FloorCount, FloorRems;
  for (int i = 0; i < NumLoops; ++i) {
    Value *TileSize = TileSizes[i];
    Value *OrigTripCount = OrigTripCounts[i];
    Type *IVType = OrigTripCount->getType();

    Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);
    Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);

    // 0 if tripcount divides the tilesize, 1 otherwise.
    // 1 means we need an additional iteration for a partial tile.
    //
    // Unfortunately we cannot just use the roundup-formula
    //   (tripcount + tilesize - 1)/tilesize
    // because the summation might overflow. We do not want introduce undefined
    // behavior when the untiled loop nest did not.
    Value *FloorTripOverflow =
        Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));

    FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);
    FloorTripCount =
        Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
                          "omp_floor" + Twine(i) + ".tripcount", true);

    // Remember some values for later use.
    FloorCount.push_back(FloorTripCount);
    FloorRems.push_back(FloorTripRem);
  }

  // Generate the new loop nest, from the outermost to the innermost.
  std::vector<CanonicalLoopInfo *> Result;
  Result.reserve(NumLoops * 2);

  // The basic block of the surrounding loop that enters the nest generated
  // loop.
  BasicBlock *Enter = OutermostLoop->getPreheader();

  // The basic block of the surrounding loop where the inner code should
  // continue.
  BasicBlock *Continue = OutermostLoop->getAfter();

  // Where the next loop basic block should be inserted.
  BasicBlock *OutroInsertBefore = InnermostLoop->getExit();

  auto EmbeddNewLoop =
      [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
          Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
    CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
        DL, TripCount, F, InnerEnter, OutroInsertBefore, Name);
    redirectTo(Enter, EmbeddedLoop->getPreheader(), DL);
    redirectTo(EmbeddedLoop->getAfter(), Continue, DL);

    // Setup the position where the next embedded loop connects to this loop.
    Enter = EmbeddedLoop->getBody();
    Continue = EmbeddedLoop->getLatch();
    OutroInsertBefore = EmbeddedLoop->getLatch();
    return EmbeddedLoop;
  };

  auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
                                                  const Twine &NameBase) {
    for (auto P : enumerate(TripCounts)) {
      CanonicalLoopInfo *EmbeddedLoop =
          EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
      Result.push_back(EmbeddedLoop);
    }
  };

  EmbeddNewLoops(FloorCount, "floor");

  // Within the innermost floor loop, emit the code that computes the tile
  // sizes.
  Builder.SetInsertPoint(Enter->getTerminator());
  SmallVector<Value *, 4> TileCounts;
  for (int i = 0; i < NumLoops; ++i) {
    CanonicalLoopInfo *FloorLoop = Result[i];
    Value *TileSize = TileSizes[i];

    Value *FloorIsEpilogue =
        Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]);
    Value *TileTripCount =
        Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);

    TileCounts.push_back(TileTripCount);
  }

  // Create the tile loops.
  EmbeddNewLoops(TileCounts, "tile");

  // Insert the inbetween code into the body.
  BasicBlock *BodyEnter = Enter;
  BasicBlock *BodyEntered = nullptr;
  for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
    BasicBlock *EnterBB = P.first;
    BasicBlock *ExitBB = P.second;

    if (BodyEnter)
      redirectTo(BodyEnter, EnterBB, DL);
    else
      redirectAllPredecessorsTo(BodyEntered, EnterBB, DL);

    BodyEnter = nullptr;
    BodyEntered = ExitBB;
  }

  // Append the original loop nest body into the generated loop nest body.
  if (BodyEnter)
    redirectTo(BodyEnter, InnerEnter, DL);
  else
    redirectAllPredecessorsTo(BodyEntered, InnerEnter, DL);
  redirectAllPredecessorsTo(InnerLatch, Continue, DL);

  // Replace the original induction variable with an induction variable computed
  // from the tile and floor induction variables.
  Builder.restoreIP(Result.back()->getBodyIP());
  for (int i = 0; i < NumLoops; ++i) {
    CanonicalLoopInfo *FloorLoop = Result[i];
    CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
    Value *OrigIndVar = OrigIndVars[i];
    Value *Size = TileSizes[i];

    Value *Scale =
        Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);
    Value *Shift =
        Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
    OrigIndVar->replaceAllUsesWith(Shift);
  }

  // Remove unused parts of the original loops.
  removeUnusedBlocksFromParent(OldControlBBs);

  for (CanonicalLoopInfo *L : Loops)
    L->invalidate();

#ifndef NDEBUG
  for (CanonicalLoopInfo *GenL : Result)
    GenL->assertOK();
#endif
  return Result;
}

/// Attach metadata \p Properties to the basic block described by \p BB. If the
/// basic block already has metadata, the basic block properties are appended.
static void addBasicBlockMetadata(BasicBlock *BB,
                                  ArrayRef<Metadata *> Properties) {
  // Nothing to do if no property to attach.
  if (Properties.empty())
    return;

  LLVMContext &Ctx = BB->getContext();
  SmallVector<Metadata *> NewProperties;
  NewProperties.push_back(nullptr);

  // If the basic block already has metadata, prepend it to the new metadata.
  MDNode *Existing = BB->getTerminator()->getMetadata(LLVMContext::MD_loop);
  if (Existing)
    append_range(NewProperties, drop_begin(Existing->operands(), 1));

  append_range(NewProperties, Properties);
  MDNode *BasicBlockID = MDNode::getDistinct(Ctx, NewProperties);
  BasicBlockID->replaceOperandWith(0, BasicBlockID);

  BB->getTerminator()->setMetadata(LLVMContext::MD_loop, BasicBlockID);
}

/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
/// loop already has metadata, the loop properties are appended.
static void addLoopMetadata(CanonicalLoopInfo *Loop,
                            ArrayRef<Metadata *> Properties) {
  assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");

  // Attach metadata to the loop's latch
  BasicBlock *Latch = Loop->getLatch();
  assert(Latch && "A valid CanonicalLoopInfo must have a unique latch");
  addBasicBlockMetadata(Latch, Properties);
}

/// Attach llvm.access.group metadata to the memref instructions of \p Block
static void addSimdMetadata(BasicBlock *Block, MDNode *AccessGroup,
                            LoopInfo &LI) {
  for (Instruction &I : *Block) {
    if (I.mayReadOrWriteMemory()) {
      // TODO: This instruction may already have access group from
      // other pragmas e.g. #pragma clang loop vectorize.  Append
      // so that the existing metadata is not overwritten.
      I.setMetadata(LLVMContext::MD_access_group, AccessGroup);
    }
  }
}

void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
  LLVMContext &Ctx = Builder.getContext();
  addLoopMetadata(
      Loop, {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))});
}

void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
  LLVMContext &Ctx = Builder.getContext();
  addLoopMetadata(
      Loop, {
                MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
            });
}

void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
                                      Value *IfCond, ValueToValueMapTy &VMap,
                                      const Twine &NamePrefix) {
  Function *F = CanonicalLoop->getFunction();

  // Define where if branch should be inserted
  Instruction *SplitBefore;
  if (Instruction::classof(IfCond)) {
    SplitBefore = dyn_cast<Instruction>(IfCond);
  } else {
    SplitBefore = CanonicalLoop->getPreheader()->getTerminator();
  }

  // TODO: We should not rely on pass manager. Currently we use pass manager
  // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
  // object. We should have a method  which returns all blocks between
  // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
  FunctionAnalysisManager FAM;
  FAM.registerPass([]() { return DominatorTreeAnalysis(); });
  FAM.registerPass([]() { return LoopAnalysis(); });
  FAM.registerPass([]() { return PassInstrumentationAnalysis(); });

  // Get the loop which needs to be cloned
  LoopAnalysis LIA;
  LoopInfo &&LI = LIA.run(*F, FAM);
  Loop *L = LI.getLoopFor(CanonicalLoop->getHeader());

  // Create additional blocks for the if statement
  BasicBlock *Head = SplitBefore->getParent();
  Instruction *HeadOldTerm = Head->getTerminator();
  llvm::LLVMContext &C = Head->getContext();
  llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
      C, NamePrefix + ".if.then", Head->getParent(), Head->getNextNode());
  llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
      C, NamePrefix + ".if.else", Head->getParent(), CanonicalLoop->getExit());

  // Create if condition branch.
  Builder.SetInsertPoint(HeadOldTerm);
  Instruction *BrInstr =
      Builder.CreateCondBr(IfCond, ThenBlock, /*ifFalse*/ ElseBlock);
  InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};
  // Then block contains branch to omp loop which needs to be vectorized
  spliceBB(IP, ThenBlock, false);
  ThenBlock->replaceSuccessorsPhiUsesWith(Head, ThenBlock);

  Builder.SetInsertPoint(ElseBlock);

  // Clone loop for the else branch
  SmallVector<BasicBlock *, 8> NewBlocks;

  VMap[CanonicalLoop->getPreheader()] = ElseBlock;
  for (BasicBlock *Block : L->getBlocks()) {
    BasicBlock *NewBB = CloneBasicBlock(Block, VMap, "", F);
    NewBB->moveBefore(CanonicalLoop->getExit());
    VMap[Block] = NewBB;
    NewBlocks.push_back(NewBB);
  }
  remapInstructionsInBlocks(NewBlocks, VMap);
  Builder.CreateBr(NewBlocks.front());
}

unsigned
OpenMPIRBuilder::getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
                                           const StringMap<bool> &Features) {
  if (TargetTriple.isX86()) {
    if (Features.lookup("avx512f"))
      return 512;
    else if (Features.lookup("avx"))
      return 256;
    return 128;
  }
  if (TargetTriple.isPPC())
    return 128;
  if (TargetTriple.isWasm())
    return 128;
  return 0;
}

void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,
                                MapVector<Value *, Value *> AlignedVars,
                                Value *IfCond, OrderKind Order,
                                ConstantInt *Simdlen, ConstantInt *Safelen) {
  LLVMContext &Ctx = Builder.getContext();

  Function *F = CanonicalLoop->getFunction();

  // TODO: We should not rely on pass manager. Currently we use pass manager
  // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
  // object. We should have a method  which returns all blocks between
  // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
  FunctionAnalysisManager FAM;
  FAM.registerPass([]() { return DominatorTreeAnalysis(); });
  FAM.registerPass([]() { return LoopAnalysis(); });
  FAM.registerPass([]() { return PassInstrumentationAnalysis(); });

  LoopAnalysis LIA;
  LoopInfo &&LI = LIA.run(*F, FAM);

  Loop *L = LI.getLoopFor(CanonicalLoop->getHeader());
  if (AlignedVars.size()) {
    InsertPointTy IP = Builder.saveIP();
    Builder.SetInsertPoint(CanonicalLoop->getPreheader()->getTerminator());
    for (auto &AlignedItem : AlignedVars) {
      Value *AlignedPtr = AlignedItem.first;
      Value *Alignment = AlignedItem.second;
      Builder.CreateAlignmentAssumption(F->getDataLayout(),
                                        AlignedPtr, Alignment);
    }
    Builder.restoreIP(IP);
  }

  if (IfCond) {
    ValueToValueMapTy VMap;
    createIfVersion(CanonicalLoop, IfCond, VMap, "simd");
    // Add metadata to the cloned loop which disables vectorization
    Value *MappedLatch = VMap.lookup(CanonicalLoop->getLatch());
    assert(MappedLatch &&
           "Cannot find value which corresponds to original loop latch");
    assert(isa<BasicBlock>(MappedLatch) &&
           "Cannot cast mapped latch block value to BasicBlock");
    BasicBlock *NewLatchBlock = dyn_cast<BasicBlock>(MappedLatch);
    ConstantAsMetadata *BoolConst =
        ConstantAsMetadata::get(ConstantInt::getFalse(Type::getInt1Ty(Ctx)));
    addBasicBlockMetadata(
        NewLatchBlock,
        {MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
                           BoolConst})});
  }

  SmallSet<BasicBlock *, 8> Reachable;

  // Get the basic blocks from the loop in which memref instructions
  // can be found.
  // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
  // preferably without running any passes.
  for (BasicBlock *Block : L->getBlocks()) {
    if (Block == CanonicalLoop->getCond() ||
        Block == CanonicalLoop->getHeader())
      continue;
    Reachable.insert(Block);
  }

  SmallVector<Metadata *> LoopMDList;

  // In presence of finite 'safelen', it may be unsafe to mark all
  // the memory instructions parallel, because loop-carried
  // dependences of 'safelen' iterations are possible.
  // If clause order(concurrent) is specified then the memory instructions
  // are marked parallel even if 'safelen' is finite.
  if ((Safelen == nullptr) || (Order == OrderKind::OMP_ORDER_concurrent)) {
    // Add access group metadata to memory-access instructions.
    MDNode *AccessGroup = MDNode::getDistinct(Ctx, {});
    for (BasicBlock *BB : Reachable)
      addSimdMetadata(BB, AccessGroup, LI);
    // TODO:  If the loop has existing parallel access metadata, have
    // to combine two lists.
    LoopMDList.push_back(MDNode::get(
        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccessGroup}));
  }

  // Use the above access group metadata to create loop level
  // metadata, which should be distinct for each loop.
  ConstantAsMetadata *BoolConst =
      ConstantAsMetadata::get(ConstantInt::getTrue(Type::getInt1Ty(Ctx)));
  LoopMDList.push_back(MDNode::get(
      Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), BoolConst}));

  if (Simdlen || Safelen) {
    // If both simdlen and safelen clauses are specified, the value of the
    // simdlen parameter must be less than or equal to the value of the safelen
    // parameter. Therefore, use safelen only in the absence of simdlen.
    ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
    LoopMDList.push_back(
        MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.width"),
                          ConstantAsMetadata::get(VectorizeWidth)}));
  }

  addLoopMetadata(CanonicalLoop, LoopMDList);
}

/// Create the TargetMachine object to query the backend for optimization
/// preferences.
///
/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
/// needed for the LLVM pass pipline. We use some default options to avoid
/// having to pass too many settings from the frontend that probably do not
/// matter.
///
/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
/// method. If we are going to use TargetMachine for more purposes, especially
/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
/// might become be worth requiring front-ends to pass on their TargetMachine,
/// or at least cache it between methods. Note that while fontends such as Clang
/// have just a single main TargetMachine per translation unit, "target-cpu" and
/// "target-features" that determine the TargetMachine are per-function and can
/// be overrided using __attribute__((target("OPTIONS"))).
static std::unique_ptr<TargetMachine>
createTargetMachine(Function *F, CodeGenOptLevel OptLevel) {
  Module *M = F->getParent();

  StringRef CPU = F->getFnAttribute("target-cpu").getValueAsString();
  StringRef Features = F->getFnAttribute("target-features").getValueAsString();
  const std::string &Triple = M->getTargetTriple();

  std::string Error;
  const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
  if (!TheTarget)
    return {};

  llvm::TargetOptions Options;
  return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
      Triple, CPU, Features, Options, /*RelocModel=*/std::nullopt,
      /*CodeModel=*/std::nullopt, OptLevel));
}

/// Heuristically determine the best-performant unroll factor for \p CLI. This
/// depends on the target processor. We are re-using the same heuristics as the
/// LoopUnrollPass.
static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
  Function *F = CLI->getFunction();

  // Assume the user requests the most aggressive unrolling, even if the rest of
  // the code is optimized using a lower setting.
  CodeGenOptLevel OptLevel = CodeGenOptLevel::Aggressive;
  std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);

  FunctionAnalysisManager FAM;
  FAM.registerPass([]() { return TargetLibraryAnalysis(); });
  FAM.registerPass([]() { return AssumptionAnalysis(); });
  FAM.registerPass([]() { return DominatorTreeAnalysis(); });
  FAM.registerPass([]() { return LoopAnalysis(); });
  FAM.registerPass([]() { return ScalarEvolutionAnalysis(); });
  FAM.registerPass([]() { return PassInstrumentationAnalysis(); });
  TargetIRAnalysis TIRA;
  if (TM)
    TIRA = TargetIRAnalysis(
        [&](const Function &F) { return TM->getTargetTransformInfo(F); });
  FAM.registerPass([&]() { return TIRA; });

  TargetIRAnalysis::Result &&TTI = TIRA.run(*F, FAM);
  ScalarEvolutionAnalysis SEA;
  ScalarEvolution &&SE = SEA.run(*F, FAM);
  DominatorTreeAnalysis DTA;
  DominatorTree &&DT = DTA.run(*F, FAM);
  LoopAnalysis LIA;
  LoopInfo &&LI = LIA.run(*F, FAM);
  AssumptionAnalysis ACT;
  AssumptionCache &&AC = ACT.run(*F, FAM);
  OptimizationRemarkEmitter ORE{F};

  Loop *L = LI.getLoopFor(CLI->getHeader());
  assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop");

  TargetTransformInfo::UnrollingPreferences UP =
      gatherUnrollingPreferences(L, SE, TTI,
                                 /*BlockFrequencyInfo=*/nullptr,
                                 /*ProfileSummaryInfo=*/nullptr, ORE, static_cast<int>(OptLevel),
                                 /*UserThreshold=*/std::nullopt,
                                 /*UserCount=*/std::nullopt,
                                 /*UserAllowPartial=*/true,
                                 /*UserAllowRuntime=*/true,
                                 /*UserUpperBound=*/std::nullopt,
                                 /*UserFullUnrollMaxCount=*/std::nullopt);

  UP.Force = true;

  // Account for additional optimizations taking place before the LoopUnrollPass
  // would unroll the loop.
  UP.Threshold *= UnrollThresholdFactor;
  UP.PartialThreshold *= UnrollThresholdFactor;

  // Use normal unroll factors even if the rest of the code is optimized for
  // size.
  UP.OptSizeThreshold = UP.Threshold;
  UP.PartialOptSizeThreshold = UP.PartialThreshold;

  LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"
                    << "  Threshold=" << UP.Threshold << "\n"
                    << "  PartialThreshold=" << UP.PartialThreshold << "\n"
                    << "  OptSizeThreshold=" << UP.OptSizeThreshold << "\n"
                    << "  PartialOptSizeThreshold="
                    << UP.PartialOptSizeThreshold << "\n");

  // Disable peeling.
  TargetTransformInfo::PeelingPreferences PP =
      gatherPeelingPreferences(L, SE, TTI,
                               /*UserAllowPeeling=*/false,
                               /*UserAllowProfileBasedPeeling=*/false,
                               /*UnrollingSpecficValues=*/false);

  SmallPtrSet<const Value *, 32> EphValues;
  CodeMetrics::collectEphemeralValues(L, &AC, EphValues);

  // Assume that reads and writes to stack variables can be eliminated by
  // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
  // size.
  for (BasicBlock *BB : L->blocks()) {
    for (Instruction &I : *BB) {
      Value *Ptr;
      if (auto *Load = dyn_cast<LoadInst>(&I)) {
        Ptr = Load->getPointerOperand();
      } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
        Ptr = Store->getPointerOperand();
      } else
        continue;

      Ptr = Ptr->stripPointerCasts();

      if (auto *Alloca = dyn_cast<AllocaInst>(Ptr)) {
        if (Alloca->getParent() == &F->getEntryBlock())
          EphValues.insert(&I);
      }
    }
  }

  UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);

  // Loop is not unrollable if the loop contains certain instructions.
  if (!UCE.canUnroll()) {
    LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n");
    return 1;
  }

  LLVM_DEBUG(dbgs() << "Estimated loop size is " << UCE.getRolledLoopSize()
                    << "\n");

  // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
  // be able to use it.
  int TripCount = 0;
  int MaxTripCount = 0;
  bool MaxOrZero = false;
  unsigned TripMultiple = 0;

  bool UseUpperBound = false;
  computeUnrollCount(L, TTI, DT, &LI, &AC, SE, EphValues, &ORE, TripCount,
                     MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP,
                     UseUpperBound);
  unsigned Factor = UP.Count;
  LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n");

  // This function returns 1 to signal to not unroll a loop.
  if (Factor == 0)
    return 1;
  return Factor;
}

void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
                                        int32_t Factor,
                                        CanonicalLoopInfo **UnrolledCLI) {
  assert(Factor >= 0 && "Unroll factor must not be negative");

  Function *F = Loop->getFunction();
  LLVMContext &Ctx = F->getContext();

  // If the unrolled loop is not used for another loop-associated directive, it
  // is sufficient to add metadata for the LoopUnrollPass.
  if (!UnrolledCLI) {
    SmallVector<Metadata *, 2> LoopMetadata;
    LoopMetadata.push_back(
        MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")));

    if (Factor >= 1) {
      ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
          ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));
      LoopMetadata.push_back(MDNode::get(
          Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst}));
    }

    addLoopMetadata(Loop, LoopMetadata);
    return;
  }

  // Heuristically determine the unroll factor.
  if (Factor == 0)
    Factor = computeHeuristicUnrollFactor(Loop);

  // No change required with unroll factor 1.
  if (Factor == 1) {
    *UnrolledCLI = Loop;
    return;
  }

  assert(Factor >= 2 &&
         "unrolling only makes sense with a factor of 2 or larger");

  Type *IndVarTy = Loop->getIndVarType();

  // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
  // unroll the inner loop.
  Value *FactorVal =
      ConstantInt::get(IndVarTy, APInt(IndVarTy->getIntegerBitWidth(), Factor,
                                       /*isSigned=*/false));
  std::vector<CanonicalLoopInfo *> LoopNest =
      tileLoops(DL, {Loop}, {FactorVal});
  assert(LoopNest.size() == 2 && "Expect 2 loops after tiling");
  *UnrolledCLI = LoopNest[0];
  CanonicalLoopInfo *InnerLoop = LoopNest[1];

  // LoopUnrollPass can only fully unroll loops with constant trip count.
  // Unroll by the unroll factor with a fallback epilog for the remainder
  // iterations if necessary.
  ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
      ConstantInt::get(Type::getInt32Ty(Ctx), APInt(32, Factor)));
  addLoopMetadata(
      InnerLoop,
      {MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.enable")),
       MDNode::get(
           Ctx, {MDString::get(Ctx, "llvm.loop.unroll.count"), FactorConst})});

#ifndef NDEBUG
  (*UnrolledCLI)->assertOK();
#endif
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
                                   llvm::Value *BufSize, llvm::Value *CpyBuf,
                                   llvm::Value *CpyFn, llvm::Value *DidIt) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);

  llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);

  Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};

  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);
  Builder.CreateCall(Fn, Args);

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle(
    const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
    FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef<llvm::Value *> CPVars,
    ArrayRef<llvm::Function *> CPFuncs) {

  if (!updateToLocation(Loc))
    return Loc.IP;

  // If needed allocate and initialize `DidIt` with 0.
  // DidIt: flag variable: 1=single thread; 0=not single thread.
  llvm::Value *DidIt = nullptr;
  if (!CPVars.empty()) {
    DidIt = Builder.CreateAlloca(llvm::Type::getInt32Ty(Builder.getContext()));
    Builder.CreateStore(Builder.getInt32(0), DidIt);
  }

  Directive OMPD = Directive::OMPD_single;
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  Value *Args[] = {Ident, ThreadId};

  Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);
  Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);

  Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
  Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);

  auto FiniCBWrapper = [&](InsertPointTy IP) {
    FiniCB(IP);

    // The thread that executes the single region must set `DidIt` to 1.
    // This is used by __kmpc_copyprivate, to know if the caller is the
    // single thread or not.
    if (DidIt)
      Builder.CreateStore(Builder.getInt32(1), DidIt);
  };

  // generates the following:
  // if (__kmpc_single()) {
  //		.... single region ...
  // 		__kmpc_end_single
  // }
  // __kmpc_copyprivate
  // __kmpc_barrier

  EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCBWrapper,
                       /*Conditional*/ true,
                       /*hasFinalize*/ true);

  if (DidIt) {
    for (size_t I = 0, E = CPVars.size(); I < E; ++I)
      // NOTE BufSize is currently unused, so just pass 0.
      createCopyPrivate(LocationDescription(Builder.saveIP(), Loc.DL),
                        /*BufSize=*/ConstantInt::get(Int64, 0), CPVars[I],
                        CPFuncs[I], DidIt);
    // NOTE __kmpc_copyprivate already inserts a barrier
  } else if (!IsNowait)
    createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
                  omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
                  /* CheckCancelFlag */ false);
  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical(
    const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
    FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {

  if (!updateToLocation(Loc))
    return Loc.IP;

  Directive OMPD = Directive::OMPD_critical;
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  Value *LockVar = getOMPCriticalRegionLock(CriticalName);
  Value *Args[] = {Ident, ThreadId, LockVar};

  SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), std::end(Args));
  Function *RTFn = nullptr;
  if (HintInst) {
    // Add Hint to entry Args and create call
    EnterArgs.push_back(HintInst);
    RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);
  } else {
    RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);
  }
  Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs);

  Function *ExitRTLFn =
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);
  Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);

  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
                              /*Conditional*/ false, /*hasFinalize*/ true);
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
                                     InsertPointTy AllocaIP, unsigned NumLoops,
                                     ArrayRef<llvm::Value *> StoreValues,
                                     const Twine &Name, bool IsDependSource) {
  assert(
      llvm::all_of(StoreValues,
                   [](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&
      "OpenMP runtime requires depend vec with i64 type");

  if (!updateToLocation(Loc))
    return Loc.IP;

  // Allocate space for vector and generate alloc instruction.
  auto *ArrI64Ty = ArrayType::get(Int64, NumLoops);
  Builder.restoreIP(AllocaIP);
  AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI64Ty, nullptr, Name);
  ArgsBase->setAlignment(Align(8));
  Builder.restoreIP(Loc.IP);

  // Store the index value with offset in depend vector.
  for (unsigned I = 0; I < NumLoops; ++I) {
    Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
        ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(I)});
    StoreInst *STInst = Builder.CreateStore(StoreValues[I], DependAddrGEPIter);
    STInst->setAlignment(Align(8));
  }

  Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
      ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(0)});

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};

  Function *RTLFn = nullptr;
  if (IsDependSource)
    RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_post);
  else
    RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_doacross_wait);
  Builder.CreateCall(RTLFn, Args);

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd(
    const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
    FinalizeCallbackTy FiniCB, bool IsThreads) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  Directive OMPD = Directive::OMPD_ordered;
  Instruction *EntryCall = nullptr;
  Instruction *ExitCall = nullptr;

  if (IsThreads) {
    uint32_t SrcLocStrSize;
    Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
    Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
    Value *ThreadId = getOrCreateThreadID(Ident);
    Value *Args[] = {Ident, ThreadId};

    Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_ordered);
    EntryCall = Builder.CreateCall(EntryRTLFn, Args);

    Function *ExitRTLFn =
        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_ordered);
    ExitCall = Builder.CreateCall(ExitRTLFn, Args);
  }

  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
                              /*Conditional*/ false, /*hasFinalize*/ true);
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
    Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
    BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
    bool HasFinalize, bool IsCancellable) {

  if (HasFinalize)
    FinalizationStack.push_back({FiniCB, OMPD, IsCancellable});

  // Create inlined region's entry and body blocks, in preparation
  // for conditional creation
  BasicBlock *EntryBB = Builder.GetInsertBlock();
  Instruction *SplitPos = EntryBB->getTerminator();
  if (!isa_and_nonnull<BranchInst>(SplitPos))
    SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
  BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end");
  BasicBlock *FiniBB =
      EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize");

  Builder.SetInsertPoint(EntryBB->getTerminator());
  emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);

  // generate body
  BodyGenCB(/* AllocaIP */ InsertPointTy(),
            /* CodeGenIP */ Builder.saveIP());

  // emit exit call and do any needed finalization.
  auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
  assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
         FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
         "Unexpected control flow graph state!!");
  emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
  assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&
         "Unexpected Control Flow State!");
  MergeBlockIntoPredecessor(FiniBB);

  // If we are skipping the region of a non conditional, remove the exit
  // block, and clear the builder's insertion point.
  assert(SplitPos->getParent() == ExitBB &&
         "Unexpected Insertion point location!");
  auto merged = MergeBlockIntoPredecessor(ExitBB);
  BasicBlock *ExitPredBB = SplitPos->getParent();
  auto InsertBB = merged ? ExitPredBB : ExitBB;
  if (!isa_and_nonnull<BranchInst>(SplitPos))
    SplitPos->eraseFromParent();
  Builder.SetInsertPoint(InsertBB);

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
    Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
  // if nothing to do, Return current insertion point.
  if (!Conditional || !EntryCall)
    return Builder.saveIP();

  BasicBlock *EntryBB = Builder.GetInsertBlock();
  Value *CallBool = Builder.CreateIsNotNull(EntryCall);
  auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body");
  auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);

  // Emit thenBB and set the Builder's insertion point there for
  // body generation next. Place the block after the current block.
  Function *CurFn = EntryBB->getParent();
  CurFn->insert(std::next(EntryBB->getIterator()), ThenBB);

  // Move Entry branch to end of ThenBB, and replace with conditional
  // branch (If-stmt)
  Instruction *EntryBBTI = EntryBB->getTerminator();
  Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
  EntryBBTI->removeFromParent();
  Builder.SetInsertPoint(UI);
  Builder.Insert(EntryBBTI);
  UI->eraseFromParent();
  Builder.SetInsertPoint(ThenBB->getTerminator());

  // return an insertion point to ExitBB.
  return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
    omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
    bool HasFinalize) {

  Builder.restoreIP(FinIP);

  // If there is finalization to do, emit it before the exit call
  if (HasFinalize) {
    assert(!FinalizationStack.empty() &&
           "Unexpected finalization stack state!");

    FinalizationInfo Fi = FinalizationStack.pop_back_val();
    assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");

    Fi.FiniCB(FinIP);

    BasicBlock *FiniBB = FinIP.getBlock();
    Instruction *FiniBBTI = FiniBB->getTerminator();

    // set Builder IP for call creation
    Builder.SetInsertPoint(FiniBBTI);
  }

  if (!ExitCall)
    return Builder.saveIP();

  // place the Exitcall as last instruction before Finalization block terminator
  ExitCall->removeFromParent();
  Builder.Insert(ExitCall);

  return IRBuilder<>::InsertPoint(ExitCall->getParent(),
                                  ExitCall->getIterator());
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
    InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
    llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
  if (!IP.isSet())
    return IP;

  IRBuilder<>::InsertPointGuard IPG(Builder);

  // creates the following CFG structure
  //	   OMP_Entry : (MasterAddr != PrivateAddr)?
  //       F     T
  //       |      \
  //       |     copin.not.master
  //       |      /
  //       v     /
  //   copyin.not.master.end
  //		     |
  //         v
  //   OMP.Entry.Next

  BasicBlock *OMP_Entry = IP.getBlock();
  Function *CurFn = OMP_Entry->getParent();
  BasicBlock *CopyBegin =
      BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);
  BasicBlock *CopyEnd = nullptr;

  // If entry block is terminated, split to preserve the branch to following
  // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
  if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {
    CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),
                                         "copyin.not.master.end");
    OMP_Entry->getTerminator()->eraseFromParent();
  } else {
    CopyEnd =
        BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);
  }

  Builder.SetInsertPoint(OMP_Entry);
  Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
  Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
  Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
  Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);

  Builder.SetInsertPoint(CopyBegin);
  if (BranchtoEnd)
    Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));

  return Builder.saveIP();
}

CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
                                          Value *Size, Value *Allocator,
                                          std::string Name) {
  IRBuilder<>::InsertPointGuard IPG(Builder);
  updateToLocation(Loc);

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  Value *Args[] = {ThreadId, Size, Allocator};

  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);

  return Builder.CreateCall(Fn, Args, Name);
}

CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
                                         Value *Addr, Value *Allocator,
                                         std::string Name) {
  IRBuilder<>::InsertPointGuard IPG(Builder);
  updateToLocation(Loc);

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  Value *Args[] = {ThreadId, Addr, Allocator};
  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
  return Builder.CreateCall(Fn, Args, Name);
}

CallInst *OpenMPIRBuilder::createOMPInteropInit(
    const LocationDescription &Loc, Value *InteropVar,
    omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
    Value *DependenceAddress, bool HaveNowaitClause) {
  IRBuilder<>::InsertPointGuard IPG(Builder);
  updateToLocation(Loc);

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  if (Device == nullptr)
    Device = Constant::getAllOnesValue(Int32);
  Constant *InteropTypeVal = ConstantInt::get(Int32, (int)InteropType);
  if (NumDependences == nullptr) {
    NumDependences = ConstantInt::get(Int32, 0);
    PointerType *PointerTypeVar = PointerType::getUnqual(M.getContext());
    DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
  }
  Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
  Value *Args[] = {
      Ident,  ThreadId,       InteropVar,        InteropTypeVal,
      Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};

  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_init);

  return Builder.CreateCall(Fn, Args);
}

CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
    const LocationDescription &Loc, Value *InteropVar, Value *Device,
    Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
  IRBuilder<>::InsertPointGuard IPG(Builder);
  updateToLocation(Loc);

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  if (Device == nullptr)
    Device = Constant::getAllOnesValue(Int32);
  if (NumDependences == nullptr) {
    NumDependences = ConstantInt::get(Int32, 0);
    PointerType *PointerTypeVar = PointerType::getUnqual(M.getContext());
    DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
  }
  Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
  Value *Args[] = {
      Ident,          ThreadId,          InteropVar,         Device,
      NumDependences, DependenceAddress, HaveNowaitClauseVal};

  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_destroy);

  return Builder.CreateCall(Fn, Args);
}

CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
                                               Value *InteropVar, Value *Device,
                                               Value *NumDependences,
                                               Value *DependenceAddress,
                                               bool HaveNowaitClause) {
  IRBuilder<>::InsertPointGuard IPG(Builder);
  updateToLocation(Loc);
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  if (Device == nullptr)
    Device = Constant::getAllOnesValue(Int32);
  if (NumDependences == nullptr) {
    NumDependences = ConstantInt::get(Int32, 0);
    PointerType *PointerTypeVar = PointerType::getUnqual(M.getContext());
    DependenceAddress = ConstantPointerNull::get(PointerTypeVar);
  }
  Value *HaveNowaitClauseVal = ConstantInt::get(Int32, HaveNowaitClause);
  Value *Args[] = {
      Ident,          ThreadId,          InteropVar,         Device,
      NumDependences, DependenceAddress, HaveNowaitClauseVal};

  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___tgt_interop_use);

  return Builder.CreateCall(Fn, Args);
}

CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
    const LocationDescription &Loc, llvm::Value *Pointer,
    llvm::ConstantInt *Size, const llvm::Twine &Name) {
  IRBuilder<>::InsertPointGuard IPG(Builder);
  updateToLocation(Loc);

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Value *ThreadId = getOrCreateThreadID(Ident);
  Constant *ThreadPrivateCache =
      getOrCreateInternalVariable(Int8PtrPtr, Name.str());
  llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};

  Function *Fn =
      getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);

  return Builder.CreateCall(Fn, Args);
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
                                  int32_t MinThreadsVal, int32_t MaxThreadsVal,
                                  int32_t MinTeamsVal, int32_t MaxTeamsVal) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Constant *IsSPMDVal = ConstantInt::getSigned(
      Int8, IsSPMD ? OMP_TGT_EXEC_MODE_SPMD : OMP_TGT_EXEC_MODE_GENERIC);
  Constant *UseGenericStateMachineVal = ConstantInt::getSigned(Int8, !IsSPMD);
  Constant *MayUseNestedParallelismVal = ConstantInt::getSigned(Int8, true);
  Constant *DebugIndentionLevelVal = ConstantInt::getSigned(Int16, 0);

  Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();
  Function *Kernel = DebugKernelWrapper;

  // We need to strip the debug prefix to get the correct kernel name.
  StringRef KernelName = Kernel->getName();
  const std::string DebugPrefix = "_debug__";
  if (KernelName.ends_with(DebugPrefix)) {
    KernelName = KernelName.drop_back(DebugPrefix.length());
    Kernel = M.getFunction(KernelName);
    assert(Kernel && "Expected the real kernel to exist");
  }

  // Manifest the launch configuration in the metadata matching the kernel
  // environment.
  if (MinTeamsVal > 1 || MaxTeamsVal > 0)
    writeTeamsForKernel(T, *Kernel, MinTeamsVal, MaxTeamsVal);

  // For max values, < 0 means unset, == 0 means set but unknown.
  if (MaxThreadsVal < 0)
    MaxThreadsVal = std::max(
        int32_t(getGridValue(T, Kernel).GV_Default_WG_Size), MinThreadsVal);

  if (MaxThreadsVal > 0)
    writeThreadBoundsForKernel(T, *Kernel, MinThreadsVal, MaxThreadsVal);

  Constant *MinThreads = ConstantInt::getSigned(Int32, MinThreadsVal);
  Constant *MaxThreads = ConstantInt::getSigned(Int32, MaxThreadsVal);
  Constant *MinTeams = ConstantInt::getSigned(Int32, MinTeamsVal);
  Constant *MaxTeams = ConstantInt::getSigned(Int32, MaxTeamsVal);
  Constant *ReductionDataSize = ConstantInt::getSigned(Int32, 0);
  Constant *ReductionBufferLength = ConstantInt::getSigned(Int32, 0);

  Function *Fn = getOrCreateRuntimeFunctionPtr(
      omp::RuntimeFunction::OMPRTL___kmpc_target_init);
  const DataLayout &DL = Fn->getDataLayout();

  Twine DynamicEnvironmentName = KernelName + "_dynamic_environment";
  Constant *DynamicEnvironmentInitializer =
      ConstantStruct::get(DynamicEnvironment, {DebugIndentionLevelVal});
  GlobalVariable *DynamicEnvironmentGV = new GlobalVariable(
      M, DynamicEnvironment, /*IsConstant=*/false, GlobalValue::WeakODRLinkage,
      DynamicEnvironmentInitializer, DynamicEnvironmentName,
      /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
      DL.getDefaultGlobalsAddressSpace());
  DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);

  Constant *DynamicEnvironment =
      DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr
          ? DynamicEnvironmentGV
          : ConstantExpr::getAddrSpaceCast(DynamicEnvironmentGV,
                                           DynamicEnvironmentPtr);

  Constant *ConfigurationEnvironmentInitializer = ConstantStruct::get(
      ConfigurationEnvironment, {
                                    UseGenericStateMachineVal,
                                    MayUseNestedParallelismVal,
                                    IsSPMDVal,
                                    MinThreads,
                                    MaxThreads,
                                    MinTeams,
                                    MaxTeams,
                                    ReductionDataSize,
                                    ReductionBufferLength,
                                });
  Constant *KernelEnvironmentInitializer = ConstantStruct::get(
      KernelEnvironment, {
                             ConfigurationEnvironmentInitializer,
                             Ident,
                             DynamicEnvironment,
                         });
  std::string KernelEnvironmentName =
      (KernelName + "_kernel_environment").str();
  GlobalVariable *KernelEnvironmentGV = new GlobalVariable(
      M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,
      KernelEnvironmentInitializer, KernelEnvironmentName,
      /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
      DL.getDefaultGlobalsAddressSpace());
  KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);

  Constant *KernelEnvironment =
      KernelEnvironmentGV->getType() == KernelEnvironmentPtr
          ? KernelEnvironmentGV
          : ConstantExpr::getAddrSpaceCast(KernelEnvironmentGV,
                                           KernelEnvironmentPtr);
  Value *KernelLaunchEnvironment = DebugKernelWrapper->getArg(0);
  CallInst *ThreadKind =
      Builder.CreateCall(Fn, {KernelEnvironment, KernelLaunchEnvironment});

  Value *ExecUserCode = Builder.CreateICmpEQ(
      ThreadKind, Constant::getAllOnesValue(ThreadKind->getType()),
      "exec_user_code");

  // ThreadKind = __kmpc_target_init(...)
  // if (ThreadKind == -1)
  //   user_code
  // else
  //   return;

  auto *UI = Builder.CreateUnreachable();
  BasicBlock *CheckBB = UI->getParent();
  BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(UI, "user_code.entry");

  BasicBlock *WorkerExitBB = BasicBlock::Create(
      CheckBB->getContext(), "worker.exit", CheckBB->getParent());
  Builder.SetInsertPoint(WorkerExitBB);
  Builder.CreateRetVoid();

  auto *CheckBBTI = CheckBB->getTerminator();
  Builder.SetInsertPoint(CheckBBTI);
  Builder.CreateCondBr(ExecUserCode, UI->getParent(), WorkerExitBB);

  CheckBBTI->eraseFromParent();
  UI->eraseFromParent();

  // Continue in the "user_code" block, see diagram above and in
  // openmp/libomptarget/deviceRTLs/common/include/target.h .
  return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());
}

void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
                                         int32_t TeamsReductionDataSize,
                                         int32_t TeamsReductionBufferLength) {
  if (!updateToLocation(Loc))
    return;

  Function *Fn = getOrCreateRuntimeFunctionPtr(
      omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);

  Builder.CreateCall(Fn, {});

  if (!TeamsReductionBufferLength || !TeamsReductionDataSize)
    return;

  Function *Kernel = Builder.GetInsertBlock()->getParent();
  // We need to strip the debug prefix to get the correct kernel name.
  StringRef KernelName = Kernel->getName();
  const std::string DebugPrefix = "_debug__";
  if (KernelName.ends_with(DebugPrefix))
    KernelName = KernelName.drop_back(DebugPrefix.length());
  auto *KernelEnvironmentGV =
      M.getNamedGlobal((KernelName + "_kernel_environment").str());
  assert(KernelEnvironmentGV && "Expected kernel environment global\n");
  auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
  auto *NewInitializer = ConstantFoldInsertValueInstruction(
      KernelEnvironmentInitializer,
      ConstantInt::get(Int32, TeamsReductionDataSize), {0, 7});
  NewInitializer = ConstantFoldInsertValueInstruction(
      NewInitializer, ConstantInt::get(Int32, TeamsReductionBufferLength),
      {0, 8});
  KernelEnvironmentGV->setInitializer(NewInitializer);
}

static MDNode *getNVPTXMDNode(Function &Kernel, StringRef Name) {
  Module &M = *Kernel.getParent();
  NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
  for (auto *Op : MD->operands()) {
    if (Op->getNumOperands() != 3)
      continue;
    auto *KernelOp = dyn_cast<ConstantAsMetadata>(Op->getOperand(0));
    if (!KernelOp || KernelOp->getValue() != &Kernel)
      continue;
    auto *Prop = dyn_cast<MDString>(Op->getOperand(1));
    if (!Prop || Prop->getString() != Name)
      continue;
    return Op;
  }
  return nullptr;
}

static void updateNVPTXMetadata(Function &Kernel, StringRef Name, int32_t Value,
                                bool Min) {
  // Update the "maxntidx" metadata for NVIDIA, or add it.
  MDNode *ExistingOp = getNVPTXMDNode(Kernel, Name);
  if (ExistingOp) {
    auto *OldVal = cast<ConstantAsMetadata>(ExistingOp->getOperand(2));
    int32_t OldLimit = cast<ConstantInt>(OldVal->getValue())->getZExtValue();
    ExistingOp->replaceOperandWith(
        2, ConstantAsMetadata::get(ConstantInt::get(
               OldVal->getValue()->getType(),
               Min ? std::min(OldLimit, Value) : std::max(OldLimit, Value))));
  } else {
    LLVMContext &Ctx = Kernel.getContext();
    Metadata *MDVals[] = {ConstantAsMetadata::get(&Kernel),
                          MDString::get(Ctx, Name),
                          ConstantAsMetadata::get(
                              ConstantInt::get(Type::getInt32Ty(Ctx), Value))};
    // Append metadata to nvvm.annotations
    Module &M = *Kernel.getParent();
    NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
    MD->addOperand(MDNode::get(Ctx, MDVals));
  }
}

std::pair<int32_t, int32_t>
OpenMPIRBuilder::readThreadBoundsForKernel(const Triple &T, Function &Kernel) {
  int32_t ThreadLimit =
      Kernel.getFnAttributeAsParsedInteger("omp_target_thread_limit");

  if (T.isAMDGPU()) {
    const auto &Attr = Kernel.getFnAttribute("amdgpu-flat-work-group-size");
    if (!Attr.isValid() || !Attr.isStringAttribute())
      return {0, ThreadLimit};
    auto [LBStr, UBStr] = Attr.getValueAsString().split(',');
    int32_t LB, UB;
    if (!llvm::to_integer(UBStr, UB, 10))
      return {0, ThreadLimit};
    UB = ThreadLimit ? std::min(ThreadLimit, UB) : UB;
    if (!llvm::to_integer(LBStr, LB, 10))
      return {0, UB};
    return {LB, UB};
  }

  if (MDNode *ExistingOp = getNVPTXMDNode(Kernel, "maxntidx")) {
    auto *OldVal = cast<ConstantAsMetadata>(ExistingOp->getOperand(2));
    int32_t UB = cast<ConstantInt>(OldVal->getValue())->getZExtValue();
    return {0, ThreadLimit ? std::min(ThreadLimit, UB) : UB};
  }
  return {0, ThreadLimit};
}

void OpenMPIRBuilder::writeThreadBoundsForKernel(const Triple &T,
                                                 Function &Kernel, int32_t LB,
                                                 int32_t UB) {
  Kernel.addFnAttr("omp_target_thread_limit", std::to_string(UB));

  if (T.isAMDGPU()) {
    Kernel.addFnAttr("amdgpu-flat-work-group-size",
                     llvm::utostr(LB) + "," + llvm::utostr(UB));
    return;
  }

  updateNVPTXMetadata(Kernel, "maxntidx", UB, true);
}

std::pair<int32_t, int32_t>
OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {
  // TODO: Read from backend annotations if available.
  return {0, Kernel.getFnAttributeAsParsedInteger("omp_target_num_teams")};
}

void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,
                                          int32_t LB, int32_t UB) {
  if (T.isNVPTX())
    if (UB > 0)
      updateNVPTXMetadata(Kernel, "maxclusterrank", UB, true);
  if (T.isAMDGPU())
    Kernel.addFnAttr("amdgpu-max-num-workgroups", llvm::utostr(LB) + ",1,1");

  Kernel.addFnAttr("omp_target_num_teams", std::to_string(LB));
}

void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
    Function *OutlinedFn) {
  if (Config.isTargetDevice()) {
    OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage);
    // TODO: Determine if DSO local can be set to true.
    OutlinedFn->setDSOLocal(false);
    OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
    if (T.isAMDGCN())
      OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
  }
}

Constant *OpenMPIRBuilder::createOutlinedFunctionID(Function *OutlinedFn,
                                                    StringRef EntryFnIDName) {
  if (Config.isTargetDevice()) {
    assert(OutlinedFn && "The outlined function must exist if embedded");
    return OutlinedFn;
  }

  return new GlobalVariable(
      M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
      Constant::getNullValue(Builder.getInt8Ty()), EntryFnIDName);
}

Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,
                                                       StringRef EntryFnName) {
  if (OutlinedFn)
    return OutlinedFn;

  assert(!M.getGlobalVariable(EntryFnName, true) &&
         "Named kernel already exists?");
  return new GlobalVariable(
      M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,
      Constant::getNullValue(Builder.getInt8Ty()), EntryFnName);
}

void OpenMPIRBuilder::emitTargetRegionFunction(
    TargetRegionEntryInfo &EntryInfo,
    FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
    Function *&OutlinedFn, Constant *&OutlinedFnID) {

  SmallString<64> EntryFnName;
  OffloadInfoManager.getTargetRegionEntryFnName(EntryFnName, EntryInfo);

  OutlinedFn = Config.isTargetDevice() || !Config.openMPOffloadMandatory()
                   ? GenerateFunctionCallback(EntryFnName)
                   : nullptr;

  // If this target outline function is not an offload entry, we don't need to
  // register it. This may be in the case of a false if clause, or if there are
  // no OpenMP targets.
  if (!IsOffloadEntry)
    return;

  std::string EntryFnIDName =
      Config.isTargetDevice()
          ? std::string(EntryFnName)
          : createPlatformSpecificName({EntryFnName, "region_id"});

  OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFn,
                                              EntryFnName, EntryFnIDName);
}

Constant *OpenMPIRBuilder::registerTargetRegionFunction(
    TargetRegionEntryInfo &EntryInfo, Function *OutlinedFn,
    StringRef EntryFnName, StringRef EntryFnIDName) {
  if (OutlinedFn)
    setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
  auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
  auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
  OffloadInfoManager.registerTargetRegionEntryInfo(
      EntryInfo, EntryAddr, OutlinedFnID,
      OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
  return OutlinedFnID;
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
    const LocationDescription &Loc, InsertPointTy AllocaIP,
    InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
    TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB,
    omp::RuntimeFunction *MapperFunc,
    function_ref<InsertPointTy(InsertPointTy CodeGenIP, BodyGenTy BodyGenType)>
        BodyGenCB,
    function_ref<void(unsigned int, Value *)> DeviceAddrCB,
    function_ref<Value *(unsigned int)> CustomMapperCB, Value *SrcLocInfo) {
  if (!updateToLocation(Loc))
    return InsertPointTy();

  // Disable TargetData CodeGen on Device pass.
  if (Config.IsTargetDevice.value_or(false)) {
    if (BodyGenCB)
      Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv));
    return Builder.saveIP();
  }

  Builder.restoreIP(CodeGenIP);
  bool IsStandAlone = !BodyGenCB;
  MapInfosTy *MapInfo;
  // Generate the code for the opening of the data environment. Capture all the
  // arguments of the runtime call by reference because they are used in the
  // closing of the region.
  auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
    MapInfo = &GenMapInfoCB(Builder.saveIP());
    emitOffloadingArrays(AllocaIP, Builder.saveIP(), *MapInfo, Info,
                         /*IsNonContiguous=*/true, DeviceAddrCB,
                         CustomMapperCB);

    TargetDataRTArgs RTArgs;
    emitOffloadingArraysArgument(Builder, RTArgs, Info);

    // Emit the number of elements in the offloading arrays.
    Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs);

    // Source location for the ident struct
    if (!SrcLocInfo) {
      uint32_t SrcLocStrSize;
      Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
      SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
    }

    Value *OffloadingArgs[] = {SrcLocInfo,           DeviceID,
                               PointerNum,           RTArgs.BasePointersArray,
                               RTArgs.PointersArray, RTArgs.SizesArray,
                               RTArgs.MapTypesArray, RTArgs.MapNamesArray,
                               RTArgs.MappersArray};

    if (IsStandAlone) {
      assert(MapperFunc && "MapperFunc missing for standalone target data");
      Builder.CreateCall(getOrCreateRuntimeFunctionPtr(*MapperFunc),
                         OffloadingArgs);
    } else {
      Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
          omp::OMPRTL___tgt_target_data_begin_mapper);

      Builder.CreateCall(BeginMapperFunc, OffloadingArgs);

      for (auto DeviceMap : Info.DevicePtrInfoMap) {
        if (isa<AllocaInst>(DeviceMap.second.second)) {
          auto *LI =
              Builder.CreateLoad(Builder.getPtrTy(), DeviceMap.second.first);
          Builder.CreateStore(LI, DeviceMap.second.second);
        }
      }

      // If device pointer privatization is required, emit the body of the
      // region here. It will have to be duplicated: with and without
      // privatization.
      Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::Priv));
    }
  };

  // If we need device pointer privatization, we need to emit the body of the
  // region with no privatization in the 'else' branch of the conditional.
  // Otherwise, we don't have to do anything.
  auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
    Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv));
  };

  // Generate code for the closing of the data region.
  auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
    TargetDataRTArgs RTArgs;
    Info.EmitDebug = !MapInfo->Names.empty();
    emitOffloadingArraysArgument(Builder, RTArgs, Info, /*ForEndCall=*/true);

    // Emit the number of elements in the offloading arrays.
    Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs);

    // Source location for the ident struct
    if (!SrcLocInfo) {
      uint32_t SrcLocStrSize;
      Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
      SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
    }

    Value *OffloadingArgs[] = {SrcLocInfo,           DeviceID,
                               PointerNum,           RTArgs.BasePointersArray,
                               RTArgs.PointersArray, RTArgs.SizesArray,
                               RTArgs.MapTypesArray, RTArgs.MapNamesArray,
                               RTArgs.MappersArray};
    Function *EndMapperFunc =
        getOrCreateRuntimeFunctionPtr(omp::OMPRTL___tgt_target_data_end_mapper);

    Builder.CreateCall(EndMapperFunc, OffloadingArgs);
  };

  // We don't have to do anything to close the region if the if clause evaluates
  // to false.
  auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};

  if (BodyGenCB) {
    if (IfCond) {
      emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP);
    } else {
      BeginThenGen(AllocaIP, Builder.saveIP());
    }

    // If we don't require privatization of device pointers, we emit the body in
    // between the runtime calls. This avoids duplicating the body code.
    Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv));

    if (IfCond) {
      emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);
    } else {
      EndThenGen(AllocaIP, Builder.saveIP());
    }
  } else {
    if (IfCond) {
      emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP);
    } else {
      BeginThenGen(AllocaIP, Builder.saveIP());
    }
  }

  return Builder.saveIP();
}

FunctionCallee
OpenMPIRBuilder::createForStaticInitFunction(unsigned IVSize, bool IVSigned,
                                             bool IsGPUDistribute) {
  assert((IVSize == 32 || IVSize == 64) &&
         "IV size is not compatible with the omp runtime");
  RuntimeFunction Name;
  if (IsGPUDistribute)
    Name = IVSize == 32
               ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
                           : omp::OMPRTL___kmpc_distribute_static_init_4u)
               : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
                           : omp::OMPRTL___kmpc_distribute_static_init_8u);
  else
    Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
                                    : omp::OMPRTL___kmpc_for_static_init_4u)
                        : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
                                    : omp::OMPRTL___kmpc_for_static_init_8u);

  return getOrCreateRuntimeFunction(M, Name);
}

FunctionCallee OpenMPIRBuilder::createDispatchInitFunction(unsigned IVSize,
                                                           bool IVSigned) {
  assert((IVSize == 32 || IVSize == 64) &&
         "IV size is not compatible with the omp runtime");
  RuntimeFunction Name = IVSize == 32
                             ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
                                         : omp::OMPRTL___kmpc_dispatch_init_4u)
                             : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
                                         : omp::OMPRTL___kmpc_dispatch_init_8u);

  return getOrCreateRuntimeFunction(M, Name);
}

FunctionCallee OpenMPIRBuilder::createDispatchNextFunction(unsigned IVSize,
                                                           bool IVSigned) {
  assert((IVSize == 32 || IVSize == 64) &&
         "IV size is not compatible with the omp runtime");
  RuntimeFunction Name = IVSize == 32
                             ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
                                         : omp::OMPRTL___kmpc_dispatch_next_4u)
                             : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
                                         : omp::OMPRTL___kmpc_dispatch_next_8u);

  return getOrCreateRuntimeFunction(M, Name);
}

FunctionCallee OpenMPIRBuilder::createDispatchFiniFunction(unsigned IVSize,
                                                           bool IVSigned) {
  assert((IVSize == 32 || IVSize == 64) &&
         "IV size is not compatible with the omp runtime");
  RuntimeFunction Name = IVSize == 32
                             ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
                                         : omp::OMPRTL___kmpc_dispatch_fini_4u)
                             : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
                                         : omp::OMPRTL___kmpc_dispatch_fini_8u);

  return getOrCreateRuntimeFunction(M, Name);
}

FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {
  return getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_dispatch_deinit);
}

static Function *createOutlinedFunction(
    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, StringRef FuncName,
    SmallVectorImpl<Value *> &Inputs,
    OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
    OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {
  SmallVector<Type *> ParameterTypes;
  if (OMPBuilder.Config.isTargetDevice()) {
    // Add the "implicit" runtime argument we use to provide launch specific
    // information for target devices.
    auto *Int8PtrTy = PointerType::getUnqual(Builder.getContext());
    ParameterTypes.push_back(Int8PtrTy);

    // All parameters to target devices are passed as pointers
    // or i64. This assumes 64-bit address spaces/pointers.
    for (auto &Arg : Inputs)
      ParameterTypes.push_back(Arg->getType()->isPointerTy()
                                   ? Arg->getType()
                                   : Type::getInt64Ty(Builder.getContext()));
  } else {
    for (auto &Arg : Inputs)
      ParameterTypes.push_back(Arg->getType());
  }

  auto FuncType = FunctionType::get(Builder.getVoidTy(), ParameterTypes,
                                    /*isVarArg*/ false);
  auto Func = Function::Create(FuncType, GlobalValue::InternalLinkage, FuncName,
                               Builder.GetInsertBlock()->getModule());

  // Save insert point.
  auto OldInsertPoint = Builder.saveIP();

  // Generate the region into the function.
  BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
  Builder.SetInsertPoint(EntryBB);

  // Insert target init call in the device compilation pass.
  if (OMPBuilder.Config.isTargetDevice())
    Builder.restoreIP(OMPBuilder.createTargetInit(Builder, /*IsSPMD*/ false));

  BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();

  // As we embed the user code in the middle of our target region after we
  // generate entry code, we must move what allocas we can into the entry
  // block to avoid possible breaking optimisations for device
  if (OMPBuilder.Config.isTargetDevice())
    OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Func);

  // Insert target deinit call in the device compilation pass.
  Builder.restoreIP(CBFunc(Builder.saveIP(), Builder.saveIP()));
  if (OMPBuilder.Config.isTargetDevice())
    OMPBuilder.createTargetDeinit(Builder);

  // Insert return instruction.
  Builder.CreateRetVoid();

  // New Alloca IP at entry point of created device function.
  Builder.SetInsertPoint(EntryBB->getFirstNonPHI());
  auto AllocaIP = Builder.saveIP();

  Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg());

  // Skip the artificial dyn_ptr on the device.
  const auto &ArgRange =
      OMPBuilder.Config.isTargetDevice()
          ? make_range(Func->arg_begin() + 1, Func->arg_end())
          : Func->args();

  auto ReplaceValue = [](Value *Input, Value *InputCopy, Function *Func) {
    // Things like GEP's can come in the form of Constants. Constants and
    // ConstantExpr's do not have access to the knowledge of what they're
    // contained in, so we must dig a little to find an instruction so we
    // can tell if they're used inside of the function we're outlining. We
    // also replace the original constant expression with a new instruction
    // equivalent; an instruction as it allows easy modification in the
    // following loop, as we can now know the constant (instruction) is
    // owned by our target function and replaceUsesOfWith can now be invoked
    // on it (cannot do this with constants it seems). A brand new one also
    // allows us to be cautious as it is perhaps possible the old expression
    // was used inside of the function but exists and is used externally
    // (unlikely by the nature of a Constant, but still).
    // NOTE: We cannot remove dead constants that have been rewritten to
    // instructions at this stage, we run the risk of breaking later lowering
    // by doing so as we could still be in the process of lowering the module
    // from MLIR to LLVM-IR and the MLIR lowering may still require the original
    // constants we have created rewritten versions of.
    if (auto *Const = dyn_cast<Constant>(Input))
      convertUsersOfConstantsToInstructions(Const, Func, false);

    // Collect all the instructions
    for (User *User : make_early_inc_range(Input->users()))
      if (auto *Instr = dyn_cast<Instruction>(User))
        if (Instr->getFunction() == Func)
          Instr->replaceUsesOfWith(Input, InputCopy);
  };

  SmallVector<std::pair<Value *, Value *>> DeferredReplacement;

  // Rewrite uses of input valus to parameters.
  for (auto InArg : zip(Inputs, ArgRange)) {
    Value *Input = std::get<0>(InArg);
    Argument &Arg = std::get<1>(InArg);
    Value *InputCopy = nullptr;

    Builder.restoreIP(
        ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP()));

    // In certain cases a Global may be set up for replacement, however, this
    // Global may be used in multiple arguments to the kernel, just segmented
    // apart, for example, if we have a global array, that is sectioned into
    // multiple mappings (technically not legal in OpenMP, but there is a case
    // in Fortran for Common Blocks where this is neccesary), we will end up
    // with GEP's into this array inside the kernel, that refer to the Global
    // but are technically seperate arguments to the kernel for all intents and
    // purposes. If we have mapped a segment that requires a GEP into the 0-th
    // index, it will fold into an referal to the Global, if we then encounter
    // this folded GEP during replacement all of the references to the
    // Global in the kernel will be replaced with the argument we have generated
    // that corresponds to it, including any other GEP's that refer to the
    // Global that may be other arguments. This will invalidate all of the other
    // preceding mapped arguments that refer to the same global that may be
    // seperate segments. To prevent this, we defer global processing until all
    // other processing has been performed.
    if (llvm::isa<llvm::GlobalValue>(std::get<0>(InArg)) ||
        llvm::isa<llvm::GlobalObject>(std::get<0>(InArg)) ||
        llvm::isa<llvm::GlobalVariable>(std::get<0>(InArg))) {
      DeferredReplacement.push_back(std::make_pair(Input, InputCopy));
      continue;
    }

    ReplaceValue(Input, InputCopy, Func);
  }

  // Replace all of our deferred Input values, currently just Globals.
  for (auto Deferred : DeferredReplacement)
    ReplaceValue(std::get<0>(Deferred), std::get<1>(Deferred), Func);

  // Restore insert point.
  Builder.restoreIP(OldInsertPoint);

  return Func;
}

/// Create an entry point for a target task with the following.
/// It'll have the following signature
/// void @.omp_target_task_proxy_func(i32 %thread.id, ptr %task)
/// This function is called from emitTargetTask once the
/// code to launch the target kernel has been outlined already.
static Function *emitTargetTaskProxyFunction(OpenMPIRBuilder &OMPBuilder,
                                             IRBuilderBase &Builder,
                                             CallInst *StaleCI) {
  Module &M = OMPBuilder.M;
  // KernelLaunchFunction is the target launch function, i.e.
  // the function that sets up kernel arguments and calls
  // __tgt_target_kernel to launch the kernel on the device.
  //
  Function *KernelLaunchFunction = StaleCI->getCalledFunction();

  // StaleCI is the CallInst which is the call to the outlined
  // target kernel launch function. If there are values that the
  // outlined function uses then these are aggregated into a structure
  // which is passed as the second argument. If not, then there's
  // only one argument, the threadID. So, StaleCI can be
  //
  // %structArg = alloca { ptr, ptr }, align 8
  // %gep_ = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 0
  // store ptr %20, ptr %gep_, align 8
  // %gep_8 = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 1
  // store ptr %21, ptr %gep_8, align 8
  // call void @_QQmain..omp_par.1(i32 %global.tid.val6, ptr %structArg)
  //
  // OR
  //
  // call void @_QQmain..omp_par.1(i32 %global.tid.val6)
  OpenMPIRBuilder::InsertPointTy IP(StaleCI->getParent(),
                                    StaleCI->getIterator());
  LLVMContext &Ctx = StaleCI->getParent()->getContext();
  Type *ThreadIDTy = Type::getInt32Ty(Ctx);
  Type *TaskPtrTy = OMPBuilder.TaskPtr;
  Type *TaskTy = OMPBuilder.Task;
  auto ProxyFnTy =
      FunctionType::get(Builder.getVoidTy(), {ThreadIDTy, TaskPtrTy},
                        /* isVarArg */ false);
  auto ProxyFn = Function::Create(ProxyFnTy, GlobalValue::InternalLinkage,
                                  ".omp_target_task_proxy_func",
                                  Builder.GetInsertBlock()->getModule());
  ProxyFn->getArg(0)->setName("thread.id");
  ProxyFn->getArg(1)->setName("task");

  BasicBlock *EntryBB =
      BasicBlock::Create(Builder.getContext(), "entry", ProxyFn);
  Builder.SetInsertPoint(EntryBB);

  bool HasShareds = StaleCI->arg_size() > 1;
  // TODO: This is a temporary assert to prove to ourselves that
  // the outlined target launch function is always going to have
  // atmost two arguments if there is any data shared between
  // host and device.
  assert((!HasShareds || (StaleCI->arg_size() == 2)) &&
         "StaleCI with shareds should have exactly two arguments.");
  if (HasShareds) {
    auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->getArgOperand(1));
    assert(ArgStructAlloca &&
           "Unable to find the alloca instruction corresponding to arguments "
           "for extracted function");
    auto *ArgStructType =
        dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());

    AllocaInst *NewArgStructAlloca =
        Builder.CreateAlloca(ArgStructType, nullptr, "structArg");
    Value *TaskT = ProxyFn->getArg(1);
    Value *ThreadId = ProxyFn->getArg(0);
    Value *SharedsSize =
        Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));

    Value *Shareds = Builder.CreateStructGEP(TaskTy, TaskT, 0);
    LoadInst *LoadShared =
        Builder.CreateLoad(PointerType::getUnqual(Ctx), Shareds);

    Builder.CreateMemCpy(
        NewArgStructAlloca, NewArgStructAlloca->getAlign(), LoadShared,
        LoadShared->getPointerAlignment(M.getDataLayout()), SharedsSize);

    Builder.CreateCall(KernelLaunchFunction, {ThreadId, NewArgStructAlloca});
  }
  Builder.CreateRetVoid();
  return ProxyFn;
}
static void emitTargetOutlinedFunction(
    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry,
    TargetRegionEntryInfo &EntryInfo, Function *&OutlinedFn,
    Constant *&OutlinedFnID, SmallVectorImpl<Value *> &Inputs,
    OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
    OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {

  OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
      [&OMPBuilder, &Builder, &Inputs, &CBFunc,
       &ArgAccessorFuncCB](StringRef EntryFnName) {
        return createOutlinedFunction(OMPBuilder, Builder, EntryFnName, Inputs,
                                      CBFunc, ArgAccessorFuncCB);
      };

  OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction,
                                      IsOffloadEntry, OutlinedFn, OutlinedFnID);
}
OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetTask(
    Function *OutlinedFn, Value *OutlinedFnID,
    EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
    Value *DeviceID, Value *RTLoc, OpenMPIRBuilder::InsertPointTy AllocaIP,
    SmallVector<llvm::OpenMPIRBuilder::DependData> &Dependencies,
    bool HasNoWait) {

  // When we arrive at this function, the target region itself has been
  // outlined into the function OutlinedFn.
  // So at ths point, for
  // --------------------------------------------------
  //   void user_code_that_offloads(...) {
  //     omp target depend(..) map(from:a) map(to:b, c)
  //        a = b + c
  //   }
  //
  // --------------------------------------------------
  //
  // we have
  //
  // --------------------------------------------------
  //
  //   void user_code_that_offloads(...) {
  //     %.offload_baseptrs = alloca [3 x ptr], align 8
  //     %.offload_ptrs = alloca [3 x ptr], align 8
  //     %.offload_mappers = alloca [3 x ptr], align 8
  //     ;; target region has been outlined and now we need to
  //     ;; offload to it via a target task.
  //   }
  //   void outlined_device_function(ptr a, ptr b, ptr c) {
  //     *a = *b + *c
  //   }
  //
  // We have to now do the following
  // (i)   Make an offloading call to outlined_device_function using the OpenMP
  //       RTL. See 'kernel_launch_function' in the pseudo code below. This is
  //       emitted by emitKernelLaunch
  // (ii)  Create a task entry point function that calls kernel_launch_function
  //       and is the entry point for the target task. See
  //       '@.omp_target_task_proxy_func in the pseudocode below.
  // (iii) Create a task with the task entry point created in (ii)
  //
  // That is we create the following
  //
  //   void user_code_that_offloads(...) {
  //     %.offload_baseptrs = alloca [3 x ptr], align 8
  //     %.offload_ptrs = alloca [3 x ptr], align 8
  //     %.offload_mappers = alloca [3 x ptr], align 8
  //
  //     %structArg = alloca { ptr, ptr, ptr }, align 8
  //     %strucArg[0] = %.offload_baseptrs
  //     %strucArg[1] = %.offload_ptrs
  //     %strucArg[2] = %.offload_mappers
  //     proxy_target_task = @__kmpc_omp_task_alloc(...,
  //                                               @.omp_target_task_proxy_func)
  //     memcpy(proxy_target_task->shareds, %structArg, sizeof(structArg))
  //     dependencies_array = ...
  //     ;; if nowait not present
  //     call @__kmpc_omp_wait_deps(..., dependencies_array)
  //     call @__kmpc_omp_task_begin_if0(...)
  //     call @ @.omp_target_task_proxy_func(i32 thread_id, ptr
  //     %proxy_target_task) call @__kmpc_omp_task_complete_if0(...)
  //   }
  //
  //   define internal void @.omp_target_task_proxy_func(i32 %thread.id,
  //                                                     ptr %task) {
  //       %structArg = alloca {ptr, ptr, ptr}
  //       %shared_data = load (getelementptr %task, 0, 0)
  //       mempcy(%structArg, %shared_data, sizeof(structArg))
  //       kernel_launch_function(%thread.id, %structArg)
  //   }
  //
  //   We need the proxy function because the signature of the task entry point
  //   expected by kmpc_omp_task is always the same and will be different from
  //   that of the kernel_launch function.
  //
  //   kernel_launch_function is generated by emitKernelLaunch and has the
  //   always_inline attribute.
  //   void kernel_launch_function(thread_id,
  //                               structArg) alwaysinline {
  //       %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
  //       offload_baseptrs = load(getelementptr structArg, 0, 0)
  //       offload_ptrs = load(getelementptr structArg, 0, 1)
  //       offload_mappers = load(getelementptr structArg, 0, 2)
  //       ; setup kernel_args using offload_baseptrs, offload_ptrs and
  //       ; offload_mappers
  //       call i32 @__tgt_target_kernel(...,
  //                                     outlined_device_function,
  //                                     ptr %kernel_args)
  //   }
  //   void outlined_device_function(ptr a, ptr b, ptr c) {
  //      *a = *b + *c
  //   }
  //
  BasicBlock *TargetTaskBodyBB =
      splitBB(Builder, /*CreateBranch=*/true, "target.task.body");
  BasicBlock *TargetTaskAllocaBB =
      splitBB(Builder, /*CreateBranch=*/true, "target.task.alloca");

  InsertPointTy TargetTaskAllocaIP(TargetTaskAllocaBB,
                                   TargetTaskAllocaBB->begin());
  InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB, TargetTaskBodyBB->begin());

  OutlineInfo OI;
  OI.EntryBB = TargetTaskAllocaBB;
  OI.OuterAllocaBB = AllocaIP.getBlock();

  // Add the thread ID argument.
  SmallVector<Instruction *, 4> ToBeDeleted;
  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(
      Builder, AllocaIP, ToBeDeleted, TargetTaskAllocaIP, "global.tid", false));

  Builder.restoreIP(TargetTaskBodyIP);

  if (OutlinedFnID) {
    // emitKernelLaunch makes the necessary runtime call to offload the kernel.
    // We then outline all that code into a separate function
    // ('kernel_launch_function' in the pseudo code above). This function is
    // then called by the target task proxy function (see
    // '@.omp_target_task_proxy_func' in the pseudo code above)
    // "@.omp_target_task_proxy_func' is generated by
    // emitTargetTaskProxyFunction.
    Builder.restoreIP(emitKernelLaunch(Builder, OutlinedFn, OutlinedFnID,
                                       EmitTargetCallFallbackCB, Args, DeviceID,
                                       RTLoc, TargetTaskAllocaIP));
  } else {
    // When OutlinedFnID is set to nullptr, then it's not an offloading call. In
    // this case, we execute the host implementation directly.
    Builder.restoreIP(EmitTargetCallFallbackCB(Builder.saveIP()));
  }

  OI.ExitBB = Builder.saveIP().getBlock();
  OI.PostOutlineCB = [this, ToBeDeleted, Dependencies,
                      HasNoWait](Function &OutlinedFn) mutable {
    assert(OutlinedFn.getNumUses() == 1 &&
           "there must be a single user for the outlined function");

    CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
    bool HasShareds = StaleCI->arg_size() > 1;

    Function *ProxyFn = emitTargetTaskProxyFunction(*this, Builder, StaleCI);

    LLVM_DEBUG(dbgs() << "Proxy task entry function created: " << *ProxyFn
                      << "\n");

    Builder.SetInsertPoint(StaleCI);

    // Gather the arguments for emitting the runtime call.
    uint32_t SrcLocStrSize;
    Constant *SrcLocStr =
        getOrCreateSrcLocStr(LocationDescription(Builder), SrcLocStrSize);
    Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);

    // @__kmpc_omp_task_alloc
    Function *TaskAllocFn =
        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);

    // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
    // call.
    Value *ThreadID = getOrCreateThreadID(Ident);

    // Argument - `sizeof_kmp_task_t` (TaskSize)
    // Tasksize refers to the size in bytes of kmp_task_t data structure
    // including private vars accessed in task.
    // TODO: add kmp_task_t_with_privates (privates)
    Value *TaskSize =
        Builder.getInt64(M.getDataLayout().getTypeStoreSize(Task));

    // Argument - `sizeof_shareds` (SharedsSize)
    // SharedsSize refers to the shareds array size in the kmp_task_t data
    // structure.
    Value *SharedsSize = Builder.getInt64(0);
    if (HasShareds) {
      auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->getArgOperand(1));
      assert(ArgStructAlloca &&
             "Unable to find the alloca instruction corresponding to arguments "
             "for extracted function");
      auto *ArgStructType =
          dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
      assert(ArgStructType && "Unable to find struct type corresponding to "
                              "arguments for extracted function");
      SharedsSize =
          Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
    }

    // Argument - `flags`
    // Task is tied iff (Flags & 1) == 1.
    // Task is untied iff (Flags & 1) == 0.
    // Task is final iff (Flags & 2) == 2.
    // Task is not final iff (Flags & 2) == 0.
    // A target task is not final and is untied.
    Value *Flags = Builder.getInt32(0);

    // Emit the @__kmpc_omp_task_alloc runtime call
    // The runtime call returns a pointer to an area where the task captured
    // variables must be copied before the task is run (TaskData)
    CallInst *TaskData = Builder.CreateCall(
        TaskAllocFn, {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
                      /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
                      /*task_func=*/ProxyFn});

    if (HasShareds) {
      Value *Shareds = StaleCI->getArgOperand(1);
      Align Alignment = TaskData->getPointerAlignment(M.getDataLayout());
      Value *TaskShareds = Builder.CreateLoad(VoidPtr, TaskData);
      Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment,
                           SharedsSize);
    }

    Value *DepArray = emitTaskDependencies(*this, Dependencies);

    // ---------------------------------------------------------------
    // V5.2 13.8 target construct
    // If the nowait clause is present, execution of the target task
    // may be deferred. If the nowait clause is not present, the target task is
    // an included task.
    // ---------------------------------------------------------------
    // The above means that the lack of a nowait on the target construct
    // translates to '#pragma omp task if(0)'
    if (!HasNoWait) {
      if (DepArray) {
        Function *TaskWaitFn =
            getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_wait_deps);
        Builder.CreateCall(
            TaskWaitFn,
            {/*loc_ref=*/Ident, /*gtid=*/ThreadID,
             /*ndeps=*/Builder.getInt32(Dependencies.size()),
             /*dep_list=*/DepArray,
             /*ndeps_noalias=*/ConstantInt::get(Builder.getInt32Ty(), 0),
             /*noalias_dep_list=*/
             ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});
      }
      // Included task.
      Function *TaskBeginFn =
          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0);
      Function *TaskCompleteFn =
          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0);
      Builder.CreateCall(TaskBeginFn, {Ident, ThreadID, TaskData});
      CallInst *CI = Builder.CreateCall(ProxyFn, {ThreadID, TaskData});
      CI->setDebugLoc(StaleCI->getDebugLoc());
      Builder.CreateCall(TaskCompleteFn, {Ident, ThreadID, TaskData});
    } else if (DepArray) {
      // HasNoWait - meaning the task may be deferred. Call
      // __kmpc_omp_task_with_deps if there are dependencies,
      // else call __kmpc_omp_task
      Function *TaskFn =
          getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps);
      Builder.CreateCall(
          TaskFn,
          {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()),
           DepArray, ConstantInt::get(Builder.getInt32Ty(), 0),
           ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});
    } else {
      // Emit the @__kmpc_omp_task runtime call to spawn the task
      Function *TaskFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task);
      Builder.CreateCall(TaskFn, {Ident, ThreadID, TaskData});
    }

    StaleCI->eraseFromParent();
    for (Instruction *I : llvm::reverse(ToBeDeleted))
      I->eraseFromParent();
  };
  addOutlineInfo(std::move(OI));

  LLVM_DEBUG(dbgs() << "Insert block after emitKernelLaunch = \n"
                    << *(Builder.GetInsertBlock()) << "\n");
  LLVM_DEBUG(dbgs() << "Module after emitKernelLaunch = \n"
                    << *(Builder.GetInsertBlock()->getParent()->getParent())
                    << "\n");
  return Builder.saveIP();
}

void OpenMPIRBuilder::emitOffloadingArraysAndArgs(
    InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info,
    TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo, bool IsNonContiguous,
    bool ForEndCall, function_ref<void(unsigned int, Value *)> DeviceAddrCB,
    function_ref<Value *(unsigned int)> CustomMapperCB) {
  emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info, IsNonContiguous,
                       DeviceAddrCB, CustomMapperCB);
  emitOffloadingArraysArgument(Builder, RTArgs, Info, ForEndCall);
}

static void emitTargetCall(
    OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
    OpenMPIRBuilder::InsertPointTy AllocaIP, Function *OutlinedFn,
    Constant *OutlinedFnID, ArrayRef<int32_t> NumTeams,
    ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Args,
    OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
    SmallVector<llvm::OpenMPIRBuilder::DependData> Dependencies = {}) {
  // Generate a function call to the host fallback implementation of the target
  // region. This is called by the host when no offload entry was generated for
  // the target region and when the offloading call fails at runtime.
  auto &&EmitTargetCallFallbackCB =
      [&](OpenMPIRBuilder::InsertPointTy IP) -> OpenMPIRBuilder::InsertPointTy {
    Builder.restoreIP(IP);
    Builder.CreateCall(OutlinedFn, Args);
    return Builder.saveIP();
  };

  bool HasNoWait = false;
  bool HasDependencies = Dependencies.size() > 0;
  bool RequiresOuterTargetTask = HasNoWait || HasDependencies;

  // If we don't have an ID for the target region, it means an offload entry
  // wasn't created. In this case we just run the host fallback directly.
  if (!OutlinedFnID) {
    if (RequiresOuterTargetTask) {
      // Arguments that are intended to be directly forwarded to an
      // emitKernelLaunch call are pased as nullptr, since OutlinedFnID=nullptr
      // results in that call not being done.
      OpenMPIRBuilder::TargetKernelArgs KArgs;
      Builder.restoreIP(OMPBuilder.emitTargetTask(
          OutlinedFn, /*OutlinedFnID=*/nullptr, EmitTargetCallFallbackCB, KArgs,
          /*DeviceID=*/nullptr, /*RTLoc=*/nullptr, AllocaIP, Dependencies,
          HasNoWait));
    } else {
      Builder.restoreIP(EmitTargetCallFallbackCB(Builder.saveIP()));
    }
    return;
  }

  OpenMPIRBuilder::TargetDataInfo Info(
      /*RequiresDevicePointerInfo=*/false,
      /*SeparateBeginEndCalls=*/true);

  OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
  OpenMPIRBuilder::TargetDataRTArgs RTArgs;
  OMPBuilder.emitOffloadingArraysAndArgs(AllocaIP, Builder.saveIP(), Info,
                                         RTArgs, MapInfo,
                                         /*IsNonContiguous=*/true,
                                         /*ForEndCall=*/false);

  SmallVector<Value *, 3> NumTeamsC;
  SmallVector<Value *, 3> NumThreadsC;
  for (auto V : NumTeams)
    NumTeamsC.push_back(llvm::ConstantInt::get(Builder.getInt32Ty(), V));
  for (auto V : NumThreads)
    NumThreadsC.push_back(llvm::ConstantInt::get(Builder.getInt32Ty(), V));

  unsigned NumTargetItems = Info.NumberOfPtrs;
  // TODO: Use correct device ID
  Value *DeviceID = Builder.getInt64(OMP_DEVICEID_UNDEF);
  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
  Value *RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
                                             llvm::omp::IdentFlag(0), 0);
  // TODO: Use correct NumIterations
  Value *NumIterations = Builder.getInt64(0);
  // TODO: Use correct DynCGGroupMem
  Value *DynCGGroupMem = Builder.getInt32(0);

  OpenMPIRBuilder::TargetKernelArgs KArgs(NumTargetItems, RTArgs, NumIterations,
                                          NumTeamsC, NumThreadsC, DynCGGroupMem,
                                          HasNoWait);

  // The presence of certain clauses on the target directive require the
  // explicit generation of the target task.
  if (RequiresOuterTargetTask) {
    Builder.restoreIP(OMPBuilder.emitTargetTask(
        OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs, DeviceID,
        RTLoc, AllocaIP, Dependencies, HasNoWait));
  } else {
    Builder.restoreIP(OMPBuilder.emitKernelLaunch(
        Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs,
        DeviceID, RTLoc, AllocaIP));
  }
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
    const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
    InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo,
    ArrayRef<int32_t> NumTeams, ArrayRef<int32_t> NumThreads,
    SmallVectorImpl<Value *> &Args, GenMapInfoCallbackTy GenMapInfoCB,
    OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
    OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
    SmallVector<DependData> Dependencies) {

  if (!updateToLocation(Loc))
    return InsertPointTy();

  Builder.restoreIP(CodeGenIP);

  Function *OutlinedFn;
  Constant *OutlinedFnID = nullptr;
  // The target region is outlined into its own function. The LLVM IR for
  // the target region itself is generated using the callbacks CBFunc
  // and ArgAccessorFuncCB
  emitTargetOutlinedFunction(*this, Builder, IsOffloadEntry, EntryInfo,
                             OutlinedFn, OutlinedFnID, Args, CBFunc,
                             ArgAccessorFuncCB);

  // If we are not on the target device, then we need to generate code
  // to make a remote call (offload) to the previously outlined function
  // that represents the target region. Do that now.
  if (!Config.isTargetDevice())
    emitTargetCall(*this, Builder, AllocaIP, OutlinedFn, OutlinedFnID, NumTeams,
                   NumThreads, Args, GenMapInfoCB, Dependencies);
  return Builder.saveIP();
}

std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
                                                   StringRef FirstSeparator,
                                                   StringRef Separator) {
  SmallString<128> Buffer;
  llvm::raw_svector_ostream OS(Buffer);
  StringRef Sep = FirstSeparator;
  for (StringRef Part : Parts) {
    OS << Sep << Part;
    Sep = Separator;
  }
  return OS.str().str();
}

std::string
OpenMPIRBuilder::createPlatformSpecificName(ArrayRef<StringRef> Parts) const {
  return OpenMPIRBuilder::getNameWithSeparators(Parts, Config.firstSeparator(),
                                                Config.separator());
}

GlobalVariable *
OpenMPIRBuilder::getOrCreateInternalVariable(Type *Ty, const StringRef &Name,
                                             unsigned AddressSpace) {
  auto &Elem = *InternalVars.try_emplace(Name, nullptr).first;
  if (Elem.second) {
    assert(Elem.second->getValueType() == Ty &&
           "OMP internal variable has different type than requested");
  } else {
    // TODO: investigate the appropriate linkage type used for the global
    // variable for possibly changing that to internal or private, or maybe
    // create different versions of the function for different OMP internal
    // variables.
    auto Linkage = this->M.getTargetTriple().rfind("wasm32") == 0
                       ? GlobalValue::InternalLinkage
                       : GlobalValue::CommonLinkage;
    auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
                                  Constant::getNullValue(Ty), Elem.first(),
                                  /*InsertBefore=*/nullptr,
                                  GlobalValue::NotThreadLocal, AddressSpace);
    const DataLayout &DL = M.getDataLayout();
    const llvm::Align TypeAlign = DL.getABITypeAlign(Ty);
    const llvm::Align PtrAlign = DL.getPointerABIAlignment(AddressSpace);
    GV->setAlignment(std::max(TypeAlign, PtrAlign));
    Elem.second = GV;
  }

  return Elem.second;
}

Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
  std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
  std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");
  return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
}

Value *OpenMPIRBuilder::getSizeInBytes(Value *BasePtr) {
  LLVMContext &Ctx = Builder.getContext();
  Value *Null =
      Constant::getNullValue(PointerType::getUnqual(BasePtr->getContext()));
  Value *SizeGep =
      Builder.CreateGEP(BasePtr->getType(), Null, Builder.getInt32(1));
  Value *SizePtrToInt = Builder.CreatePtrToInt(SizeGep, Type::getInt64Ty(Ctx));
  return SizePtrToInt;
}

GlobalVariable *
OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
                                       std::string VarName) {
  llvm::Constant *MaptypesArrayInit =
      llvm::ConstantDataArray::get(M.getContext(), Mappings);
  auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
      M, MaptypesArrayInit->getType(),
      /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
      VarName);
  MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
  return MaptypesArrayGlobal;
}

void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
                                          InsertPointTy AllocaIP,
                                          unsigned NumOperands,
                                          struct MapperAllocas &MapperAllocas) {
  if (!updateToLocation(Loc))
    return;

  auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);
  auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);
  Builder.restoreIP(AllocaIP);
  AllocaInst *ArgsBase = Builder.CreateAlloca(
      ArrI8PtrTy, /* ArraySize = */ nullptr, ".offload_baseptrs");
  AllocaInst *Args = Builder.CreateAlloca(ArrI8PtrTy, /* ArraySize = */ nullptr,
                                          ".offload_ptrs");
  AllocaInst *ArgSizes = Builder.CreateAlloca(
      ArrI64Ty, /* ArraySize = */ nullptr, ".offload_sizes");
  Builder.restoreIP(Loc.IP);
  MapperAllocas.ArgsBase = ArgsBase;
  MapperAllocas.Args = Args;
  MapperAllocas.ArgSizes = ArgSizes;
}

void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
                                     Function *MapperFunc, Value *SrcLocInfo,
                                     Value *MaptypesArg, Value *MapnamesArg,
                                     struct MapperAllocas &MapperAllocas,
                                     int64_t DeviceID, unsigned NumOperands) {
  if (!updateToLocation(Loc))
    return;

  auto *ArrI8PtrTy = ArrayType::get(Int8Ptr, NumOperands);
  auto *ArrI64Ty = ArrayType::get(Int64, NumOperands);
  Value *ArgsBaseGEP =
      Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,
                                {Builder.getInt32(0), Builder.getInt32(0)});
  Value *ArgsGEP =
      Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,
                                {Builder.getInt32(0), Builder.getInt32(0)});
  Value *ArgSizesGEP =
      Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,
                                {Builder.getInt32(0), Builder.getInt32(0)});
  Value *NullPtr =
      Constant::getNullValue(PointerType::getUnqual(Int8Ptr->getContext()));
  Builder.CreateCall(MapperFunc,
                     {SrcLocInfo, Builder.getInt64(DeviceID),
                      Builder.getInt32(NumOperands), ArgsBaseGEP, ArgsGEP,
                      ArgSizesGEP, MaptypesArg, MapnamesArg, NullPtr});
}

void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
                                                   TargetDataRTArgs &RTArgs,
                                                   TargetDataInfo &Info,
                                                   bool ForEndCall) {
  assert((!ForEndCall || Info.separateBeginEndCalls()) &&
         "expected region end call to runtime only when end call is separate");
  auto UnqualPtrTy = PointerType::getUnqual(M.getContext());
  auto VoidPtrTy = UnqualPtrTy;
  auto VoidPtrPtrTy = UnqualPtrTy;
  auto Int64Ty = Type::getInt64Ty(M.getContext());
  auto Int64PtrTy = UnqualPtrTy;

  if (!Info.NumberOfPtrs) {
    RTArgs.BasePointersArray = ConstantPointerNull::get(VoidPtrPtrTy);
    RTArgs.PointersArray = ConstantPointerNull::get(VoidPtrPtrTy);
    RTArgs.SizesArray = ConstantPointerNull::get(Int64PtrTy);
    RTArgs.MapTypesArray = ConstantPointerNull::get(Int64PtrTy);
    RTArgs.MapNamesArray = ConstantPointerNull::get(VoidPtrPtrTy);
    RTArgs.MappersArray = ConstantPointerNull::get(VoidPtrPtrTy);
    return;
  }

  RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(
      ArrayType::get(VoidPtrTy, Info.NumberOfPtrs),
      Info.RTArgs.BasePointersArray,
      /*Idx0=*/0, /*Idx1=*/0);
  RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(
      ArrayType::get(VoidPtrTy, Info.NumberOfPtrs), Info.RTArgs.PointersArray,
      /*Idx0=*/0,
      /*Idx1=*/0);
  RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(
      ArrayType::get(Int64Ty, Info.NumberOfPtrs), Info.RTArgs.SizesArray,
      /*Idx0=*/0, /*Idx1=*/0);
  RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(
      ArrayType::get(Int64Ty, Info.NumberOfPtrs),
      ForEndCall && Info.RTArgs.MapTypesArrayEnd ? Info.RTArgs.MapTypesArrayEnd
                                                 : Info.RTArgs.MapTypesArray,
      /*Idx0=*/0,
      /*Idx1=*/0);

  // Only emit the mapper information arrays if debug information is
  // requested.
  if (!Info.EmitDebug)
    RTArgs.MapNamesArray = ConstantPointerNull::get(VoidPtrPtrTy);
  else
    RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(
        ArrayType::get(VoidPtrTy, Info.NumberOfPtrs), Info.RTArgs.MapNamesArray,
        /*Idx0=*/0,
        /*Idx1=*/0);
  // If there is no user-defined mapper, set the mapper array to nullptr to
  // avoid an unnecessary data privatization
  if (!Info.HasMapper)
    RTArgs.MappersArray = ConstantPointerNull::get(VoidPtrPtrTy);
  else
    RTArgs.MappersArray =
        Builder.CreatePointerCast(Info.RTArgs.MappersArray, VoidPtrPtrTy);
}

void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,
                                                  InsertPointTy CodeGenIP,
                                                  MapInfosTy &CombinedInfo,
                                                  TargetDataInfo &Info) {
  MapInfosTy::StructNonContiguousInfo &NonContigInfo =
      CombinedInfo.NonContigInfo;

  // Build an array of struct descriptor_dim and then assign it to
  // offload_args.
  //
  // struct descriptor_dim {
  //  uint64_t offset;
  //  uint64_t count;
  //  uint64_t stride
  // };
  Type *Int64Ty = Builder.getInt64Ty();
  StructType *DimTy = StructType::create(
      M.getContext(), ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),
      "struct.descriptor_dim");

  enum { OffsetFD = 0, CountFD, StrideFD };
  // We need two index variable here since the size of "Dims" is the same as
  // the size of Components, however, the size of offset, count, and stride is
  // equal to the size of base declaration that is non-contiguous.
  for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
    // Skip emitting ir if dimension size is 1 since it cannot be
    // non-contiguous.
    if (NonContigInfo.Dims[I] == 1)
      continue;
    Builder.restoreIP(AllocaIP);
    ArrayType *ArrayTy = ArrayType::get(DimTy, NonContigInfo.Dims[I]);
    AllocaInst *DimsAddr =
        Builder.CreateAlloca(ArrayTy, /* ArraySize = */ nullptr, "dims");
    Builder.restoreIP(CodeGenIP);
    for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
      unsigned RevIdx = EE - II - 1;
      Value *DimsLVal = Builder.CreateInBoundsGEP(
          DimsAddr->getAllocatedType(), DimsAddr,
          {Builder.getInt64(0), Builder.getInt64(II)});
      // Offset
      Value *OffsetLVal = Builder.CreateStructGEP(DimTy, DimsLVal, OffsetFD);
      Builder.CreateAlignedStore(
          NonContigInfo.Offsets[L][RevIdx], OffsetLVal,
          M.getDataLayout().getPrefTypeAlign(OffsetLVal->getType()));
      // Count
      Value *CountLVal = Builder.CreateStructGEP(DimTy, DimsLVal, CountFD);
      Builder.CreateAlignedStore(
          NonContigInfo.Counts[L][RevIdx], CountLVal,
          M.getDataLayout().getPrefTypeAlign(CountLVal->getType()));
      // Stride
      Value *StrideLVal = Builder.CreateStructGEP(DimTy, DimsLVal, StrideFD);
      Builder.CreateAlignedStore(
          NonContigInfo.Strides[L][RevIdx], StrideLVal,
          M.getDataLayout().getPrefTypeAlign(CountLVal->getType()));
    }
    // args[I] = &dims
    Builder.restoreIP(CodeGenIP);
    Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
        DimsAddr, Builder.getPtrTy());
    Value *P = Builder.CreateConstInBoundsGEP2_32(
        ArrayType::get(Builder.getPtrTy(), Info.NumberOfPtrs),
        Info.RTArgs.PointersArray, 0, I);
    Builder.CreateAlignedStore(
        DAddr, P, M.getDataLayout().getPrefTypeAlign(Builder.getPtrTy()));
    ++L;
  }
}

void OpenMPIRBuilder::emitOffloadingArrays(
    InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
    TargetDataInfo &Info, bool IsNonContiguous,
    function_ref<void(unsigned int, Value *)> DeviceAddrCB,
    function_ref<Value *(unsigned int)> CustomMapperCB) {

  // Reset the array information.
  Info.clearArrayInfo();
  Info.NumberOfPtrs = CombinedInfo.BasePointers.size();

  if (Info.NumberOfPtrs == 0)
    return;

  Builder.restoreIP(AllocaIP);
  // Detect if we have any capture size requiring runtime evaluation of the
  // size so that a constant array could be eventually used.
  ArrayType *PointerArrayType =
      ArrayType::get(Builder.getPtrTy(), Info.NumberOfPtrs);

  Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
      PointerArrayType, /* ArraySize = */ nullptr, ".offload_baseptrs");

  Info.RTArgs.PointersArray = Builder.CreateAlloca(
      PointerArrayType, /* ArraySize = */ nullptr, ".offload_ptrs");
  AllocaInst *MappersArray = Builder.CreateAlloca(
      PointerArrayType, /* ArraySize = */ nullptr, ".offload_mappers");
  Info.RTArgs.MappersArray = MappersArray;

  // If we don't have any VLA types or other types that require runtime
  // evaluation, we can use a constant array for the map sizes, otherwise we
  // need to fill up the arrays as we do for the pointers.
  Type *Int64Ty = Builder.getInt64Ty();
  SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),
                                     ConstantInt::get(Int64Ty, 0));
  SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
  for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
    if (auto *CI = dyn_cast<Constant>(CombinedInfo.Sizes[I])) {
      if (!isa<ConstantExpr>(CI) && !isa<GlobalValue>(CI)) {
        if (IsNonContiguous &&
            static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
                CombinedInfo.Types[I] &
                OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG))
          ConstSizes[I] =
              ConstantInt::get(Int64Ty, CombinedInfo.NonContigInfo.Dims[I]);
        else
          ConstSizes[I] = CI;
        continue;
      }
    }
    RuntimeSizes.set(I);
  }

  if (RuntimeSizes.all()) {
    ArrayType *SizeArrayType = ArrayType::get(Int64Ty, Info.NumberOfPtrs);
    Info.RTArgs.SizesArray = Builder.CreateAlloca(
        SizeArrayType, /* ArraySize = */ nullptr, ".offload_sizes");
    Builder.restoreIP(CodeGenIP);
  } else {
    auto *SizesArrayInit = ConstantArray::get(
        ArrayType::get(Int64Ty, ConstSizes.size()), ConstSizes);
    std::string Name = createPlatformSpecificName({"offload_sizes"});
    auto *SizesArrayGbl =
        new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,
                           GlobalValue::PrivateLinkage, SizesArrayInit, Name);
    SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);

    if (!RuntimeSizes.any()) {
      Info.RTArgs.SizesArray = SizesArrayGbl;
    } else {
      unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(0);
      Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(64);
      ArrayType *SizeArrayType = ArrayType::get(Int64Ty, Info.NumberOfPtrs);
      AllocaInst *Buffer = Builder.CreateAlloca(
          SizeArrayType, /* ArraySize = */ nullptr, ".offload_sizes");
      Buffer->setAlignment(OffloadSizeAlign);
      Builder.restoreIP(CodeGenIP);
      Builder.CreateMemCpy(
          Buffer, M.getDataLayout().getPrefTypeAlign(Buffer->getType()),
          SizesArrayGbl, OffloadSizeAlign,
          Builder.getIntN(
              IndexSize,
              Buffer->getAllocationSize(M.getDataLayout())->getFixedValue()));

      Info.RTArgs.SizesArray = Buffer;
    }
    Builder.restoreIP(CodeGenIP);
  }

  // The map types are always constant so we don't need to generate code to
  // fill arrays. Instead, we create an array constant.
  SmallVector<uint64_t, 4> Mapping;
  for (auto mapFlag : CombinedInfo.Types)
    Mapping.push_back(
        static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
            mapFlag));
  std::string MaptypesName = createPlatformSpecificName({"offload_maptypes"});
  auto *MapTypesArrayGbl = createOffloadMaptypes(Mapping, MaptypesName);
  Info.RTArgs.MapTypesArray = MapTypesArrayGbl;

  // The information types are only built if provided.
  if (!CombinedInfo.Names.empty()) {
    std::string MapnamesName = createPlatformSpecificName({"offload_mapnames"});
    auto *MapNamesArrayGbl =
        createOffloadMapnames(CombinedInfo.Names, MapnamesName);
    Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
    Info.EmitDebug = true;
  } else {
    Info.RTArgs.MapNamesArray =
        Constant::getNullValue(PointerType::getUnqual(Builder.getContext()));
    Info.EmitDebug = false;
  }

  // If there's a present map type modifier, it must not be applied to the end
  // of a region, so generate a separate map type array in that case.
  if (Info.separateBeginEndCalls()) {
    bool EndMapTypesDiffer = false;
    for (uint64_t &Type : Mapping) {
      if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
                     OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
        Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
            OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
        EndMapTypesDiffer = true;
      }
    }
    if (EndMapTypesDiffer) {
      MapTypesArrayGbl = createOffloadMaptypes(Mapping, MaptypesName);
      Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
    }
  }

  PointerType *PtrTy = Builder.getPtrTy();
  for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
    Value *BPVal = CombinedInfo.BasePointers[I];
    Value *BP = Builder.CreateConstInBoundsGEP2_32(
        ArrayType::get(PtrTy, Info.NumberOfPtrs), Info.RTArgs.BasePointersArray,
        0, I);
    Builder.CreateAlignedStore(BPVal, BP,
                               M.getDataLayout().getPrefTypeAlign(PtrTy));

    if (Info.requiresDevicePointerInfo()) {
      if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Pointer) {
        CodeGenIP = Builder.saveIP();
        Builder.restoreIP(AllocaIP);
        Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(PtrTy)};
        Builder.restoreIP(CodeGenIP);
        if (DeviceAddrCB)
          DeviceAddrCB(I, Info.DevicePtrInfoMap[BPVal].second);
      } else if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Address) {
        Info.DevicePtrInfoMap[BPVal] = {BP, BP};
        if (DeviceAddrCB)
          DeviceAddrCB(I, BP);
      }
    }

    Value *PVal = CombinedInfo.Pointers[I];
    Value *P = Builder.CreateConstInBoundsGEP2_32(
        ArrayType::get(PtrTy, Info.NumberOfPtrs), Info.RTArgs.PointersArray, 0,
        I);
    // TODO: Check alignment correct.
    Builder.CreateAlignedStore(PVal, P,
                               M.getDataLayout().getPrefTypeAlign(PtrTy));

    if (RuntimeSizes.test(I)) {
      Value *S = Builder.CreateConstInBoundsGEP2_32(
          ArrayType::get(Int64Ty, Info.NumberOfPtrs), Info.RTArgs.SizesArray,
          /*Idx0=*/0,
          /*Idx1=*/I);
      Builder.CreateAlignedStore(Builder.CreateIntCast(CombinedInfo.Sizes[I],
                                                       Int64Ty,
                                                       /*isSigned=*/true),
                                 S, M.getDataLayout().getPrefTypeAlign(PtrTy));
    }
    // Fill up the mapper array.
    unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(0);
    Value *MFunc = ConstantPointerNull::get(PtrTy);
    if (CustomMapperCB)
      if (Value *CustomMFunc = CustomMapperCB(I))
        MFunc = Builder.CreatePointerCast(CustomMFunc, PtrTy);
    Value *MAddr = Builder.CreateInBoundsGEP(
        MappersArray->getAllocatedType(), MappersArray,
        {Builder.getIntN(IndexSize, 0), Builder.getIntN(IndexSize, I)});
    Builder.CreateAlignedStore(
        MFunc, MAddr, M.getDataLayout().getPrefTypeAlign(MAddr->getType()));
  }

  if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
      Info.NumberOfPtrs == 0)
    return;
  emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
}

void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {
  BasicBlock *CurBB = Builder.GetInsertBlock();

  if (!CurBB || CurBB->getTerminator()) {
    // If there is no insert point or the previous block is already
    // terminated, don't touch it.
  } else {
    // Otherwise, create a fall-through branch.
    Builder.CreateBr(Target);
  }

  Builder.ClearInsertionPoint();
}

void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
                                bool IsFinished) {
  BasicBlock *CurBB = Builder.GetInsertBlock();

  // Fall out of the current block (if necessary).
  emitBranch(BB);

  if (IsFinished && BB->use_empty()) {
    BB->eraseFromParent();
    return;
  }

  // Place the block after the current block, if possible, or else at
  // the end of the function.
  if (CurBB && CurBB->getParent())
    CurFn->insert(std::next(CurBB->getIterator()), BB);
  else
    CurFn->insert(CurFn->end(), BB);
  Builder.SetInsertPoint(BB);
}

void OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
                                   BodyGenCallbackTy ElseGen,
                                   InsertPointTy AllocaIP) {
  // If the condition constant folds and can be elided, try to avoid emitting
  // the condition and the dead arm of the if/else.
  if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
    auto CondConstant = CI->getSExtValue();
    if (CondConstant)
      ThenGen(AllocaIP, Builder.saveIP());
    else
      ElseGen(AllocaIP, Builder.saveIP());
    return;
  }

  Function *CurFn = Builder.GetInsertBlock()->getParent();

  // Otherwise, the condition did not fold, or we couldn't elide it.  Just
  // emit the conditional branch.
  BasicBlock *ThenBlock = BasicBlock::Create(M.getContext(), "omp_if.then");
  BasicBlock *ElseBlock = BasicBlock::Create(M.getContext(), "omp_if.else");
  BasicBlock *ContBlock = BasicBlock::Create(M.getContext(), "omp_if.end");
  Builder.CreateCondBr(Cond, ThenBlock, ElseBlock);
  // Emit the 'then' code.
  emitBlock(ThenBlock, CurFn);
  ThenGen(AllocaIP, Builder.saveIP());
  emitBranch(ContBlock);
  // Emit the 'else' code if present.
  // There is no need to emit line number for unconditional branch.
  emitBlock(ElseBlock, CurFn);
  ElseGen(AllocaIP, Builder.saveIP());
  // There is no need to emit line number for unconditional branch.
  emitBranch(ContBlock);
  // Emit the continuation block for code after the if.
  emitBlock(ContBlock, CurFn, /*IsFinished=*/true);
}

bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
    const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
  assert(!(AO == AtomicOrdering::NotAtomic ||
           AO == llvm::AtomicOrdering::Unordered) &&
         "Unexpected Atomic Ordering.");

  bool Flush = false;
  llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;

  switch (AK) {
  case Read:
    if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
        AO == AtomicOrdering::SequentiallyConsistent) {
      FlushAO = AtomicOrdering::Acquire;
      Flush = true;
    }
    break;
  case Write:
  case Compare:
  case Update:
    if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
        AO == AtomicOrdering::SequentiallyConsistent) {
      FlushAO = AtomicOrdering::Release;
      Flush = true;
    }
    break;
  case Capture:
    switch (AO) {
    case AtomicOrdering::Acquire:
      FlushAO = AtomicOrdering::Acquire;
      Flush = true;
      break;
    case AtomicOrdering::Release:
      FlushAO = AtomicOrdering::Release;
      Flush = true;
      break;
    case AtomicOrdering::AcquireRelease:
    case AtomicOrdering::SequentiallyConsistent:
      FlushAO = AtomicOrdering::AcquireRelease;
      Flush = true;
      break;
    default:
      // do nothing - leave silently.
      break;
    }
  }

  if (Flush) {
    // Currently Flush RT call still doesn't take memory_ordering, so for when
    // that happens, this tries to do the resolution of which atomic ordering
    // to use with but issue the flush call
    // TODO: pass `FlushAO` after memory ordering support is added
    (void)FlushAO;
    emitFlush(Loc);
  }

  // for AO == AtomicOrdering::Monotonic and  all other case combinations
  // do nothing
  return Flush;
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
                                  AtomicOpValue &X, AtomicOpValue &V,
                                  AtomicOrdering AO) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  assert(X.Var->getType()->isPointerTy() &&
         "OMP Atomic expects a pointer to target memory");
  Type *XElemTy = X.ElemTy;
  assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
          XElemTy->isPointerTy()) &&
         "OMP atomic read expected a scalar type");

  Value *XRead = nullptr;

  if (XElemTy->isIntegerTy()) {
    LoadInst *XLD =
        Builder.CreateLoad(XElemTy, X.Var, X.IsVolatile, "omp.atomic.read");
    XLD->setAtomic(AO);
    XRead = cast<Value>(XLD);
  } else {
    // We need to perform atomic op as integer
    IntegerType *IntCastTy =
        IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
    LoadInst *XLoad =
        Builder.CreateLoad(IntCastTy, X.Var, X.IsVolatile, "omp.atomic.load");
    XLoad->setAtomic(AO);
    if (XElemTy->isFloatingPointTy()) {
      XRead = Builder.CreateBitCast(XLoad, XElemTy, "atomic.flt.cast");
    } else {
      XRead = Builder.CreateIntToPtr(XLoad, XElemTy, "atomic.ptr.cast");
    }
  }
  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Read);
  Builder.CreateStore(XRead, V.Var, V.IsVolatile);
  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
                                   AtomicOpValue &X, Value *Expr,
                                   AtomicOrdering AO) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  assert(X.Var->getType()->isPointerTy() &&
         "OMP Atomic expects a pointer to target memory");
  Type *XElemTy = X.ElemTy;
  assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
          XElemTy->isPointerTy()) &&
         "OMP atomic write expected a scalar type");

  if (XElemTy->isIntegerTy()) {
    StoreInst *XSt = Builder.CreateStore(Expr, X.Var, X.IsVolatile);
    XSt->setAtomic(AO);
  } else {
    // We need to bitcast and perform atomic op as integers
    IntegerType *IntCastTy =
        IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
    Value *ExprCast =
        Builder.CreateBitCast(Expr, IntCastTy, "atomic.src.int.cast");
    StoreInst *XSt = Builder.CreateStore(ExprCast, X.Var, X.IsVolatile);
    XSt->setAtomic(AO);
  }

  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Write);
  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
    const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
    Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
  assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
  if (!updateToLocation(Loc))
    return Loc.IP;

  LLVM_DEBUG({
    Type *XTy = X.Var->getType();
    assert(XTy->isPointerTy() &&
           "OMP Atomic expects a pointer to target memory");
    Type *XElemTy = X.ElemTy;
    assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
            XElemTy->isPointerTy()) &&
           "OMP atomic update expected a scalar type");
    assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
           (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&
           "OpenMP atomic does not support LT or GT operations");
  });

  emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp,
                   X.IsVolatile, IsXBinopExpr);
  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
  return Builder.saveIP();
}

// FIXME: Duplicating AtomicExpand
Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
                                               AtomicRMWInst::BinOp RMWOp) {
  switch (RMWOp) {
  case AtomicRMWInst::Add:
    return Builder.CreateAdd(Src1, Src2);
  case AtomicRMWInst::Sub:
    return Builder.CreateSub(Src1, Src2);
  case AtomicRMWInst::And:
    return Builder.CreateAnd(Src1, Src2);
  case AtomicRMWInst::Nand:
    return Builder.CreateNeg(Builder.CreateAnd(Src1, Src2));
  case AtomicRMWInst::Or:
    return Builder.CreateOr(Src1, Src2);
  case AtomicRMWInst::Xor:
    return Builder.CreateXor(Src1, Src2);
  case AtomicRMWInst::Xchg:
  case AtomicRMWInst::FAdd:
  case AtomicRMWInst::FSub:
  case AtomicRMWInst::BAD_BINOP:
  case AtomicRMWInst::Max:
  case AtomicRMWInst::Min:
  case AtomicRMWInst::UMax:
  case AtomicRMWInst::UMin:
  case AtomicRMWInst::FMax:
  case AtomicRMWInst::FMin:
  case AtomicRMWInst::UIncWrap:
  case AtomicRMWInst::UDecWrap:
    llvm_unreachable("Unsupported atomic update operation");
  }
  llvm_unreachable("Unsupported atomic update operation");
}

std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
    InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
    AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) {
  // TODO: handle the case where XElemTy is not byte-sized or not a power of 2
  // or a complex datatype.
  bool emitRMWOp = false;
  switch (RMWOp) {
  case AtomicRMWInst::Add:
  case AtomicRMWInst::And:
  case AtomicRMWInst::Nand:
  case AtomicRMWInst::Or:
  case AtomicRMWInst::Xor:
  case AtomicRMWInst::Xchg:
    emitRMWOp = XElemTy;
    break;
  case AtomicRMWInst::Sub:
    emitRMWOp = (IsXBinopExpr && XElemTy);
    break;
  default:
    emitRMWOp = false;
  }
  emitRMWOp &= XElemTy->isIntegerTy();

  std::pair<Value *, Value *> Res;
  if (emitRMWOp) {
    Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
    // not needed except in case of postfix captures. Generate anyway for
    // consistency with the else part. Will be removed with any DCE pass.
    // AtomicRMWInst::Xchg does not have a coressponding instruction.
    if (RMWOp == AtomicRMWInst::Xchg)
      Res.second = Res.first;
    else
      Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);
  } else {
    IntegerType *IntCastTy =
        IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
    LoadInst *OldVal =
        Builder.CreateLoad(IntCastTy, X, X->getName() + ".atomic.load");
    OldVal->setAtomic(AO);
    // CurBB
    // |     /---\
		// ContBB    |
    // |     \---/
    // ExitBB
    BasicBlock *CurBB = Builder.GetInsertBlock();
    Instruction *CurBBTI = CurBB->getTerminator();
    CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
    BasicBlock *ExitBB =
        CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit");
    BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(),
                                                X->getName() + ".atomic.cont");
    ContBB->getTerminator()->eraseFromParent();
    Builder.restoreIP(AllocaIP);
    AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);
    NewAtomicAddr->setName(X->getName() + "x.new.val");
    Builder.SetInsertPoint(ContBB);
    llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);
    PHI->addIncoming(OldVal, CurBB);
    bool IsIntTy = XElemTy->isIntegerTy();
    Value *OldExprVal = PHI;
    if (!IsIntTy) {
      if (XElemTy->isFloatingPointTy()) {
        OldExprVal = Builder.CreateBitCast(PHI, XElemTy,
                                           X->getName() + ".atomic.fltCast");
      } else {
        OldExprVal = Builder.CreateIntToPtr(PHI, XElemTy,
                                            X->getName() + ".atomic.ptrCast");
      }
    }

    Value *Upd = UpdateOp(OldExprVal, Builder);
    Builder.CreateStore(Upd, NewAtomicAddr);
    LoadInst *DesiredVal = Builder.CreateLoad(IntCastTy, NewAtomicAddr);
    AtomicOrdering Failure =
        llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
    AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
        X, PHI, DesiredVal, llvm::MaybeAlign(), AO, Failure);
    Result->setVolatile(VolatileX);
    Value *PreviousVal = Builder.CreateExtractValue(Result, /*Idxs=*/0);
    Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
    PHI->addIncoming(PreviousVal, Builder.GetInsertBlock());
    Builder.CreateCondBr(SuccessFailureVal, ExitBB, ContBB);

    Res.first = OldExprVal;
    Res.second = Upd;

    // set Insertion point in exit block
    if (UnreachableInst *ExitTI =
            dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
      CurBBTI->eraseFromParent();
      Builder.SetInsertPoint(ExitBB);
    } else {
      Builder.SetInsertPoint(ExitTI);
    }
  }

  return Res;
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
    const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
    AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
    AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
  if (!updateToLocation(Loc))
    return Loc.IP;

  LLVM_DEBUG({
    Type *XTy = X.Var->getType();
    assert(XTy->isPointerTy() &&
           "OMP Atomic expects a pointer to target memory");
    Type *XElemTy = X.ElemTy;
    assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
            XElemTy->isPointerTy()) &&
           "OMP atomic capture expected a scalar type");
    assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
           "OpenMP atomic does not support LT or GT operations");
  });

  // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
  // 'x' is simply atomically rewritten with 'expr'.
  AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
  std::pair<Value *, Value *> Result =
      emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
                       X.IsVolatile, IsXBinopExpr);

  Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second);
  Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile);

  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture);
  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
    const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
    AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
    omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
    bool IsFailOnly) {

  AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
  return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,
                             IsPostfixUpdate, IsFailOnly, Failure);
}

OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
    const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
    AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
    omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
    bool IsFailOnly, AtomicOrdering Failure) {

  if (!updateToLocation(Loc))
    return Loc.IP;

  assert(X.Var->getType()->isPointerTy() &&
         "OMP atomic expects a pointer to target memory");
  // compare capture
  if (V.Var) {
    assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type");
    assert(V.ElemTy == X.ElemTy && "x and v must be of same type");
  }

  bool IsInteger = E->getType()->isIntegerTy();

  if (Op == OMPAtomicCompareOp::EQ) {
    AtomicCmpXchgInst *Result = nullptr;
    if (!IsInteger) {
      IntegerType *IntCastTy =
          IntegerType::get(M.getContext(), X.ElemTy->getScalarSizeInBits());
      Value *EBCast = Builder.CreateBitCast(E, IntCastTy);
      Value *DBCast = Builder.CreateBitCast(D, IntCastTy);
      Result = Builder.CreateAtomicCmpXchg(X.Var, EBCast, DBCast, MaybeAlign(),
                                           AO, Failure);
    } else {
      Result =
          Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure);
    }

    if (V.Var) {
      Value *OldValue = Builder.CreateExtractValue(Result, /*Idxs=*/0);
      if (!IsInteger)
        OldValue = Builder.CreateBitCast(OldValue, X.ElemTy);
      assert(OldValue->getType() == V.ElemTy &&
             "OldValue and V must be of same type");
      if (IsPostfixUpdate) {
        Builder.CreateStore(OldValue, V.Var, V.IsVolatile);
      } else {
        Value *SuccessOrFail = Builder.CreateExtractValue(Result, /*Idxs=*/1);
        if (IsFailOnly) {
          // CurBB----
          //   |     |
          //   v     |
          // ContBB  |
          //   |     |
          //   v     |
          // ExitBB <-
          //
          // where ContBB only contains the store of old value to 'v'.
          BasicBlock *CurBB = Builder.GetInsertBlock();
          Instruction *CurBBTI = CurBB->getTerminator();
          CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
          BasicBlock *ExitBB = CurBB->splitBasicBlock(
              CurBBTI, X.Var->getName() + ".atomic.exit");
          BasicBlock *ContBB = CurBB->splitBasicBlock(
              CurBB->getTerminator(), X.Var->getName() + ".atomic.cont");
          ContBB->getTerminator()->eraseFromParent();
          CurBB->getTerminator()->eraseFromParent();

          Builder.CreateCondBr(SuccessOrFail, ExitBB, ContBB);

          Builder.SetInsertPoint(ContBB);
          Builder.CreateStore(OldValue, V.Var);
          Builder.CreateBr(ExitBB);

          if (UnreachableInst *ExitTI =
                  dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
            CurBBTI->eraseFromParent();
            Builder.SetInsertPoint(ExitBB);
          } else {
            Builder.SetInsertPoint(ExitTI);
          }
        } else {
          Value *CapturedValue =
              Builder.CreateSelect(SuccessOrFail, E, OldValue);
          Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);
        }
      }
    }
    // The comparison result has to be stored.
    if (R.Var) {
      assert(R.Var->getType()->isPointerTy() &&
             "r.var must be of pointer type");
      assert(R.ElemTy->isIntegerTy() && "r must be of integral type");

      Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1);
      Value *ResultCast = R.IsSigned
                              ? Builder.CreateSExt(SuccessFailureVal, R.ElemTy)
                              : Builder.CreateZExt(SuccessFailureVal, R.ElemTy);
      Builder.CreateStore(ResultCast, R.Var, R.IsVolatile);
    }
  } else {
    assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&
           "Op should be either max or min at this point");
    assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==");

    // Reverse the ordop as the OpenMP forms are different from LLVM forms.
    // Let's take max as example.
    // OpenMP form:
    // x = x > expr ? expr : x;
    // LLVM form:
    // *ptr = *ptr > val ? *ptr : val;
    // We need to transform to LLVM form.
    // x = x <= expr ? x : expr;
    AtomicRMWInst::BinOp NewOp;
    if (IsXBinopExpr) {
      if (IsInteger) {
        if (X.IsSigned)
          NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
                                                : AtomicRMWInst::Max;
        else
          NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
                                                : AtomicRMWInst::UMax;
      } else {
        NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
                                              : AtomicRMWInst::FMax;
      }
    } else {
      if (IsInteger) {
        if (X.IsSigned)
          NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
                                                : AtomicRMWInst::Min;
        else
          NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
                                                : AtomicRMWInst::UMin;
      } else {
        NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
                                              : AtomicRMWInst::FMin;
      }
    }

    AtomicRMWInst *OldValue =
        Builder.CreateAtomicRMW(NewOp, X.Var, E, MaybeAlign(), AO);
    if (V.Var) {
      Value *CapturedValue = nullptr;
      if (IsPostfixUpdate) {
        CapturedValue = OldValue;
      } else {
        CmpInst::Predicate Pred;
        switch (NewOp) {
        case AtomicRMWInst::Max:
          Pred = CmpInst::ICMP_SGT;
          break;
        case AtomicRMWInst::UMax:
          Pred = CmpInst::ICMP_UGT;
          break;
        case AtomicRMWInst::FMax:
          Pred = CmpInst::FCMP_OGT;
          break;
        case AtomicRMWInst::Min:
          Pred = CmpInst::ICMP_SLT;
          break;
        case AtomicRMWInst::UMin:
          Pred = CmpInst::ICMP_ULT;
          break;
        case AtomicRMWInst::FMin:
          Pred = CmpInst::FCMP_OLT;
          break;
        default:
          llvm_unreachable("unexpected comparison op");
        }
        Value *NonAtomicCmp = Builder.CreateCmp(Pred, OldValue, E);
        CapturedValue = Builder.CreateSelect(NonAtomicCmp, E, OldValue);
      }
      Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile);
    }
  }

  checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Compare);

  return Builder.saveIP();
}

OpenMPIRBuilder::InsertPointTy
OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
                             BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
                             Value *NumTeamsUpper, Value *ThreadLimit,
                             Value *IfExpr) {
  if (!updateToLocation(Loc))
    return InsertPointTy();

  uint32_t SrcLocStrSize;
  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
  Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
  Function *CurrentFunction = Builder.GetInsertBlock()->getParent();

  // Outer allocation basicblock is the entry block of the current function.
  BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock();
  if (&OuterAllocaBB == Builder.GetInsertBlock()) {
    BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "teams.entry");
    Builder.SetInsertPoint(BodyBB, BodyBB->begin());
  }

  // The current basic block is split into four basic blocks. After outlining,
  // they will be mapped as follows:
  // ```
  // def current_fn() {
  //   current_basic_block:
  //     br label %teams.exit
  //   teams.exit:
  //     ; instructions after teams
  // }
  //
  // def outlined_fn() {
  //   teams.alloca:
  //     br label %teams.body
  //   teams.body:
  //     ; instructions within teams body
  // }
  // ```
  BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, "teams.exit");
  BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "teams.body");
  BasicBlock *AllocaBB =
      splitBB(Builder, /*CreateBranch=*/true, "teams.alloca");

  bool SubClausesPresent =
      (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
  // Push num_teams
  if (!Config.isTargetDevice() && SubClausesPresent) {
    assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&
           "if lowerbound is non-null, then upperbound must also be non-null "
           "for bounds on num_teams");

    if (NumTeamsUpper == nullptr)
      NumTeamsUpper = Builder.getInt32(0);

    if (NumTeamsLower == nullptr)
      NumTeamsLower = NumTeamsUpper;

    if (IfExpr) {
      assert(IfExpr->getType()->isIntegerTy() &&
             "argument to if clause must be an integer value");

      // upper = ifexpr ? upper : 1
      if (IfExpr->getType() != Int1)
        IfExpr = Builder.CreateICmpNE(IfExpr,
                                      ConstantInt::get(IfExpr->getType(), 0));
      NumTeamsUpper = Builder.CreateSelect(
          IfExpr, NumTeamsUpper, Builder.getInt32(1), "numTeamsUpper");

      // lower = ifexpr ? lower : 1
      NumTeamsLower = Builder.CreateSelect(
          IfExpr, NumTeamsLower, Builder.getInt32(1), "numTeamsLower");
    }

    if (ThreadLimit == nullptr)
      ThreadLimit = Builder.getInt32(0);

    Value *ThreadNum = getOrCreateThreadID(Ident);
    Builder.CreateCall(
        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_teams_51),
        {Ident, ThreadNum, NumTeamsLower, NumTeamsUpper, ThreadLimit});
  }
  // Generate the body of teams.
  InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
  InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
  BodyGenCB(AllocaIP, CodeGenIP);

  OutlineInfo OI;
  OI.EntryBB = AllocaBB;
  OI.ExitBB = ExitBB;
  OI.OuterAllocaBB = &OuterAllocaBB;

  // Insert fake values for global tid and bound tid.
  SmallVector<Instruction *, 8> ToBeDeleted;
  InsertPointTy OuterAllocaIP(&OuterAllocaBB, OuterAllocaBB.begin());
  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(
      Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "gid", true));
  OI.ExcludeArgsFromAggregate.push_back(createFakeIntVal(
      Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "tid", true));

  auto HostPostOutlineCB = [this, Ident,
                            ToBeDeleted](Function &OutlinedFn) mutable {
    // The stale call instruction will be replaced with a new call instruction
    // for runtime call with the outlined function.

    assert(OutlinedFn.getNumUses() == 1 &&
           "there must be a single user for the outlined function");
    CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
    ToBeDeleted.push_back(StaleCI);

    assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&
           "Outlined function must have two or three arguments only");

    bool HasShared = OutlinedFn.arg_size() == 3;

    OutlinedFn.getArg(0)->setName("global.tid.ptr");
    OutlinedFn.getArg(1)->setName("bound.tid.ptr");
    if (HasShared)
      OutlinedFn.getArg(2)->setName("data");

    // Call to the runtime function for teams in the current function.
    assert(StaleCI && "Error while outlining - no CallInst user found for the "
                      "outlined function.");
    Builder.SetInsertPoint(StaleCI);
    SmallVector<Value *> Args = {
        Ident, Builder.getInt32(StaleCI->arg_size() - 2), &OutlinedFn};
    if (HasShared)
      Args.push_back(StaleCI->getArgOperand(2));
    Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
                           omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
                       Args);

    for (Instruction *I : llvm::reverse(ToBeDeleted))
      I->eraseFromParent();
  };

  if (!Config.isTargetDevice())
    OI.PostOutlineCB = HostPostOutlineCB;

  addOutlineInfo(std::move(OI));

  Builder.SetInsertPoint(ExitBB, ExitBB->begin());

  return Builder.saveIP();
}

GlobalVariable *
OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
                                       std::string VarName) {
  llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
      llvm::ArrayType::get(llvm::PointerType::getUnqual(M.getContext()),
                           Names.size()),
      Names);
  auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
      M, MapNamesArrayInit->getType(),
      /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
      VarName);
  return MapNamesArrayGlobal;
}

// Create all simple and struct types exposed by the runtime and remember
// the llvm::PointerTypes of them for easy access later.
void OpenMPIRBuilder::initializeTypes(Module &M) {
  LLVMContext &Ctx = M.getContext();
  StructType *T;
#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize)                             \
  VarName##Ty = ArrayType::get(ElemTy, ArraySize);                             \
  VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...)                  \
  VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg);            \
  VarName##Ptr = PointerType::getUnqual(VarName);
#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...)                      \
  T = StructType::getTypeByName(Ctx, StructName);                              \
  if (!T)                                                                      \
    T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed);            \
  VarName = T;                                                                 \
  VarName##Ptr = PointerType::getUnqual(T);
#include "llvm/Frontend/OpenMP/OMPKinds.def"
}

void OpenMPIRBuilder::OutlineInfo::collectBlocks(
    SmallPtrSetImpl<BasicBlock *> &BlockSet,
    SmallVectorImpl<BasicBlock *> &BlockVector) {
  SmallVector<BasicBlock *, 32> Worklist;
  BlockSet.insert(EntryBB);
  BlockSet.insert(ExitBB);

  Worklist.push_back(EntryBB);
  while (!Worklist.empty()) {
    BasicBlock *BB = Worklist.pop_back_val();
    BlockVector.push_back(BB);
    for (BasicBlock *SuccBB : successors(BB))
      if (BlockSet.insert(SuccBB).second)
        Worklist.push_back(SuccBB);
  }
}

void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
                                         uint64_t Size, int32_t Flags,
                                         GlobalValue::LinkageTypes,
                                         StringRef Name) {
  if (!Config.isGPU()) {
    llvm::offloading::emitOffloadingEntry(
        M, ID, Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0,
        "omp_offloading_entries");
    return;
  }
  // TODO: Add support for global variables on the device after declare target
  // support.
  Function *Fn = dyn_cast<Function>(Addr);
  if (!Fn)
    return;

  Module &M = *(Fn->getParent());
  LLVMContext &Ctx = M.getContext();

  // Get "nvvm.annotations" metadata node.
  NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");

  Metadata *MDVals[] = {
      ConstantAsMetadata::get(Fn), MDString::get(Ctx, "kernel"),
      ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(Ctx), 1))};
  // Append metadata to nvvm.annotations.
  MD->addOperand(MDNode::get(Ctx, MDVals));

  // Add a function attribute for the kernel.
  Fn->addFnAttr(Attribute::get(Ctx, "kernel"));
  if (T.isAMDGCN())
    Fn->addFnAttr("uniform-work-group-size", "true");
  Fn->addFnAttr(Attribute::MustProgress);
}

// We only generate metadata for function that contain target regions.
void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(
    EmitMetadataErrorReportFunctionTy &ErrorFn) {

  // If there are no entries, we don't need to do anything.
  if (OffloadInfoManager.empty())
    return;

  LLVMContext &C = M.getContext();
  SmallVector<std::pair<const OffloadEntriesInfoManager::OffloadEntryInfo *,
                        TargetRegionEntryInfo>,
              16>
      OrderedEntries(OffloadInfoManager.size());

  // Auxiliary methods to create metadata values and strings.
  auto &&GetMDInt = [this](unsigned V) {
    return ConstantAsMetadata::get(ConstantInt::get(Builder.getInt32Ty(), V));
  };

  auto &&GetMDString = [&C](StringRef V) { return MDString::get(C, V); };

  // Create the offloading info metadata node.
  NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
  auto &&TargetRegionMetadataEmitter =
      [&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
          const TargetRegionEntryInfo &EntryInfo,
          const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) {
        // Generate metadata for target regions. Each entry of this metadata
        // contains:
        // - Entry 0 -> Kind of this type of metadata (0).
        // - Entry 1 -> Device ID of the file where the entry was identified.
        // - Entry 2 -> File ID of the file where the entry was identified.
        // - Entry 3 -> Mangled name of the function where the entry was
        // identified.
        // - Entry 4 -> Line in the file where the entry was identified.
        // - Entry 5 -> Count of regions at this DeviceID/FilesID/Line.
        // - Entry 6 -> Order the entry was created.
        // The first element of the metadata node is the kind.
        Metadata *Ops[] = {
            GetMDInt(E.getKind()),      GetMDInt(EntryInfo.DeviceID),
            GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
            GetMDInt(EntryInfo.Line),   GetMDInt(EntryInfo.Count),
            GetMDInt(E.getOrder())};

        // Save this entry in the right position of the ordered entries array.
        OrderedEntries[E.getOrder()] = std::make_pair(&E, EntryInfo);

        // Add metadata to the named metadata node.
        MD->addOperand(MDNode::get(C, Ops));
      };

  OffloadInfoManager.actOnTargetRegionEntriesInfo(TargetRegionMetadataEmitter);

  // Create function that emits metadata for each device global variable entry;
  auto &&DeviceGlobalVarMetadataEmitter =
      [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
          StringRef MangledName,
          const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) {
        // Generate metadata for global variables. Each entry of this metadata
        // contains:
        // - Entry 0 -> Kind of this type of metadata (1).
        // - Entry 1 -> Mangled name of the variable.
        // - Entry 2 -> Declare target kind.
        // - Entry 3 -> Order the entry was created.
        // The first element of the metadata node is the kind.
        Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
                           GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};

        // Save this entry in the right position of the ordered entries array.
        TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0);
        OrderedEntries[E.getOrder()] = std::make_pair(&E, varInfo);

        // Add metadata to the named metadata node.
        MD->addOperand(MDNode::get(C, Ops));
      };

  OffloadInfoManager.actOnDeviceGlobalVarEntriesInfo(
      DeviceGlobalVarMetadataEmitter);

  for (const auto &E : OrderedEntries) {
    assert(E.first && "All ordered entries must exist!");
    if (const auto *CE =
            dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
                E.first)) {
      if (!CE->getID() || !CE->getAddress()) {
        // Do not blame the entry if the parent funtion is not emitted.
        TargetRegionEntryInfo EntryInfo = E.second;
        StringRef FnName = EntryInfo.ParentName;
        if (!M.getNamedValue(FnName))
          continue;
        ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo);
        continue;
      }
      createOffloadEntry(CE->getID(), CE->getAddress(),
                         /*Size=*/0, CE->getFlags(),
                         GlobalValue::WeakAnyLinkage);
    } else if (const auto *CE = dyn_cast<
                   OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>(
                   E.first)) {
      OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags =
          static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
              CE->getFlags());
      switch (Flags) {
      case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter:
      case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo:
        if (Config.isTargetDevice() && Config.hasRequiresUnifiedSharedMemory())
          continue;
        if (!CE->getAddress()) {
          ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second);
          continue;
        }
        // The vaiable has no definition - no need to add the entry.
        if (CE->getVarSize() == 0)
          continue;
        break;
      case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink:
        assert(((Config.isTargetDevice() && !CE->getAddress()) ||
                (!Config.isTargetDevice() && CE->getAddress())) &&
               "Declaret target link address is set.");
        if (Config.isTargetDevice())
          continue;
        if (!CE->getAddress()) {
          ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo());
          continue;
        }
        break;
      default:
        break;
      }

      // Hidden or internal symbols on the device are not externally visible.
      // We should not attempt to register them by creating an offloading
      // entry. Indirect variables are handled separately on the device.
      if (auto *GV = dyn_cast<GlobalValue>(CE->getAddress()))
        if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
            Flags != OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
          continue;

      // Indirect globals need to use a special name that doesn't match the name
      // of the associated host global.
      if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
        createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
                           Flags, CE->getLinkage(), CE->getVarName());
      else
        createOffloadEntry(CE->getAddress(), CE->getAddress(), CE->getVarSize(),
                           Flags, CE->getLinkage());

    } else {
      llvm_unreachable("Unsupported entry kind.");
    }
  }

  // Emit requires directive globals to a special entry so the runtime can
  // register them when the device image is loaded.
  // TODO: This reduces the offloading entries to a 32-bit integer. Offloading
  //       entries should be redesigned to better suit this use-case.
  if (Config.hasRequiresFlags() && !Config.isTargetDevice())
    offloading::emitOffloadingEntry(
        M, Constant::getNullValue(PointerType::getUnqual(M.getContext())),
        /*Name=*/"",
        /*Size=*/0, OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,
        Config.getRequiresFlags(), "omp_offloading_entries");
}

void TargetRegionEntryInfo::getTargetRegionEntryFnName(
    SmallVectorImpl<char> &Name, StringRef ParentName, unsigned DeviceID,
    unsigned FileID, unsigned Line, unsigned Count) {
  raw_svector_ostream OS(Name);
  OS << KernelNamePrefix << llvm::format("%x", DeviceID)
     << llvm::format("_%x_", FileID) << ParentName << "_l" << Line;
  if (Count)
    OS << "_" << Count;
}

void OffloadEntriesInfoManager::getTargetRegionEntryFnName(
    SmallVectorImpl<char> &Name, const TargetRegionEntryInfo &EntryInfo) {
  unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
  TargetRegionEntryInfo::getTargetRegionEntryFnName(
      Name, EntryInfo.ParentName, EntryInfo.DeviceID, EntryInfo.FileID,
      EntryInfo.Line, NewCount);
}

TargetRegionEntryInfo
OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
                                          StringRef ParentName) {
  sys::fs::UniqueID ID;
  auto FileIDInfo = CallBack();
  if (auto EC = sys::fs::getUniqueID(std::get<0>(FileIDInfo), ID)) {
    report_fatal_error(("Unable to get unique ID for file, during "
                        "getTargetEntryUniqueInfo, error message: " +
                        EC.message())
                           .c_str());
  }

  return TargetRegionEntryInfo(ParentName, ID.getDevice(), ID.getFile(),
                               std::get<1>(FileIDInfo));
}

unsigned OpenMPIRBuilder::getFlagMemberOffset() {
  unsigned Offset = 0;
  for (uint64_t Remain =
           static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
               omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
       !(Remain & 1); Remain = Remain >> 1)
    Offset++;
  return Offset;
}

omp::OpenMPOffloadMappingFlags
OpenMPIRBuilder::getMemberOfFlag(unsigned Position) {
  // Rotate by getFlagMemberOffset() bits.
  return static_cast<omp::OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
                                                     << getFlagMemberOffset());
}

void OpenMPIRBuilder::setCorrectMemberOfFlag(
    omp::OpenMPOffloadMappingFlags &Flags,
    omp::OpenMPOffloadMappingFlags MemberOfFlag) {
  // If the entry is PTR_AND_OBJ but has not been marked with the special
  // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
  // marked as MEMBER_OF.
  if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
          Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ) &&
      static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
          (Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
          omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF))
    return;

  // Reset the placeholder value to prepare the flag for the assignment of the
  // proper MEMBER_OF value.
  Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
  Flags |= MemberOfFlag;
}

Constant *OpenMPIRBuilder::getAddrOfDeclareTargetVar(
    OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
    OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
    bool IsDeclaration, bool IsExternallyVisible,
    TargetRegionEntryInfo EntryInfo, StringRef MangledName,
    std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
    std::vector<Triple> TargetTriple, Type *LlvmPtrTy,
    std::function<Constant *()> GlobalInitializer,
    std::function<GlobalValue::LinkageTypes()> VariableLinkage) {
  // TODO: convert this to utilise the IRBuilder Config rather than
  // a passed down argument.
  if (OpenMPSIMD)
    return nullptr;

  if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink ||
      ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
        CaptureClause ==
            OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
       Config.hasRequiresUnifiedSharedMemory())) {
    SmallString<64> PtrName;
    {
      raw_svector_ostream OS(PtrName);
      OS << MangledName;
      if (!IsExternallyVisible)
        OS << format("_%x", EntryInfo.FileID);
      OS << "_decl_tgt_ref_ptr";
    }

    Value *Ptr = M.getNamedValue(PtrName);

    if (!Ptr) {
      GlobalValue *GlobalValue = M.getNamedValue(MangledName);
      Ptr = getOrCreateInternalVariable(LlvmPtrTy, PtrName);

      auto *GV = cast<GlobalVariable>(Ptr);
      GV->setLinkage(GlobalValue::WeakAnyLinkage);

      if (!Config.isTargetDevice()) {
        if (GlobalInitializer)
          GV->setInitializer(GlobalInitializer());
        else
          GV->setInitializer(GlobalValue);
      }

      registerTargetGlobalVariable(
          CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
          EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
          GlobalInitializer, VariableLinkage, LlvmPtrTy, cast<Constant>(Ptr));
    }

    return cast<Constant>(Ptr);
  }

  return nullptr;
}

void OpenMPIRBuilder::registerTargetGlobalVariable(
    OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
    OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
    bool IsDeclaration, bool IsExternallyVisible,
    TargetRegionEntryInfo EntryInfo, StringRef MangledName,
    std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
    std::vector<Triple> TargetTriple,
    std::function<Constant *()> GlobalInitializer,
    std::function<GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy,
    Constant *Addr) {
  if (DeviceClause != OffloadEntriesInfoManager::OMPTargetDeviceClauseAny ||
      (TargetTriple.empty() && !Config.isTargetDevice()))
    return;

  OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags;
  StringRef VarName;
  int64_t VarSize;
  GlobalValue::LinkageTypes Linkage;

  if ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
       CaptureClause ==
           OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
      !Config.hasRequiresUnifiedSharedMemory()) {
    Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
    VarName = MangledName;
    GlobalValue *LlvmVal = M.getNamedValue(VarName);

    if (!IsDeclaration)
      VarSize = divideCeil(
          M.getDataLayout().getTypeSizeInBits(LlvmVal->getValueType()), 8);
    else
      VarSize = 0;
    Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage();

    // This is a workaround carried over from Clang which prevents undesired
    // optimisation of internal variables.
    if (Config.isTargetDevice() &&
        (!IsExternallyVisible || Linkage == GlobalValue::LinkOnceODRLinkage)) {
      // Do not create a "ref-variable" if the original is not also available
      // on the host.
      if (!OffloadInfoManager.hasDeviceGlobalVarEntryInfo(VarName))
        return;

      std::string RefName = createPlatformSpecificName({VarName, "ref"});

      if (!M.getNamedValue(RefName)) {
        Constant *AddrRef =
            getOrCreateInternalVariable(Addr->getType(), RefName);
        auto *GvAddrRef = cast<GlobalVariable>(AddrRef);
        GvAddrRef->setConstant(true);
        GvAddrRef->setLinkage(GlobalValue::InternalLinkage);
        GvAddrRef->setInitializer(Addr);
        GeneratedRefs.push_back(GvAddrRef);
      }
    }
  } else {
    if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink)
      Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
    else
      Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;

    if (Config.isTargetDevice()) {
      VarName = (Addr) ? Addr->getName() : "";
      Addr = nullptr;
    } else {
      Addr = getAddrOfDeclareTargetVar(
          CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
          EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
          LlvmPtrTy, GlobalInitializer, VariableLinkage);
      VarName = (Addr) ? Addr->getName() : "";
    }
    VarSize = M.getDataLayout().getPointerSize();
    Linkage = GlobalValue::WeakAnyLinkage;
  }

  OffloadInfoManager.registerDeviceGlobalVarEntryInfo(VarName, Addr, VarSize,
                                                      Flags, Linkage);
}

/// Loads all the offload entries information from the host IR
/// metadata.
void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) {
  // If we are in target mode, load the metadata from the host IR. This code has
  // to match the metadata creation in createOffloadEntriesAndInfoMetadata().

  NamedMDNode *MD = M.getNamedMetadata(ompOffloadInfoName);
  if (!MD)
    return;

  for (MDNode *MN : MD->operands()) {
    auto &&GetMDInt = [MN](unsigned Idx) {
      auto *V = cast<ConstantAsMetadata>(MN->getOperand(Idx));
      return cast<ConstantInt>(V->getValue())->getZExtValue();
    };

    auto &&GetMDString = [MN](unsigned Idx) {
      auto *V = cast<MDString>(MN->getOperand(Idx));
      return V->getString();
    };

    switch (GetMDInt(0)) {
    default:
      llvm_unreachable("Unexpected metadata!");
      break;
    case OffloadEntriesInfoManager::OffloadEntryInfo::
        OffloadingEntryInfoTargetRegion: {
      TargetRegionEntryInfo EntryInfo(/*ParentName=*/GetMDString(3),
                                      /*DeviceID=*/GetMDInt(1),
                                      /*FileID=*/GetMDInt(2),
                                      /*Line=*/GetMDInt(4),
                                      /*Count=*/GetMDInt(5));
      OffloadInfoManager.initializeTargetRegionEntryInfo(EntryInfo,
                                                         /*Order=*/GetMDInt(6));
      break;
    }
    case OffloadEntriesInfoManager::OffloadEntryInfo::
        OffloadingEntryInfoDeviceGlobalVar:
      OffloadInfoManager.initializeDeviceGlobalVarEntryInfo(
          /*MangledName=*/GetMDString(1),
          static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
              /*Flags=*/GetMDInt(2)),
          /*Order=*/GetMDInt(3));
      break;
    }
  }
}

void OpenMPIRBuilder::loadOffloadInfoMetadata(StringRef HostFilePath) {
  if (HostFilePath.empty())
    return;

  auto Buf = MemoryBuffer::getFile(HostFilePath);
  if (std::error_code Err = Buf.getError()) {
    report_fatal_error(("error opening host file from host file path inside of "
                        "OpenMPIRBuilder: " +
                        Err.message())
                           .c_str());
  }

  LLVMContext Ctx;
  auto M = expectedToErrorOrAndEmitErrors(
      Ctx, parseBitcodeFile(Buf.get()->getMemBufferRef(), Ctx));
  if (std::error_code Err = M.getError()) {
    report_fatal_error(
        ("error parsing host file inside of OpenMPIRBuilder: " + Err.message())
            .c_str());
  }

  loadOffloadInfoMetadata(*M.get());
}

//===----------------------------------------------------------------------===//
// OffloadEntriesInfoManager
//===----------------------------------------------------------------------===//

bool OffloadEntriesInfoManager::empty() const {
  return OffloadEntriesTargetRegion.empty() &&
         OffloadEntriesDeviceGlobalVar.empty();
}

unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
    const TargetRegionEntryInfo &EntryInfo) const {
  auto It = OffloadEntriesTargetRegionCount.find(
      getTargetRegionEntryCountKey(EntryInfo));
  if (It == OffloadEntriesTargetRegionCount.end())
    return 0;
  return It->second;
}

void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
    const TargetRegionEntryInfo &EntryInfo) {
  OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
      EntryInfo.Count + 1;
}

/// Initialize target region entry.
void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo(
    const TargetRegionEntryInfo &EntryInfo, unsigned Order) {
  OffloadEntriesTargetRegion[EntryInfo] =
      OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
                                   OMPTargetRegionEntryTargetRegion);
  ++OffloadingEntriesNum;
}

void OffloadEntriesInfoManager::registerTargetRegionEntryInfo(
    TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID,
    OMPTargetRegionEntryKind Flags) {
  assert(EntryInfo.Count == 0 && "expected default EntryInfo");

  // Update the EntryInfo with the next available count for this location.
  EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);

  // If we are emitting code for a target, the entry is already initialized,
  // only has to be registered.
  if (OMPBuilder->Config.isTargetDevice()) {
    // This could happen if the device compilation is invoked standalone.
    if (!hasTargetRegionEntryInfo(EntryInfo)) {
      return;
    }
    auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
    Entry.setAddress(Addr);
    Entry.setID(ID);
    Entry.setFlags(Flags);
  } else {
    if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion &&
        hasTargetRegionEntryInfo(EntryInfo, /*IgnoreAddressId*/ true))
      return;
    assert(!hasTargetRegionEntryInfo(EntryInfo) &&
           "Target region entry already registered!");
    OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
    OffloadEntriesTargetRegion[EntryInfo] = Entry;
    ++OffloadingEntriesNum;
  }
  incrementTargetRegionEntryInfoCount(EntryInfo);
}

bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo(
    TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId) const {

  // Update the EntryInfo with the next available count for this location.
  EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);

  auto It = OffloadEntriesTargetRegion.find(EntryInfo);
  if (It == OffloadEntriesTargetRegion.end()) {
    return false;
  }
  // Fail if this entry is already registered.
  if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
    return false;
  return true;
}

void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo(
    const OffloadTargetRegionEntryInfoActTy &Action) {
  // Scan all target region entries and perform the provided action.
  for (const auto &It : OffloadEntriesTargetRegion) {
    Action(It.first, It.second);
  }
}

void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo(
    StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) {
  OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
  ++OffloadingEntriesNum;
}

void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo(
    StringRef VarName, Constant *Addr, int64_t VarSize,
    OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage) {
  if (OMPBuilder->Config.isTargetDevice()) {
    // This could happen if the device compilation is invoked standalone.
    if (!hasDeviceGlobalVarEntryInfo(VarName))
      return;
    auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
    if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
      if (Entry.getVarSize() == 0) {
        Entry.setVarSize(VarSize);
        Entry.setLinkage(Linkage);
      }
      return;
    }
    Entry.setVarSize(VarSize);
    Entry.setLinkage(Linkage);
    Entry.setAddress(Addr);
  } else {
    if (hasDeviceGlobalVarEntryInfo(VarName)) {
      auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
      assert(Entry.isValid() && Entry.getFlags() == Flags &&
             "Entry not initialized!");
      if (Entry.getVarSize() == 0) {
        Entry.setVarSize(VarSize);
        Entry.setLinkage(Linkage);
      }
      return;
    }
    if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
      OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
                                                Addr, VarSize, Flags, Linkage,
                                                VarName.str());
    else
      OffloadEntriesDeviceGlobalVar.try_emplace(
          VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage, "");
    ++OffloadingEntriesNum;
  }
}

void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo(
    const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
  // Scan all target region entries and perform the provided action.
  for (const auto &E : OffloadEntriesDeviceGlobalVar)
    Action(E.getKey(), E.getValue());
}

//===----------------------------------------------------------------------===//
// CanonicalLoopInfo
//===----------------------------------------------------------------------===//

void CanonicalLoopInfo::collectControlBlocks(
    SmallVectorImpl<BasicBlock *> &BBs) {
  // We only count those BBs as control block for which we do not need to
  // reverse the CFG, i.e. not the loop body which can contain arbitrary control
  // flow. For consistency, this also means we do not add the Body block, which
  // is just the entry to the body code.
  BBs.reserve(BBs.size() + 6);
  BBs.append({getPreheader(), Header, Cond, Latch, Exit, getAfter()});
}

BasicBlock *CanonicalLoopInfo::getPreheader() const {
  assert(isValid() && "Requires a valid canonical loop");
  for (BasicBlock *Pred : predecessors(Header)) {
    if (Pred != Latch)
      return Pred;
  }
  llvm_unreachable("Missing preheader");
}

void CanonicalLoopInfo::setTripCount(Value *TripCount) {
  assert(isValid() && "Requires a valid canonical loop");

  Instruction *CmpI = &getCond()->front();
  assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
  CmpI->setOperand(1, TripCount);

#ifndef NDEBUG
  assertOK();
#endif
}

void CanonicalLoopInfo::mapIndVar(
    llvm::function_ref<Value *(Instruction *)> Updater) {
  assert(isValid() && "Requires a valid canonical loop");

  Instruction *OldIV = getIndVar();

  // Record all uses excluding those introduced by the updater. Uses by the
  // CanonicalLoopInfo itself to keep track of the number of iterations are
  // excluded.
  SmallVector<Use *> ReplacableUses;
  for (Use &U : OldIV->uses()) {
    auto *User = dyn_cast<Instruction>(U.getUser());
    if (!User)
      continue;
    if (User->getParent() == getCond())
      continue;
    if (User->getParent() == getLatch())
      continue;
    ReplacableUses.push_back(&U);
  }

  // Run the updater that may introduce new uses
  Value *NewIV = Updater(OldIV);

  // Replace the old uses with the value returned by the updater.
  for (Use *U : ReplacableUses)
    U->set(NewIV);

#ifndef NDEBUG
  assertOK();
#endif
}

void CanonicalLoopInfo::assertOK() const {
#ifndef NDEBUG
  // No constraints if this object currently does not describe a loop.
  if (!isValid())
    return;

  BasicBlock *Preheader = getPreheader();
  BasicBlock *Body = getBody();
  BasicBlock *After = getAfter();

  // Verify standard control-flow we use for OpenMP loops.
  assert(Preheader);
  assert(isa<BranchInst>(Preheader->getTerminator()) &&
         "Preheader must terminate with unconditional branch");
  assert(Preheader->getSingleSuccessor() == Header &&
         "Preheader must jump to header");

  assert(Header);
  assert(isa<BranchInst>(Header->getTerminator()) &&
         "Header must terminate with unconditional branch");
  assert(Header->getSingleSuccessor() == Cond &&
         "Header must jump to exiting block");

  assert(Cond);
  assert(Cond->getSinglePredecessor() == Header &&
         "Exiting block only reachable from header");

  assert(isa<BranchInst>(Cond->getTerminator()) &&
         "Exiting block must terminate with conditional branch");
  assert(size(successors(Cond)) == 2 &&
         "Exiting block must have two successors");
  assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&
         "Exiting block's first successor jump to the body");
  assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&
         "Exiting block's second successor must exit the loop");

  assert(Body);
  assert(Body->getSinglePredecessor() == Cond &&
         "Body only reachable from exiting block");
  assert(!isa<PHINode>(Body->front()));

  assert(Latch);
  assert(isa<BranchInst>(Latch->getTerminator()) &&
         "Latch must terminate with unconditional branch");
  assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");
  // TODO: To support simple redirecting of the end of the body code that has
  // multiple; introduce another auxiliary basic block like preheader and after.
  assert(Latch->getSinglePredecessor() != nullptr);
  assert(!isa<PHINode>(Latch->front()));

  assert(Exit);
  assert(isa<BranchInst>(Exit->getTerminator()) &&
         "Exit block must terminate with unconditional branch");
  assert(Exit->getSingleSuccessor() == After &&
         "Exit block must jump to after block");

  assert(After);
  assert(After->getSinglePredecessor() == Exit &&
         "After block only reachable from exit block");
  assert(After->empty() || !isa<PHINode>(After->front()));

  Instruction *IndVar = getIndVar();
  assert(IndVar && "Canonical induction variable not found?");
  assert(isa<IntegerType>(IndVar->getType()) &&
         "Induction variable must be an integer");
  assert(cast<PHINode>(IndVar)->getParent() == Header &&
         "Induction variable must be a PHI in the loop header");
  assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
  assert(
      cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero());
  assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);

  auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
  assert(cast<Instruction>(NextIndVar)->getParent() == Latch);
  assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);
  assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
  assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
             ->isOne());

  Value *TripCount = getTripCount();
  assert(TripCount && "Loop trip count not found?");
  assert(IndVar->getType() == TripCount->getType() &&
         "Trip count and induction variable must have the same type");

  auto *CmpI = cast<CmpInst>(&Cond->front());
  assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&
         "Exit condition must be a signed less-than comparison");
  assert(CmpI->getOperand(0) == IndVar &&
         "Exit condition must compare the induction variable");
  assert(CmpI->getOperand(1) == TripCount &&
         "Exit condition must compare with the trip count");
#endif
}

void CanonicalLoopInfo::invalidate() {
  Header = nullptr;
  Cond = nullptr;
  Latch = nullptr;
  Exit = nullptr;
}