xref: /llvm-project/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp (revision 4b17a8b10ebb69d3bd30ee7714b5ca24f7e944dc)

//===- SimplifyIntrinsics.cpp -- replace intrinsics with simpler form -----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

//===----------------------------------------------------------------------===//
/// \file
/// This pass looks for suitable calls to runtime library for intrinsics that
/// can be simplified/specialized and replaces with a specialized function.
///
/// For example, SUM(arr) can be specialized as a simple function with one loop,
/// compared to the three arguments (plus file & line info) that the runtime
/// call has - when the argument is a 1D-array (multiple loops may be needed
//  for higher dimension arrays, of course)
///
/// The general idea is that besides making the call simpler, it can also be
/// inlined by other passes that run after this pass, which further improves
/// performance, particularly when the work done in the function is trivial
/// and small in size.
//===----------------------------------------------------------------------===//

#include "flang/Common/Fortran.h"
#include "flang/Optimizer/Builder/BoxValue.h"
#include "flang/Optimizer/Builder/CUFCommon.h"
#include "flang/Optimizer/Builder/FIRBuilder.h"
#include "flang/Optimizer/Builder/LowLevelIntrinsics.h"
#include "flang/Optimizer/Builder/Todo.h"
#include "flang/Optimizer/Dialect/FIROps.h"
#include "flang/Optimizer/Dialect/FIRType.h"
#include "flang/Optimizer/Dialect/Support/FIRContext.h"
#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
#include "flang/Optimizer/Transforms/Passes.h"
#include "flang/Optimizer/Transforms/Utils.h"
#include "flang/Runtime/entry-names.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/Operation.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/DialectConversion.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/RegionUtils.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <llvm/Support/ErrorHandling.h>
#include <mlir/Dialect/Arith/IR/Arith.h>
#include <mlir/IR/BuiltinTypes.h>
#include <mlir/IR/Location.h>
#include <mlir/IR/MLIRContext.h>
#include <mlir/IR/Value.h>
#include <mlir/Support/LLVM.h>
#include <optional>

namespace fir {
#define GEN_PASS_DEF_SIMPLIFYINTRINSICS
#include "flang/Optimizer/Transforms/Passes.h.inc"
} // namespace fir

#define DEBUG_TYPE "flang-simplify-intrinsics"

namespace {

class SimplifyIntrinsicsPass
    : public fir::impl::SimplifyIntrinsicsBase<SimplifyIntrinsicsPass> {
  using FunctionTypeGeneratorTy =
      llvm::function_ref<mlir::FunctionType(fir::FirOpBuilder &)>;
  using FunctionBodyGeneratorTy =
      llvm::function_ref<void(fir::FirOpBuilder &, mlir::func::FuncOp &)>;
  using GenReductionBodyTy = llvm::function_ref<void(
      fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp, unsigned rank,
      mlir::Type elementType)>;

public:
  using fir::impl::SimplifyIntrinsicsBase<
      SimplifyIntrinsicsPass>::SimplifyIntrinsicsBase;

  /// Generate a new function implementing a simplified version
  /// of a Fortran runtime function defined by \p basename name.
  /// \p typeGenerator is a callback that generates the new function's type.
  /// \p bodyGenerator is a callback that generates the new function's body.
  /// The new function is created in the \p builder's Module.
  mlir::func::FuncOp getOrCreateFunction(fir::FirOpBuilder &builder,
                                         const mlir::StringRef &basename,
                                         FunctionTypeGeneratorTy typeGenerator,
                                         FunctionBodyGeneratorTy bodyGenerator);
  void runOnOperation() override;
  void getDependentDialects(mlir::DialectRegistry &registry) const override;

private:
  /// Helper functions to replace a reduction type of call with its
  /// simplified form. The actual function is generated using a callback
  /// function.
  /// \p call is the call to be replaced
  /// \p kindMap is used to create FIROpBuilder
  /// \p genBodyFunc is the callback that builds the replacement function
  void simplifyIntOrFloatReduction(fir::CallOp call,
                                   const fir::KindMapping &kindMap,
                                   GenReductionBodyTy genBodyFunc);
  void simplifyLogicalDim0Reduction(fir::CallOp call,
                                    const fir::KindMapping &kindMap,
                                    GenReductionBodyTy genBodyFunc);
  void simplifyLogicalDim1Reduction(fir::CallOp call,
                                    const fir::KindMapping &kindMap,
                                    GenReductionBodyTy genBodyFunc);
  void simplifyMinMaxlocReduction(fir::CallOp call,
                                  const fir::KindMapping &kindMap, bool isMax);
  void simplifyReductionBody(fir::CallOp call, const fir::KindMapping &kindMap,
                             GenReductionBodyTy genBodyFunc,
                             fir::FirOpBuilder &builder,
                             const mlir::StringRef &basename,
                             mlir::Type elementType);
};

} // namespace

/// Create FirOpBuilder with the provided \p op insertion point
/// and \p kindMap additionally inheriting FastMathFlags from \p op.
static fir::FirOpBuilder
getSimplificationBuilder(mlir::Operation *op, const fir::KindMapping &kindMap) {
  fir::FirOpBuilder builder{op, kindMap};
  auto fmi = mlir::dyn_cast<mlir::arith::ArithFastMathInterface>(*op);
  if (!fmi)
    return builder;

  // Regardless of what default FastMathFlags are used by FirOpBuilder,
  // override them with FastMathFlags attached to the operation.
  builder.setFastMathFlags(fmi.getFastMathFlagsAttr().getValue());
  return builder;
}

/// Generate function type for the simplified version of RTNAME(Sum) and
/// similar functions with a fir.box<none> type returning \p elementType.
static mlir::FunctionType genNoneBoxType(fir::FirOpBuilder &builder,
                                         const mlir::Type &elementType) {
  mlir::Type boxType = fir::BoxType::get(builder.getNoneType());
  return mlir::FunctionType::get(builder.getContext(), {boxType},
                                 {elementType});
}

template <typename Op>
Op expectOp(mlir::Value val) {
  if (Op op = mlir::dyn_cast_or_null<Op>(val.getDefiningOp()))
    return op;
  LLVM_DEBUG(llvm::dbgs() << "Didn't find expected " << Op::getOperationName()
                          << '\n');
  return nullptr;
}

template <typename Op>
static mlir::Value findDefSingle(fir::ConvertOp op) {
  if (auto defOp = expectOp<Op>(op->getOperand(0))) {
    return defOp.getResult();
  }
  return {};
}

template <typename... Ops>
static mlir::Value findDef(fir::ConvertOp op) {
  mlir::Value defOp;
  // Loop over the operation types given to see if any match, exiting once
  // a match is found. Cast to void is needed to avoid compiler complaining
  // that the result of expression is unused
  (void)((defOp = findDefSingle<Ops>(op), (defOp)) || ...);
  return defOp;
}

static bool isOperandAbsent(mlir::Value val) {
  if (auto op = expectOp<fir::ConvertOp>(val)) {
    assert(op->getOperands().size() != 0);
    return mlir::isa_and_nonnull<fir::AbsentOp>(
        op->getOperand(0).getDefiningOp());
  }
  return false;
}

static bool isTrueOrNotConstant(mlir::Value val) {
  if (auto op = expectOp<mlir::arith::ConstantOp>(val)) {
    return !mlir::matchPattern(val, mlir::m_Zero());
  }
  return true;
}

static bool isZero(mlir::Value val) {
  if (auto op = expectOp<fir::ConvertOp>(val)) {
    assert(op->getOperands().size() != 0);
    if (mlir::Operation *defOp = op->getOperand(0).getDefiningOp())
      return mlir::matchPattern(defOp, mlir::m_Zero());
  }
  return false;
}

static mlir::Value findBoxDef(mlir::Value val) {
  if (auto op = expectOp<fir::ConvertOp>(val)) {
    assert(op->getOperands().size() != 0);
    return findDef<fir::EmboxOp, fir::ReboxOp>(op);
  }
  return {};
}

static mlir::Value findMaskDef(mlir::Value val) {
  if (auto op = expectOp<fir::ConvertOp>(val)) {
    assert(op->getOperands().size() != 0);
    return findDef<fir::EmboxOp, fir::ReboxOp, fir::AbsentOp>(op);
  }
  return {};
}

static unsigned getDimCount(mlir::Value val) {
  // In order to find the dimensions count, we look for EmboxOp/ReboxOp
  // and take the count from its *result* type. Note that in case
  // of sliced emboxing the operand and the result of EmboxOp/ReboxOp
  // have different types.
  // Actually, we can take the box type from the operand of
  // the first ConvertOp that has non-opaque box type that we meet
  // going through the ConvertOp chain.
  if (mlir::Value emboxVal = findBoxDef(val))
    if (auto boxTy = mlir::dyn_cast<fir::BoxType>(emboxVal.getType()))
      if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(boxTy.getEleTy()))
        return seqTy.getDimension();
  return 0;
}

/// Given the call operation's box argument \p val, discover
/// the element type of the underlying array object.
/// \returns the element type or std::nullopt if the type cannot
/// be reliably found.
/// We expect that the argument is a result of fir.convert
/// with the destination type of !fir.box<none>.
static std::optional<mlir::Type> getArgElementType(mlir::Value val) {
  mlir::Operation *defOp;
  do {
    defOp = val.getDefiningOp();
    // Analyze only sequences of convert operations.
    if (!mlir::isa<fir::ConvertOp>(defOp))
      return std::nullopt;
    val = defOp->getOperand(0);
    // The convert operation is expected to convert from one
    // box type to another box type.
    auto boxType = mlir::cast<fir::BoxType>(val.getType());
    auto elementType = fir::unwrapSeqOrBoxedSeqType(boxType);
    if (!mlir::isa<mlir::NoneType>(elementType))
      return elementType;
  } while (true);
}

using BodyOpGeneratorTy = llvm::function_ref<mlir::Value(
    fir::FirOpBuilder &, mlir::Location, const mlir::Type &, mlir::Value,
    mlir::Value)>;
using ContinueLoopGenTy = llvm::function_ref<llvm::SmallVector<mlir::Value>(
    fir::FirOpBuilder &, mlir::Location, mlir::Value)>;

/// Generate the reduction loop into \p funcOp.
///
/// \p initVal is a function, called to get the initial value for
///    the reduction value
/// \p genBody is called to fill in the actual reduciton operation
///    for example add for SUM, MAX for MAXVAL, etc.
/// \p rank is the rank of the input argument.
/// \p elementType is the type of the elements in the input array,
///    which may be different to the return type.
/// \p loopCond is called to generate the condition to continue or
///    not for IterWhile loops
/// \p unorderedOrInitalLoopCond contains either a boolean or bool
///    mlir constant, and controls the inital value for while loops
///    or if DoLoop is ordered/unordered.

template <typename OP, typename T, int resultIndex>
static void
genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp,
                 fir::InitValGeneratorTy initVal, ContinueLoopGenTy loopCond,
                 T unorderedOrInitialLoopCond, BodyOpGeneratorTy genBody,
                 unsigned rank, mlir::Type elementType, mlir::Location loc) {

  mlir::IndexType idxTy = builder.getIndexType();

  mlir::Block::BlockArgListType args = funcOp.front().getArguments();
  mlir::Value arg = args[0];

  mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0);

  fir::SequenceType::Shape flatShape(rank,
                                     fir::SequenceType::getUnknownExtent());
  mlir::Type arrTy = fir::SequenceType::get(flatShape, elementType);
  mlir::Type boxArrTy = fir::BoxType::get(arrTy);
  mlir::Value array = builder.create<fir::ConvertOp>(loc, boxArrTy, arg);
  mlir::Type resultType = funcOp.getResultTypes()[0];
  mlir::Value init = initVal(builder, loc, resultType);

  llvm::SmallVector<mlir::Value, Fortran::common::maxRank> bounds;

  assert(rank > 0 && "rank cannot be zero");
  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);

  // Compute all the upper bounds before the loop nest.
  // It is not strictly necessary for performance, since the loop nest
  // does not have any store operations and any LICM optimization
  // should be able to optimize the redundancy.
  for (unsigned i = 0; i < rank; ++i) {
    mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i);
    auto dims =
        builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, array, dimIdx);
    mlir::Value len = dims.getResult(1);
    // We use C indexing here, so len-1 as loopcount
    mlir::Value loopCount = builder.create<mlir::arith::SubIOp>(loc, len, one);
    bounds.push_back(loopCount);
  }
  // Create a loop nest consisting of OP operations.
  // Collect the loops' induction variables into indices array,
  // which will be used in the innermost loop to load the input
  // array's element.
  // The loops are generated such that the innermost loop processes
  // the 0 dimension.
  llvm::SmallVector<mlir::Value, Fortran::common::maxRank> indices;
  for (unsigned i = rank; 0 < i; --i) {
    mlir::Value step = one;
    mlir::Value loopCount = bounds[i - 1];
    auto loop = builder.create<OP>(loc, zeroIdx, loopCount, step,
                                   unorderedOrInitialLoopCond,
                                   /*finalCountValue=*/false, init);
    init = loop.getRegionIterArgs()[resultIndex];
    indices.push_back(loop.getInductionVar());
    // Set insertion point to the loop body so that the next loop
    // is inserted inside the current one.
    builder.setInsertionPointToStart(loop.getBody());
  }

  // Reverse the indices such that they are ordered as:
  //   <dim-0-idx, dim-1-idx, ...>
  std::reverse(indices.begin(), indices.end());
  // We are in the innermost loop: generate the reduction body.
  mlir::Type eleRefTy = builder.getRefType(elementType);
  mlir::Value addr =
      builder.create<fir::CoordinateOp>(loc, eleRefTy, array, indices);
  mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
  mlir::Value reductionVal = genBody(builder, loc, elementType, elem, init);
  // Generate vector with condition to continue while loop at [0] and result
  // from current loop at [1] for IterWhileOp loops, just result at [0] for
  // DoLoopOp loops.
  llvm::SmallVector<mlir::Value> results = loopCond(builder, loc, reductionVal);

  // Unwind the loop nest and insert ResultOp on each level
  // to return the updated value of the reduction to the enclosing
  // loops.
  for (unsigned i = 0; i < rank; ++i) {
    auto result = builder.create<fir::ResultOp>(loc, results);
    // Proceed to the outer loop.
    auto loop = mlir::cast<OP>(result->getParentOp());
    results = loop.getResults();
    // Set insertion point after the loop operation that we have
    // just processed.
    builder.setInsertionPointAfter(loop.getOperation());
  }
  // End of loop nest. The insertion point is after the outermost loop.
  // Return the reduction value from the function.
  builder.create<mlir::func::ReturnOp>(loc, results[resultIndex]);
}

static llvm::SmallVector<mlir::Value> nopLoopCond(fir::FirOpBuilder &builder,
                                                  mlir::Location loc,
                                                  mlir::Value reductionVal) {
  return {reductionVal};
}

/// Generate function body of the simplified version of RTNAME(Sum)
/// with signature provided by \p funcOp. The caller is responsible
/// for saving/restoring the original insertion point of \p builder.
/// \p funcOp is expected to be empty on entry to this function.
/// \p rank specifies the rank of the input argument.
static void genRuntimeSumBody(fir::FirOpBuilder &builder,
                              mlir::func::FuncOp &funcOp, unsigned rank,
                              mlir::Type elementType) {
  // function RTNAME(Sum)<T>x<rank>_simplified(arr)
  //   T, dimension(:) :: arr
  //   T sum = 0
  //   integer iter
  //   do iter = 0, extent(arr)
  //     sum = sum + arr[iter]
  //   end do
  //   RTNAME(Sum)<T>x<rank>_simplified = sum
  // end function RTNAME(Sum)<T>x<rank>_simplified
  auto zero = [](fir::FirOpBuilder builder, mlir::Location loc,
                 mlir::Type elementType) {
    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
      const llvm::fltSemantics &sem = ty.getFloatSemantics();
      return builder.createRealConstant(loc, elementType,
                                        llvm::APFloat::getZero(sem));
    }
    return builder.createIntegerConstant(loc, elementType, 0);
  };

  auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,
                      mlir::Type elementType, mlir::Value elem1,
                      mlir::Value elem2) -> mlir::Value {
    if (mlir::isa<mlir::FloatType>(elementType))
      return builder.create<mlir::arith::AddFOp>(loc, elem1, elem2);
    if (mlir::isa<mlir::IntegerType>(elementType))
      return builder.create<mlir::arith::AddIOp>(loc, elem1, elem2);

    llvm_unreachable("unsupported type");
    return {};
  };

  mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());
  builder.setInsertionPointToEnd(funcOp.addEntryBlock());

  genReductionLoop<fir::DoLoopOp, bool, 0>(builder, funcOp, zero, nopLoopCond,
                                           false, genBodyOp, rank, elementType,
                                           loc);
}

static void genRuntimeMaxvalBody(fir::FirOpBuilder &builder,
                                 mlir::func::FuncOp &funcOp, unsigned rank,
                                 mlir::Type elementType) {
  auto init = [](fir::FirOpBuilder builder, mlir::Location loc,
                 mlir::Type elementType) {
    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
      const llvm::fltSemantics &sem = ty.getFloatSemantics();
      return builder.createRealConstant(
          loc, elementType, llvm::APFloat::getLargest(sem, /*Negative=*/true));
    }
    unsigned bits = elementType.getIntOrFloatBitWidth();
    int64_t minInt = llvm::APInt::getSignedMinValue(bits).getSExtValue();
    return builder.createIntegerConstant(loc, elementType, minInt);
  };

  auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,
                      mlir::Type elementType, mlir::Value elem1,
                      mlir::Value elem2) -> mlir::Value {
    if (mlir::isa<mlir::FloatType>(elementType)) {
      // arith.maxf later converted to llvm.intr.maxnum does not work
      // correctly for NaNs and -0.0 (see maxnum/minnum pattern matching
      // in LLVM's InstCombine pass). Moreover, llvm.intr.maxnum
      // for F128 operands is lowered into fmaxl call by LLVM.
      // This libm function may not work properly for F128 arguments
      // on targets where long double is not F128. It is an LLVM issue,
      // but we just use normal select here to resolve all the cases.
      auto compare = builder.create<mlir::arith::CmpFOp>(
          loc, mlir::arith::CmpFPredicate::OGT, elem1, elem2);
      return builder.create<mlir::arith::SelectOp>(loc, compare, elem1, elem2);
    }
    if (mlir::isa<mlir::IntegerType>(elementType))
      return builder.create<mlir::arith::MaxSIOp>(loc, elem1, elem2);

    llvm_unreachable("unsupported type");
    return {};
  };

  mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());
  builder.setInsertionPointToEnd(funcOp.addEntryBlock());

  genReductionLoop<fir::DoLoopOp, bool, 0>(builder, funcOp, init, nopLoopCond,
                                           false, genBodyOp, rank, elementType,
                                           loc);
}

static void genRuntimeCountBody(fir::FirOpBuilder &builder,
                                mlir::func::FuncOp &funcOp, unsigned rank,
                                mlir::Type elementType) {
  auto zero = [](fir::FirOpBuilder builder, mlir::Location loc,
                 mlir::Type elementType) {
    unsigned bits = elementType.getIntOrFloatBitWidth();
    int64_t zeroInt = llvm::APInt::getZero(bits).getSExtValue();
    return builder.createIntegerConstant(loc, elementType, zeroInt);
  };

  auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,
                      mlir::Type elementType, mlir::Value elem1,
                      mlir::Value elem2) -> mlir::Value {
    auto zero32 = builder.createIntegerConstant(loc, elementType, 0);
    auto zero64 = builder.createIntegerConstant(loc, builder.getI64Type(), 0);
    auto one64 = builder.createIntegerConstant(loc, builder.getI64Type(), 1);

    auto compare = builder.create<mlir::arith::CmpIOp>(
        loc, mlir::arith::CmpIPredicate::eq, elem1, zero32);
    auto select =
        builder.create<mlir::arith::SelectOp>(loc, compare, zero64, one64);
    return builder.create<mlir::arith::AddIOp>(loc, select, elem2);
  };

  // Count always gets I32 for elementType as it converts logical input to
  // logical<4> before passing to the function.
  mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());
  builder.setInsertionPointToEnd(funcOp.addEntryBlock());

  genReductionLoop<fir::DoLoopOp, bool, 0>(builder, funcOp, zero, nopLoopCond,
                                           false, genBodyOp, rank, elementType,
                                           loc);
}

static void genRuntimeAnyBody(fir::FirOpBuilder &builder,
                              mlir::func::FuncOp &funcOp, unsigned rank,
                              mlir::Type elementType) {
  auto zero = [](fir::FirOpBuilder builder, mlir::Location loc,
                 mlir::Type elementType) {
    return builder.createIntegerConstant(loc, elementType, 0);
  };

  auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,
                      mlir::Type elementType, mlir::Value elem1,
                      mlir::Value elem2) -> mlir::Value {
    auto zero = builder.createIntegerConstant(loc, elementType, 0);
    return builder.create<mlir::arith::CmpIOp>(
        loc, mlir::arith::CmpIPredicate::ne, elem1, zero);
  };

  auto continueCond = [](fir::FirOpBuilder builder, mlir::Location loc,
                         mlir::Value reductionVal) {
    auto one1 = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
    auto eor = builder.create<mlir::arith::XOrIOp>(loc, reductionVal, one1);
    llvm::SmallVector<mlir::Value> results = {eor, reductionVal};
    return results;
  };

  mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());
  builder.setInsertionPointToEnd(funcOp.addEntryBlock());
  mlir::Value ok = builder.createBool(loc, true);

  genReductionLoop<fir::IterWhileOp, mlir::Value, 1>(
      builder, funcOp, zero, continueCond, ok, genBodyOp, rank, elementType,
      loc);
}

static void genRuntimeAllBody(fir::FirOpBuilder &builder,
                              mlir::func::FuncOp &funcOp, unsigned rank,
                              mlir::Type elementType) {
  auto one = [](fir::FirOpBuilder builder, mlir::Location loc,
                mlir::Type elementType) {
    return builder.createIntegerConstant(loc, elementType, 1);
  };

  auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,
                      mlir::Type elementType, mlir::Value elem1,
                      mlir::Value elem2) -> mlir::Value {
    auto zero = builder.createIntegerConstant(loc, elementType, 0);
    return builder.create<mlir::arith::CmpIOp>(
        loc, mlir::arith::CmpIPredicate::ne, elem1, zero);
  };

  auto continueCond = [](fir::FirOpBuilder builder, mlir::Location loc,
                         mlir::Value reductionVal) {
    llvm::SmallVector<mlir::Value> results = {reductionVal, reductionVal};
    return results;
  };

  mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());
  builder.setInsertionPointToEnd(funcOp.addEntryBlock());
  mlir::Value ok = builder.createBool(loc, true);

  genReductionLoop<fir::IterWhileOp, mlir::Value, 1>(
      builder, funcOp, one, continueCond, ok, genBodyOp, rank, elementType,
      loc);
}

static mlir::FunctionType genRuntimeMinlocType(fir::FirOpBuilder &builder,
                                               unsigned int rank) {
  mlir::Type boxType = fir::BoxType::get(builder.getNoneType());
  mlir::Type boxRefType = builder.getRefType(boxType);

  return mlir::FunctionType::get(builder.getContext(),
                                 {boxRefType, boxType, boxType}, {});
}

// Produces a loop nest for a Minloc intrinsic.
void fir::genMinMaxlocReductionLoop(
    fir::FirOpBuilder &builder, mlir::Value array,
    fir::InitValGeneratorTy initVal, fir::MinlocBodyOpGeneratorTy genBody,
    fir::AddrGeneratorTy getAddrFn, unsigned rank, mlir::Type elementType,
    mlir::Location loc, mlir::Type maskElemType, mlir::Value resultArr,
    bool maskMayBeLogicalScalar) {
  mlir::IndexType idxTy = builder.getIndexType();

  mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0);

  fir::SequenceType::Shape flatShape(rank,
                                     fir::SequenceType::getUnknownExtent());
  mlir::Type arrTy = fir::SequenceType::get(flatShape, elementType);
  mlir::Type boxArrTy = fir::BoxType::get(arrTy);
  array = builder.create<fir::ConvertOp>(loc, boxArrTy, array);

  mlir::Type resultElemType = hlfir::getFortranElementType(resultArr.getType());
  mlir::Value flagSet = builder.createIntegerConstant(loc, resultElemType, 1);
  mlir::Value zero = builder.createIntegerConstant(loc, resultElemType, 0);
  mlir::Value flagRef = builder.createTemporary(loc, resultElemType);
  builder.create<fir::StoreOp>(loc, zero, flagRef);

  mlir::Value init = initVal(builder, loc, elementType);
  llvm::SmallVector<mlir::Value, Fortran::common::maxRank> bounds;

  assert(rank > 0 && "rank cannot be zero");
  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);

  // Compute all the upper bounds before the loop nest.
  // It is not strictly necessary for performance, since the loop nest
  // does not have any store operations and any LICM optimization
  // should be able to optimize the redundancy.
  for (unsigned i = 0; i < rank; ++i) {
    mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i);
    auto dims =
        builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, array, dimIdx);
    mlir::Value len = dims.getResult(1);
    // We use C indexing here, so len-1 as loopcount
    mlir::Value loopCount = builder.create<mlir::arith::SubIOp>(loc, len, one);
    bounds.push_back(loopCount);
  }
  // Create a loop nest consisting of OP operations.
  // Collect the loops' induction variables into indices array,
  // which will be used in the innermost loop to load the input
  // array's element.
  // The loops are generated such that the innermost loop processes
  // the 0 dimension.
  llvm::SmallVector<mlir::Value, Fortran::common::maxRank> indices;
  for (unsigned i = rank; 0 < i; --i) {
    mlir::Value step = one;
    mlir::Value loopCount = bounds[i - 1];
    auto loop =
        builder.create<fir::DoLoopOp>(loc, zeroIdx, loopCount, step, false,
                                      /*finalCountValue=*/false, init);
    init = loop.getRegionIterArgs()[0];
    indices.push_back(loop.getInductionVar());
    // Set insertion point to the loop body so that the next loop
    // is inserted inside the current one.
    builder.setInsertionPointToStart(loop.getBody());
  }

  // Reverse the indices such that they are ordered as:
  //   <dim-0-idx, dim-1-idx, ...>
  std::reverse(indices.begin(), indices.end());
  mlir::Value reductionVal =
      genBody(builder, loc, elementType, array, flagRef, init, indices);

  // Unwind the loop nest and insert ResultOp on each level
  // to return the updated value of the reduction to the enclosing
  // loops.
  for (unsigned i = 0; i < rank; ++i) {
    auto result = builder.create<fir::ResultOp>(loc, reductionVal);
    // Proceed to the outer loop.
    auto loop = mlir::cast<fir::DoLoopOp>(result->getParentOp());
    reductionVal = loop.getResult(0);
    // Set insertion point after the loop operation that we have
    // just processed.
    builder.setInsertionPointAfter(loop.getOperation());
  }
  // End of loop nest. The insertion point is after the outermost loop.
  if (maskMayBeLogicalScalar) {
    if (fir::IfOp ifOp =
            mlir::dyn_cast<fir::IfOp>(builder.getBlock()->getParentOp())) {
      builder.create<fir::ResultOp>(loc, reductionVal);
      builder.setInsertionPointAfter(ifOp);
      // Redefine flagSet to escape scope of ifOp
      flagSet = builder.createIntegerConstant(loc, resultElemType, 1);
      reductionVal = ifOp.getResult(0);
    }
  }
}

static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,
                                    mlir::func::FuncOp &funcOp, bool isMax,
                                    unsigned rank, int maskRank,
                                    mlir::Type elementType,
                                    mlir::Type maskElemType,
                                    mlir::Type resultElemTy, bool isDim) {
  auto init = [isMax](fir::FirOpBuilder builder, mlir::Location loc,
                      mlir::Type elementType) {
    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
      const llvm::fltSemantics &sem = ty.getFloatSemantics();
      llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
      return builder.createRealConstant(loc, elementType, limit);
    }
    unsigned bits = elementType.getIntOrFloatBitWidth();
    int64_t initValue = (isMax ? llvm::APInt::getSignedMinValue(bits)
                               : llvm::APInt::getSignedMaxValue(bits))
                            .getSExtValue();
    return builder.createIntegerConstant(loc, elementType, initValue);
  };

  mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());
  builder.setInsertionPointToEnd(funcOp.addEntryBlock());

  mlir::Value mask = funcOp.front().getArgument(2);

  // Set up result array in case of early exit / 0 length array
  mlir::IndexType idxTy = builder.getIndexType();
  mlir::Type resultTy = fir::SequenceType::get(rank, resultElemTy);
  mlir::Type resultHeapTy = fir::HeapType::get(resultTy);
  mlir::Type resultBoxTy = fir::BoxType::get(resultHeapTy);

  mlir::Value returnValue = builder.createIntegerConstant(loc, resultElemTy, 0);
  mlir::Value resultArrSize = builder.createIntegerConstant(loc, idxTy, rank);

  mlir::Value resultArrInit = builder.create<fir::AllocMemOp>(loc, resultTy);
  mlir::Value resultArrShape = builder.create<fir::ShapeOp>(loc, resultArrSize);
  mlir::Value resultArr = builder.create<fir::EmboxOp>(
      loc, resultBoxTy, resultArrInit, resultArrShape);

  mlir::Type resultRefTy = builder.getRefType(resultElemTy);

  if (maskRank > 0) {
    fir::SequenceType::Shape flatShape(rank,
                                       fir::SequenceType::getUnknownExtent());
    mlir::Type maskTy = fir::SequenceType::get(flatShape, maskElemType);
    mlir::Type boxMaskTy = fir::BoxType::get(maskTy);
    mask = builder.create<fir::ConvertOp>(loc, boxMaskTy, mask);
  }

  for (unsigned int i = 0; i < rank; ++i) {
    mlir::Value index = builder.createIntegerConstant(loc, idxTy, i);
    mlir::Value resultElemAddr =
        builder.create<fir::CoordinateOp>(loc, resultRefTy, resultArr, index);
    builder.create<fir::StoreOp>(loc, returnValue, resultElemAddr);
  }

  auto genBodyOp =
      [&rank, &resultArr, isMax, &mask, &maskElemType, &maskRank](
          fir::FirOpBuilder builder, mlir::Location loc, mlir::Type elementType,
          mlir::Value array, mlir::Value flagRef, mlir::Value reduction,
          const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {
    // We are in the innermost loop: generate the reduction body.
    if (maskRank > 0) {
      mlir::Type logicalRef = builder.getRefType(maskElemType);
      mlir::Value maskAddr =
          builder.create<fir::CoordinateOp>(loc, logicalRef, mask, indices);
      mlir::Value maskElem = builder.create<fir::LoadOp>(loc, maskAddr);

      // fir::IfOp requires argument to be I1 - won't accept logical or any
      // other Integer.
      mlir::Type ifCompatType = builder.getI1Type();
      mlir::Value ifCompatElem =
          builder.create<fir::ConvertOp>(loc, ifCompatType, maskElem);

      llvm::SmallVector<mlir::Type> resultsTy = {elementType, elementType};
      fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, ifCompatElem,
                                                 /*withElseRegion=*/true);
      builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
    }

    // Set flag that mask was true at some point
    mlir::Value flagSet = builder.createIntegerConstant(
        loc, mlir::cast<fir::ReferenceType>(flagRef.getType()).getEleTy(), 1);
    mlir::Value isFirst = builder.create<fir::LoadOp>(loc, flagRef);
    mlir::Type eleRefTy = builder.getRefType(elementType);
    mlir::Value addr =
        builder.create<fir::CoordinateOp>(loc, eleRefTy, array, indices);
    mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);

    mlir::Value cmp;
    if (mlir::isa<mlir::FloatType>(elementType)) {
      // For FP reductions we want the first smallest value to be used, that
      // is not NaN. A OGL/OLT condition will usually work for this unless all
      // the values are Nan or Inf. This follows the same logic as
      // NumericCompare for Minloc/Maxlox in extrema.cpp.
      cmp = builder.create<mlir::arith::CmpFOp>(
          loc,
          isMax ? mlir::arith::CmpFPredicate::OGT
                : mlir::arith::CmpFPredicate::OLT,
          elem, reduction);

      mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
          loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
      mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
          loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
      cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
      cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
    } else if (mlir::isa<mlir::IntegerType>(elementType)) {
      cmp = builder.create<mlir::arith::CmpIOp>(
          loc,
          isMax ? mlir::arith::CmpIPredicate::sgt
                : mlir::arith::CmpIPredicate::slt,
          elem, reduction);
    } else {
      llvm_unreachable("unsupported type");
    }

    // The condition used for the loop is isFirst || <the condition above>.
    isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
    isFirst = builder.create<mlir::arith::XOrIOp>(
        loc, isFirst, builder.createIntegerConstant(loc, cmp.getType(), 1));
    cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, isFirst);
    fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, cmp,
                                               /*withElseRegion*/ true);

    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
    builder.create<fir::StoreOp>(loc, flagSet, flagRef);
    mlir::Type resultElemTy = hlfir::getFortranElementType(resultArr.getType());
    mlir::Type returnRefTy = builder.getRefType(resultElemTy);
    mlir::IndexType idxTy = builder.getIndexType();

    mlir::Value one = builder.createIntegerConstant(loc, resultElemTy, 1);

    for (unsigned int i = 0; i < rank; ++i) {
      mlir::Value index = builder.createIntegerConstant(loc, idxTy, i);
      mlir::Value resultElemAddr =
          builder.create<fir::CoordinateOp>(loc, returnRefTy, resultArr, index);
      mlir::Value convert =
          builder.create<fir::ConvertOp>(loc, resultElemTy, indices[i]);
      mlir::Value fortranIndex =
          builder.create<mlir::arith::AddIOp>(loc, convert, one);
      builder.create<fir::StoreOp>(loc, fortranIndex, resultElemAddr);
    }
    builder.create<fir::ResultOp>(loc, elem);
    builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
    builder.create<fir::ResultOp>(loc, reduction);
    builder.setInsertionPointAfter(ifOp);
    mlir::Value reductionVal = ifOp.getResult(0);

    // Close the mask if needed
    if (maskRank > 0) {
      fir::IfOp ifOp =
          mlir::dyn_cast<fir::IfOp>(builder.getBlock()->getParentOp());
      builder.create<fir::ResultOp>(loc, reductionVal);
      builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
      builder.create<fir::ResultOp>(loc, reduction);
      reductionVal = ifOp.getResult(0);
      builder.setInsertionPointAfter(ifOp);
    }

    return reductionVal;
  };

  // if mask is a logical scalar, we can check its value before the main loop
  // and either ignore the fact it is there or exit early.
  if (maskRank == 0) {
    mlir::Type logical = builder.getI1Type();
    mlir::IndexType idxTy = builder.getIndexType();

    fir::SequenceType::Shape singleElement(1, 1);
    mlir::Type arrTy = fir::SequenceType::get(singleElement, logical);
    mlir::Type boxArrTy = fir::BoxType::get(arrTy);
    mlir::Value array = builder.create<fir::ConvertOp>(loc, boxArrTy, mask);

    mlir::Value indx = builder.createIntegerConstant(loc, idxTy, 0);
    mlir::Type logicalRefTy = builder.getRefType(logical);
    mlir::Value condAddr =
        builder.create<fir::CoordinateOp>(loc, logicalRefTy, array, indx);
    mlir::Value cond = builder.create<fir::LoadOp>(loc, condAddr);

    fir::IfOp ifOp = builder.create<fir::IfOp>(loc, elementType, cond,
                                               /*withElseRegion=*/true);

    builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
    mlir::Value basicValue;
    if (mlir::isa<mlir::IntegerType>(elementType)) {
      basicValue = builder.createIntegerConstant(loc, elementType, 0);
    } else {
      basicValue = builder.createRealConstant(loc, elementType, 0);
    }
    builder.create<fir::ResultOp>(loc, basicValue);

    builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
  }
  auto getAddrFn = [](fir::FirOpBuilder builder, mlir::Location loc,
                      const mlir::Type &resultElemType, mlir::Value resultArr,
                      mlir::Value index) {
    mlir::Type resultRefTy = builder.getRefType(resultElemType);
    return builder.create<fir::CoordinateOp>(loc, resultRefTy, resultArr,
                                             index);
  };

  genMinMaxlocReductionLoop(builder, funcOp.front().getArgument(1), init,
                            genBodyOp, getAddrFn, rank, elementType, loc,
                            maskElemType, resultArr, maskRank == 0);

  // Store newly created output array to the reference passed in
  if (isDim) {
    mlir::Type resultBoxTy =
        fir::BoxType::get(fir::HeapType::get(resultElemTy));
    mlir::Value outputArr = builder.create<fir::ConvertOp>(
        loc, builder.getRefType(resultBoxTy), funcOp.front().getArgument(0));
    mlir::Value resultArrScalar = builder.create<fir::ConvertOp>(
        loc, fir::HeapType::get(resultElemTy), resultArrInit);
    mlir::Value resultBox =
        builder.create<fir::EmboxOp>(loc, resultBoxTy, resultArrScalar);
    builder.create<fir::StoreOp>(loc, resultBox, outputArr);
  } else {
    fir::SequenceType::Shape resultShape(1, rank);
    mlir::Type outputArrTy = fir::SequenceType::get(resultShape, resultElemTy);
    mlir::Type outputHeapTy = fir::HeapType::get(outputArrTy);
    mlir::Type outputBoxTy = fir::BoxType::get(outputHeapTy);
    mlir::Type outputRefTy = builder.getRefType(outputBoxTy);
    mlir::Value outputArr = builder.create<fir::ConvertOp>(
        loc, outputRefTy, funcOp.front().getArgument(0));
    builder.create<fir::StoreOp>(loc, resultArr, outputArr);
  }

  builder.create<mlir::func::ReturnOp>(loc);
}

/// Generate function type for the simplified version of RTNAME(DotProduct)
/// operating on the given \p elementType.
static mlir::FunctionType genRuntimeDotType(fir::FirOpBuilder &builder,
                                            const mlir::Type &elementType) {
  mlir::Type boxType = fir::BoxType::get(builder.getNoneType());
  return mlir::FunctionType::get(builder.getContext(), {boxType, boxType},
                                 {elementType});
}

/// Generate function body of the simplified version of RTNAME(DotProduct)
/// with signature provided by \p funcOp. The caller is responsible
/// for saving/restoring the original insertion point of \p builder.
/// \p funcOp is expected to be empty on entry to this function.
/// \p arg1ElementTy and \p arg2ElementTy specify elements types
/// of the underlying array objects - they are used to generate proper
/// element accesses.
static void genRuntimeDotBody(fir::FirOpBuilder &builder,
                              mlir::func::FuncOp &funcOp,
                              mlir::Type arg1ElementTy,
                              mlir::Type arg2ElementTy) {
  // function RTNAME(DotProduct)<T>_simplified(arr1, arr2)
  //   T, dimension(:) :: arr1, arr2
  //   T product = 0
  //   integer iter
  //   do iter = 0, extent(arr1)
  //     product = product + arr1[iter] * arr2[iter]
  //   end do
  //   RTNAME(ADotProduct)<T>_simplified = product
  // end function RTNAME(DotProduct)<T>_simplified
  auto loc = mlir::UnknownLoc::get(builder.getContext());
  mlir::Type resultElementType = funcOp.getResultTypes()[0];
  builder.setInsertionPointToEnd(funcOp.addEntryBlock());

  mlir::IndexType idxTy = builder.getIndexType();

  mlir::Value zero =
      mlir::isa<mlir::FloatType>(resultElementType)
          ? builder.createRealConstant(loc, resultElementType, 0.0)
          : builder.createIntegerConstant(loc, resultElementType, 0);

  mlir::Block::BlockArgListType args = funcOp.front().getArguments();
  mlir::Value arg1 = args[0];
  mlir::Value arg2 = args[1];

  mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0);

  fir::SequenceType::Shape flatShape = {fir::SequenceType::getUnknownExtent()};
  mlir::Type arrTy1 = fir::SequenceType::get(flatShape, arg1ElementTy);
  mlir::Type boxArrTy1 = fir::BoxType::get(arrTy1);
  mlir::Value array1 = builder.create<fir::ConvertOp>(loc, boxArrTy1, arg1);
  mlir::Type arrTy2 = fir::SequenceType::get(flatShape, arg2ElementTy);
  mlir::Type boxArrTy2 = fir::BoxType::get(arrTy2);
  mlir::Value array2 = builder.create<fir::ConvertOp>(loc, boxArrTy2, arg2);
  // This version takes the loop trip count from the first argument.
  // If the first argument's box has unknown (at compilation time)
  // extent, then it may be better to take the extent from the second
  // argument - so that after inlining the loop may be better optimized, e.g.
  // fully unrolled. This requires generating two versions of the simplified
  // function and some analysis at the call site to choose which version
  // is more profitable to call.
  // Note that we can assume that both arguments have the same extent.
  auto dims =
      builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, array1, zeroIdx);
  mlir::Value len = dims.getResult(1);
  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
  mlir::Value step = one;

  // We use C indexing here, so len-1 as loopcount
  mlir::Value loopCount = builder.create<mlir::arith::SubIOp>(loc, len, one);
  auto loop = builder.create<fir::DoLoopOp>(loc, zeroIdx, loopCount, step,
                                            /*unordered=*/false,
                                            /*finalCountValue=*/false, zero);
  mlir::Value sumVal = loop.getRegionIterArgs()[0];

  // Begin loop code
  mlir::OpBuilder::InsertPoint loopEndPt = builder.saveInsertionPoint();
  builder.setInsertionPointToStart(loop.getBody());

  mlir::Type eleRef1Ty = builder.getRefType(arg1ElementTy);
  mlir::Value index = loop.getInductionVar();
  mlir::Value addr1 =
      builder.create<fir::CoordinateOp>(loc, eleRef1Ty, array1, index);
  mlir::Value elem1 = builder.create<fir::LoadOp>(loc, addr1);
  // Convert to the result type.
  elem1 = builder.create<fir::ConvertOp>(loc, resultElementType, elem1);

  mlir::Type eleRef2Ty = builder.getRefType(arg2ElementTy);
  mlir::Value addr2 =
      builder.create<fir::CoordinateOp>(loc, eleRef2Ty, array2, index);
  mlir::Value elem2 = builder.create<fir::LoadOp>(loc, addr2);
  // Convert to the result type.
  elem2 = builder.create<fir::ConvertOp>(loc, resultElementType, elem2);

  if (mlir::isa<mlir::FloatType>(resultElementType))
    sumVal = builder.create<mlir::arith::AddFOp>(
        loc, builder.create<mlir::arith::MulFOp>(loc, elem1, elem2), sumVal);
  else if (mlir::isa<mlir::IntegerType>(resultElementType))
    sumVal = builder.create<mlir::arith::AddIOp>(
        loc, builder.create<mlir::arith::MulIOp>(loc, elem1, elem2), sumVal);
  else
    llvm_unreachable("unsupported type");

  builder.create<fir::ResultOp>(loc, sumVal);
  // End of loop.
  builder.restoreInsertionPoint(loopEndPt);

  mlir::Value resultVal = loop.getResult(0);
  builder.create<mlir::func::ReturnOp>(loc, resultVal);
}

mlir::func::FuncOp SimplifyIntrinsicsPass::getOrCreateFunction(
    fir::FirOpBuilder &builder, const mlir::StringRef &baseName,
    FunctionTypeGeneratorTy typeGenerator,
    FunctionBodyGeneratorTy bodyGenerator) {
  // WARNING: if the function generated here changes its signature
  //          or behavior (the body code), we should probably embed some
  //          versioning information into its name, otherwise libraries
  //          statically linked with older versions of Flang may stop
  //          working with object files created with newer Flang.
  //          We can also avoid this by using internal linkage, but
  //          this may increase the size of final executable/shared library.
  std::string replacementName = mlir::Twine{baseName, "_simplified"}.str();
  // If we already have a function, just return it.
  mlir::func::FuncOp newFunc = builder.getNamedFunction(replacementName);
  mlir::FunctionType fType = typeGenerator(builder);
  if (newFunc) {
    assert(newFunc.getFunctionType() == fType &&
           "type mismatch for simplified function");
    return newFunc;
  }

  // Need to build the function!
  auto loc = mlir::UnknownLoc::get(builder.getContext());
  newFunc = builder.createFunction(loc, replacementName, fType);
  auto inlineLinkage = mlir::LLVM::linkage::Linkage::LinkonceODR;
  auto linkage =
      mlir::LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);
  newFunc->setAttr("llvm.linkage", linkage);

  // Save the position of the original call.
  mlir::OpBuilder::InsertPoint insertPt = builder.saveInsertionPoint();

  bodyGenerator(builder, newFunc);

  // Now back to where we were adding code earlier...
  builder.restoreInsertionPoint(insertPt);

  return newFunc;
}

void SimplifyIntrinsicsPass::simplifyIntOrFloatReduction(
    fir::CallOp call, const fir::KindMapping &kindMap,
    GenReductionBodyTy genBodyFunc) {
  // args[1] and args[2] are source filename and line number, ignored.
  mlir::Operation::operand_range args = call.getArgs();

  const mlir::Value &dim = args[3];
  const mlir::Value &mask = args[4];
  // dim is zero when it is absent, which is an implementation
  // detail in the runtime library.

  bool dimAndMaskAbsent = isZero(dim) && isOperandAbsent(mask);
  unsigned rank = getDimCount(args[0]);

  // Rank is set to 0 for assumed shape arrays, don't simplify
  // in these cases
  if (!(dimAndMaskAbsent && rank > 0))
    return;

  mlir::Type resultType = call.getResult(0).getType();

  if (!mlir::isa<mlir::FloatType>(resultType) &&
      !mlir::isa<mlir::IntegerType>(resultType))
    return;

  auto argType = getArgElementType(args[0]);
  if (!argType)
    return;
  assert(*argType == resultType &&
         "Argument/result types mismatch in reduction");

  mlir::SymbolRefAttr callee = call.getCalleeAttr();

  fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)};
  std::string fmfString{builder.getFastMathFlagsString()};
  std::string funcName =
      (mlir::Twine{callee.getLeafReference().getValue(), "x"} +
       mlir::Twine{rank} +
       // We must mangle the generated function name with FastMathFlags
       // value.
       (fmfString.empty() ? mlir::Twine{} : mlir::Twine{"_", fmfString}))
          .str();

  simplifyReductionBody(call, kindMap, genBodyFunc, builder, funcName,
                        resultType);
}

void SimplifyIntrinsicsPass::simplifyLogicalDim0Reduction(
    fir::CallOp call, const fir::KindMapping &kindMap,
    GenReductionBodyTy genBodyFunc) {

  mlir::Operation::operand_range args = call.getArgs();
  const mlir::Value &dim = args[3];
  unsigned rank = getDimCount(args[0]);

  // getDimCount returns a rank of 0 for assumed shape arrays, don't simplify in
  // these cases.
  if (!(isZero(dim) && rank > 0))
    return;

  mlir::Value inputBox = findBoxDef(args[0]);

  mlir::Type elementType = hlfir::getFortranElementType(inputBox.getType());
  mlir::SymbolRefAttr callee = call.getCalleeAttr();

  fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)};

  // Treating logicals as integers makes things a lot easier
  fir::LogicalType logicalType = {
      mlir::dyn_cast<fir::LogicalType>(elementType)};
  fir::KindTy kind = logicalType.getFKind();
  mlir::Type intElementType = builder.getIntegerType(kind * 8);

  // Mangle kind into function name as it is not done by default
  std::string funcName =
      (mlir::Twine{callee.getLeafReference().getValue(), "Logical"} +
       mlir::Twine{kind} + "x" + mlir::Twine{rank})
          .str();

  simplifyReductionBody(call, kindMap, genBodyFunc, builder, funcName,
                        intElementType);
}

void SimplifyIntrinsicsPass::simplifyLogicalDim1Reduction(
    fir::CallOp call, const fir::KindMapping &kindMap,
    GenReductionBodyTy genBodyFunc) {

  mlir::Operation::operand_range args = call.getArgs();
  mlir::SymbolRefAttr callee = call.getCalleeAttr();
  mlir::StringRef funcNameBase = callee.getLeafReference().getValue();
  unsigned rank = getDimCount(args[0]);

  // getDimCount returns a rank of 0 for assumed shape arrays, don't simplify in
  // these cases. We check for Dim at the end as some logical functions (Any,
  // All) set dim to 1 instead of 0 when the argument is not present.
  if (funcNameBase.ends_with("Dim") || !(rank > 0))
    return;

  mlir::Value inputBox = findBoxDef(args[0]);
  mlir::Type elementType = hlfir::getFortranElementType(inputBox.getType());

  fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)};

  // Treating logicals as integers makes things a lot easier
  fir::LogicalType logicalType = {
      mlir::dyn_cast<fir::LogicalType>(elementType)};
  fir::KindTy kind = logicalType.getFKind();
  mlir::Type intElementType = builder.getIntegerType(kind * 8);

  // Mangle kind into function name as it is not done by default
  std::string funcName =
      (mlir::Twine{callee.getLeafReference().getValue(), "Logical"} +
       mlir::Twine{kind} + "x" + mlir::Twine{rank})
          .str();

  simplifyReductionBody(call, kindMap, genBodyFunc, builder, funcName,
                        intElementType);
}

void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction(
    fir::CallOp call, const fir::KindMapping &kindMap, bool isMax) {

  mlir::Operation::operand_range args = call.getArgs();

  mlir::SymbolRefAttr callee = call.getCalleeAttr();
  mlir::StringRef funcNameBase = callee.getLeafReference().getValue();
  bool isDim = funcNameBase.ends_with("Dim");
  mlir::Value back = args[isDim ? 7 : 6];
  if (isTrueOrNotConstant(back))
    return;

  mlir::Value mask = args[isDim ? 6 : 5];
  mlir::Value maskDef = findMaskDef(mask);

  // maskDef is set to NULL when the defining op is not one we accept.
  // This tends to be because it is a selectOp, in which case let the
  // runtime deal with it.
  if (maskDef == NULL)
    return;

  unsigned rank = getDimCount(args[1]);
  if ((isDim && rank != 1) || !(rank > 0))
    return;

  fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)};
  mlir::Location loc = call.getLoc();
  auto inputBox = findBoxDef(args[1]);
  mlir::Type inputType = hlfir::getFortranElementType(inputBox.getType());

  if (mlir::isa<fir::CharacterType>(inputType))
    return;

  int maskRank;
  fir::KindTy kind = 0;
  mlir::Type logicalElemType = builder.getI1Type();
  if (isOperandAbsent(mask)) {
    maskRank = -1;
  } else {
    maskRank = getDimCount(mask);
    mlir::Type maskElemTy = hlfir::getFortranElementType(maskDef.getType());
    fir::LogicalType logicalFirType = {
        mlir::dyn_cast<fir::LogicalType>(maskElemTy)};
    kind = logicalFirType.getFKind();
    // Convert fir::LogicalType to mlir::Type
    logicalElemType = logicalFirType;
  }

  mlir::Operation *outputDef = args[0].getDefiningOp();
  mlir::Value outputAlloc = outputDef->getOperand(0);
  mlir::Type outType = hlfir::getFortranElementType(outputAlloc.getType());

  std::string fmfString{builder.getFastMathFlagsString()};
  std::string funcName =
      (mlir::Twine{callee.getLeafReference().getValue(), "x"} +
       mlir::Twine{rank} +
       (maskRank >= 0
            ? "_Logical" + mlir::Twine{kind} + "x" + mlir::Twine{maskRank}
            : "") +
       "_")
          .str();

  llvm::raw_string_ostream nameOS(funcName);
  outType.print(nameOS);
  if (isDim)
    nameOS << '_' << inputType;
  nameOS << '_' << fmfString;

  auto typeGenerator = [rank](fir::FirOpBuilder &builder) {
    return genRuntimeMinlocType(builder, rank);
  };
  auto bodyGenerator = [rank, maskRank, inputType, logicalElemType, outType,
                        isMax, isDim](fir::FirOpBuilder &builder,
                                      mlir::func::FuncOp &funcOp) {
    genRuntimeMinMaxlocBody(builder, funcOp, isMax, rank, maskRank, inputType,
                            logicalElemType, outType, isDim);
  };

  mlir::func::FuncOp newFunc =
      getOrCreateFunction(builder, funcName, typeGenerator, bodyGenerator);
  builder.create<fir::CallOp>(loc, newFunc,
                              mlir::ValueRange{args[0], args[1], mask});
  call->dropAllReferences();
  call->erase();
}

void SimplifyIntrinsicsPass::simplifyReductionBody(
    fir::CallOp call, const fir::KindMapping &kindMap,
    GenReductionBodyTy genBodyFunc, fir::FirOpBuilder &builder,
    const mlir::StringRef &funcName, mlir::Type elementType) {

  mlir::Operation::operand_range args = call.getArgs();

  mlir::Type resultType = call.getResult(0).getType();
  unsigned rank = getDimCount(args[0]);

  mlir::Location loc = call.getLoc();

  auto typeGenerator = [&resultType](fir::FirOpBuilder &builder) {
    return genNoneBoxType(builder, resultType);
  };
  auto bodyGenerator = [&rank, &genBodyFunc,
                        &elementType](fir::FirOpBuilder &builder,
                                      mlir::func::FuncOp &funcOp) {
    genBodyFunc(builder, funcOp, rank, elementType);
  };
  // Mangle the function name with the rank value as "x<rank>".
  mlir::func::FuncOp newFunc =
      getOrCreateFunction(builder, funcName, typeGenerator, bodyGenerator);
  auto newCall =
      builder.create<fir::CallOp>(loc, newFunc, mlir::ValueRange{args[0]});
  call->replaceAllUsesWith(newCall.getResults());
  call->dropAllReferences();
  call->erase();
}

void SimplifyIntrinsicsPass::runOnOperation() {
  LLVM_DEBUG(llvm::dbgs() << "=== Begin " DEBUG_TYPE " ===\n");
  mlir::ModuleOp module = getOperation();
  fir::KindMapping kindMap = fir::getKindMapping(module);
  module.walk([&](mlir::Operation *op) {
    if (auto call = mlir::dyn_cast<fir::CallOp>(op)) {
      if (cuf::isInCUDADeviceContext(op))
        return;
      if (mlir::SymbolRefAttr callee = call.getCalleeAttr()) {
        mlir::StringRef funcName = callee.getLeafReference().getValue();
        // Replace call to runtime function for SUM when it has single
        // argument (no dim or mask argument) for 1D arrays with either
        // Integer4 or Real8 types. Other forms are ignored.
        // The new function is added to the module.
        //
        // Prototype for runtime call (from sum.cpp):
        // RTNAME(Sum<T>)(const Descriptor &x, const char *source, int line,
        //                int dim, const Descriptor *mask)
        //
        if (funcName.starts_with(RTNAME_STRING(Sum))) {
          simplifyIntOrFloatReduction(call, kindMap, genRuntimeSumBody);
          return;
        }
        if (funcName.starts_with(RTNAME_STRING(DotProduct))) {
          LLVM_DEBUG(llvm::dbgs() << "Handling " << funcName << "\n");
          LLVM_DEBUG(llvm::dbgs() << "Call operation:\n"; op->dump();
                     llvm::dbgs() << "\n");
          mlir::Operation::operand_range args = call.getArgs();
          const mlir::Value &v1 = args[0];
          const mlir::Value &v2 = args[1];
          mlir::Location loc = call.getLoc();
          fir::FirOpBuilder builder{getSimplificationBuilder(op, kindMap)};
          // Stringize the builder's FastMathFlags flags for mangling
          // the generated function name.
          std::string fmfString{builder.getFastMathFlagsString()};

          mlir::Type type = call.getResult(0).getType();
          if (!mlir::isa<mlir::FloatType>(type) &&
              !mlir::isa<mlir::IntegerType>(type))
            return;

          // Try to find the element types of the boxed arguments.
          auto arg1Type = getArgElementType(v1);
          auto arg2Type = getArgElementType(v2);

          if (!arg1Type || !arg2Type)
            return;

          // Support only floating point and integer arguments
          // now (e.g. logical is skipped here).
          if (!mlir::isa<mlir::FloatType, mlir::IntegerType>(*arg1Type))
            return;
          if (!mlir::isa<mlir::FloatType, mlir::IntegerType>(*arg2Type))
            return;

          auto typeGenerator = [&type](fir::FirOpBuilder &builder) {
            return genRuntimeDotType(builder, type);
          };
          auto bodyGenerator = [&arg1Type,
                                &arg2Type](fir::FirOpBuilder &builder,
                                           mlir::func::FuncOp &funcOp) {
            genRuntimeDotBody(builder, funcOp, *arg1Type, *arg2Type);
          };

          // Suffix the function name with the element types
          // of the arguments.
          std::string typedFuncName(funcName);
          llvm::raw_string_ostream nameOS(typedFuncName);
          // We must mangle the generated function name with FastMathFlags
          // value.
          if (!fmfString.empty())
            nameOS << '_' << fmfString;
          nameOS << '_';
          arg1Type->print(nameOS);
          nameOS << '_';
          arg2Type->print(nameOS);

          mlir::func::FuncOp newFunc = getOrCreateFunction(
              builder, typedFuncName, typeGenerator, bodyGenerator);
          auto newCall = builder.create<fir::CallOp>(loc, newFunc,
                                                     mlir::ValueRange{v1, v2});
          call->replaceAllUsesWith(newCall.getResults());
          call->dropAllReferences();
          call->erase();

          LLVM_DEBUG(llvm::dbgs() << "Replaced with:\n"; newCall.dump();
                     llvm::dbgs() << "\n");
          return;
        }
        if (funcName.starts_with(RTNAME_STRING(Maxval))) {
          simplifyIntOrFloatReduction(call, kindMap, genRuntimeMaxvalBody);
          return;
        }
        if (funcName.starts_with(RTNAME_STRING(Count))) {
          simplifyLogicalDim0Reduction(call, kindMap, genRuntimeCountBody);
          return;
        }
        if (funcName.starts_with(RTNAME_STRING(Any))) {
          simplifyLogicalDim1Reduction(call, kindMap, genRuntimeAnyBody);
          return;
        }
        if (funcName.ends_with(RTNAME_STRING(All))) {
          simplifyLogicalDim1Reduction(call, kindMap, genRuntimeAllBody);
          return;
        }
        if (funcName.starts_with(RTNAME_STRING(Minloc))) {
          simplifyMinMaxlocReduction(call, kindMap, false);
          return;
        }
        if (funcName.starts_with(RTNAME_STRING(Maxloc))) {
          simplifyMinMaxlocReduction(call, kindMap, true);
          return;
        }
      }
    }
  });
  LLVM_DEBUG(llvm::dbgs() << "=== End " DEBUG_TYPE " ===\n");
}

void SimplifyIntrinsicsPass::getDependentDialects(
    mlir::DialectRegistry &registry) const {
  // LLVM::LinkageAttr creation requires that LLVM dialect is loaded.
  registry.insert<mlir::LLVM::LLVMDialect>();
}