xref: /llvm-project/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td (revision 402efa733c64bd20b54dbc5b7057868cbb938d07) 
1#ifndef LLVM_INTRINSIC_OPS
2#define LLVM_INTRINSIC_OPS
3
4include "mlir/IR/OpBase.td"
5include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
6include "mlir/Dialect/LLVMIR/LLVMEnums.td"
7include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
8include "mlir/Interfaces/InferTypeOpInterface.td"
9include "mlir/Interfaces/MemorySlotInterfaces.td"
10
11// Operations that correspond to LLVM intrinsics. With MLIR operation set being
12// extendable, there is no reason to introduce a hard boundary between "core"
13// operations and intrinsics. However, we systematically prefix them with
14// "intr." to avoid potential name clashes.
15
16class LLVM_UnaryIntrOpBase<string func, Type element,
17                           list<Trait> traits = [], bit requiresFastmath = 0> :
18    LLVM_OneResultIntrOp<func, [], [0],
19           !listconcat([Pure, SameOperandsAndResultType], traits),
20           requiresFastmath> {
21  dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$in);
22  let assemblyFormat = "`(` operands `)` attr-dict `:` "
23      "functional-type(operands, results)";
24}
25
26class LLVM_UnaryIntrOpI<string func, list<Trait> traits = []> :
27    LLVM_UnaryIntrOpBase<func, AnySignlessInteger, traits> {
28  let arguments = commonArgs;
29}
30
31class LLVM_UnaryIntrOpF<string func, list<Trait> traits = []> :
32    LLVM_UnaryIntrOpBase<func, LLVM_AnyFloat, traits, /*requiresFastmath=*/1> {
33  dag fmfArg = (
34    ins DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags);
35  let arguments = !con(commonArgs, fmfArg);
36}
37
38class LLVM_BinarySameArgsIntrOpBase<string func, Type element,
39              list<Trait> traits = [], bit requiresFastmath = 0> :
40    LLVM_OneResultIntrOp<func, [], [0],
41           !listconcat([Pure, SameOperandsAndResultType], traits),
42           requiresFastmath> {
43  dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$a,
44                        LLVM_ScalarOrVectorOf<element>:$b);
45  let assemblyFormat = "`(` operands `)` attr-dict `:` "
46      "functional-type(operands, results)";
47}
48
49class LLVM_BinarySameArgsIntrOpI<string func, list<Trait> traits = []> :
50    LLVM_BinarySameArgsIntrOpBase<func, AnySignlessInteger, traits> {
51  let arguments = commonArgs;
52}
53
54class LLVM_BinarySameArgsIntrOpF<string func, list<Trait> traits = []> :
55    LLVM_BinarySameArgsIntrOpBase<func, LLVM_AnyFloat, traits,
56                                  /*requiresFastmath=*/1> {
57  dag fmfArg = (
58    ins DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags);
59  let arguments = !con(commonArgs, fmfArg);
60}
61
62class LLVM_TernarySameArgsIntrOpBase<string func, Type element,
63              list<Trait> traits = [], bit requiresFastmath = 0> :
64    LLVM_OneResultIntrOp<func, [], [0],
65           !listconcat([Pure, SameOperandsAndResultType], traits),
66           requiresFastmath> {
67  dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$a,
68                       LLVM_ScalarOrVectorOf<element>:$b,
69                       LLVM_ScalarOrVectorOf<element>:$c);
70  let assemblyFormat = "`(` operands `)` attr-dict `:` "
71      "functional-type(operands, results)";
72}
73
74class LLVM_TernarySameArgsIntrOpI<string func, list<Trait> traits = []> :
75    LLVM_TernarySameArgsIntrOpBase<func, AnySignlessInteger, traits> {
76  let arguments = commonArgs;
77}
78
79class LLVM_TernarySameArgsIntrOpF<string func, list<Trait> traits = []> :
80    LLVM_TernarySameArgsIntrOpBase<func, LLVM_AnyFloat, traits,
81                                  /*requiresFastmath=*/1> {
82  dag fmfArg = (
83    ins DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags);
84  let arguments = !con(commonArgs, fmfArg);
85}
86
87class LLVM_CountZerosIntrOp<string func, list<Trait> traits = []> :
88    LLVM_OneResultIntrOp<func, [], [0],
89           !listconcat([Pure], traits),
90            /*requiresFastmath=*/0,
91            /*immArgPositions=*/[1], /*immArgAttrNames=*/["is_zero_poison"]> {
92  let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
93                   I1Attr:$is_zero_poison);
94}
95
96def LLVM_AbsOp : LLVM_OneResultIntrOp<"abs", [], [0], [Pure],
97    /*requiresFastmath=*/0,
98    /*immArgPositions=*/[1], /*immArgAttrNames=*/["is_int_min_poison"]> {
99  let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
100                   I1Attr:$is_int_min_poison);
101}
102
103def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure],
104  /*requiresFastmath=*/0,
105  /*immArgPositions=*/[1], /*immArgAttrNames=*/["bit"]> {
106  let arguments = (ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$in, I32Attr:$bit);
107}
108
109def LLVM_CopySignOp : LLVM_BinarySameArgsIntrOpF<"copysign">;
110def LLVM_CosOp : LLVM_UnaryIntrOpF<"cos">;
111def LLVM_ExpOp : LLVM_UnaryIntrOpF<"exp">;
112def LLVM_Exp2Op : LLVM_UnaryIntrOpF<"exp2">;
113def LLVM_FAbsOp : LLVM_UnaryIntrOpF<"fabs">;
114def LLVM_FCeilOp : LLVM_UnaryIntrOpF<"ceil">;
115def LLVM_FFloorOp : LLVM_UnaryIntrOpF<"floor">;
116def LLVM_FMAOp : LLVM_TernarySameArgsIntrOpF<"fma">;
117def LLVM_FMulAddOp : LLVM_TernarySameArgsIntrOpF<"fmuladd">;
118def LLVM_Log10Op : LLVM_UnaryIntrOpF<"log10">;
119def LLVM_Log2Op : LLVM_UnaryIntrOpF<"log2">;
120def LLVM_LogOp : LLVM_UnaryIntrOpF<"log">;
121def LLVM_Prefetch : LLVM_ZeroResultIntrOp<"prefetch", [0],
122  /*traits=*/[], /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
123  /*requiresOpBundles=*/0, /*immArgPositions=*/[1, 2, 3],
124  /*immArgAttrNames=*/["rw", "hint", "cache"]
125> {
126  let arguments = (ins LLVM_AnyPointer:$addr, I32Attr:$rw, I32Attr:$hint, I32Attr:$cache);
127}
128def LLVM_SinOp : LLVM_UnaryIntrOpF<"sin">;
129def LLVM_RoundEvenOp : LLVM_UnaryIntrOpF<"roundeven">;
130def LLVM_RoundOp : LLVM_UnaryIntrOpF<"round">;
131def LLVM_FTruncOp : LLVM_UnaryIntrOpF<"trunc">;
132def LLVM_SqrtOp : LLVM_UnaryIntrOpF<"sqrt">;
133def LLVM_PowOp : LLVM_BinarySameArgsIntrOpF<"pow">;
134def LLVM_PowIOp : LLVM_OneResultIntrOp<"powi", [], [0,1],
135                                       [Pure], /*requiresFastmath=*/1> {
136  let arguments =
137      (ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$val,
138           AnySignlessInteger:$power,
139           DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags);
140  let assemblyFormat = "`(` operands `)` attr-dict `:` "
141      "functional-type(operands, results)";
142}
143def LLVM_RintOp : LLVM_UnaryIntrOpF<"rint">;
144def LLVM_NearbyintOp : LLVM_UnaryIntrOpF<"nearbyint">;
145class LLVM_IntRoundIntrOpBase<string func> :
146        LLVM_OneResultIntrOp<func, [0], [0], [Pure]> {
147  let arguments = (ins LLVM_AnyFloat:$val);
148  let assemblyFormat = "`(` operands `)` attr-dict `:` "
149      "functional-type(operands, results)";
150}
151def LLVM_LroundOp : LLVM_IntRoundIntrOpBase<"lround">;
152def LLVM_LlroundOp : LLVM_IntRoundIntrOpBase<"llround">;
153def LLVM_LrintOp : LLVM_IntRoundIntrOpBase<"lrint">;
154def LLVM_LlrintOp : LLVM_IntRoundIntrOpBase<"llrint">;
155def LLVM_BitReverseOp : LLVM_UnaryIntrOpI<"bitreverse">;
156def LLVM_ByteSwapOp : LLVM_UnaryIntrOpI<"bswap">;
157def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrOp<"ctlz">;
158def LLVM_CountTrailingZerosOp : LLVM_CountZerosIntrOp<"cttz">;
159def LLVM_CtPopOp : LLVM_UnaryIntrOpI<"ctpop">;
160def LLVM_FshlOp : LLVM_TernarySameArgsIntrOpI<"fshl">;
161def LLVM_FshrOp : LLVM_TernarySameArgsIntrOpI<"fshr">;
162def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrOpF<"maxnum">;
163def LLVM_MinNumOp : LLVM_BinarySameArgsIntrOpF<"minnum">;
164def LLVM_MaximumOp : LLVM_BinarySameArgsIntrOpF<"maximum">;
165def LLVM_MinimumOp : LLVM_BinarySameArgsIntrOpF<"minimum">;
166def LLVM_SMaxOp : LLVM_BinarySameArgsIntrOpI<"smax">;
167def LLVM_SMinOp : LLVM_BinarySameArgsIntrOpI<"smin">;
168def LLVM_UMaxOp : LLVM_BinarySameArgsIntrOpI<"umax">;
169def LLVM_UMinOp : LLVM_BinarySameArgsIntrOpI<"umin">;
170def LLVM_SinhOp : LLVM_UnaryIntrOpF<"sinh">;
171def LLVM_CoshOp : LLVM_UnaryIntrOpF<"cosh">;
172def LLVM_TanhOp : LLVM_UnaryIntrOpF<"tanh">;
173
174class LLVM_MemcpyIntrOpBase<string name> :
175    LLVM_ZeroResultIntrOp<name, [0, 1, 2],
176    [DeclareOpInterfaceMethods<PromotableMemOpInterface>,
177     DeclareOpInterfaceMethods<DestructurableAccessorOpInterface>,
178     DeclareOpInterfaceMethods<SafeMemorySlotAccessOpInterface>],
179    /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1,
180    /*requiresOpBundles=*/0, /*immArgPositions=*/[3],
181    /*immArgAttrNames=*/["isVolatile"]> {
182  dag args = (ins Arg<LLVM_AnyPointer,"",[MemWrite]>:$dst,
183                  Arg<LLVM_AnyPointer,"",[MemRead]>:$src,
184                  AnySignlessInteger:$len, I1Attr:$isVolatile);
185  // Append the alias attributes defined by LLVM_IntrOpBase.
186  let arguments = !con(args, aliasAttrs);
187  let builders = [
188    OpBuilder<(ins "Value":$dst, "Value":$src, "Value":$len,
189                   "bool":$isVolatile), [{
190      build($_builder, $_state, dst, src, len,
191            $_builder.getBoolAttr(isVolatile));
192    }]>,
193    OpBuilder<(ins "Value":$dst, "Value":$src, "Value":$len,
194                   "IntegerAttr":$isVolatile), [{
195      build($_builder, $_state, dst, src, len, isVolatile,
196            /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
197            /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
198    }]>
199  ];
200}
201
202def LLVM_MemcpyOp : LLVM_MemcpyIntrOpBase<"memcpy">;
203def LLVM_MemmoveOp : LLVM_MemcpyIntrOpBase<"memmove">;
204
205def LLVM_MemcpyInlineOp :
206    LLVM_ZeroResultIntrOp<"memcpy.inline", [0, 1, 2],
207    [DeclareOpInterfaceMethods<PromotableMemOpInterface>,
208     DeclareOpInterfaceMethods<DestructurableAccessorOpInterface>,
209     DeclareOpInterfaceMethods<SafeMemorySlotAccessOpInterface>],
210    /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1,
211    /*requiresOpBundles=*/0, /*immArgPositions=*/[2, 3],
212    /*immArgAttrNames=*/["len", "isVolatile"]> {
213  dag args = (ins Arg<LLVM_AnyPointer,"",[MemWrite]>:$dst,
214                  Arg<LLVM_AnyPointer,"",[MemRead]>:$src,
215                  APIntAttr:$len, I1Attr:$isVolatile);
216  // Append the alias attributes defined by LLVM_IntrOpBase.
217  let arguments = !con(args, aliasAttrs);
218  let builders = [
219    OpBuilder<(ins "Value":$dst, "Value":$src, "IntegerAttr":$len,
220                   "bool":$isVolatile), [{
221      build($_builder, $_state, dst, src, len,
222            $_builder.getBoolAttr(isVolatile));
223    }]>,
224    OpBuilder<(ins "Value":$dst, "Value":$src, "IntegerAttr":$len,
225                   "IntegerAttr":$isVolatile), [{
226      build($_builder, $_state, dst, src, len, isVolatile,
227            /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
228            /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
229    }]>
230  ];
231}
232
233def LLVM_MemsetOp : LLVM_ZeroResultIntrOp<"memset", [0, 2],
234    [DeclareOpInterfaceMethods<PromotableMemOpInterface>,
235     DeclareOpInterfaceMethods<DestructurableAccessorOpInterface>,
236     DeclareOpInterfaceMethods<SafeMemorySlotAccessOpInterface>],
237    /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1,
238    /*requiresOpBundles=*/0, /*immArgPositions=*/[3],
239    /*immArgAttrNames=*/["isVolatile"]> {
240  dag args = (ins Arg<LLVM_AnyPointer,"",[MemWrite]>:$dst,
241                  I8:$val, AnySignlessInteger:$len, I1Attr:$isVolatile);
242  // Append the alias attributes defined by LLVM_IntrOpBase.
243  let arguments = !con(args, aliasAttrs);
244  let builders = [
245    OpBuilder<(ins "Value":$dst, "Value":$val, "Value":$len,
246                    "bool":$isVolatile), [{
247      build($_builder, $_state, dst, val, len,
248            $_builder.getBoolAttr(isVolatile));
249    }]>,
250    OpBuilder<(ins "Value":$dst, "Value":$val, "Value":$len,
251                    "IntegerAttr":$isVolatile), [{
252      build($_builder, $_state, dst, val, len, isVolatile,
253            /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
254            /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
255    }]>
256  ];
257}
258
259def LLVM_MemsetInlineOp : LLVM_ZeroResultIntrOp<"memset.inline", [0, 2],
260    [DeclareOpInterfaceMethods<PromotableMemOpInterface>,
261     DeclareOpInterfaceMethods<DestructurableAccessorOpInterface>,
262     DeclareOpInterfaceMethods<SafeMemorySlotAccessOpInterface>],
263    /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1,
264    /*requiresOpBundles=*/0, /*immArgPositions=*/[2, 3],
265    /*immArgAttrNames=*/["len", "isVolatile"]> {
266  dag args = (ins Arg<LLVM_AnyPointer,"",[MemWrite]>:$dst,
267                  I8:$val, APIntAttr:$len, I1Attr:$isVolatile);
268  // Append the alias attributes defined by LLVM_IntrOpBase.
269  let arguments = !con(args, aliasAttrs);
270  let builders = [
271    OpBuilder<(ins "Value":$dst, "Value":$val, "IntegerAttr":$len,
272                    "bool":$isVolatile), [{
273      build($_builder, $_state, dst, val, len,
274            $_builder.getBoolAttr(isVolatile));
275    }]>,
276    OpBuilder<(ins "Value":$dst, "Value":$val, "IntegerAttr":$len,
277                    "IntegerAttr":$isVolatile), [{
278      build($_builder, $_state, dst, val, len, isVolatile,
279            /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
280            /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
281    }]>
282  ];
283}
284
285def LLVM_NoAliasScopeDeclOp
286    : LLVM_ZeroResultIntrOp<"experimental.noalias.scope.decl"> {
287  let arguments = (ins LLVM_AliasScopeAttr:$scope);
288  string llvmBuilder = [{
289    // Wrap the scope argument into a list since the LLVM IR intrinsic takes
290    // a list containing exactly one scope rather than a scope itself.
291    llvm::MDNode* node = moduleTranslation.getOrCreateAliasScopes({$scope});
292    builder.CreateNoAliasScopeDeclaration(node);
293  }];
294  string mlirBuilder = [{
295    FailureOr<SmallVector<LLVM::AliasScopeAttr>> scopeAttrs =
296      moduleImport.matchAliasScopeAttrs(llvmOperands[0]);
297    // Drop the intrinsic if the alias scope translation fails since the scope
298    // is not used by an aliasing operation, such as a load or store, that is
299    // used to convert the alias scope metadata.
300    if (failed(scopeAttrs))
301      return success();
302    if (scopeAttrs->size() != 1)
303      return failure();
304    $_op = $_builder.create<LLVM::NoAliasScopeDeclOp>(
305      $_location, (*scopeAttrs)[0]);
306  }];
307  let assemblyFormat = "$scope attr-dict";
308}
309
310//
311// Memory marker intrinsics.
312//
313
314/// Base operation for lifetime markers. The LLVM intrinsics require the size
315/// operand to be an immediate. In MLIR it is encoded as an attribute.
316class LLVM_LifetimeBaseOp<string opName> : LLVM_ZeroResultIntrOp<opName, [1],
317    [DeclareOpInterfaceMethods<PromotableOpInterface>],
318    /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
319    /*requiresOpBundles=*/0, /*immArgPositions=*/[0],
320    /*immArgAttrNames=*/["size"]> {
321  let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
322  let assemblyFormat = "$size `,` $ptr attr-dict `:` qualified(type($ptr))";
323}
324
325def LLVM_LifetimeStartOp : LLVM_LifetimeBaseOp<"lifetime.start">;
326def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">;
327
328def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1],
329    [DeclareOpInterfaceMethods<PromotableOpInterface>],
330    /*requiresFastmath=*/0, /*immArgPositions=*/[0],
331    /*immArgAttrNames=*/["size"]> {
332  let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
333  let results = (outs LLVM_DefaultPointer:$res);
334  let assemblyFormat = "$size `,` $ptr attr-dict `:` qualified(type($ptr))";
335}
336
337def LLVM_InvariantEndOp : LLVM_ZeroResultIntrOp<"invariant.end", [2],
338    [DeclareOpInterfaceMethods<PromotableOpInterface>],
339    /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
340    /*requiresOpBundles=*/0, /*immArgPositions=*/[1],
341    /*immArgAttrNames=*/["size"]> {
342  let arguments = (ins LLVM_DefaultPointer:$start,
343                       I64Attr:$size,
344                       LLVM_AnyPointer:$ptr);
345  let assemblyFormat = "$start `,` $size `,` $ptr attr-dict `:` "
346      "qualified(type($ptr))";
347}
348
349def LLVM_LaunderInvariantGroupOp
350    : LLVM_OneResultIntrOp<"launder.invariant.group", [], [0],
351        [DeclareOpInterfaceMethods<PromotableOpInterface>,
352         SameOperandsAndResultType]> {
353  let arguments = (ins LLVM_AnyPointer:$ptr);
354  let results = (outs LLVM_AnyPointer:$res);
355  let assemblyFormat = "$ptr attr-dict `:` qualified(type($ptr))";
356}
357
358def LLVM_StripInvariantGroupOp
359    : LLVM_OneResultIntrOp<"strip.invariant.group", [], [0],
360        [DeclareOpInterfaceMethods<PromotableOpInterface>,
361         SameOperandsAndResultType]> {
362  let arguments = (ins LLVM_AnyPointer:$ptr);
363  let results = (outs LLVM_AnyPointer:$res);
364  let assemblyFormat = "$ptr attr-dict `:` qualified(type($ptr))";
365}
366
367// Constrained Floating-Point Intrinsics.
368
369class LLVM_ConstrainedIntr<string mnem, int numArgs,
370                           bit overloadedResult, list<int> overloadedOperands,
371                           bit hasRoundingMode>
372    : LLVM_OneResultIntrOp<"experimental.constrained." # mnem,
373                           /*overloadedResults=*/
374                           !cond(!gt(overloadedResult, 0) : [0],
375                                 true : []),
376                           overloadedOperands,
377                           /*traits=*/[Pure, DeclareOpInterfaceMethods<FPExceptionBehaviorOpInterface>]
378                           # !cond(
379                               !gt(hasRoundingMode, 0) : [DeclareOpInterfaceMethods<RoundingModeOpInterface>],
380                               true : []),
381                           /*requiresFastmath=*/0,
382                           /*immArgPositions=*/[],
383                           /*immArgAttrNames=*/[]> {
384  dag regularArgs = !dag(ins, !listsplat(LLVM_Type, numArgs), !foreach(i, !range(numArgs), "arg_" #i));
385  dag attrArgs = !con(!cond(!gt(hasRoundingMode, 0) : (ins ValidRoundingModeAttr:$roundingmode),
386                            true : (ins)),
387                      (ins FPExceptionBehaviorAttr:$fpExceptionBehavior));
388  let arguments = !con(regularArgs, attrArgs);
389  let llvmBuilder = [{
390    SmallVector<llvm::Value *> args =
391      moduleTranslation.lookupValues(opInst.getOperands());
392    SmallVector<llvm::Type *> overloadedTypes; }] #
393    !cond(!gt(overloadedResult, 0) : [{
394    // Take into account overloaded result type.
395    overloadedTypes.push_back($_resultType); }],
396    // No overloaded result type.
397          true : "") # [{
398    llvm::transform(ArrayRef<unsigned>}] # overloadedOperandsCpp # [{,
399                    std::back_inserter(overloadedTypes),
400                    [&args](unsigned index) { return args[index]->getType(); });
401    llvm::Module *module = builder.GetInsertBlock()->getModule();
402    llvm::Function *callee =
403      llvm::Intrinsic::getOrInsertDeclaration(module,
404        llvm::Intrinsic::experimental_constrained_}] #
405    mnem # [{, overloadedTypes); }] #
406    !cond(!gt(hasRoundingMode, 0) : [{
407    // Get rounding mode using interface.
408    llvm::RoundingMode rounding =
409        moduleTranslation.translateRoundingMode($roundingmode); }],
410          true : [{
411    // No rounding mode.
412    std::optional<llvm::RoundingMode> rounding; }]) # [{
413    llvm::fp::ExceptionBehavior except =
414      moduleTranslation.translateFPExceptionBehavior($fpExceptionBehavior);
415    $res = builder.CreateConstrainedFPCall(callee, args, "", rounding, except);
416  }];
417  let mlirBuilder = [{
418    SmallVector<Value> mlirOperands;
419    SmallVector<NamedAttribute> mlirAttrs;
420    if (failed(moduleImport.convertIntrinsicArguments(
421        llvmOperands.take_front( }] # numArgs # [{), {}, false,
422        {}, {}, mlirOperands, mlirAttrs))) {
423      return failure();
424    }
425
426    FPExceptionBehaviorAttr fpExceptionBehaviorAttr =
427        $_fpExceptionBehavior_attr($fpExceptionBehavior);
428    mlirAttrs.push_back(
429        $_builder.getNamedAttr(
430            $_qualCppClassName::getFPExceptionBehaviorAttrName(),
431            fpExceptionBehaviorAttr)); }] #
432    !cond(!gt(hasRoundingMode, 0) : [{
433    RoundingModeAttr roundingModeAttr = $_roundingMode_attr($roundingmode);
434    mlirAttrs.push_back(
435        $_builder.getNamedAttr($_qualCppClassName::getRoundingModeAttrName(),
436                               roundingModeAttr));
437    }], true : "") # [{
438    $res = $_builder.create<$_qualCppClassName>($_location,
439      $_resultType, mlirOperands, mlirAttrs);
440  }];
441}
442
443def LLVM_ConstrainedFPTruncIntr
444    : LLVM_ConstrainedIntr<"fptrunc", /*numArgs=*/1,
445        /*overloadedResult=*/1, /*overloadedOperands=*/[0],
446        /*hasRoundingMode=*/1> {
447  let assemblyFormat = [{
448    $arg_0 $roundingmode $fpExceptionBehavior attr-dict `:` type($arg_0) `to` type(results)
449  }];
450}
451
452// Intrinsics with multiple returns.
453
454class LLVM_ArithWithOverflowOp<string mnem>
455    : LLVM_IntrOp<mnem, [0], [], [Pure, SameOperandsElementType], 2>,
456      Arguments<(ins LLVM_ScalarOrVectorOf<AnySignlessInteger>,
457                 LLVM_ScalarOrVectorOf<AnySignlessInteger>)>;
458
459def LLVM_SAddWithOverflowOp : LLVM_ArithWithOverflowOp<"sadd.with.overflow">;
460def LLVM_UAddWithOverflowOp : LLVM_ArithWithOverflowOp<"uadd.with.overflow">;
461def LLVM_SSubWithOverflowOp : LLVM_ArithWithOverflowOp<"ssub.with.overflow">;
462def LLVM_USubWithOverflowOp : LLVM_ArithWithOverflowOp<"usub.with.overflow">;
463def LLVM_SMulWithOverflowOp : LLVM_ArithWithOverflowOp<"smul.with.overflow">;
464def LLVM_UMulWithOverflowOp : LLVM_ArithWithOverflowOp<"umul.with.overflow">;
465
466//
467// Saturation Arithmetic Intrinsics.
468//
469
470def LLVM_SAddSat : LLVM_BinarySameArgsIntrOpI<"sadd.sat">;
471def LLVM_UAddSat : LLVM_BinarySameArgsIntrOpI<"uadd.sat">;
472def LLVM_SSubSat : LLVM_BinarySameArgsIntrOpI<"ssub.sat">;
473def LLVM_USubSat : LLVM_BinarySameArgsIntrOpI<"usub.sat">;
474def LLVM_SSHLSat : LLVM_BinarySameArgsIntrOpI<"sshl.sat">;
475def LLVM_USHLSat : LLVM_BinarySameArgsIntrOpI<"ushl.sat">;
476
477//
478// Optimization hint intrinsics.
479//
480
481def LLVM_AssumeOp
482    : LLVM_ZeroResultIntrOp<"assume", /*overloadedOperands=*/[], /*traits=*/[],
483                            /*requiresAccessGroup=*/0,
484                            /*requiresAliasAnalysis=*/0,
485                            /*requiresOpBundles=*/1> {
486  dag args = (ins I1:$cond);
487  let arguments = !con(args, opBundleArgs);
488
489  let assemblyFormat = [{
490    $cond
491    ( custom<OpBundles>($op_bundle_operands, type($op_bundle_operands),
492                        $op_bundle_tags)^ )?
493    `:` type($cond) attr-dict
494  }];
495
496  let builders = [
497    OpBuilder<(ins "Value":$cond)>,
498    OpBuilder<(ins "Value":$cond,
499                   "ArrayRef<llvm::OperandBundleDefT<Value>>":$opBundles)>,
500    OpBuilder<(ins "Value":$cond, "llvm::StringRef":$tag, "ValueRange":$args)>,
501    OpBuilder<(ins "Value":$cond, "AssumeAlignTag":$tag, "Value":$ptr,
502                   "Value":$align)>,
503    OpBuilder<(ins "Value":$cond, "AssumeSeparateStorageTag":$tag,
504                   "Value":$ptr1, "Value":$ptr2)>
505  ];
506
507  let hasVerifier = 1;
508}
509
510def LLVM_SSACopyOp : LLVM_OneResultIntrOp<"ssa.copy", [], [0],
511                                            [Pure, SameOperandsAndResultType]> {
512  let arguments = (ins AnyType:$operand);
513
514  let assemblyFormat = "$operand attr-dict `:` type($operand)";
515}
516
517def LLVM_IsConstantOp : LLVM_IntrOp<"is.constant", [], [0], [Pure], 1> {
518  let arguments = (ins LLVM_Type:$val);
519  let results = (outs I1:$res);
520}
521
522def LLVM_ExpectOp
523  : LLVM_OneResultIntrOp<"expect", [], [0],
524                         [Pure, SameOperandsAndResultType]> {
525  let arguments = (ins AnySignlessInteger:$val,
526                       AnySignlessInteger:$expected);
527  let assemblyFormat = "$val `,` $expected attr-dict `:` type($val)";
528}
529
530def LLVM_ExpectWithProbabilityOp
531  : LLVM_OneResultIntrOp<"expect.with.probability", [], [0],
532                         [Pure, AllTypesMatch<["val", "expected", "res"]>],
533                         /*requiresFastmath=*/0,
534                         /*immArgPositions=*/[2], /*immArgAttrNames=*/["prob"]> {
535  let arguments = (ins AnySignlessInteger:$val,
536                       AnySignlessInteger:$expected,
537                       F64Attr:$prob);
538  let assemblyFormat = "$val `,` $expected `,` $prob attr-dict `:` type($val)";
539}
540
541def LLVM_ThreadlocalAddressOp : LLVM_OneResultIntrOp<"threadlocal.address", [],
542                                [0], [Pure]> {
543  let arguments = (ins LLVM_AnyPointer:$global);
544}
545
546//
547// Coroutine intrinsics.
548//
549
550def LLVM_CoroIdOp : LLVM_IntrOp<"coro.id", [], [], [], 1> {
551  let arguments = (ins I32:$align,
552                       LLVM_AnyPointer:$promise,
553                       LLVM_AnyPointer:$coroaddr,
554                       LLVM_AnyPointer:$fnaddrs);
555  let assemblyFormat = "$align `,` $promise `,` $coroaddr `,` $fnaddrs"
556    " attr-dict `:` functional-type(operands, results)";
557}
558
559def LLVM_CoroBeginOp : LLVM_IntrOp<"coro.begin", [], [], [], 1> {
560  let arguments = (ins LLVM_TokenType:$token,
561                       LLVM_AnyPointer:$mem);
562  let assemblyFormat = "$token `,` $mem attr-dict `:` functional-type(operands, results)";
563}
564
565def LLVM_CoroSizeOp : LLVM_IntrOp<"coro.size", [0], [], [], 1> {
566  let assemblyFormat = "attr-dict `:` type($res)";
567}
568
569def LLVM_CoroAlignOp : LLVM_IntrOp<"coro.align", [0], [], [], 1> {
570  let assemblyFormat = "attr-dict `:` type($res)";
571}
572
573def LLVM_CoroSaveOp : LLVM_IntrOp<"coro.save", [], [], [], 1> {
574  let arguments = (ins LLVM_AnyPointer:$handle);
575  let assemblyFormat = "$handle attr-dict `:` functional-type(operands, results)";
576}
577
578def LLVM_CoroSuspendOp : LLVM_IntrOp<"coro.suspend", [], [], [], 1> {
579  let arguments = (ins LLVM_TokenType:$save,
580                       I1:$final);
581  let assemblyFormat = "$save `,` $final attr-dict `:` type($res)";
582}
583
584def LLVM_CoroEndOp : LLVM_IntrOp<"coro.end", [], [], [], 1> {
585  let arguments = (ins LLVM_AnyPointer:$handle,
586                       I1:$unwind,
587                       LLVM_TokenType:$retvals);
588  let assemblyFormat = "$handle `,` $unwind `,` $retvals attr-dict `:` functional-type(operands, results)";
589}
590
591def LLVM_CoroFreeOp : LLVM_IntrOp<"coro.free", [], [], [], 1> {
592  let arguments = (ins LLVM_TokenType:$id,
593                       LLVM_AnyPointer:$handle);
594  let assemblyFormat = "$id `,` $handle attr-dict `:` functional-type(operands, results)";
595}
596
597def LLVM_CoroResumeOp : LLVM_IntrOp<"coro.resume", [], [], [], 0> {
598  let arguments = (ins LLVM_AnyPointer:$handle);
599  let assemblyFormat = "$handle attr-dict `:` qualified(type($handle))";
600}
601
602def LLVM_CoroPromiseOp : LLVM_IntrOp<"coro.promise", [], [], [], 1> {
603  let arguments = (ins LLVM_AnyPointer:$handle,
604                       I32:$align,
605                       I1:$from);
606  let results = (outs LLVM_AnyPointer:$res);
607  let assemblyFormat = "$handle `,` $align `,` $from attr-dict `:` functional-type(operands, results)";
608}
609
610//
611// Debug function intrinsics.
612//
613
614class LLVM_DbgIntrOp<string name, string argName, list<Trait> traits = []>
615    : LLVM_IntrOp<name, [], [], traits, 0> {
616  let llvmBuilder = [{
617    // Debug intrinsics without debug locations are invalid.
618    if(!builder.getCurrentDebugLocation())
619      return success();
620    llvm::Module *module = builder.GetInsertBlock()->getModule();
621    llvm::LLVMContext &ctx = module->getContext();
622    llvm::Function *fn =
623      llvm::Intrinsic::getOrInsertDeclaration(module, llvm::Intrinsic::}]
624       # !subst(".", "_", name) # [{);
625    builder.CreateCall(fn, {
626        llvm::MetadataAsValue::get(ctx,
627            llvm::ValueAsMetadata::get(moduleTranslation.lookupValue(opInst.getOperand(0)))),
628        llvm::MetadataAsValue::get(ctx, moduleTranslation.translateDebugInfo($varInfo)),
629        llvm::MetadataAsValue::get(ctx, moduleTranslation.translateExpression($locationExpr)),
630      });
631  }];
632  let mlirBuilder = [{
633    // Add debug intrindic to the list of intrinsics that need to be converted once the
634    // full function was converted.
635    moduleImport.addDebugIntrinsic(inst);
636    return success();
637  }];
638  let assemblyFormat = [{
639    qualified($varInfo) (qualified($locationExpr)^)? `=` $}] # argName #
640      [{ `:` qualified(type($}] # argName # [{)) attr-dict
641  }];
642}
643
644def LLVM_DbgDeclareOp : LLVM_DbgIntrOp<"dbg.declare", "addr", [
645    DeclareOpInterfaceMethods<PromotableOpInterface, [
646      "requiresReplacedValues", "visitReplacedValues"
647    ]>]> {
648  let summary = "Describes how the address relates to a source language variable.";
649  let arguments = (ins
650    LLVM_AnyPointer:$addr,
651    LLVM_DILocalVariableAttr:$varInfo,
652    DefaultValuedAttr<LLVM_DIExpressionAttr, "{}">:$locationExpr
653  );
654}
655
656def LLVM_DbgValueOp : LLVM_DbgIntrOp<"dbg.value", "value",
657    [DeclareOpInterfaceMethods<PromotableOpInterface>]> {
658  let summary = "Describes how the value relates to a source language variable.";
659  let arguments = (ins
660    LLVM_Type:$value,
661    LLVM_DILocalVariableAttr:$varInfo,
662    DefaultValuedAttr<LLVM_DIExpressionAttr, "{}">:$locationExpr
663  );
664}
665
666def LLVM_DbgLabelOp : LLVM_IntrOp<"dbg.label", [], [], [], 0> {
667  let summary = "Relates the program to a debug information label.";
668  let arguments = (ins LLVM_DILabelAttr:$label);
669  let llvmBuilder = [{
670    // Debug intrinsics without debug locations are invalid.
671    if(!builder.getCurrentDebugLocation())
672      return success();
673    llvm::Module *module = builder.GetInsertBlock()->getModule();
674    llvm::LLVMContext &ctx = module->getContext();
675    llvm::Function *fn =
676      llvm::Intrinsic::getOrInsertDeclaration(module, llvm::Intrinsic::dbg_label);
677    builder.CreateCall(fn, {
678        llvm::MetadataAsValue::get(ctx, moduleTranslation.translateDebugInfo($label))
679      });
680  }];
681  let mlirBuilder = [{
682    DILabelAttr labelAttr = $_label_attr($label);
683    // Drop the intrinsic if the label translation fails due to cylic metadata.
684    if (!labelAttr)
685      return success();
686    $_op = $_builder.create<$_qualCppClassName>($_location, labelAttr);
687  }];
688  let assemblyFormat = "$label attr-dict";
689}
690
691//
692// Variadic function intrinsics.
693//
694
695def LLVM_VaStartOp : LLVM_ZeroResultIntrOp<"vastart", [0]>,
696                     Arguments<(ins LLVM_AnyPointer:$arg_list)> {
697  let assemblyFormat = "$arg_list attr-dict `:` qualified(type($arg_list))";
698  let summary = "Initializes `arg_list` for subsequent variadic argument extractions.";
699}
700
701def LLVM_VaCopyOp : LLVM_ZeroResultIntrOp<"vacopy", [0]>,
702                    Arguments<(ins LLVM_AnyPointer:$dest_list, LLVM_AnyPointer:$src_list)> {
703  let assemblyFormat = "$src_list `to` $dest_list attr-dict `:` type(operands)";
704  let summary = "Copies the current argument position from `src_list` to `dest_list`.";
705}
706
707def LLVM_VaEndOp : LLVM_ZeroResultIntrOp<"vaend", [0]>,
708                   Arguments<(ins LLVM_AnyPointer:$arg_list)> {
709  let assemblyFormat = "$arg_list attr-dict `:` qualified(type($arg_list))";
710  let summary = "Destroys `arg_list`, which has been initialized by `intr.vastart` or `intr.vacopy`.";
711}
712
713//
714// Exception handling intrinsics.
715//
716
717def LLVM_EhTypeidForOp : LLVM_OneResultIntrOp<"eh.typeid.for", [], [0]> {
718    let arguments = (ins LLVM_AnyPointer:$type_info);
719    let assemblyFormat = "$type_info attr-dict `:` functional-type(operands, results)";
720}
721
722//
723// Stack save/restore intrinsics.
724//
725
726def LLVM_StackSaveOp : LLVM_OneResultIntrOp<"stacksave", [0]> {
727  let assemblyFormat = "attr-dict `:` qualified(type($res))";
728}
729
730def LLVM_StackRestoreOp : LLVM_ZeroResultIntrOp<"stackrestore", [0]> {
731  let arguments = (ins LLVM_AnyPointer:$ptr);
732  let assemblyFormat = "$ptr attr-dict `:` qualified(type($ptr))";
733}
734
735//
736// Vector Reductions.
737//
738
739// LLVM vector reduction over a single vector.
740class LLVM_VecReductionBase<string mnem, Type element, bit requiresFastmath=0>
741    : LLVM_OneResultIntrOp<"vector.reduce." # mnem, [], [0],
742                           [Pure, SameOperandsAndResultElementType],
743                           requiresFastmath> {
744      dag commonArgs = (ins LLVM_VectorOf<element>:$in);
745}
746
747class LLVM_VecReductionF<string mnem>
748    : LLVM_VecReductionBase<mnem, AnyFloat, /*requiresFastmath=*/1> {
749  dag fmfArg = (
750    ins DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags);
751  let arguments = !con(commonArgs, fmfArg);
752
753  let assemblyFormat = "`(` operands `)` attr-dict `:` "
754      "functional-type(operands, results)";
755}
756
757class LLVM_VecReductionI<string mnem>
758    : LLVM_VecReductionBase<mnem, AnySignlessInteger> {
759      let arguments = commonArgs;
760}
761
762// LLVM vector reduction over a single vector, with an initial value,
763// and with permission to reassociate the reduction operations.
764class LLVM_VecReductionAccBase<string mnem, Type element>
765    : LLVM_OneResultIntrOp</*mnem=*/"vector.reduce." # mnem,
766                           /*overloadedResults=*/[],
767                           /*overloadedOperands=*/[1],
768                           /*traits=*/[Pure, SameOperandsAndResultElementType],
769                           /*equiresFastmath=*/1>,
770      Arguments<(ins element:$start_value,
771                     LLVM_VectorOf<element>:$input,
772                     DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags)>;
773
774class LLVM_VecReductionAccF<string mnem>
775    : LLVM_VecReductionAccBase<mnem, AnyFloat>;
776
777def LLVM_vector_reduce_add : LLVM_VecReductionI<"add">;
778def LLVM_vector_reduce_and : LLVM_VecReductionI<"and">;
779def LLVM_vector_reduce_mul : LLVM_VecReductionI<"mul">;
780def LLVM_vector_reduce_or : LLVM_VecReductionI<"or">;
781def LLVM_vector_reduce_smax : LLVM_VecReductionI<"smax">;
782def LLVM_vector_reduce_smin : LLVM_VecReductionI<"smin">;
783def LLVM_vector_reduce_umax : LLVM_VecReductionI<"umax">;
784def LLVM_vector_reduce_umin : LLVM_VecReductionI<"umin">;
785def LLVM_vector_reduce_xor : LLVM_VecReductionI<"xor">;
786
787def LLVM_vector_reduce_fmax : LLVM_VecReductionF<"fmax">;
788def LLVM_vector_reduce_fmin : LLVM_VecReductionF<"fmin">;
789def LLVM_vector_reduce_fmaximum : LLVM_VecReductionF<"fmaximum">;
790def LLVM_vector_reduce_fminimum : LLVM_VecReductionF<"fminimum">;
791
792def LLVM_vector_reduce_fadd : LLVM_VecReductionAccF<"fadd">;
793def LLVM_vector_reduce_fmul : LLVM_VecReductionAccF<"fmul">;
794
795//
796// LLVM Matrix operations.
797//
798
799/// Create a column major, strided 2-D matrix load, as specified in the LLVM
800/// MatrixBuilder.
801/// data       - Start address of the matrix read
802/// rows       - Number of rows in matrix (must be a constant)
803/// isVolatile - True if the load operation is marked as volatile.
804/// columns    - Number of columns in matrix (must be a constant)
805/// stride     - Space between columns
806def LLVM_MatrixColumnMajorLoadOp : LLVM_OneResultIntrOp<"matrix.column.major.load"> {
807  let arguments = (ins LLVM_AnyPointer:$data, AnySignlessInteger:$stride, I1Attr:$isVolatile,
808                   I32Attr:$rows, I32Attr:$columns);
809  let results = (outs LLVM_AnyVector:$res);
810  let builders = [LLVM_OneResultOpBuilder];
811  let assemblyFormat = "$data `,` `<` `stride` `=` $stride `>` attr-dict"
812    "`:` type($res) `from` qualified(type($data)) `stride` type($stride)";
813
814  string llvmBuilder = [{
815    llvm::MatrixBuilder mb(builder);
816    const llvm::DataLayout &dl =
817      builder.GetInsertBlock()->getModule()->getDataLayout();
818    llvm::Type *ElemTy = moduleTranslation.convertType(
819        getVectorElementType(op.getType()));
820    llvm::Align align = dl.getABITypeAlign(ElemTy);
821    $res = mb.CreateColumnMajorLoad(
822      ElemTy, $data, align, $stride, $isVolatile, $rows,
823      $columns);
824  }];
825  string mlirBuilder = [{
826    $res = $_builder.create<LLVM::MatrixColumnMajorLoadOp>(
827      $_location, $_resultType, $data, $stride,
828      $_int_attr($isVolatile), $_int_attr($rows), $_int_attr($columns));
829  }];
830}
831
832/// Create a column major, strided 2-D matrix store, as specified in the LLVM
833/// MatrixBuilder.
834/// matrix     - Matrix to store
835/// ptr        - Pointer to write back to
836/// isVolatile - True if the load operation is marked as volatile.
837/// rows       - Number of rows in matrix (must be a constant)
838/// columns    - Number of columns in matrix (must be a constant)
839/// stride     - Space between columns
840def LLVM_MatrixColumnMajorStoreOp : LLVM_ZeroResultIntrOp<"matrix.column.major.store"> {
841  let arguments = (ins LLVM_AnyVector:$matrix, LLVM_AnyPointer:$data,
842                   AnySignlessInteger:$stride, I1Attr:$isVolatile, I32Attr:$rows,
843                   I32Attr:$columns);
844  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
845  let assemblyFormat = "$matrix `,` $data `,` `<` `stride` `=` $stride `>` "
846    "attr-dict`:` type($matrix) `to` qualified(type($data)) `stride` type($stride)";
847
848  string llvmBuilder = [{
849    llvm::MatrixBuilder mb(builder);
850    const llvm::DataLayout &dl =
851      builder.GetInsertBlock()->getModule()->getDataLayout();
852    Type elementType = getVectorElementType(op.getMatrix().getType());
853    llvm::Align align = dl.getABITypeAlign(
854      moduleTranslation.convertType(elementType));
855    mb.CreateColumnMajorStore(
856      $matrix, $data, align, $stride, $isVolatile,
857      $rows, $columns);
858  }];
859  string mlirBuilder = [{
860    $_op = $_builder.create<LLVM::MatrixColumnMajorStoreOp>(
861      $_location, $matrix, $data, $stride,
862      $_int_attr($isVolatile), $_int_attr($rows), $_int_attr($columns));
863  }];
864}
865
866/// Create a llvm.matrix.multiply call, multiplying 2-D matrices LHS and RHS, as
867/// specified in the LLVM MatrixBuilder.
868def LLVM_MatrixMultiplyOp : LLVM_OneResultIntrOp<"matrix.multiply"> {
869  let arguments = (ins LLVM_AnyVector:$lhs, LLVM_AnyVector:$rhs, I32Attr:$lhs_rows,
870                   I32Attr:$lhs_columns, I32Attr:$rhs_columns);
871  let results = (outs LLVM_AnyVector:$res);
872  let builders = [LLVM_OneResultOpBuilder];
873  let assemblyFormat = "$lhs `,` $rhs attr-dict "
874    "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)";
875
876  string llvmBuilder = [{
877    llvm::MatrixBuilder mb(builder);
878    $res = mb.CreateMatrixMultiply(
879      $lhs, $rhs, $lhs_rows, $lhs_columns,
880      $rhs_columns);
881  }];
882  string mlirBuilder = [{
883    $res = $_builder.create<LLVM::MatrixMultiplyOp>(
884      $_location, $_resultType, $lhs, $rhs,
885      $_int_attr($lhs_rows), $_int_attr($lhs_columns), $_int_attr($rhs_columns));
886  }];
887}
888
889/// Create a llvm.matrix.transpose call, transposing a `rows` x `columns` 2-D
890/// `matrix`, as specified in the LLVM MatrixBuilder.
891def LLVM_MatrixTransposeOp : LLVM_OneResultIntrOp<"matrix.transpose"> {
892  let arguments = (ins LLVM_AnyVector:$matrix, I32Attr:$rows, I32Attr:$columns);
893  let results = (outs LLVM_AnyVector:$res);
894  let builders = [LLVM_OneResultOpBuilder];
895  let assemblyFormat = "$matrix attr-dict `:` type($matrix) `into` type($res)";
896
897  string llvmBuilder = [{
898    llvm::MatrixBuilder mb(builder);
899    $res = mb.CreateMatrixTranspose(
900      $matrix, $rows, $columns);
901  }];
902  string mlirBuilder = [{
903    $res = $_builder.create<LLVM::MatrixTransposeOp>(
904      $_location, $_resultType, $matrix,
905      $_int_attr($rows), $_int_attr($columns));
906  }];
907}
908
909//
910// LLVM masked operations.
911//
912
913/// Create a llvm.get.active.lane.mask to set a mask up to a given position.
914def LLVM_GetActiveLaneMaskOp
915    : LLVM_OneResultIntrOp<"get.active.lane.mask", [0], [0], [Pure]> {
916  let arguments = (ins AnySignlessInteger:$base, AnySignlessInteger:$n);
917  let assemblyFormat = "$base `,` $n attr-dict `:` "
918    "type($base) `,` type($n) `to` type($res)";
919}
920
921/// Create a call to Masked Load intrinsic.
922def LLVM_MaskedLoadOp : LLVM_OneResultIntrOp<"masked.load"> {
923  let arguments = (ins LLVM_AnyPointer:$data, LLVM_VectorOf<I1>:$mask,
924                   Variadic<LLVM_AnyVector>:$pass_thru, I32Attr:$alignment,
925                   UnitAttr:$nontemporal);
926  let results = (outs LLVM_AnyVector:$res);
927  let assemblyFormat =
928    "operands attr-dict `:` functional-type(operands, results)";
929
930  string llvmBuilder = [{
931    auto *inst = $pass_thru.empty() ? builder.CreateMaskedLoad(
932        $_resultType, $data, llvm::Align($alignment), $mask) :
933      builder.CreateMaskedLoad(
934        $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru[0]);
935    $res = inst;
936  }] #setNonTemporalMetadataCode;
937  string mlirBuilder = [{
938    auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
939    bool nontemporal = intrinInst->hasMetadata(llvm::LLVMContext::MD_nontemporal);
940    $res = $_builder.create<LLVM::MaskedLoadOp>($_location,
941      $_resultType, $data, $mask, $pass_thru, $_int_attr($alignment),
942        nontemporal ? $_builder.getUnitAttr() : nullptr);
943  }];
944  list<int> llvmArgIndices = [0, 2, 3, 1, -1];
945}
946
947/// Create a call to Masked Store intrinsic.
948def LLVM_MaskedStoreOp : LLVM_ZeroResultIntrOp<"masked.store"> {
949  let arguments = (ins LLVM_AnyVector:$value, LLVM_AnyPointer:$data,
950                   LLVM_VectorOf<I1>:$mask, I32Attr:$alignment);
951  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
952  let assemblyFormat = "$value `,` $data `,` $mask attr-dict `:` "
953    "type($value) `,` type($mask) `into` qualified(type($data))";
954
955  string llvmBuilder = [{
956    builder.CreateMaskedStore(
957      $value, $data, llvm::Align($alignment), $mask);
958  }];
959  string mlirBuilder = [{
960    $_op = $_builder.create<LLVM::MaskedStoreOp>($_location,
961      $value, $data, $mask, $_int_attr($alignment));
962  }];
963  list<int> llvmArgIndices = [0, 1, 3, 2];
964}
965
966/// Create a call to Masked Gather intrinsic.
967def LLVM_masked_gather : LLVM_OneResultIntrOp<"masked.gather"> {
968  let arguments = (ins LLVM_VectorOf<LLVM_AnyPointer>:$ptrs,
969                   LLVM_VectorOf<I1>:$mask, Variadic<LLVM_AnyVector>:$pass_thru,
970                   I32Attr:$alignment);
971  let results = (outs LLVM_AnyVector:$res);
972  let builders = [LLVM_OneResultOpBuilder];
973  let assemblyFormat =
974    "operands attr-dict `:` functional-type(operands, results)";
975
976  string llvmBuilder = [{
977    $res = $pass_thru.empty() ? builder.CreateMaskedGather(
978        $_resultType, $ptrs, llvm::Align($alignment), $mask) :
979      builder.CreateMaskedGather(
980        $_resultType, $ptrs, llvm::Align($alignment), $mask, $pass_thru[0]);
981  }];
982  string mlirBuilder = [{
983    $res = $_builder.create<LLVM::masked_gather>($_location,
984      $_resultType, $ptrs, $mask, $pass_thru, $_int_attr($alignment));
985  }];
986  list<int> llvmArgIndices = [0, 2, 3, 1];
987
988  let hasVerifier = 1;
989}
990
991/// Create a call to Masked Scatter intrinsic.
992def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
993  let arguments = (ins LLVM_AnyVector:$value, LLVM_VectorOf<LLVM_AnyPointer>:$ptrs,
994                   LLVM_VectorOf<I1>:$mask, I32Attr:$alignment);
995  let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
996  let assemblyFormat = "$value `,` $ptrs `,` $mask attr-dict `:` "
997    "type($value) `,` type($mask) `into` type($ptrs)";
998
999  string llvmBuilder = [{
1000    builder.CreateMaskedScatter(
1001      $value, $ptrs, llvm::Align($alignment), $mask);
1002  }];
1003  string mlirBuilder = [{
1004    $_op = $_builder.create<LLVM::masked_scatter>($_location,
1005      $value, $ptrs, $mask, $_int_attr($alignment));
1006  }];
1007  list<int> llvmArgIndices = [0, 1, 3, 2];
1008
1009  let hasVerifier = 1;
1010}
1011
1012/// Create a call to Masked Expand Load intrinsic.
1013def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
1014  let arguments = (ins LLVM_AnyPointer, LLVM_VectorOf<I1>, LLVM_AnyVector);
1015}
1016
1017/// Create a call to Masked Compress Store intrinsic.
1018def LLVM_masked_compressstore
1019    : LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
1020  let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf<I1>);
1021}
1022
1023//
1024// Annotate intrinsics.
1025//
1026
1027def LLVM_VarAnnotation
1028    : LLVM_ZeroResultIntrOp<"var.annotation", [0, 1],
1029        [AllTypesMatch<["annotation", "fileName", "attr"]>]> {
1030  let arguments = (ins LLVM_AnyPointer:$val,
1031                       LLVM_AnyPointer:$annotation,
1032                       LLVM_AnyPointer:$fileName,
1033                       I32:$line,
1034                       LLVM_AnyPointer:$attr);
1035}
1036
1037def LLVM_PtrAnnotation
1038    : LLVM_OneResultIntrOp<"ptr.annotation", [0], [2],
1039        [AllTypesMatch<["res", "ptr"]>,
1040         AllTypesMatch<["annotation", "fileName", "attr"]>]> {
1041  let arguments = (ins LLVM_AnyPointer:$ptr,
1042                       LLVM_AnyPointer:$annotation,
1043                       LLVM_AnyPointer:$fileName,
1044                       I32:$line,
1045                       LLVM_AnyPointer:$attr);
1046  let results = (outs LLVM_AnyPointer:$res);
1047}
1048
1049def LLVM_Annotation
1050    : LLVM_OneResultIntrOp<"annotation", [0], [2],
1051        [AllTypesMatch<["res", "integer"]>,
1052         AllTypesMatch<["annotation", "fileName"]>]> {
1053  let arguments = (ins AnySignlessInteger:$integer,
1054                       LLVM_AnyPointer:$annotation,
1055                       LLVM_AnyPointer:$fileName,
1056                       I32:$line);
1057  let results = (outs AnySignlessInteger:$res);
1058}
1059
1060//
1061// Trap intrinsics.
1062//
1063
1064def LLVM_Trap : LLVM_ZeroResultIntrOp<"trap">;
1065
1066def LLVM_DebugTrap : LLVM_ZeroResultIntrOp<"debugtrap">;
1067
1068def LLVM_UBSanTrap : LLVM_ZeroResultIntrOp<"ubsantrap",
1069  /*overloadedOperands=*/[], /*traits=*/[],
1070  /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
1071  /*requiresOpBundles=*/0, /*immArgPositions=*/[0],
1072  /*immArgAttrNames=*/["failureKind"]> {
1073  let arguments = (ins I8Attr:$failureKind);
1074}
1075
1076/// Create a call to vscale intrinsic.
1077def LLVM_vscale : LLVM_IntrOp<"vscale", [0], [], [], 1>;
1078
1079/// Create a call to stepvector intrinsic.
1080def LLVM_StepVectorOp
1081    : LLVM_IntrOp<"stepvector", [0], [], [Pure], 1> {
1082  let arguments = (ins);
1083  let results = (outs LLVM_VectorOf<AnySignlessInteger>:$res);
1084  let assemblyFormat = "attr-dict `:` type($res)";
1085}
1086
1087/// Create a call to vector.insert intrinsic
1088def LLVM_vector_insert
1089    : LLVM_OneResultIntrOp<"vector.insert",
1090                  /*overloadedResults=*/[0], /*overloadedOperands=*/[1],
1091                  /*traits=*/[Pure, AllTypesMatch<["dstvec", "res"]>,
1092                  PredOpTrait<"vectors are not bigger than 2^17 bits.", And<[
1093                    CPred<"getSrcVectorBitWidth() <= 131072">,
1094                    CPred<"getDstVectorBitWidth() <= 131072">
1095                  ]>>,
1096                  PredOpTrait<"it is not inserting scalable into fixed-length vectors.",
1097                    CPred<"!isScalableVectorType($srcvec.getType()) || "
1098                          "isScalableVectorType($dstvec.getType())">>],
1099                  /*requiresFastmath=*/0,
1100                  /*immArgPositions=*/[2], /*immArgAttrNames=*/["pos"]> {
1101  let arguments = (ins LLVM_AnyVector:$dstvec, LLVM_AnyVector:$srcvec,
1102                       I64Attr:$pos);
1103  let results = (outs LLVM_AnyVector:$res);
1104  let assemblyFormat = "$srcvec `,` $dstvec `[` $pos `]` attr-dict `:` "
1105    "type($srcvec) `into` type($res)";
1106  let extraClassDeclaration = [{
1107    uint64_t getVectorBitWidth(Type vector) {
1108      return getVectorNumElements(vector).getKnownMinValue() *
1109             getVectorElementType(vector).getIntOrFloatBitWidth();
1110    }
1111    uint64_t getSrcVectorBitWidth() {
1112      return getVectorBitWidth(getSrcvec().getType());
1113    }
1114    uint64_t getDstVectorBitWidth() {
1115      return getVectorBitWidth(getDstvec().getType());
1116    }
1117  }];
1118}
1119
1120/// Create a call to vector.extract intrinsic
1121def LLVM_vector_extract
1122    : LLVM_OneResultIntrOp<"vector.extract",
1123                 /*overloadedResults=*/[0], /*overloadedOperands=*/[0],
1124                 /*traits=*/[Pure,
1125                  PredOpTrait<"vectors are not bigger than 2^17 bits.", And<[
1126                    CPred<"getSrcVectorBitWidth() <= 131072">,
1127                    CPred<"getResVectorBitWidth() <= 131072">
1128                  ]>>,
1129                  PredOpTrait<"it is not extracting scalable from fixed-length vectors.",
1130                    CPred<"!isScalableVectorType($res.getType()) || "
1131                          "isScalableVectorType($srcvec.getType())">>],
1132                  /*requiresFastmath=*/0,
1133                  /*immArgPositions=*/[1], /*immArgAttrNames=*/["pos"]> {
1134  let arguments = (ins LLVM_AnyVector:$srcvec, I64Attr:$pos);
1135  let results = (outs LLVM_AnyVector:$res);
1136  let assemblyFormat = "$srcvec `[` $pos `]` attr-dict `:` "
1137    "type($res) `from` type($srcvec)";
1138  let extraClassDeclaration = [{
1139    uint64_t getVectorBitWidth(Type vector) {
1140      return getVectorNumElements(vector).getKnownMinValue() *
1141             getVectorElementType(vector).getIntOrFloatBitWidth();
1142    }
1143    uint64_t getSrcVectorBitWidth() {
1144      return getVectorBitWidth(getSrcvec().getType());
1145    }
1146    uint64_t getResVectorBitWidth() {
1147      return getVectorBitWidth(getRes().getType());
1148    }
1149  }];
1150}
1151
1152def LLVM_vector_interleave2
1153    : LLVM_OneResultIntrOp<"vector.interleave2",
1154        /*overloadedResults=*/[0], /*overloadedOperands=*/[],
1155        /*traits=*/[
1156          Pure, AllTypesMatch<["vec1", "vec2"]>,
1157          PredOpTrait<
1158            "result has twice as many elements as 'vec1'",
1159            And<[CPred<"getVectorNumElements($res.getType()) == "
1160                       "getVectorNumElements($vec1.getType()) * 2">,
1161                 CPred<"getVectorElementType($vec1.getType()) == "
1162                       "getVectorElementType($res.getType())">]>>,
1163        ]>,
1164        Arguments<(ins LLVM_AnyVector:$vec1, LLVM_AnyVector:$vec2)>;
1165
1166def LLVM_vector_deinterleave2
1167    : LLVM_OneResultIntrOp<"vector.deinterleave2",
1168        /*overloadedResults=*/[], /*overloadedOperands=*/[0],
1169        /*traits=*/[Pure]>,
1170        Arguments<(ins LLVM_AnyVector:$vec)>;
1171
1172//
1173// LLVM Vector Predication operations.
1174//
1175
1176class LLVM_VPBinaryBase<string mnem, Type element>
1177    : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [Pure]>,
1178      Arguments<(ins LLVM_VectorOf<element>:$lhs, LLVM_VectorOf<element>:$rhs,
1179                     LLVM_VectorOf<I1>:$mask, I32:$evl)>;
1180
1181class LLVM_VPBinaryI<string mnem> : LLVM_VPBinaryBase<mnem, AnySignlessInteger>;
1182
1183class LLVM_VPBinaryF<string mnem> : LLVM_VPBinaryBase<mnem, AnyFloat>;
1184
1185class LLVM_VPUnaryBase<string mnem, Type element>
1186    : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [Pure]>,
1187      Arguments<(ins LLVM_VectorOf<element>:$op,
1188                     LLVM_VectorOf<I1>:$mask, I32:$evl)>;
1189
1190class LLVM_VPUnaryF<string mnem> : LLVM_VPUnaryBase<mnem, AnyFloat>;
1191
1192class LLVM_VPTernaryBase<string mnem, Type element>
1193    : LLVM_OneResultIntrOp<"vp." # mnem, [0], [], [Pure]>,
1194      Arguments<(ins LLVM_VectorOf<element>:$op1, LLVM_VectorOf<element>:$op2,
1195                     LLVM_VectorOf<element>:$op3, LLVM_VectorOf<I1>:$mask,
1196                     I32:$evl)>;
1197
1198class LLVM_VPTernaryF<string mnem> : LLVM_VPTernaryBase<mnem, AnyFloat>;
1199
1200class LLVM_VPReductionBase<string mnem, Type element>
1201    : LLVM_OneResultIntrOp<"vp.reduce." # mnem, [], [1], [Pure]>,
1202      Arguments<(ins element:$satrt_value, LLVM_VectorOf<element>:$val,
1203                     LLVM_VectorOf<I1>:$mask, I32:$evl)>;
1204
1205class LLVM_VPReductionI<string mnem> : LLVM_VPReductionBase<mnem, AnySignlessInteger>;
1206
1207class LLVM_VPReductionF<string mnem> : LLVM_VPReductionBase<mnem, AnyFloat>;
1208
1209class LLVM_VPSelectBase<string mnem>
1210    : LLVM_OneResultIntrOp<"vp." # mnem, [], [1], [Pure]>,
1211      Arguments<(ins LLVM_VectorOf<I1>:$cond, LLVM_AnyVector:$true_val,
1212                     LLVM_AnyVector:$false_val, I32:$evl)>;
1213
1214class LLVM_VPCastBase<string mnem, Type element>
1215    : LLVM_OneResultIntrOp<"vp." # mnem, [0], [0], [Pure]>,
1216      Arguments<(ins LLVM_VectorOf<element>:$src,
1217                     LLVM_VectorOf<I1>:$mask, I32:$evl)>;
1218
1219class LLVM_VPCastI<string mnem>   : LLVM_VPCastBase<mnem, AnySignlessInteger>;
1220
1221class LLVM_VPCastF<string mnem>   : LLVM_VPCastBase<mnem, AnyFloat>;
1222
1223class LLVM_VPCastPtr<string mnem> : LLVM_VPCastBase<mnem, LLVM_AnyPointer>;
1224
1225// Integer Binary
1226def LLVM_VPAddOp  : LLVM_VPBinaryI<"add">;
1227def LLVM_VPSubOp  : LLVM_VPBinaryI<"sub">;
1228def LLVM_VPMulOp  : LLVM_VPBinaryI<"mul">;
1229def LLVM_VPSDivOp : LLVM_VPBinaryI<"sdiv">;
1230def LLVM_VPUDivOp : LLVM_VPBinaryI<"udiv">;
1231def LLVM_VPSRemOp : LLVM_VPBinaryI<"srem">;
1232def LLVM_VPURemOp : LLVM_VPBinaryI<"urem">;
1233def LLVM_VPAShrOp : LLVM_VPBinaryI<"ashr">;
1234def LLVM_VPLShrOp : LLVM_VPBinaryI<"lshr">;
1235def LLVM_VPShlOp  : LLVM_VPBinaryI<"shl">;
1236def LLVM_VPOrOp   : LLVM_VPBinaryI<"or">;
1237def LLVM_VPAndOp  : LLVM_VPBinaryI<"and">;
1238def LLVM_VPXorOp  : LLVM_VPBinaryI<"xor">;
1239def LLVM_VPSMaxOp : LLVM_VPBinaryI<"smax">;
1240def LLVM_VPSMinOp : LLVM_VPBinaryI<"smin">;
1241def LLVM_VPUMaxOp : LLVM_VPBinaryI<"umax">;
1242def LLVM_VPUMinOp : LLVM_VPBinaryI<"umin">;
1243
1244// Float Binary
1245def LLVM_VPFAddOp : LLVM_VPBinaryF<"fadd">;
1246def LLVM_VPFSubOp : LLVM_VPBinaryF<"fsub">;
1247def LLVM_VPFMulOp : LLVM_VPBinaryF<"fmul">;
1248def LLVM_VPFDivOp : LLVM_VPBinaryF<"fdiv">;
1249def LLVM_VPFRemOp : LLVM_VPBinaryF<"frem">;
1250
1251// Float Unary
1252def LLVM_VPFNegOp : LLVM_VPUnaryF<"fneg">;
1253
1254// Float Ternary
1255def LLVM_VPFMulAddOp  : LLVM_VPTernaryF<"fmuladd">;
1256def LLVM_VPFmaOp      : LLVM_VPTernaryF<"fma">;
1257
1258// Integer Reduction
1259def LLVM_VPReduceAddOp  : LLVM_VPReductionI<"add">;
1260def LLVM_VPReduceMulOp  : LLVM_VPReductionI<"mul">;
1261def LLVM_VPReduceAndOp  : LLVM_VPReductionI<"and">;
1262def LLVM_VPReduceOrOp   : LLVM_VPReductionI<"or">;
1263def LLVM_VPReduceXorOp  : LLVM_VPReductionI<"xor">;
1264def LLVM_VPReduceSMaxOp : LLVM_VPReductionI<"smax">;
1265def LLVM_VPReduceSMinOp : LLVM_VPReductionI<"smin">;
1266def LLVM_VPReduceUMaxOp : LLVM_VPReductionI<"umax">;
1267def LLVM_VPReduceUMinOp : LLVM_VPReductionI<"umin">;
1268
1269// Float Reduction
1270def LLVM_VPReduceFAddOp : LLVM_VPReductionF<"fadd">;
1271def LLVM_VPReduceFMulOp : LLVM_VPReductionF<"fmul">;
1272def LLVM_VPReduceFMaxOp : LLVM_VPReductionF<"fmax">;
1273def LLVM_VPReduceFMinOp : LLVM_VPReductionF<"fmin">;
1274
1275def LLVM_VPSelectMinOp : LLVM_VPSelectBase<"select">;
1276def LLVM_VPMergeMinOp  : LLVM_VPSelectBase<"merge">;
1277
1278// Load/store
1279def LLVM_VPLoadOp
1280    : LLVM_OneResultIntrOp<"vp.load", [0], [0], []>,
1281      Arguments<(ins LLVM_AnyPointer:$ptr,
1282                     LLVM_VectorOf<I1>:$mask, I32:$evl)>;
1283
1284def LLVM_VPStoreOp
1285    : LLVM_ZeroResultIntrOp<"vp.store", [0, 1], []>,
1286      Arguments<(ins LLVM_AnyVector:$val,
1287                     LLVM_AnyPointer:$ptr,
1288                     LLVM_VectorOf<I1>:$mask, I32:$evl)>;
1289
1290// Strided load/store
1291def LLVM_VPStridedLoadOp
1292    : LLVM_OneResultIntrOp<"experimental.vp.strided.load", [0], [0, 1], []>,
1293      Arguments<(ins LLVM_AnyPointer:$ptr, AnySignlessInteger:$stride,
1294                     LLVM_VectorOf<I1>:$mask, I32:$evl)>;
1295
1296def LLVM_VPStridedStoreOp
1297    : LLVM_ZeroResultIntrOp<"experimental.vp.strided.store",[0, 1, 2], []>,
1298      Arguments<(ins LLVM_AnyVector:$val, LLVM_AnyPointer:$ptr,
1299                     AnySignlessInteger:$stride, LLVM_VectorOf<I1>:$mask, I32:$evl)>;
1300
1301def LLVM_VPTruncOp : LLVM_VPCastI<"trunc">;
1302def LLVM_VPZExtOp  : LLVM_VPCastI<"zext">;
1303def LLVM_VPSExtOp  : LLVM_VPCastI<"sext">;
1304
1305def LLVM_VPFPTruncOp : LLVM_VPCastF<"fptrunc">;
1306def LLVM_VPFPExtOp   : LLVM_VPCastF<"fpext">;
1307
1308def LLVM_VPFPToUIOp : LLVM_VPCastF<"fptoui">;
1309def LLVM_VPFPToSIOp : LLVM_VPCastF<"fptosi">;
1310
1311def LLVM_VPUIToFPOp : LLVM_VPCastI<"uitofp">;
1312def LLVM_VPSIToFPOp : LLVM_VPCastI<"sitofp">;
1313
1314def LLVM_VPPtrToIntOp : LLVM_VPCastPtr<"ptrtoint">;
1315def LLVM_VPIntToPtrOp : LLVM_VPCastI<"inttoptr">;
1316
1317#endif // LLVM_INTRINSIC_OP
1318