https://github.com/aobolensk created https://github.com/llvm/llvm-project/pull/237732
Classic codegen keeps fast-math flags on these ops, but CIR dropped them Carry the flags in an optional attribute and forward it to the LLVM dialect >From 8d375464fd8a540dd09817712714418f6cf32bc5 Mon Sep 17 00:00:00 2001 From: Arseniy Obolenskiy <[email protected]> Date: Sun, 1 Nov 2026 14:42:52 +0100 Subject: [PATCH] [CIR] Add fast-math flags to cir.fma, cir.fmuladd and cir.fneg Classic codegen keeps fast-math flags on these ops, but CIR dropped them Carry the flags in an optional attribute and forward it to the LLVM dialect --- .../CIR/Dialect/Builder/CIRBaseBuilder.h | 3 +- clang/include/clang/CIR/Dialect/IR/CIROps.td | 26 +++++++++++++-- clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 3 +- .../lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp | 3 +- clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp | 13 +++++--- .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 33 ++++++++++++------- .../CIR/Lowering/DirectToLLVM/LowerToLLVM.h | 6 ++++ clang/test/CIR/CodeGen/fast-math-flags.c | 16 +++++++++ clang/test/CIR/CodeGen/fp-contract.c | 22 +++++++++++++ clang/utils/TableGen/CIRLoweringEmitter.cpp | 11 +++++++ 10 files changed, 113 insertions(+), 23 deletions(-) diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h index 489ed57695d299..cd1e950a665399 100644 --- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h +++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h @@ -947,10 +947,9 @@ class CIRBaseBuilderTy : public mlir::OpBuilder { assert(cir::isFPOrVectorOfFPType(operand.getType()) && "expected floating-point or vector-of-float type"); assert(!cir::MissingFeatures::metaDataNode()); - assert(!cir::MissingFeatures::fastMathFlags()); // fneg does not raise FP exceptions or depend on the rounding mode, so it // never carries an fenv attribute. - return cir::FNegOp::create(*this, loc, operand); + return cir::FNegOp::create(*this, loc, operand, getFastMathFlagsAttr()); } mlir::Value createXor(mlir::Location loc, mlir::Value lhs, mlir::Value rhs) { diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index bd762aae3e1d7e..990a84da651de4 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -2196,6 +2196,9 @@ def CIR_FNegOp : CIR_UnaryOp<"fneg", CIR_AnyFloatOrVecOfFloatType> { The `cir.fneg` operation negates the operand. The operand and result must have the same type. + The optional `fastmath_flags` attribute holds the fast-math flags of the + operation. + Example: ``` @@ -2205,6 +2208,19 @@ def CIR_FNegOp : CIR_UnaryOp<"fneg", CIR_AnyFloatOrVecOfFloatType> { ``` }]; + let arguments = !con(commonArgs, (ins + OptionalAttr<CIR_FastMathFlagsAttr>:$fastmath_flags)); + + let assemblyFormat = [{ + $input `:` type($input) prop-dict attr-dict + }]; + + let builders = [ + OpBuilder<(ins "mlir::Value":$input), [{ + build($_builder, $_state, input, cir::FastMathFlagsAttr{}); + }]> + ]; + let hasFolder = 1; let llvmOp = "FNegOp"; @@ -8802,6 +8818,8 @@ def CIR_ModfOp : CIR_Op<"modf", [Pure]> { }]; } +// The optional `fastmath_flags` attribute holds the fast-math flags of the +// operation. class CIR_TernaryFPToFPBuiltinOp<string mnemonic, string llvmOpName> : CIR_Op<mnemonic, !listconcat([SameOperandsAndResultType], CIR_FenvOpTraits)> @@ -8810,19 +8828,21 @@ class CIR_TernaryFPToFPBuiltinOp<string mnemonic, string llvmOpName> CIR_AnyFloatOrVecOfFloatType:$a, CIR_AnyFloatOrVecOfFloatType:$b, CIR_AnyFloatOrVecOfFloatType:$c, - OptionalAttr<CIR_FenvAttr>:$fenv + OptionalAttr<CIR_FenvAttr>:$fenv, + OptionalAttr<CIR_FastMathFlagsAttr>:$fastmath_flags ); let results = (outs CIR_AnyFloatOrVecOfFloatType:$result); let assemblyFormat = [{ - $a `,` $b `,` $c `:` type($a) (`fenv` `` $fenv^)? attr-dict + $a `,` $b `,` $c `:` type($a) (`fenv` `` $fenv^)? prop-dict attr-dict }]; let builders = [ OpBuilder<(ins "mlir::Type":$result, "mlir::Value":$a, "mlir::Value":$b, "mlir::Value":$c), [{ - build($_builder, $_state, result, a, b, c, cir::FenvAttr{}); + build($_builder, $_state, result, a, b, c, cir::FenvAttr{}, + cir::FastMathFlagsAttr{}); }]> ]; diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp index 55941a8480cf1d..e1b50240da8ff2 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp @@ -632,7 +632,8 @@ static RValue emitTernaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf, mlir::Type ty = cgf.convertType(e.getType()); auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1, arg2, - cgf.getBuilder().getConstrainedFPAttr()); + cgf.getBuilder().getConstrainedFPAttr(), + cgf.getBuilder().getFastMathFlagsAttr()); return RValue::get(call->getResult(0)); } diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp index 9f9b34f80ec3ef..591625def330ed 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp @@ -172,7 +172,8 @@ emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder, if constexpr (std::is_same_v<Operation, cir::FMAOp>) { assert(args.size() == 3 && "fma expects three operands"); return Operation::create(builder, loc, funcResTy, args[0], args[1], args[2], - builder.getConstrainedFPAttr()) + builder.getConstrainedFPAttr(), + builder.getFastMathFlagsAttr()) .getResult(); } if constexpr (std::is_same_v<Operation, cir::SqrtOp>) { diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp index 8e6ded0527cc90..0801ed6c4ab71a 100644 --- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp @@ -887,8 +887,11 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> { mlir::Location loc = cgf.getLoc(e->getSourceRange().getBegin()); - if (cir::isFPOrVectorOfFPType(operand.getType())) - return builder.createOrFold<cir::FNegOp>(loc, operand); + if (cir::isFPOrVectorOfFPType(operand.getType())) { + CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, e); + return builder.createOrFold<cir::FNegOp>(loc, operand, + builder.getFastMathFlagsAttr()); + } // TODO(cir): We might have to change this to support overflow trapping. // Classic codegen routes unary minus through emitSub to ensure @@ -2110,9 +2113,9 @@ static mlir::Value buildFMulAdd(mlir::Location addLoc, cir::FMulOp mulOp, // Carry the mul's fenv attribute so a constrained fmul yields a constrained // fmuladd; the builder is under the add's FP options, not the mul's. - mlir::Value fmuladd = - cir::FMulAddOp::create(builder, loc, addend.getType(), mulOp0, mulOp1, - addend, mulOp.getFenvAttr()); + mlir::Value fmuladd = cir::FMulAddOp::create( + builder, loc, addend.getType(), mulOp0, mulOp1, addend, + mulOp.getFenvAttr(), builder.getFastMathFlagsAttr()); mulOp.erase(); return fmuladd; } diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index 9d74696bde0d39..8338ad1ea9ac4b 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -680,12 +680,34 @@ static mlir::Value createConstrainedFPIntrinsicCall( return intrinsic->getResult(0); } +template <typename LLVMOp> +mlir::LogicalResult lowerFPOp(mlir::Operation *op, mlir::ValueRange operands, + cir::FastMathFlagsAttr fastmath, + const mlir::TypeConverter &typeConverter, + mlir::ConversionPatternRewriter &rewriter) { + mlir::Type llvmResTy = typeConverter.convertType(op->getResultTypes()[0]); + if (!llvmResTy) + return op->emitError("expected LLVM result type for floating-point op"); + + auto newOp = rewriter.replaceOpWithNewOp<LLVMOp>( + op, mlir::TypeRange{llvmResTy}, operands, + cir::getDefaultProperties<LLVMOp>(op->getContext())); + if (fastmath) + mlir::cast<mlir::LLVM::FastmathFlagsInterface>(newOp.getOperation()) + .setFastmathAttr(mlir::LLVM::FastmathFlagsAttr::get( + rewriter.getContext(), convertFastMathFlags(fastmath.getValue()))); + return mlir::success(); +} + template <typename LLVMOp> mlir::LogicalResult lowerConstrainableFPOp( mlir::Operation *op, mlir::ValueRange operands, cir::FenvAttr fenv, cir::FastMathFlagsAttr fastmath, const mlir::TypeConverter &typeConverter, mlir::ConversionPatternRewriter &rewriter, llvm::StringRef constrainedMnemonic, bool hasRoundingMode) { + if (!fenv) + return lowerFPOp<LLVMOp>(op, operands, fastmath, typeConverter, rewriter); + mlir::Type llvmResTy = typeConverter.convertType(op->getResultTypes()[0]); if (!llvmResTy) return op->emitError("expected LLVM result type for floating-point op"); @@ -694,17 +716,6 @@ mlir::LogicalResult lowerConstrainableFPOp( if (fastmath) fastmathFlags = convertFastMathFlags(fastmath.getValue()); - if (!fenv) { - auto newOp = rewriter.replaceOpWithNewOp<LLVMOp>( - op, mlir::TypeRange{llvmResTy}, operands, - cir::getDefaultProperties<LLVMOp>(op->getContext())); - if (fastmathFlags != mlir::LLVM::FastmathFlags::none) - mlir::cast<mlir::LLVM::FastmathFlagsInterface>(newOp.getOperation()) - .setFastmathAttr(mlir::LLVM::FastmathFlagsAttr::get( - rewriter.getContext(), fastmathFlags)); - return mlir::success(); - } - return lowerToConstrainedFPIntrinsic(op, operands, fenv, llvmResTy, rewriter, constrainedMnemonic, hasRoundingMode, fastmathFlags); diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h index d3ef1d968f3856..426e71f8a5561e 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h @@ -93,6 +93,12 @@ mlir::LogicalResult lowerToConstrainedFPIntrinsic( llvm::StringRef constrainedMnemonic, bool hasRoundingMode, mlir::LLVM::FastmathFlags fastmathFlags = {}); +template <typename LLVMOp> +mlir::LogicalResult lowerFPOp(mlir::Operation *op, mlir::ValueRange operands, + cir::FastMathFlagsAttr fastmath, + const mlir::TypeConverter &typeConverter, + mlir::ConversionPatternRewriter &rewriter); + template <typename LLVMOp> mlir::LogicalResult lowerConstrainableFPOp( mlir::Operation *op, mlir::ValueRange operands, cir::FenvAttr fenv, diff --git a/clang/test/CIR/CodeGen/fast-math-flags.c b/clang/test/CIR/CodeGen/fast-math-flags.c index cd2e74b4e5ab42..1cf3dc53fb00bb 100644 --- a/clang/test/CIR/CodeGen/fast-math-flags.c +++ b/clang/test/CIR/CodeGen/fast-math-flags.c @@ -224,3 +224,19 @@ float4 vector(float4 a, float4 b, float4 c) { // LLVM-FAST: fsub fast <4 x float> %{{.*}}, %{{.*}} // LLVM-FAST: fmul fast <4 x float> %{{.*}}, %{{.*}} // LLVM-FAST: fdiv fast <4 x float> %{{.*}}, %{{.*}} + +float fma_builtin(float a, float b, float c) { + return __builtin_fmaf(a, b, c); +} +// CIR-LABEL: cir.func {{.*}}@fma_builtin( +// LLVM-LABEL: define {{.*}}@fma_builtin( +// CIR-FAST-NO-CONTRACT: cir.fma %{{.*}}, %{{.*}}, %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn, reassoc]>{{( loc.*)?$}} +// LLVM-FAST-NO-CONTRACT: call reassoc nnan ninf nsz arcp afn float @llvm.fma.f32( + +float unary_minus(float a) { + return -a; +} +// CIR-LABEL: cir.func {{.*}}@unary_minus( +// LLVM-LABEL: define {{.*}}@unary_minus( +// CIR-FAST-NO-CONTRACT: cir.fneg %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn, reassoc]>{{( loc.*)?$}} +// LLVM-FAST-NO-CONTRACT: fneg reassoc nnan ninf nsz arcp afn float %{{.*}} diff --git a/clang/test/CIR/CodeGen/fp-contract.c b/clang/test/CIR/CodeGen/fp-contract.c index 746d6e991f691a..0ec886e686652b 100644 --- a/clang/test/CIR/CodeGen/fp-contract.c +++ b/clang/test/CIR/CodeGen/fp-contract.c @@ -51,6 +51,14 @@ // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -ffp-contract=fast -fexperimental-strict-floating-point -ffp-exception-behavior=strict -emit-llvm %s -o %t-strict-fast-ogcg.ll // RUN: FileCheck --input-file=%t-strict-fast-ogcg.ll %s -check-prefix=LLVM-STRICT-FAST +// The fused op carries the remaining fast-math flags. +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -ffp-contract=on -menable-no-infs -menable-no-nans -fno-signed-zeros -freciprocal-math -fapprox-func -emit-cir %s -o %t-fmf.cir +// RUN: FileCheck --input-file=%t-fmf.cir %s -check-prefix=CIR-FMF +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -ffp-contract=on -menable-no-infs -menable-no-nans -fno-signed-zeros -freciprocal-math -fapprox-func -emit-llvm %s -o %t-fmf.ll +// RUN: FileCheck --input-file=%t-fmf.ll %s -check-prefix=LLVM-FMF +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -ffp-contract=on -menable-no-infs -menable-no-nans -fno-signed-zeros -freciprocal-math -fapprox-func -emit-llvm %s -o %t-fmf-ogcg.ll +// RUN: FileCheck --input-file=%t-fmf-ogcg.ll %s -check-prefix=LLVM-FMF + // a * b + c => fmuladd(a, b, c) float fmuladd_add(float a, float b, float c) { return a * b + c; @@ -79,6 +87,12 @@ float fmuladd_add(float a, float b, float c) { // LLVM-FAST: fmul contract float // LLVM-FAST: fadd contract float +// CIR-FMF-LABEL: cir.func {{.*}}@fmuladd_add +// CIR-FMF: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn]> + +// LLVM-FMF-LABEL: @fmuladd_add +// LLVM-FMF: call nnan ninf nsz arcp afn float @llvm.fmuladd.f32 + // c + a * b => fmuladd(a, b, c) (mul on the RHS) float fmuladd_add_rhs(float a, float b, float c) { @@ -102,6 +116,14 @@ float fmuladd_sub(float a, float b, float c) { // LLVM-ON: %[[NEG:.*]] = fneg float // LLVM-ON: call float @llvm.fmuladd.f32(float %{{.*}}, float %{{.*}}, float %[[NEG]]) +// CIR-FMF-LABEL: cir.func {{.*}}@fmuladd_sub +// CIR-FMF: cir.fneg %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn]> +// CIR-FMF: cir.fmuladd %{{.*}}, %{{.*}}, %{{.*}} : !cir.float <fastmath_flags = [nnan, ninf, nsz, arcp, afn]> + +// LLVM-FMF-LABEL: @fmuladd_sub +// LLVM-FMF: fneg nnan ninf nsz arcp afn float +// LLVM-FMF: call nnan ninf nsz arcp afn float @llvm.fmuladd.f32 + // If the mul result is used elsewhere, it must NOT be fused. float no_fmuladd_reused_mul(float a, float b, float c, float *p) { float m = a * b; diff --git a/clang/utils/TableGen/CIRLoweringEmitter.cpp b/clang/utils/TableGen/CIRLoweringEmitter.cpp index 46a8dd8bd6ba0d..44fe1143938af5 100644 --- a/clang/utils/TableGen/CIRLoweringEmitter.cpp +++ b/clang/utils/TableGen/CIRLoweringEmitter.cpp @@ -223,6 +223,17 @@ void GenerateLLVMLoweringPattern(llvm::StringRef OpName, Code << " /*hasRoundingMode=*/" << (ConstrainedHasRoundingMode ? "true" : "false") << ");\n"; Code << " }\n"; + } else if (!LLVMOp.empty() && HasFastMathFlags) { + assert(!HasZeroResult && "fast-math op must have a result"); + Code + << " mlir::LogicalResult matchAndRewrite(cir::" << OpName + << " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) " + "const override {\n"; + Code << " return lowerFPOp<mlir::LLVM::" << LLVMOp + << ">(op, adaptor.getOperands(),\n"; + Code << " op.getFastmathFlagsAttr(), *getTypeConverter(), " + "rewriter);\n"; + Code << " }\n"; } else if (!LLVMOp.empty()) { // Generate the matchAndRewrite body automatically. Code _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
