https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/177427
>From 0498d8771edfdea0ac8c4104b382875c0c79e127 Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Thu, 22 Jan 2026 19:10:00 +0100 Subject: [PATCH 1/2] DAG: Remove softPromoteHalfType Remove the now unimplemented target hook and associated DAG machinery for the old half legalization path. Really fixes #97975 --- llvm/include/llvm/CodeGen/TargetLowering.h | 29 +++++-------------- .../SelectionDAG/LegalizeFloatTypes.cpp | 20 ------------- .../SelectionDAG/LegalizeIntegerTypes.cpp | 11 ------- .../CodeGen/SelectionDAG/LegalizeTypes.cpp | 8 ----- .../SelectionDAG/LegalizeTypesGeneric.cpp | 1 - .../SelectionDAG/LegalizeVectorTypes.cpp | 2 -- llvm/lib/CodeGen/TargetLoweringBase.cpp | 9 ++---- 7 files changed, 9 insertions(+), 71 deletions(-) diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index c512dfa063131..df0fced112f73 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -218,15 +218,14 @@ class LLVM_ABI TargetLoweringBase { TypeScalarizeVector, // Replace this one-element vector with its element. TypeSplitVector, // Split this vector into two of half the size. TypeWidenVector, // This vector should be widened into a larger vector. - TypePromoteFloat, // Replace this float with a larger one. TypeSoftPromoteHalf, // Soften half to i16 and use float to do arithmetic. - TypeScalarizeScalableVector, // This action is explicitly left unimplemented. - // While it is theoretically possible to - // legalize operations on scalable types with a - // loop that handles the vscale * #lanes of the - // vector, this is non-trivial at SelectionDAG - // level and these types are better to be - // widened or promoted. + TypeScalarizeScalableVector, // This action is explicitly left + // unimplemented. While it is theoretically + // possible to legalize operations on scalable + // types with a loop that handles the vscale * + // #lanes of the vector, this is non-trivial at + // SelectionDAG level and these types are + // better to be widened or promoted. }; /// LegalizeKind holds the legalization kind that needs to happen to EVT @@ -546,20 +545,6 @@ class LLVM_ABI TargetLoweringBase { return TypePromoteInteger; } - /// Warning: this option is problem-prone and tends to introduce - /// float miscompilations: - /// - /// - https://github.com/llvm/llvm-project/issues/97975 - /// - https://github.com/llvm/llvm-project/issues/97981 - /// - /// It should not be overridden to `false` except for special cases. - /// - /// Return true if the half type should be promoted using soft promotion rules - /// where each operation is promoted to f32 individually, then converted to - /// fp16. The default behavior is to promote chains of operations, keeping - /// intermediate results in f32 precision and range. - virtual bool softPromoteHalfType() const { return true; } - // Return true if, for soft-promoted half, the half type should be passed to // and returned from functions as f32. The default behavior is to pass as // i16. If soft-promoted half is not used, this function is ignored and diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 1e7bc757d2c58..ce952d50c287c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -634,17 +634,6 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); - if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { - Op = GetPromotedFloat(Op); - // If the promotion did the FP_EXTEND to the destination type for us, - // there's nothing left to do here. - if (Op.getValueType() == N->getValueType(0)) { - if (IsStrict) - ReplaceValueWith(SDValue(N, 1), Chain); - return BitConvertToInteger(Op); - } - } - // There's only a libcall for f16 -> f32 and shifting is only valid for bf16 // -> f32, so proceed in two stages. Also, it's entirely possible for both // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather @@ -3298,8 +3287,6 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { } SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { - EVT VT = N->getValueType(0); - AtomicSDNode *AM = cast<AtomicSDNode>(N); SDLoc SL(N); @@ -3314,18 +3301,11 @@ SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { SDValue Result = NewAtomic; - if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { - EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, - NewAtomic); - } - // Legalize the chain result by replacing uses of the old value chain with the // new one ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); return Result; - } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index faa6a1d0d14a3..8ce41df6be69b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -534,12 +534,6 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { case TargetLowering::TypeSoftPromoteHalf: // Promote the integer operand by hand. return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); - case TargetLowering::TypePromoteFloat: { - // Convert the promoted float by hand. - if (!NOutVT.isVector()) - return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); - break; - } case TargetLowering::TypeExpandInteger: case TargetLowering::TypeExpandFloat: break; @@ -4250,8 +4244,6 @@ void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo, bool IsStrict = N->isStrictFPOpcode(); SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); SDValue Op = N->getOperand(IsStrict ? 1 : 0); - if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) - Op = GetPromotedFloat(Op); // If the input is bf16 or needs to be soft promoted, extend to f32. if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf || @@ -4292,9 +4284,6 @@ void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo, SDValue Op = N->getOperand(IsStrict ? 1 : 0); SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); - assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && - "Input type needs to be promoted!"); - EVT VT = Op.getValueType(); if (VT == MVT::f16) { diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index f14eeda639e71..9634e5afdc738 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -288,10 +288,6 @@ bool DAGTypeLegalizer::run() { WidenVectorResult(N, i); Changed = true; goto NodeDone; - case TargetLowering::TypePromoteFloat: - PromoteFloatResult(N, i); - Changed = true; - goto NodeDone; case TargetLowering::TypeSoftPromoteHalf: SoftPromoteHalfResult(N, i); Changed = true; @@ -351,10 +347,6 @@ bool DAGTypeLegalizer::run() { NeedsReanalyzing = WidenVectorOperand(N, i); Changed = true; break; - case TargetLowering::TypePromoteFloat: - NeedsReanalyzing = PromoteFloatOperand(N, i); - Changed = true; - break; case TargetLowering::TypeSoftPromoteHalf: NeedsReanalyzing = SoftPromoteHalfOperand(N, i); Changed = true; diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index 88c1af20a321e..6ded0bf0a92c0 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -49,7 +49,6 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { case TargetLowering::TypeLegal: case TargetLowering::TypePromoteInteger: break; - case TargetLowering::TypePromoteFloat: case TargetLowering::TypeSoftPromoteHalf: llvm_unreachable("Bitcast of a promotion-needing float should never need" "expansion"); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index e1e6252fd8abc..75a0cc8f03739 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1660,7 +1660,6 @@ void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, switch (getTypeAction(InVT)) { case TargetLowering::TypeLegal: case TargetLowering::TypePromoteInteger: - case TargetLowering::TypePromoteFloat: case TargetLowering::TypeSoftPromoteHalf: case TargetLowering::TypeSoftenFloat: case TargetLowering::TypeScalarizeVector: @@ -6023,7 +6022,6 @@ SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { break; } case TargetLowering::TypeSoftenFloat: - case TargetLowering::TypePromoteFloat: case TargetLowering::TypeSoftPromoteHalf: case TargetLowering::TypeExpandInteger: case TargetLowering::TypeExpandFloat: diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index 922b3fa40518d..c6707162d9b9e 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -1749,8 +1749,7 @@ void TargetLoweringBase::computeRegisterProperties( // conversions). if (!isTypeLegal(MVT::f16)) { // Allow targets to control how we legalize half. - bool SoftPromoteHalfType = softPromoteHalfType(); - bool UseFPRegsForHalfType = !SoftPromoteHalfType || useFPRegsForHalfType(); + bool UseFPRegsForHalfType = useFPRegsForHalfType(); if (!UseFPRegsForHalfType) { NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16]; @@ -1760,11 +1759,7 @@ void TargetLoweringBase::computeRegisterProperties( RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32]; } TransformToType[MVT::f16] = MVT::f32; - if (SoftPromoteHalfType) { - ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf); - } else { - ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat); - } + ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf); } // Decide how to handle bf16. If the target does not have native bf16 support, >From 10ef13559eb14a7310ffeec081b0b930520d8cc4 Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Fri, 23 Jan 2026 12:11:44 +0100 Subject: [PATCH 2/2] Delete the implementation functions --- .../SelectionDAG/LegalizeFloatTypes.cpp | 655 ------------------ llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 37 - 2 files changed, 692 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index ce952d50c287c..16453f220bb50 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -2631,661 +2631,6 @@ static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) { report_fatal_error("Attempt at an invalid promotion-related conversion"); } -bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { - LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG)); - SDValue R = SDValue(); - - if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { - LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); - return false; - } - - // Nodes that use a promotion-requiring floating point operand, but doesn't - // produce a promotion-requiring floating point result, need to be legalized - // to use the promoted float operand. Nodes that produce at least one - // promotion-requiring floating point result have their operands legalized as - // a part of PromoteFloatResult. - // clang-format off - switch (N->getOpcode()) { - default: - #ifndef NDEBUG - dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; - N->dump(&DAG); dbgs() << "\n"; - #endif - report_fatal_error("Do not know how to promote this operator's operand!"); - - case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; - case ISD::FAKE_USE: - R = PromoteFloatOp_FAKE_USE(N, OpNo); - break; - case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; - case ISD::FP_TO_SINT: - case ISD::FP_TO_UINT: - case ISD::LROUND: - case ISD::LLROUND: - case ISD::LRINT: - case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break; - case ISD::AssertNoFPClass: R = PromoteFloatOp_AssertNoFPClass(N, OpNo); break; - case ISD::FP_TO_SINT_SAT: - case ISD::FP_TO_UINT_SAT: - R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; - case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; - case ISD::STRICT_FP_EXTEND: - R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo); - break; - case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; - case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; - case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; - case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break; - } - // clang-format on - - if (R.getNode()) - ReplaceValueWith(SDValue(N, 0), R); - return false; -} - -SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { - SDValue Op = N->getOperand(0); - EVT OpVT = Op->getValueType(0); - - SDValue Promoted = GetPromotedFloat(N->getOperand(0)); - EVT PromotedVT = Promoted->getValueType(0); - - // Convert the promoted float value to the desired IVT. - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); - SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), - IVT, Promoted); - // The final result type might not be an scalar so we need a bitcast. The - // bitcast will be further legalized if needed. - return DAG.getBitcast(N->getValueType(0), Convert); -} - -SDValue DAGTypeLegalizer::PromoteFloatOp_FAKE_USE(SDNode *N, unsigned OpNo) { - assert(OpNo == 1 && "Only Operand 1 must need promotion here"); - SDValue Op = GetPromotedFloat(N->getOperand(OpNo)); - return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::Other, N->getOperand(0), - Op); -} - -// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by -// PromoteFloatRes_FCOPYSIGN. -SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { - assert (OpNo == 1 && "Only Operand 1 must need promotion here"); - SDValue Op1 = GetPromotedFloat(N->getOperand(1)); - - return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), - N->getOperand(0), Op1); -} - -// Convert the promoted float value to the desired integer type -SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) { - SDValue Op = GetPromotedFloat(N->getOperand(0)); - return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); -} - -// Convert the promoted float value to the desired integer type -SDValue DAGTypeLegalizer::PromoteFloatOp_AssertNoFPClass(SDNode *N, - unsigned OpNo) { - return GetPromotedFloat(N->getOperand(0)); -} - -SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, - unsigned OpNo) { - SDValue Op = GetPromotedFloat(N->getOperand(0)); - return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, - N->getOperand(1)); -} - -SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { - SDValue Op = GetPromotedFloat(N->getOperand(0)); - EVT VT = N->getValueType(0); - - // Desired VT is same as promoted type. Use promoted float directly. - if (VT == Op->getValueType(0)) - return Op; - - // Else, extend the promoted float value to the desired VT. - return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); -} - -SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, - unsigned OpNo) { - assert(OpNo == 1 && "Promoting unpromotable operand"); - - SDValue Op = GetPromotedFloat(N->getOperand(1)); - EVT VT = N->getValueType(0); - - // Desired VT is same as promoted type. Use promoted float directly. - if (VT == Op->getValueType(0)) { - ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); - return Op; - } - - // Else, extend the promoted float value to the desired VT. - SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(), - N->getOperand(0), Op); - ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); - return Res; -} - -// Promote the float operands used for comparison. The true- and false- -// operands have the same type as the result and are promoted, if needed, by -// PromoteFloatRes_SELECT_CC -SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { - SDValue LHS = GetPromotedFloat(N->getOperand(0)); - SDValue RHS = GetPromotedFloat(N->getOperand(1)); - - return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), - LHS, RHS, N->getOperand(2), N->getOperand(3), - N->getOperand(4)); -} - -// Construct a SETCC that compares the promoted values and sets the conditional -// code. -SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { - EVT VT = N->getValueType(0); - SDValue Op0 = GetPromotedFloat(N->getOperand(0)); - SDValue Op1 = GetPromotedFloat(N->getOperand(1)); - ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); - - return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); - -} - -// Lower the promoted Float down to the integer value of same size and construct -// a STORE of the integer value. -SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { - StoreSDNode *ST = cast<StoreSDNode>(N); - SDValue Val = ST->getValue(); - SDLoc DL(N); - - SDValue Promoted = GetPromotedFloat(Val); - EVT VT = ST->getOperand(1).getValueType(); - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); - - SDValue NewVal; - NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, - IVT, Promoted); - - return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), - ST->getMemOperand()); -} - -SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N, - unsigned OpNo) { - AtomicSDNode *ST = cast<AtomicSDNode>(N); - SDValue Val = ST->getVal(); - SDLoc DL(N); - - SDValue Promoted = GetPromotedFloat(Val); - EVT VT = ST->getOperand(1).getValueType(); - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); - - SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), - DL, IVT, Promoted); - - return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal, - ST->getBasePtr(), ST->getMemOperand()); -} - -//===----------------------------------------------------------------------===// -// Float Result Promotion -//===----------------------------------------------------------------------===// - -void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { - LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG)); - SDValue R = SDValue(); - - // See if the target wants to custom expand this node. - if (CustomLowerNode(N, N->getValueType(ResNo), true)) { - LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); - return; - } - - switch (N->getOpcode()) { - // These opcodes cannot appear if promotion of FP16 is done in the backend - // instead of Clang - case ISD::FP16_TO_FP: - case ISD::FP_TO_FP16: - default: -#ifndef NDEBUG - dbgs() << "PromoteFloatResult #" << ResNo << ": "; - N->dump(&DAG); dbgs() << "\n"; -#endif - report_fatal_error("Do not know how to promote this operator's result!"); - - case ISD::BITCAST: - R = PromoteFloatRes_BITCAST(N); - break; - case ISD::FREEZE: - R = PromoteFloatRes_FREEZE(N); - break; - case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; - case ISD::EXTRACT_VECTOR_ELT: - R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; - case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; - - // Unary FP Operations - case ISD::FABS: - case ISD::FACOS: - case ISD::FASIN: - case ISD::FATAN: - case ISD::FCBRT: - case ISD::FCEIL: - case ISD::FCOS: - case ISD::FCOSH: - case ISD::FEXP: - case ISD::FEXP2: - case ISD::FEXP10: - case ISD::FFLOOR: - case ISD::FLOG: - case ISD::FLOG2: - case ISD::FLOG10: - case ISD::FNEARBYINT: - case ISD::FNEG: - case ISD::FRINT: - case ISD::FROUND: - case ISD::FROUNDEVEN: - case ISD::FSIN: - case ISD::FSINH: - case ISD::FSQRT: - case ISD::FTRUNC: - case ISD::FTAN: - case ISD::FTANH: - case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; - case ISD::AssertNoFPClass: - R = PromoteFloatRes_AssertNoFPClass(N); - break; - - // Binary FP Operations - case ISD::FADD: - case ISD::FDIV: - case ISD::FMAXIMUM: - case ISD::FMINIMUM: - case ISD::FMAXIMUMNUM: - case ISD::FMINIMUMNUM: - case ISD::FMAXNUM: - case ISD::FMINNUM: - case ISD::FMAXNUM_IEEE: - case ISD::FMINNUM_IEEE: - case ISD::FMUL: - case ISD::FPOW: - case ISD::FATAN2: - case ISD::FREM: - case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; - - case ISD::FMA: // FMA is same as FMAD - case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; - - case ISD::FPOWI: - case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break; - case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break; - - case ISD::FMODF: - case ISD::FSINCOS: - case ISD::FSINCOSPI: - R = PromoteFloatRes_UnaryWithTwoFPResults(N); - break; - case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; - case ISD::STRICT_FP_ROUND: - R = PromoteFloatRes_STRICT_FP_ROUND(N); - break; - case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; - case ISD::ATOMIC_LOAD: - R = PromoteFloatRes_ATOMIC_LOAD(N); - break; - case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; - case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; - - case ISD::SINT_TO_FP: - case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; - case ISD::POISON: - case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; - case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; - case ISD::VECREDUCE_FADD: - case ISD::VECREDUCE_FMUL: - case ISD::VECREDUCE_FMIN: - case ISD::VECREDUCE_FMAX: - case ISD::VECREDUCE_FMAXIMUM: - case ISD::VECREDUCE_FMINIMUM: - R = PromoteFloatRes_VECREDUCE(N); - break; - case ISD::VECREDUCE_SEQ_FADD: - case ISD::VECREDUCE_SEQ_FMUL: - R = PromoteFloatRes_VECREDUCE_SEQ(N); - break; - } - - if (R.getNode()) - SetPromotedFloat(SDValue(N, ResNo), R); -} - -// Bitcast from i16 to f16: convert the i16 to a f32 value instead. -// At this point, it is not possible to determine if the bitcast value is -// eventually stored to memory or promoted to f32 or promoted to a floating -// point at a higher precision. Some of these cases are handled by FP_EXTEND, -// STORE promotion handlers. -SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - // Input type isn't guaranteed to be a scalar int so bitcast if not. The - // bitcast will be legalized further if necessary. - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), - N->getOperand(0).getValueType().getSizeInBits()); - SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); - return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_FREEZE(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - // Input type isn't guaranteed to be a scalar int so bitcast if not. The - // bitcast will be legalized further if necessary. - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), - N->getOperand(0).getValueType().getSizeInBits()); - SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); - return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, - DAG.getFreeze(Cast)); -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { - ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); - EVT VT = N->getValueType(0); - SDLoc DL(N); - - // Get the (bit-cast) APInt of the APFloat and build an integer constant - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); - SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, - IVT); - - // Convert the Constant to the desired FP type - // FIXME We might be able to do the conversion during compilation and get rid - // of it from the object code - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); -} - -// If the Index operand is a constant, try to redirect the extract operation to -// the correct legalized vector. If not, bit-convert the input vector to -// equivalent integer vector. Extract the element as an (bit-cast) integer -// value and convert it to the promoted type. -SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { - SDLoc DL(N); - - // If the index is constant, try to extract the value from the legalized - // vector type. - if (isa<ConstantSDNode>(N->getOperand(1))) { - SDValue Vec = N->getOperand(0); - SDValue Idx = N->getOperand(1); - EVT VecVT = Vec->getValueType(0); - EVT EltVT = VecVT.getVectorElementType(); - - uint64_t IdxVal = Idx->getAsZExtVal(); - - switch (getTypeAction(VecVT)) { - default: break; - case TargetLowering::TypeScalarizeVector: { - SDValue Res = GetScalarizedVector(N->getOperand(0)); - ReplaceValueWith(SDValue(N, 0), Res); - return SDValue(); - } - case TargetLowering::TypeWidenVector: { - Vec = GetWidenedVector(Vec); - SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); - ReplaceValueWith(SDValue(N, 0), Res); - return SDValue(); - } - case TargetLowering::TypeSplitVector: { - SDValue Lo, Hi; - GetSplitVector(Vec, Lo, Hi); - - uint64_t LoElts = Lo.getValueType().getVectorNumElements(); - SDValue Res; - if (IdxVal < LoElts) - Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); - else - Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, - DAG.getConstant(IdxVal - LoElts, DL, - Idx.getValueType())); - ReplaceValueWith(SDValue(N, 0), Res); - return SDValue(); - } - - } - } - - // Bit-convert the input vector to the equivalent integer vector - SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); - EVT IVT = NewOp.getValueType().getVectorElementType(); - - // Extract the element as an (bit-cast) integer value - SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, - NewOp, N->getOperand(1)); - - // Convert the element to the desired FP type - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); -} - -// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result -// needs promotion, so does the argument X. Note that Y, if needed, will be -// handled during operand promotion. -SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - SDValue Op0 = GetPromotedFloat(N->getOperand(0)); - - SDValue Op1 = N->getOperand(1); - - return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); -} - -// Unary operation where the result and the operand have PromoteFloat type -// action. Construct a new SDNode with the promoted float value of the old -// operand. -SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - SDValue Op = GetPromotedFloat(N->getOperand(0)); - return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); -} - -// Unary operation with a more non-float operand where the result and the -// operand have PromoteFloat type action. Construct a new SDNode with the -// promoted float value of the old operand. -SDValue DAGTypeLegalizer::PromoteFloatRes_AssertNoFPClass(SDNode *N) { - return GetPromotedFloat(N->getOperand(0)); -} - -// Binary operations where the result and both operands have PromoteFloat type -// action. Construct a new SDNode with the promoted float values of the old -// operands. -SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - SDValue Op0 = GetPromotedFloat(N->getOperand(0)); - SDValue Op1 = GetPromotedFloat(N->getOperand(1)); - return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - SDValue Op0 = GetPromotedFloat(N->getOperand(0)); - SDValue Op1 = GetPromotedFloat(N->getOperand(1)); - SDValue Op2 = GetPromotedFloat(N->getOperand(2)); - - return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); -} - -// Promote the Float (first) operand and retain the Integer (second) operand -SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - SDValue Op0 = GetPromotedFloat(N->getOperand(0)); - SDValue Op1 = N->getOperand(1); - - return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - SDValue Op = GetPromotedFloat(N->getOperand(0)); - SDValue Res = - DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op); - - ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); - return Res; -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryWithTwoFPResults(SDNode *N) { - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - SDValue Op = GetPromotedFloat(N->getOperand(0)); - SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, NVT}, Op); - - for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues; - ++ResNum) { - SetPromotedFloat(SDValue(N, ResNum), Res.getValue(ResNum)); - } - - return SDValue(); -} - -// Explicit operation to reduce precision. Reduce the value to half precision -// and promote it back to the legal type. -SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { - SDLoc DL(N); - - SDValue Op = N->getOperand(0); - EVT VT = N->getValueType(0); - EVT OpVT = Op->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); - - // Round promoted float to desired precision - SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); - // Promote it back to the legal output type - return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); -} - -// Explicit operation to reduce precision. Reduce the value to half precision -// and promote it back to the legal type. -SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) { - SDLoc DL(N); - - SDValue Chain = N->getOperand(0); - SDValue Op = N->getOperand(1); - EVT VT = N->getValueType(0); - EVT OpVT = Op->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); - - // Round promoted float to desired precision - SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL, - DAG.getVTList(IVT, MVT::Other), Chain, Op); - // Promote it back to the legal output type - SDValue Res = - DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL, - DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round); - ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); - return Res; -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { - LoadSDNode *L = cast<LoadSDNode>(N); - EVT VT = N->getValueType(0); - - // Load the value as an integer value with the same number of bits. - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); - SDValue newL = DAG.getLoad( - L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), - L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, - L->getBaseAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); - // Legalize the chain result by replacing uses of the old value chain with the - // new one - ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); - - // Convert the integer value to the desired FP type - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) { - AtomicSDNode *AM = cast<AtomicSDNode>(N); - EVT VT = AM->getValueType(0); - - // Load the value as an integer value with the same number of bits. - EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); - SDValue newL = DAG.getAtomic( - ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other), - {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand()); - - // Legalize the chain result by replacing uses of the old value chain with the - // new one - ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); - - // Convert the integer value to the desired FP type - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL); -} - -// Construct a new SELECT node with the promoted true- and false- values. -SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { - SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); - SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); - - return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), - N->getOperand(0), TrueVal, FalseVal); -} - -// Construct a new SELECT_CC node with the promoted true- and false- values. -// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. -SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { - SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); - SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); - - return DAG.getNode(ISD::SELECT_CC, SDLoc(N), - TrueVal.getNode()->getValueType(0), N->getOperand(0), - N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); -} - -// Construct a SDNode that transforms the SINT or UINT operand to the promoted -// float type. -SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { - SDLoc DL(N); - EVT VT = N->getValueType(0); - EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); - // Round the value to the desired precision (that of the source type). - return DAG.getNode( - ISD::FP_EXTEND, DL, NVT, - DAG.getNode(ISD::FP_ROUND, DL, VT, NV, - DAG.getIntPtrConstant(0, DL, /*isTarget=*/true))); -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { - return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), - N->getValueType(0))); -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { - // Expand and promote recursively. - // TODO: This is non-optimal, but dealing with the concurrently happening - // vector-legalization is non-trivial. We could do something similar to - // PromoteFloatRes_EXTRACT_VECTOR_ELT here. - ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); - return SDValue(); -} - -SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { - ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); - return SDValue(); -} - SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { AtomicSDNode *AM = cast<AtomicSDNode>(N); SDLoc SL(N); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index a5efffa2b441c..a39e419e5ad1c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -770,44 +770,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { } void SetPromotedFloat(SDValue Op, SDValue Result); - void PromoteFloatResult(SDNode *N, unsigned ResNo); - SDValue PromoteFloatRes_BITCAST(SDNode *N); - SDValue PromoteFloatRes_FREEZE(SDNode *N); - SDValue PromoteFloatRes_BinOp(SDNode *N); - SDValue PromoteFloatRes_UnaryWithTwoFPResults(SDNode *N); - SDValue PromoteFloatRes_ConstantFP(SDNode *N); - SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N); - SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N); - SDValue PromoteFloatRes_FMAD(SDNode *N); - SDValue PromoteFloatRes_ExpOp(SDNode *N); - SDValue PromoteFloatRes_FFREXP(SDNode *N); - SDValue PromoteFloatRes_FP_ROUND(SDNode *N); - SDValue PromoteFloatRes_STRICT_FP_ROUND(SDNode *N); - SDValue PromoteFloatRes_LOAD(SDNode *N); - SDValue PromoteFloatRes_ATOMIC_LOAD(SDNode *N); - SDValue PromoteFloatRes_SELECT(SDNode *N); - SDValue PromoteFloatRes_SELECT_CC(SDNode *N); - SDValue PromoteFloatRes_UnaryOp(SDNode *N); - SDValue PromoteFloatRes_AssertNoFPClass(SDNode *N); - SDValue PromoteFloatRes_UNDEF(SDNode *N); SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N); - SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N); - SDValue PromoteFloatRes_VECREDUCE(SDNode *N); - SDValue PromoteFloatRes_VECREDUCE_SEQ(SDNode *N); - - bool PromoteFloatOperand(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_FAKE_USE(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_AssertNoFPClass(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo); - SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo); //===--------------------------------------------------------------------===// // Half soft promotion support: LegalizeFloatTypes.cpp _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
