[X86] Implement llvm.isnan(x86_fp80) as unordered comparison

x86_fp80 format allows values that do not fit any of IEEE-754 category.
Previously they were recognized by intrinsic __builtin_isnan as NaNs.
Now this intrinsic is implemented using instruction FXAM, which
distinguish between NaNs and unsupported values. It can make some
programs behave differently.

As a solution, this fix changes lowering of the intrinsic. If floating
point exceptions are ignored, llvm.isnan is lowered into unordered
comparison, as __buildtin_isnan was implemented earlier. In strictfp
functions the intrinsic is lowered using FXAM, which does not raise
exceptions even for signaling NaN, as required by IEEE-754 and C
standards.

Differential Revision: https://reviews.llvm.org/D108037

2 files changed, 18 insertions, 14 deletions

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 2d48e714c2fc..bda849db9f59 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -22617,6 +22617,11 @@ static SDValue lowerISNAN(SDValue Op, SelectionDAG &DAG) {
   MVT ArgVT = Arg.getSimpleValueType();
   MVT ResultVT = Op.getSimpleValueType();
 
+  // If exceptions are ignored, use unordered comparison for fp80. It recognizes
+  // unsupported values as NaNs.
+  if (ArgVT == MVT::f80 && Op->getFlags().hasNoFPExcept())
+    return DAG.getSetCC(DL, ResultVT, Arg, Arg, ISD::CondCode::SETUNE);
+
   // Determine classification of argument using instruction FXAM.
   unsigned Opc;
   switch (ArgVT.SimpleTy) {
@@ -22647,7 +22652,7 @@ static SDValue lowerISNAN(SDValue Op, SelectionDAG &DAG) {
                         DAG.getConstant(0x45, DL, MVT::i8));
 
   return DAG.getSetCC(DL, ResultVT, Extract, DAG.getConstant(1, DL, MVT::i8),
-                      ISD::CondCode::SETEQ);
+                      ISD::CondCode::SETLE);
 }
 
 /// Helper for creating a X86ISD::SETCC node.
diff --git a/llvm/test/CodeGen/X86/x86-fpclass.ll b/llvm/test/CodeGen/X86/x86-fpclass.ll
index 601ce1f09229..6e67bc202353 100644
--- a/llvm/test/CodeGen/X86/x86-fpclass.ll
+++ b/llvm/test/CodeGen/X86/x86-fpclass.ll
@@ -90,23 +90,22 @@ define i1 @isnan_ldouble(x86_fp80 %x) nounwind {
 ; CHECK-32-LABEL: isnan_ldouble:
 ; CHECK-32:       # %bb.0: # %entry
 ; CHECK-32-NEXT:    fldt {{[0-9]+}}(%esp)
-; CHECK-32-NEXT:    fxam
-; CHECK-32-NEXT:    fstp %st(0)
+; CHECK-32-NEXT:    fucomp %st(0)
 ; CHECK-32-NEXT:    fnstsw %ax
-; CHECK-32-NEXT:    andb $69, %ah
-; CHECK-32-NEXT:    cmpb $1, %ah
-; CHECK-32-NEXT:    sete %al
+; CHECK-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; CHECK-32-NEXT:    sahf
+; CHECK-32-NEXT:    setp %cl
+; CHECK-32-NEXT:    setne %al
+; CHECK-32-NEXT:    orb %cl, %al
 ; CHECK-32-NEXT:    retl
 ;
 ; CHECK-64-LABEL: isnan_ldouble:
 ; CHECK-64:       # %bb.0: # %entry
 ; CHECK-64-NEXT:    fldt {{[0-9]+}}(%rsp)
-; CHECK-64-NEXT:    fxam
-; CHECK-64-NEXT:    fstp %st(0)
-; CHECK-64-NEXT:    fnstsw %ax
-; CHECK-64-NEXT:    andb $69, %ah
-; CHECK-64-NEXT:    cmpb $1, %ah
-; CHECK-64-NEXT:    sete %al
+; CHECK-64-NEXT:    fucompi %st(0), %st
+; CHECK-64-NEXT:    setp %cl
+; CHECK-64-NEXT:    setne %al
+; CHECK-64-NEXT:    orb %cl, %al
 ; CHECK-64-NEXT:    retq
 entry:
   %0 = tail call i1 @llvm.isnan.f80(x86_fp80 %x)
@@ -212,7 +211,7 @@ define i1 @isnan_ldouble_strictfp(x86_fp80 %x) strictfp nounwind {
 ; CHECK-32-NEXT:    fnstsw %ax
 ; CHECK-32-NEXT:    andb $69, %ah
 ; CHECK-32-NEXT:    cmpb $1, %ah
-; CHECK-32-NEXT:    sete %al
+; CHECK-32-NEXT:    setle %al
 ; CHECK-32-NEXT:    retl
 ;
 ; CHECK-64-LABEL: isnan_ldouble_strictfp:
@@ -224,7 +223,7 @@ define i1 @isnan_ldouble_strictfp(x86_fp80 %x) strictfp nounwind {
 ; CHECK-64-NEXT:    fnstsw %ax
 ; CHECK-64-NEXT:    andb $69, %ah
 ; CHECK-64-NEXT:    cmpb $1, %ah
-; CHECK-64-NEXT:    sete %al
+; CHECK-64-NEXT:    setle %al
 ; CHECK-64-NEXT:    retq
 entry:
   %0 = tail call i1 @llvm.isnan.f80(x86_fp80 %x)