[Flang] Change complex divide lowering

Currently complex division is lowered to a fir.divc operation and the
fir.divc is later converted to a sequence of llvm operations to perform
complex division, however this causes issues for extreme values when
the calculations overflow.

This patch changes the lowering of complex division to use the Intrinsic
Call functionality to lower into library calls (for single, double,
extended and quad precisions) or an MLIR complex dialect division operation
(for half and bfloat precisions).

Note 1: If the Complex To Standard conversion of division operation
matures then we can use it for all precisions. Currently it has the
same issues as the conversion of fir.divc.
Note 2: A previous patch (D145808) did the same but during conversion of
the fir.divc operation. But using function calls at that stage leads to
ABI issues since the conversion to LLVM is not aware of the complex target
rewrite.
Note 3: If the patch is accepted, fir.divc can be removed from FIR.

Reviewed By: vzakhari, PeteSteinfeld, DavidTruby

Differential Revision: https://reviews.llvm.org/D149546

5 files changed, 118 insertions, 10 deletions

diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 2cc797f9b3f2..0d32976b6f60 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -98,6 +98,11 @@ mlir::Value genMax(fir::FirOpBuilder &, mlir::Location,
 mlir::Value genMin(fir::FirOpBuilder &, mlir::Location,
                    llvm::ArrayRef<mlir::Value> args);
 
+/// Generate Complex divide with the given expected
+/// result type.
+mlir::Value genDivC(fir::FirOpBuilder &builder, mlir::Location loc,
+                    mlir::Type resultType, mlir::Value x, mlir::Value y);
+
 /// Generate power function x**y with the given expected
 /// result type.
 mlir::Value genPow(fir::FirOpBuilder &, mlir::Location, mlir::Type resultType,
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 7f89fcf88f87..ba084d1efb27 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1075,7 +1075,16 @@ public:
   GENBIN(Multiply, Complex, fir::MulcOp)
   GENBIN(Divide, Integer, mlir::arith::DivSIOp)
   GENBIN(Divide, Real, mlir::arith::DivFOp)
-  GENBIN(Divide, Complex, fir::DivcOp)
+
+  template <int KIND>
+  ExtValue genval(const Fortran::evaluate::Divide<Fortran::evaluate::Type<
+                      Fortran::common::TypeCategory::Complex, KIND>> &op) {
+    mlir::Type ty =
+        converter.genType(Fortran::common::TypeCategory::Complex, KIND);
+    mlir::Value lhs = genunbox(op.left());
+    mlir::Value rhs = genunbox(op.right());
+    return fir::genDivC(builder, getLoc(), ty, lhs, rhs);
+  }
 
   template <Fortran::common::TypeCategory TC, int KIND>
   ExtValue genval(
@@ -5077,7 +5086,21 @@ private:
   GENBIN(Multiply, Complex, fir::MulcOp)
   GENBIN(Divide, Integer, mlir::arith::DivSIOp)
   GENBIN(Divide, Real, mlir::arith::DivFOp)
-  GENBIN(Divide, Complex, fir::DivcOp)
+
+  template <int KIND>
+  CC genarr(const Fortran::evaluate::Divide<Fortran::evaluate::Type<
+                Fortran::common::TypeCategory::Complex, KIND>> &x) {
+    mlir::Location loc = getLoc();
+    mlir::Type ty =
+        converter.genType(Fortran::common::TypeCategory::Complex, KIND);
+    auto lf = genarr(x.left());
+    auto rf = genarr(x.right());
+    return [=](IterSpace iters) -> ExtValue {
+      mlir::Value lhs = fir::getBase(lf(iters));
+      mlir::Value rhs = fir::getBase(rf(iters));
+      return fir::genDivC(builder, loc, ty, lhs, rhs);
+    };
+  }
 
   template <Fortran::common::TypeCategory TC, int KIND>
   CC genarr(
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d877be0cbc0a..ca3214d67dee 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -1345,6 +1345,18 @@ static constexpr MathOperation mathOperations[] = {
     {"cosh", "cosh", genF64F64FuncType, genLibCall},
     {"cosh", "ccoshf", genComplexComplexFuncType<4>, genLibCall},
     {"cosh", "ccosh", genComplexComplexFuncType<8>, genLibCall},
+    {"divc",
+     {},
+     genComplexComplexComplexFuncType<2>,
+     genComplexMathOp<mlir::complex::DivOp>},
+    {"divc",
+     {},
+     genComplexComplexComplexFuncType<3>,
+     genComplexMathOp<mlir::complex::DivOp>},
+    {"divc", "__divsc3", genComplexComplexComplexFuncType<4>, genLibCall},
+    {"divc", "__divdc3", genComplexComplexComplexFuncType<8>, genLibCall},
+    {"divc", "__divxc3", genComplexComplexComplexFuncType<10>, genLibCall},
+    {"divc", "__divtc3", genComplexComplexComplexFuncType<16>, genLibCall},
     {"erf", "erff", genF32F32FuncType, genMathOp<mlir::math::ErfOp>},
     {"erf", "erf", genF64F64FuncType, genMathOp<mlir::math::ErfOp>},
     {"erfc", "erfcf", genF32F32FuncType, genLibCall},
@@ -5661,6 +5673,11 @@ mlir::Value fir::genMin(fir::FirOpBuilder &builder, mlir::Location loc,
                                                               args);
 }
 
+mlir::Value fir::genDivC(fir::FirOpBuilder &builder, mlir::Location loc,
+                         mlir::Type type, mlir::Value x, mlir::Value y) {
+  return IntrinsicLibrary{builder, loc}.genRuntimeCall("divc", type, {x, y});
+}
+
 mlir::Value fir::genPow(fir::FirOpBuilder &builder, mlir::Location loc,
                         mlir::Type type, mlir::Value x, mlir::Value y) {
   // TODO: since there is no libm version of pow with integer exponent,
diff --git a/flang/test/Lower/assignment.f90 b/flang/test/Lower/assignment.f90
index 8e836ce30cdb..69c77fdcfb7c 100644
--- a/flang/test/Lower/assignment.f90
+++ b/flang/test/Lower/assignment.f90
@@ -251,7 +251,7 @@ end
 ! CHECK:         %[[FCTRES:.*]] = fir.alloca !fir.complex<4>
 ! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<!fir.complex<4>>
 ! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[DIV:.*]] = fir.divc %[[A_VAL]], %[[B_VAL]] : !fir.complex<4>
+! CHECK:         %[[DIV:.*]] = fir.call @__divsc3(%[[A_VAL]], %[[B_VAL]]) fastmath<contract> : (!fir.complex<4>, !fir.complex<4>) -> !fir.complex<4>
 ! CHECK:         fir.store %[[DIV]] to %[[FCTRES]] : !fir.ref<!fir.complex<4>>
 ! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<!fir.complex<4>>
 ! CHECK:         return %[[RET]] : !fir.complex<4>
diff --git a/flang/test/Lower/complex-operations.f90 b/flang/test/Lower/complex-operations.f90
index b11ce6ee7f21..ee4b6c1bcb21 100644
--- a/flang/test/Lower/complex-operations.f90
+++ b/flang/test/Lower/complex-operations.f90
@@ -27,11 +27,74 @@ subroutine mul_test(a,b,c)
   a = b * c
 end subroutine mul_test
 
-! CHECK-LABEL: @_QPdiv_test
-subroutine div_test(a,b,c)
-  complex :: a, b, c
-  ! CHECK-NOT: fir.extract_value
-  ! CHECK-NOT: fir.insert_value
-  ! CHECK: fir.divc {{.*}}: !fir.complex
+! CHECK-LABEL: @_QPdiv_test_half
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<2>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<2>>
+! CHECK: %[[BVAL_CVT:.*]] = fir.convert %[[BVAL]] : (!fir.complex<2>) -> complex<f16>
+! CHECK: %[[CVAL_CVT:.*]] = fir.convert %[[CVAL]] : (!fir.complex<2>) -> complex<f16>
+! CHECK: %[[AVAL_CVT:.*]] = complex.div %[[BVAL_CVT]], %[[CVAL_CVT]] : complex<f16>
+! CHECK: %[[AVAL:.*]] = fir.convert %[[AVAL_CVT]] : (complex<f16>) -> !fir.complex<2>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<2>>
+subroutine div_test_half(a,b,c)
+  complex(kind=2) :: a, b, c
+  a = b / c
+end subroutine div_test_half
+
+! CHECK-LABEL: @_QPdiv_test_bfloat
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<3>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<3>>
+! CHECK: %[[BVAL_CVT:.*]] = fir.convert %[[BVAL]] : (!fir.complex<3>) -> complex<bf16>
+! CHECK: %[[CVAL_CVT:.*]] = fir.convert %[[CVAL]] : (!fir.complex<3>) -> complex<bf16>
+! CHECK: %[[AVAL_CVT:.*]] = complex.div %[[BVAL_CVT]], %[[CVAL_CVT]] : complex<bf16>
+! CHECK: %[[AVAL:.*]] = fir.convert %[[AVAL_CVT]] : (complex<bf16>) -> !fir.complex<3>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<3>>
+subroutine div_test_bfloat(a,b,c)
+  complex(kind=3) :: a, b, c
+  a = b / c
+end subroutine div_test_bfloat
+
+! CHECK-LABEL: @_QPdiv_test_single
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<4>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<4>>
+! CHECK: %[[AVAL:.*]] = fir.call @__divsc3(%[[BVAL]], %[[CVAL]]) fastmath<contract> : (!fir.complex<4>, !fir.complex<4>) -> !fir.complex<4>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<4>>
+subroutine div_test_single(a,b,c)
+  complex(kind=4) :: a, b, c
+  a = b / c
+end subroutine div_test_single
+
+! CHECK-LABEL: @_QPdiv_test_double
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<8>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<8>>
+! CHECK: %[[AVAL:.*]] = fir.call @__divdc3(%[[BVAL]], %[[CVAL]]) fastmath<contract> : (!fir.complex<8>, !fir.complex<8>) -> !fir.complex<8>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<8>>
+subroutine div_test_double(a,b,c)
+  complex(kind=8) :: a, b, c
+  a = b / c
+end subroutine div_test_double
+
+! CHECK-LABEL: @_QPdiv_test_extended
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<10>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<10>>
+! CHECK: %[[AVAL:.*]] = fir.call @__divxc3(%[[BVAL]], %[[CVAL]]) fastmath<contract> : (!fir.complex<10>, !fir.complex<10>) -> !fir.complex<10>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<10>>
+subroutine div_test_extended(a,b,c)
+  complex(kind=10) :: a, b, c
+  a = b / c
+end subroutine div_test_extended
+
+! CHECK-LABEL: @_QPdiv_test_quad
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<16>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<16>>
+! CHECK: %[[AVAL:.*]] = fir.call @__divtc3(%[[BVAL]], %[[CVAL]]) fastmath<contract> : (!fir.complex<16>, !fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<16>>
+subroutine div_test_quad(a,b,c)
+  complex(kind=16) :: a, b, c
   a = b / c
-end subroutine div_test
+end subroutine div_test_quad