[InstCombine] Bail out of casting calls when a conversion from/to byval is involved.

Fixes #58307

Reviewed By: nikic

Differential Revision: https://reviews.llvm.org/D135738

(cherry picked from commit 86e57e66da9380eaa90a9d0830d7f2c5fe87af99)

3 files changed, 36 insertions, 2 deletions

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index bc01d2ef7fe2..52596b30494f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3289,6 +3289,10 @@ bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {
     if (CallerPAL.hasParamAttr(i, Attribute::SwiftError))
       return false;
 
+    if (CallerPAL.hasParamAttr(i, Attribute::ByVal) !=
+        Callee->getAttributes().hasParamAttr(i, Attribute::ByVal))
+      return false; // Cannot transform to or from byval.
+
     // If the parameter is passed as a byval argument, then we have to have a
     // sized type and the sized type has to have the same size as the old type.
     if (ParamTy != ActTy && CallerPAL.hasParamAttr(i, Attribute::ByVal)) {
diff --git a/llvm/test/Transforms/InstCombine/byval.ll b/llvm/test/Transforms/InstCombine/byval.ll
index e62bbe21c806..45750869524b 100644
--- a/llvm/test/Transforms/InstCombine/byval.ll
+++ b/llvm/test/Transforms/InstCombine/byval.ll
@@ -7,8 +7,7 @@ declare void @add_byval_callee_2(double* byval(double))
 
 define void @add_byval(i64* %in) {
 ; CHECK-LABEL: @add_byval(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[IN:%.*]] to double*
-; CHECK-NEXT:    call void @add_byval_callee(double* byval(double) [[TMP1]])
+; CHECK-NEXT:    call void bitcast (void (double*)* @add_byval_callee to void (i64*)*)(i64* byval(i64) [[IN:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   %tmp = bitcast void (double*)* @add_byval_callee to void (i64*)*
diff --git a/llvm/test/Transforms/InstCombine/cast-byval.ll b/llvm/test/Transforms/InstCombine/cast-byval.ll
new file mode 100644
index 000000000000..b3e3055837c2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-byval.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Check that function calls involving conversion from/to byval aren't transformed.
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+%Foo = type { i64 }
+define i64 @foo (ptr byval(%Foo) %foo) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[FOO:%.*]], align 4
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = load i64, ptr %foo, align 4
+  ret i64 %1
+}
+
+define i64 @bar(i64 %0) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i64 @foo(i64 [[TMP0:%.*]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %2 = tail call i64 @foo(i64 %0)
+  ret i64 %2
+}
+
+define i64 @qux(ptr byval(%Foo) %qux) {
+; CHECK-LABEL: @qux(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @bar(ptr nonnull byval([[FOO:%.*]]) [[QUX:%.*]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = tail call i64 @bar(ptr byval(%Foo) %qux)
+  ret i64 %1
+}