Fix type variable substitution in gen_Newtype_fam_insts

Previously, `gen_Newtype_fam_insts` was substituting the type variable binders
of a type family instance using `substTyVars`, which failed to take type
variable dependencies into account. There is similar code in
`GHC.Tc.TyCl.Class.tcATDefault` that _does_ perform this substitution properly,
so this patch:

1. Factors out this code into a top-level `substATBndrs` function, and
2. Uses `substATBndrs` in `gen_Newtype_fam_insts`.

Fixes #23329.

3 files changed, 27 insertions, 0 deletions

diff --git a/testsuite/tests/deriving/should_compile/T23329.hs b/testsuite/tests/deriving/should_compile/T23329.hs
new file mode 100644
index 0000000000..7b4cd922f8
--- /dev/null
+++ b/testsuite/tests/deriving/should_compile/T23329.hs
@@ -0,0 +1,9 @@
+module T23329 where
+
+import Data.Kind (Type)
+import Data.Proxy (Proxy(Proxy))
+
+import T23329_M
+
+foo :: ()
+foo = myMethod @Type @MyMaybe @() () Proxy Proxy
diff --git a/testsuite/tests/deriving/should_compile/T23329_M.hs b/testsuite/tests/deriving/should_compile/T23329_M.hs
new file mode 100644
index 0000000000..a451a2b828
--- /dev/null
+++ b/testsuite/tests/deriving/should_compile/T23329_M.hs
@@ -0,0 +1,17 @@
+{-# LANGUAGE TypeFamilies #-}
+{-# LANGUAGE UndecidableInstances #-}
+module T23329_M where
+
+import Data.Kind (Type)
+import Data.Proxy (Proxy)
+
+class MyClass (f :: k -> Type) where
+  type MyTypeFamily f (i :: k) :: Type
+  myMethod :: MyTypeFamily f i -> Proxy f -> Proxy i -> ()
+
+instance MyClass Maybe where
+  type MyTypeFamily Maybe i = ()
+  myMethod = undefined
+
+newtype MyMaybe a = MyMaybe (Maybe a)
+  deriving MyClass
diff --git a/testsuite/tests/deriving/should_compile/all.T b/testsuite/tests/deriving/should_compile/all.T
index dd1fe9cd3d..729371b658 100644
--- a/testsuite/tests/deriving/should_compile/all.T
+++ b/testsuite/tests/deriving/should_compile/all.T
@@ -141,3 +141,4 @@ test('T20994', normal, compile, [''])
 test('T22167', normal, compile, [''])
 test('T22696a', normal, compile, [''])
 test('T22696c', normal, compile, [''])
+test('T23329', normal, multimod_compile, ['T23329', '-v0'])