Reject nested foralls/contexts in instance types more consistently

GHC is very wishy-washy about rejecting instance declarations with
nested `forall`s or contexts that are surrounded by outermost
parentheses. This can even lead to some strange interactions with
`ScopedTypeVariables`, as demonstrated in #18240. This patch makes
GHC more consistently reject instance types with nested
`forall`s/contexts so as to prevent these strange interactions.

On the implementation side, this patch tweaks `splitLHsInstDeclTy`
and `getLHsInstDeclHead` to not look through parentheses, which can
be semantically significant. I've added a
`Note [No nested foralls or contexts in instance types]` in
`GHC.Hs.Type` to explain why. This also introduces a
`no_nested_foralls_contexts_err` function in `GHC.Rename.HsType` to
catch nested `forall`s/contexts in instance types. This function is
now used in `rnClsInstDecl` (for ordinary instance declarations) and
`rnSrcDerivDecl` (for standalone `deriving` declarations), the latter
of which fixes #18271.

On the documentation side, this adds a new
"Formal syntax for instance declaration types" section to the GHC
User's Guide that presents a BNF-style grammar for what is and isn't
allowed in instance types.

Fixes #18240. Fixes #18271.

17 files changed, 169 insertions, 37 deletions

diff --git a/testsuite/tests/dependent/should_fail/T16326_Fail8.stderr b/testsuite/tests/dependent/should_fail/T16326_Fail8.stderr
index 16c2aa617e..d7666cf84d 100644
--- a/testsuite/tests/dependent/should_fail/T16326_Fail8.stderr
+++ b/testsuite/tests/dependent/should_fail/T16326_Fail8.stderr
@@ -1,6 +1,5 @@
 
 T16326_Fail8.hs:7:10: error:
-    Illegal class instance: ‘forall a -> C (Blah a)’
-      Class instances must be of the form
-        context => C ty_1 ... ty_n
-      where ‘C’ is a class
+    Illegal visible, dependent quantification in the type of a term
+    (GHC does not yet support this)
+    In an instance declaration
diff --git a/testsuite/tests/dependent/should_fail/T18271.hs b/testsuite/tests/dependent/should_fail/T18271.hs
new file mode 100644
index 0000000000..2441fa7318
--- /dev/null
+++ b/testsuite/tests/dependent/should_fail/T18271.hs
@@ -0,0 +1,7 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE StandaloneDeriving #-}
+module T18271 where
+
+class C a
+deriving instance forall a -> C (Maybe a)
diff --git a/testsuite/tests/dependent/should_fail/T18271.stderr b/testsuite/tests/dependent/should_fail/T18271.stderr
new file mode 100644
index 0000000000..0bc21f394d
--- /dev/null
+++ b/testsuite/tests/dependent/should_fail/T18271.stderr
@@ -0,0 +1,5 @@
+
+T18271.hs:7:19: error:
+    Illegal visible, dependent quantification in the type of a term
+    (GHC does not yet support this)
+    In a deriving declaration
diff --git a/testsuite/tests/dependent/should_fail/all.T b/testsuite/tests/dependent/should_fail/all.T
index d3d155f163..38c7f45d55 100644
--- a/testsuite/tests/dependent/should_fail/all.T
+++ b/testsuite/tests/dependent/should_fail/all.T
@@ -65,3 +65,4 @@ test('T14880-2', normal, compile_fail, [''])
 test('T15076', normal, compile_fail, [''])
 test('T15076b', normal, compile_fail, [''])
 test('T17687', normal, compile_fail, [''])
+test('T18271', normal, compile_fail, [''])
diff --git a/testsuite/tests/deriving/should_compile/T15831.hs b/testsuite/tests/deriving/should_compile/T15831.hs
index 309c8a8e3a..da157aa106 100644
--- a/testsuite/tests/deriving/should_compile/T15831.hs
+++ b/testsuite/tests/deriving/should_compile/T15831.hs
@@ -13,21 +13,21 @@ newtype Age = MkAge Int
   deriving Ord
     via Const Int (Any :: k)
   deriving Read
-    via (forall k. Const Int (Any :: k))
+    via forall k. Const Int (Any :: k)
   deriving Show
     via Const Int a
   deriving Enum
     via Const Int (a :: k)
   deriving Bounded
-    via (forall a. Const Int a)
+    via forall a. Const Int a
   deriving Num
-    via (forall k (a :: k). Const Int a)
+    via forall k (a :: k). Const Int a
 
 newtype Age2 = MkAge2 Int
-deriving via                     Const Int Any         instance Eq      Age2
-deriving via                     Const Int (Any :: k)  instance Ord     Age2
-deriving via (forall k.          Const Int (Any :: k)) instance Read    Age2
-deriving via                     Const Int a           instance Show    Age2
-deriving via                     Const Int (a :: k)    instance Enum    Age2
-deriving via (forall a.          Const Int a)          instance Bounded Age2
-deriving via (forall k (a :: k). Const Int a)          instance Num     Age2
+deriving via                    Const Int Any        instance Eq      Age2
+deriving via                    Const Int (Any :: k) instance Ord     Age2
+deriving via forall k.          Const Int (Any :: k) instance Read    Age2
+deriving via                    Const Int a          instance Show    Age2
+deriving via                    Const Int (a :: k)   instance Enum    Age2
+deriving via forall a.          Const Int a          instance Bounded Age2
+deriving via forall k (a :: k). Const Int a          instance Num     Age2
diff --git a/testsuite/tests/deriving/should_compile/deriving-via-standalone.hs b/testsuite/tests/deriving/should_compile/deriving-via-standalone.hs
index 26484a2df2..99da015bbd 100644
--- a/testsuite/tests/deriving/should_compile/deriving-via-standalone.hs
+++ b/testsuite/tests/deriving/should_compile/deriving-via-standalone.hs
@@ -37,6 +37,6 @@ data X1 a
 data X2 a
 data X3 a
 
-deriving via (forall a. T a) instance           Z a (X1 b)
-deriving via           (T a) instance forall b. Z a (X2 b)
-deriving via (forall a. T a) instance forall b. Z a (X3 b)
+deriving via forall a. T a instance           Z a (X1 b)
+deriving via           T a instance forall b. Z a (X2 b)
+deriving via forall a. T a instance forall b. Z a (X3 b)
diff --git a/testsuite/tests/deriving/should_fail/deriving-via-fail.hs b/testsuite/tests/deriving/should_fail/deriving-via-fail.hs
index 3fa8009638..0cb521de78 100644
--- a/testsuite/tests/deriving/should_fail/deriving-via-fail.hs
+++ b/testsuite/tests/deriving/should_fail/deriving-via-fail.hs
@@ -12,4 +12,4 @@ newtype Foo2 a b = Foo2 (a -> b)
   deriving Category
     via fooo
 
-data Foo3 deriving Eq via (forall a. a)
+data Foo3 deriving Eq via forall a. a
diff --git a/testsuite/tests/deriving/should_fail/deriving-via-fail4.hs b/testsuite/tests/deriving/should_fail/deriving-via-fail4.hs
index 1436d994c0..2e16abfa20 100644
--- a/testsuite/tests/deriving/should_fail/deriving-via-fail4.hs
+++ b/testsuite/tests/deriving/should_fail/deriving-via-fail4.hs
@@ -14,4 +14,4 @@ newtype F1 = F1 Int
   deriving Eq via Char
 
 newtype F2 a = MkF2 a
-  deriving (C a) via (forall a. a)
+  deriving (C a) via forall a. a
diff --git a/testsuite/tests/parser/should_fail/T3811c.stderr b/testsuite/tests/parser/should_fail/T3811c.stderr
index 52f081bbe6..6d662a9b16 100644
--- a/testsuite/tests/parser/should_fail/T3811c.stderr
+++ b/testsuite/tests/parser/should_fail/T3811c.stderr
@@ -1,6 +1,7 @@
 
 T3811c.hs:6:10: error:
-    Illegal class instance: ‘!Show D’
-      Class instances must be of the form
-        context => C ty_1 ... ty_n
+    Illegal head of an instance declaration: ‘!Show D’
+      Instance heads must be of the form
+        C ty_1 ... ty_n
       where ‘C’ is a class
+    In an instance declaration
diff --git a/testsuite/tests/rename/should_fail/T16114.stderr b/testsuite/tests/rename/should_fail/T16114.stderr
index aec0e3e3e0..adbaffc0ef 100644
--- a/testsuite/tests/rename/should_fail/T16114.stderr
+++ b/testsuite/tests/rename/should_fail/T16114.stderr
@@ -1,6 +1,4 @@
 
-T16114.hs:4:10: error:
-    Illegal class instance: ‘Eq a => Eq a => Eq (T a)’
-      Class instances must be of the form
-        context => C ty_1 ... ty_n
-      where ‘C’ is a class
+T16114.hs:4:18: error:
+    Instance head cannot contain nested ‘forall’s or contexts
+    In an instance declaration
diff --git a/testsuite/tests/rename/should_fail/T18240a.hs b/testsuite/tests/rename/should_fail/T18240a.hs
new file mode 100644
index 0000000000..cf3b8796d6
--- /dev/null
+++ b/testsuite/tests/rename/should_fail/T18240a.hs
@@ -0,0 +1,29 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TypeApplications #-}
+module T18240a where
+
+import Data.Proxy
+
+class C a where
+  m :: Proxy a
+
+instance (forall a. C [a]) where
+  m = Proxy @[a]
+
+instance (Eq a => C [a]) where
+  m = Proxy @[a]
+
+instance (forall a. C (Either a b)) where
+  m = Proxy @(Either a b)
+
+instance forall a. (forall b. C (Either a b)) where
+  m = Proxy @(Either a b)
+
+instance Eq a => (Eq b => C (Either a b)) where
+  m = Proxy @(Either a b)
+
+-- Some other nonsensical instance types
+
+instance 42
+instance Int -> Int
diff --git a/testsuite/tests/rename/should_fail/T18240a.stderr b/testsuite/tests/rename/should_fail/T18240a.stderr
new file mode 100644
index 0000000000..641bee5003
--- /dev/null
+++ b/testsuite/tests/rename/should_fail/T18240a.stderr
@@ -0,0 +1,40 @@
+
+T18240a.hs:11:11: error:
+    Instance head cannot contain nested ‘forall’s or contexts
+    In an instance declaration
+
+T18240a.hs:12:15: error: Not in scope: type variable ‘a’
+
+T18240a.hs:14:11: error:
+    Instance head cannot contain nested ‘forall’s or contexts
+    In an instance declaration
+
+T18240a.hs:17:11: error:
+    Instance head cannot contain nested ‘forall’s or contexts
+    In an instance declaration
+
+T18240a.hs:18:22: error: Not in scope: type variable ‘a’
+
+T18240a.hs:20:21: error:
+    Instance head cannot contain nested ‘forall’s or contexts
+    In an instance declaration
+
+T18240a.hs:21:24: error: Not in scope: type variable ‘b’
+
+T18240a.hs:23:19: error:
+    Instance head cannot contain nested ‘forall’s or contexts
+    In an instance declaration
+
+T18240a.hs:28:10: error:
+    Illegal head of an instance declaration: ‘42’
+      Instance heads must be of the form
+        C ty_1 ... ty_n
+      where ‘C’ is a class
+    In an instance declaration
+
+T18240a.hs:29:10: error:
+    Illegal head of an instance declaration: ‘Int -> Int’
+      Instance heads must be of the form
+        C ty_1 ... ty_n
+      where ‘C’ is a class
+    In an instance declaration
diff --git a/testsuite/tests/rename/should_fail/T18240b.hs b/testsuite/tests/rename/should_fail/T18240b.hs
new file mode 100644
index 0000000000..cbadf3d3e5
--- /dev/null
+++ b/testsuite/tests/rename/should_fail/T18240b.hs
@@ -0,0 +1,29 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DerivingVia #-}
+{-# LANGUAGE ExplicitForAll #-}
+{-# LANGUAGE FlexibleInstances #-}
+{-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE StandaloneDeriving #-}
+module T18240b where
+
+import Data.Proxy
+
+data T a b
+
+class W x y z
+instance W x y (T a b)
+
+newtype Foo a b = MkFoo (T a b)
+deriving via (forall x. T x y)           instance W x y (Foo a b)
+deriving via forall x. forall y. T x y   instance W x y (Foo a b)
+deriving via forall x. (forall y. T x y) instance W x y (Foo a b)
+
+class C1 x
+class C2 x y z
+
+data Bar = MkBar
+  deriving anyclass ( C1
+                    , (forall x. C2 x y)
+                    , forall x. forall y. C2 x y
+                    , forall x. (forall y. C2 x y)
+                    )
diff --git a/testsuite/tests/rename/should_fail/T18240b.stderr b/testsuite/tests/rename/should_fail/T18240b.stderr
new file mode 100644
index 0000000000..330e5cc72f
--- /dev/null
+++ b/testsuite/tests/rename/should_fail/T18240b.stderr
@@ -0,0 +1,24 @@
+
+T18240b.hs:17:15: error:
+    ‘via’ type cannot contain nested ‘forall’s or contexts
+    In a deriving declaration
+
+T18240b.hs:18:24: error:
+    ‘via’ type cannot contain nested ‘forall’s or contexts
+    In a deriving declaration
+
+T18240b.hs:19:25: error:
+    ‘via’ type cannot contain nested ‘forall’s or contexts
+    In a deriving declaration
+
+T18240b.hs:26:24: error:
+    Derived class type cannot contain nested ‘forall’s or contexts
+    In the data type declaration for ‘Bar’
+
+T18240b.hs:27:33: error:
+    Derived class type cannot contain nested ‘forall’s or contexts
+    In the data type declaration for ‘Bar’
+
+T18240b.hs:28:34: error:
+    Derived class type cannot contain nested ‘forall’s or contexts
+    In the data type declaration for ‘Bar’
diff --git a/testsuite/tests/rename/should_fail/T5951.stderr b/testsuite/tests/rename/should_fail/T5951.stderr
index b325493f35..f98fb501ed 100644
--- a/testsuite/tests/rename/should_fail/T5951.stderr
+++ b/testsuite/tests/rename/should_fail/T5951.stderr
@@ -1,6 +1,4 @@
 
-T5951.hs:8:8: error:
-    Illegal class instance: ‘A => B => C’
-      Class instances must be of the form
-        context => C ty_1 ... ty_n
-      where ‘C’ is a class
+T5951.hs:9:8: error:
+    Instance head cannot contain nested ‘forall’s or contexts
+    In an instance declaration
diff --git a/testsuite/tests/rename/should_fail/all.T b/testsuite/tests/rename/should_fail/all.T
index 27b359dec1..2647ac706b 100644
--- a/testsuite/tests/rename/should_fail/all.T
+++ b/testsuite/tests/rename/should_fail/all.T
@@ -154,3 +154,5 @@ test('T14548', normal, compile_fail, [''])
 test('T16610', normal, compile_fail, [''])
 test('T17593', normal, compile_fail, [''])
 test('T18145', normal, compile_fail, [''])
+test('T18240a', normal, compile_fail, [''])
+test('T18240b', normal, compile_fail, [''])
diff --git a/testsuite/tests/typecheck/should_fail/T16394.stderr b/testsuite/tests/typecheck/should_fail/T16394.stderr
index fff51a6e39..74e5c25ef5 100644
--- a/testsuite/tests/typecheck/should_fail/T16394.stderr
+++ b/testsuite/tests/typecheck/should_fail/T16394.stderr
@@ -1,5 +1,4 @@
-T16394.hs:6:10: error:
-    Illegal class instance: ‘C a => C b => C (a, b)’
-      Class instances must be of the form
-        context => C ty_1 ... ty_n
-      where ‘C’ is a class
+
+T16394.hs:6:17: error:
+    Instance head cannot contain nested ‘forall’s or contexts
+    In an instance declaration