API Annotations documentation update, parsing issue, add example test

Summary:
Add a reference note to each AnnKeywordId haddock comment so GHC
developers will have an idea why they are there.

Add a new test to ghc-api/annotations to serve as a template for other
GHC developers when they need to update the parser. It provides output
which checks that each SrcSpan that an annotation is attached to
actually appears in the `ParsedSource`, and lists the individual
annotations. The idea is that a developer writes a version of this
which parses a sample file using whatever syntax is changed in
Parser.y, and can then check that all the annotations come through.

Depends on D538

Test Plan: ./validate

Reviewers: simonpj, hvr, austin

Reviewed By: austin

Subscribers: thomie, jstolarek

Differential Revision: https://phabricator.haskell.org/D620

5 files changed, 243 insertions, 1 deletions

diff --git a/testsuite/tests/ghc-api/annotations/.gitignore b/testsuite/tests/ghc-api/annotations/.gitignore
index 61d9b24b9e..fe31fad9b9 100644
--- a/testsuite/tests/ghc-api/annotations/.gitignore
+++ b/testsuite/tests/ghc-api/annotations/.gitignore
@@ -1,6 +1,7 @@
 annotations
 parseTree
 comments
+exampleTest
 *.hi
 *.o
 *.run.*
diff --git a/testsuite/tests/ghc-api/annotations/Makefile b/testsuite/tests/ghc-api/annotations/Makefile
index 421154ea25..61474e9b0e 100644
--- a/testsuite/tests/ghc-api/annotations/Makefile
+++ b/testsuite/tests/ghc-api/annotations/Makefile
@@ -4,7 +4,7 @@ include $(TOP)/mk/test.mk
 
 clean:
 	rm -f *.o *.hi
-	rm -f annotations comments parseTree
+	rm -f annotations comments parseTree exampleTest
 
 annotations: 
 	rm -f annotations.o annotations.hi
@@ -21,5 +21,9 @@ comments:
 	'$(TEST_HC)' $(TEST_HC_OPTS) --make -v0 -package ghc comments
 	./comments "`'$(TEST_HC)' $(TEST_HC_OPTS) --print-libdir | tr -d '\r'`"
 
+exampleTest:
+	rm -f exampleTest.o exampleTest.hi
+	'$(TEST_HC)' $(TEST_HC_OPTS) --make -v0 -package ghc exampleTest
+	./exampleTest "`'$(TEST_HC)' $(TEST_HC_OPTS) --print-libdir | tr -d '\r'`"
 
 .PHONY: clean
diff --git a/testsuite/tests/ghc-api/annotations/all.T b/testsuite/tests/ghc-api/annotations/all.T
index 54da2efda4..cb075cb185 100644
--- a/testsuite/tests/ghc-api/annotations/all.T
+++ b/testsuite/tests/ghc-api/annotations/all.T
@@ -1,4 +1,5 @@
 test('annotations', normal, run_command, ['$MAKE -s --no-print-directory annotations'])
 test('parseTree',   normal, run_command, ['$MAKE -s --no-print-directory parseTree'])
 test('comments',    normal, run_command, ['$MAKE -s --no-print-directory comments'])
+test('exampleTest', normal, run_command, ['$MAKE -s --no-print-directory exampleTest'])
 
diff --git a/testsuite/tests/ghc-api/annotations/exampleTest.hs b/testsuite/tests/ghc-api/annotations/exampleTest.hs
new file mode 100644
index 0000000000..0b6c22464c
--- /dev/null
+++ b/testsuite/tests/ghc-api/annotations/exampleTest.hs
@@ -0,0 +1,112 @@
+{-# LANGUAGE RankNTypes #-}
+
+-- This program must be called with GHC's libdir as the single command line
+-- argument.
+module Main where
+
+-- import Data.Generics
+import Data.Data
+import Data.List
+import System.IO
+import GHC
+import BasicTypes
+import DynFlags
+import MonadUtils
+import Outputable
+import ApiAnnotation
+import Bag (filterBag,isEmptyBag)
+import System.Directory (removeFile)
+import System.Environment( getArgs )
+import qualified Data.Map as Map
+import qualified Data.Set as Set
+import Data.Dynamic ( fromDynamic,Dynamic )
+
+main::IO()
+main = do
+        [libdir] <- getArgs
+        testOneFile libdir "AnnotationTuple"
+
+testOneFile libdir fileName = do
+       ((anns,cs),p) <- runGhc (Just libdir) $ do
+                        dflags <- getSessionDynFlags
+                        setSessionDynFlags dflags
+                        let mn =mkModuleName fileName
+                        addTarget Target { targetId = TargetModule mn
+                                         , targetAllowObjCode = True
+                                         , targetContents = Nothing }
+                        load LoadAllTargets
+                        modSum <- getModSummary mn
+                        p <- parseModule modSum
+                        t <- typecheckModule p
+                        d <- desugarModule t
+                        l <- loadModule d
+                        let ts=typecheckedSource l
+                            r =renamedSource l
+                        return (pm_annotations p,p)
+
+       let spans = Set.fromList $ getAllSrcSpans (pm_parsed_source p)
+
+       -- putStrLn (pp spans)
+           problems = filter (\(s,a) -> not (Set.member s spans))
+                             $ getAnnSrcSpans (anns,cs)
+       putStrLn "---Problems---------------------"
+       putStrLn (intercalate "\n" [showAnns $ Map.fromList $ map snd problems])
+       putStrLn "--------------------------------"
+       putStrLn (intercalate "\n" [showAnns anns])
+
+    where
+      getAnnSrcSpans :: ApiAnns -> [(SrcSpan,(ApiAnnKey,[SrcSpan]))]
+      getAnnSrcSpans (anns,_) = map (\a@((ss,_),_) -> (ss,a)) $ Map.toList anns
+
+      getAllSrcSpans :: (Data t) => t -> [SrcSpan]
+      getAllSrcSpans ast = everything (++) ([] `mkQ` getSrcSpan) ast
+        where
+          getSrcSpan :: SrcSpan -> [SrcSpan]
+          getSrcSpan ss = [ss]
+
+
+showAnns anns = "[\n" ++ (intercalate "\n"
+   $ map (\((s,k),v)
+              -> ("(AK " ++ pp s ++ " " ++ show k ++" = " ++ pp v ++ ")\n"))
+   $ Map.toList anns)
+    ++ "]\n"
+
+pp a = showPpr unsafeGlobalDynFlags a
+
+
+-- ---------------------------------------------------------------------
+
+-- Copied from syb for the test
+
+
+-- | Generic queries of type \"r\",
+--   i.e., take any \"a\" and return an \"r\"
+--
+type GenericQ r = forall a. Data a => a -> r
+
+
+-- | Make a generic query;
+--   start from a type-specific case;
+--   return a constant otherwise
+--
+mkQ :: ( Typeable a
+       , Typeable b
+       )
+    => r
+    -> (b -> r)
+    -> a
+    -> r
+(r `mkQ` br) a = case cast a of
+                        Just b  -> br b
+                        Nothing -> r
+
+
+
+-- | Summarise all nodes in top-down, left-to-right order
+everything :: (r -> r -> r) -> GenericQ r -> GenericQ r
+
+-- Apply f to x to summarise top-level node;
+-- use gmapQ to recurse into immediate subterms;
+-- use ordinary foldl to reduce list of intermediate results
+
+everything k f x = foldl k (f x) (gmapQ (everything k f) x)
diff --git a/testsuite/tests/ghc-api/annotations/exampleTest.stdout b/testsuite/tests/ghc-api/annotations/exampleTest.stdout
new file mode 100644
index 0000000000..42da538cc7
--- /dev/null
+++ b/testsuite/tests/ghc-api/annotations/exampleTest.stdout
@@ -0,0 +1,124 @@
+---Problems---------------------
+[
+(AK AnnotationTuple.hs:13:39 AnnComma = [AnnotationTuple.hs:13:39])
+
+(AK <no location info> AnnEofPos = [AnnotationTuple.hs:21:1])
+]
+
+--------------------------------
+[
+(AK AnnotationTuple.hs:1:1 AnnCloseC = [AnnotationTuple.hs:16:1])
+
+(AK AnnotationTuple.hs:1:1 AnnModule = [AnnotationTuple.hs:2:1-6])
+
+(AK AnnotationTuple.hs:1:1 AnnOpenC = [AnnotationTuple.hs:4:1])
+
+(AK AnnotationTuple.hs:1:1 AnnWhere = [AnnotationTuple.hs:2:30-34])
+
+(AK AnnotationTuple.hs:2:24-28 AnnCloseP = [AnnotationTuple.hs:2:28])
+
+(AK AnnotationTuple.hs:2:24-28 AnnOpenP = [AnnotationTuple.hs:2:24])
+
+(AK AnnotationTuple.hs:5:1-32 AnnAs = [AnnotationTuple.hs:5:28-29])
+
+(AK AnnotationTuple.hs:5:1-32 AnnImport = [AnnotationTuple.hs:5:1-6])
+
+(AK AnnotationTuple.hs:5:1-32 AnnQualified = [AnnotationTuple.hs:5:8-16])
+
+(AK AnnotationTuple.hs:5:1-32 AnnSemi = [AnnotationTuple.hs:6:1])
+
+(AK AnnotationTuple.hs:5:1-32 AnnVal = [AnnotationTuple.hs:5:31-32])
+
+(AK AnnotationTuple.hs:(7,1)-(10,14) AnnEqual = [AnnotationTuple.hs:7:5])
+
+(AK AnnotationTuple.hs:(7,1)-(10,14) AnnFunId = [AnnotationTuple.hs:7:1-3])
+
+(AK AnnotationTuple.hs:(7,1)-(10,14) AnnSemi = [AnnotationTuple.hs:12:1])
+
+(AK AnnotationTuple.hs:(7,7)-(10,14) AnnIn = [AnnotationTuple.hs:10:7-8])
+
+(AK AnnotationTuple.hs:(7,7)-(10,14) AnnLet = [AnnotationTuple.hs:7:7-9])
+
+(AK AnnotationTuple.hs:8:9-13 AnnEqual = [AnnotationTuple.hs:8:11])
+
+(AK AnnotationTuple.hs:8:9-13 AnnFunId = [AnnotationTuple.hs:8:9])
+
+(AK AnnotationTuple.hs:8:9-13 AnnSemi = [AnnotationTuple.hs:9:9])
+
+(AK AnnotationTuple.hs:9:9-13 AnnEqual = [AnnotationTuple.hs:9:11])
+
+(AK AnnotationTuple.hs:9:9-13 AnnFunId = [AnnotationTuple.hs:9:9])
+
+(AK AnnotationTuple.hs:10:10-14 AnnVal = [AnnotationTuple.hs:10:12])
+
+(AK AnnotationTuple.hs:13:1-72 AnnEqual = [AnnotationTuple.hs:13:5])
+
+(AK AnnotationTuple.hs:13:1-72 AnnFunId = [AnnotationTuple.hs:13:1-3])
+
+(AK AnnotationTuple.hs:13:1-72 AnnSemi = [AnnotationTuple.hs:14:1])
+
+(AK AnnotationTuple.hs:13:7-72 AnnVal = [AnnotationTuple.hs:13:13])
+
+(AK AnnotationTuple.hs:13:19-53 AnnCloseP = [AnnotationTuple.hs:13:53])
+
+(AK AnnotationTuple.hs:13:19-53 AnnOpenP = [AnnotationTuple.hs:13:19])
+
+(AK AnnotationTuple.hs:13:20 AnnComma = [AnnotationTuple.hs:13:21])
+
+(AK AnnotationTuple.hs:13:23-29 AnnComma = [AnnotationTuple.hs:13:33])
+
+(AK AnnotationTuple.hs:13:35-37 AnnComma = [AnnotationTuple.hs:13:38])
+
+(AK AnnotationTuple.hs:13:39 AnnComma = [AnnotationTuple.hs:13:39])
+
+(AK AnnotationTuple.hs:13:41-52 AnnCloseS = [AnnotationTuple.hs:13:52])
+
+(AK AnnotationTuple.hs:13:41-52 AnnOpenS = [AnnotationTuple.hs:13:41])
+
+(AK AnnotationTuple.hs:13:42 AnnComma = [AnnotationTuple.hs:13:43])
+
+(AK AnnotationTuple.hs:13:45 AnnComma = [AnnotationTuple.hs:13:46])
+
+(AK AnnotationTuple.hs:13:48 AnnComma = [AnnotationTuple.hs:13:49])
+
+(AK AnnotationTuple.hs:13:55-72 AnnCloseS = [AnnotationTuple.hs:13:72])
+
+(AK AnnotationTuple.hs:13:55-72 AnnOpenS = [AnnotationTuple.hs:13:55])
+
+(AK AnnotationTuple.hs:13:56-62 AnnComma = [AnnotationTuple.hs:13:63])
+
+(AK AnnotationTuple.hs:13:61-62 AnnCloseP = [AnnotationTuple.hs:13:62])
+
+(AK AnnotationTuple.hs:13:61-62 AnnOpenP = [AnnotationTuple.hs:13:61])
+
+(AK AnnotationTuple.hs:15:1-41 AnnEqual = [AnnotationTuple.hs:15:5])
+
+(AK AnnotationTuple.hs:15:1-41 AnnFunId = [AnnotationTuple.hs:15:1-3])
+
+(AK AnnotationTuple.hs:15:7-27 AnnCloseP = [AnnotationTuple.hs:15:27])
+
+(AK AnnotationTuple.hs:15:7-27 AnnOpenP = [AnnotationTuple.hs:15:7])
+
+(AK AnnotationTuple.hs:15:8 AnnComma = [AnnotationTuple.hs:15:9])
+
+(AK AnnotationTuple.hs:15:11-17 AnnComma = [AnnotationTuple.hs:15:18])
+
+(AK AnnotationTuple.hs:15:20-22 AnnComma = [AnnotationTuple.hs:15:23])
+
+(AK AnnotationTuple.hs:15:24 AnnComma = [AnnotationTuple.hs:15:24])
+
+(AK AnnotationTuple.hs:15:25 AnnComma = [AnnotationTuple.hs:15:25])
+
+(AK AnnotationTuple.hs:15:26 AnnComma = [AnnotationTuple.hs:15:26])
+
+(AK AnnotationTuple.hs:15:33-41 AnnCloseP = [AnnotationTuple.hs:15:41])
+
+(AK AnnotationTuple.hs:15:33-41 AnnOpenP = [AnnotationTuple.hs:15:33])
+
+(AK AnnotationTuple.hs:15:39-40 AnnCloseP = [AnnotationTuple.hs:15:40])
+
+(AK AnnotationTuple.hs:15:39-40 AnnOpenP = [AnnotationTuple.hs:15:39])
+
+(AK <no location info> AnnEofPos = [AnnotationTuple.hs:21:1])
+]
+