Update doc gen to fail more definitively [skip appveyor]

Now the scons "product" build (including generating and
building docs) is done in CI, it's useful to fail more
distinctly on xml problems.  Modify scripts to do that,
and do a bit of reformatting along the way.

Signed-off-by: Mats Wichmann <mats@linux.com>

5 files changed, 53 insertions, 38 deletions

diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py
index a2daa5501..ba9c923ce 100644
--- a/bin/SConsDoc.py
+++ b/bin/SConsDoc.py
@@ -571,12 +571,12 @@ class SConsDocHandler:
                                xpath_context, nsmap, include_entities)
 
     def parseContent(self, content, include_entities=True):
-        """ Parses the given content as XML file. This method
-            is used when we generate the basic lists of entities
-            for the builders, tools and functions.
-            So we usually don't bother about namespaces and resolving
-            entities here...this is handled in parseXmlFile below
-            (step 2 of the overall process).
+        """Parse the given content as XML.
+
+        This method is used when we generate the basic lists of entities
+        for the builders, tools and functions.  So we usually don't
+        bother about namespaces and resolving entities here...
+        this is handled in parseXmlFile below (step 2 of the overall process).
         """
         # Create doctree
         t = SConsDocTree()
diff --git a/bin/docs-create-example-outputs.py b/bin/docs-create-example-outputs.py
index 012443533..e5d40ce4c 100644
--- a/bin/docs-create-example-outputs.py
+++ b/bin/docs-create-example-outputs.py
@@ -9,10 +9,13 @@ import SConsExamples
 
 if __name__ == "__main__":
     print("Checking whether all example names are unique...")
-    if SConsExamples.exampleNamesAreUnique(os.path.join('doc','user')):
+    if SConsExamples.exampleNamesAreUnique(os.path.join('doc', 'user')):
         print("OK")
     else:
-        print("Not all example names and suffixes are unique! Please correct the errors listed above and try again.")
+        print(
+            "Not all example names and suffixes are unique! "
+            "Please correct the errors listed above and try again."
+        )
         sys.exit(1)
 
-    SConsExamples.createAllExampleOutputs(os.path.join('doc','user'))
+    SConsExamples.createAllExampleOutputs(os.path.join('doc', 'user'))
diff --git a/bin/docs-update-generated.py b/bin/docs-update-generated.py
index 36878965e..4c96b214e 100644
--- a/bin/docs-update-generated.py
+++ b/bin/docs-update-generated.py
@@ -13,18 +13,22 @@ import subprocess
 import SConsDoc
 
 # Directory where all generated files are stored
-gen_folder = os.path.join('doc','generated')
+gen_folder = os.path.join('doc', 'generated')
 
 def argpair(key):
     """ Return the argument pair *.gen,*.mod for the given key. """
-    arg = '%s,%s' % (os.path.join(gen_folder, '%s.gen' % key),
-                     os.path.join(gen_folder, '%s.mod' % key))
-    
+    arg = '%s,%s' % (
+        os.path.join(gen_folder, '%s.gen' % key),
+        os.path.join(gen_folder, '%s.mod' % key),
+    )
+
     return arg
 
 def generate_all():
-    """ Scan for XML files in the SCons directory and call scons-proc.py
-        to generate the *.gen/*.mod files from it.
+    """Generate the entity files.
+
+    Scan for XML files in the SCons directory and call scons-proc.py
+    to generate the *.gen/*.mod files from it.
     """
     flist = []
     for path, dirs, files in os.walk('SCons'):
@@ -36,12 +40,12 @@ def generate_all():
 
     if flist:
         # Does the destination folder exist
-        if not os.path.isdir(gen_folder):
-            try:
-                os.makedirs(gen_folder)
-            except:
-                print("Couldn't create destination folder %s! Exiting..." % gen_folder)
-                return
+        try:
+            os.makedirs(gen_folder, exist_ok=True)
+        except Exception:
+            print("Couldn't create destination folder %s! Exiting..." % gen_folder, file=sys.stdout))
+            return False
+
         # Call scons-proc.py
         cp = subprocess.run(
             [
@@ -55,10 +59,11 @@ def generate_all():
             shell=False,
         )
 
-        # No-op: scons-proc doesn't actually set an exit code at the moment.
         if cp.returncode:
             print("Generation failed", file=sys.stderr)
+            return False
     
     
 if __name__ == "__main__":
-    generate_all()
+    if not generate_all():
+        sys.exit(1)
diff --git a/bin/docs-validate.py b/bin/docs-validate.py
index c4dd3b7f3..6d8fd770a 100644
--- a/bin/docs-validate.py
+++ b/bin/docs-validate.py
@@ -7,20 +7,24 @@ import sys,os
 import SConsDoc
 
 if __name__ == "__main__":
-    if len(sys.argv)>1:
+    if len(sys.argv) > 1:
         if SConsDoc.validate_all_xml((sys.argv[1],)):
             print("OK")
         else:
             print("Validation failed! Please correct the errors above and try again.")
+            sys.exit(1)
     else:
-        if SConsDoc.validate_all_xml(['src',
-                                      os.path.join('doc','design'),
-                                      os.path.join('doc','developer'),
-                                      os.path.join('doc','man'),
-                                      os.path.join('doc','python10'),
-                                      os.path.join('doc','reference'),
-                                      os.path.join('doc','user')
-                                      ]):
+        if SConsDoc.validate_all_xml(
+            [
+                'SCons',
+                os.path.join('doc', 'design'),
+                os.path.join('doc', 'developer'),
+                os.path.join('doc', 'man'),
+                os.path.join('doc', 'python10'),
+                os.path.join('doc', 'reference'),
+                os.path.join('doc', 'user'),
+            ]
+        ):
             print("OK")
         else:
             print("Validation failed! Please correct the errors above and try again.")
diff --git a/bin/scons-proc.py b/bin/scons-proc.py
index fb9e6df3c..799b12a14 100644
--- a/bin/scons-proc.py
+++ b/bin/scons-proc.py
@@ -61,9 +61,10 @@ def parse_docs(args, include_entities=True):
         if include_entities:
             try:
                 h.parseXmlFile(f)
-            except:
-                sys.stderr.write("error in %s\n" % f)
-                raise
+            except Exception as e:
+                print("error parsing %s\n" % f, file=sys.stderr)
+                print(str(e), file=sys.stderr)
+                sys.exit(1)
         else:
             # mode we read (text/bytes) has to match handling in SConsDoc
             with open(f, 'r') as fp:
@@ -71,9 +72,10 @@ def parse_docs(args, include_entities=True):
             if content:
                 try:
                     h.parseContent(content, include_entities)
-                except:
-                    sys.stderr.write("error in %s\n" % f)
-                    raise
+                except Exception as e:
+                    print("error parsing %s\n" % f, file=sys.stderr)
+                    print(str(e), file=sys.stderr)
+                    sys.exit(1)
     return h
 
 Warning = """\
@@ -408,6 +410,7 @@ if SConsDoc.validate_all_xml(['SCons']):
     print("OK")
 else:
     print("Validation failed! Please correct the errors above and try again.")
+    sys.exit(1)
 
 # Step 3: Creating actual documentation snippets, using the
 #         fully resolved and updated entities from the *.mod files.