doctool: Output formatter for DevDocs

In order to generate HTML output that DevDocs can easily scrape and
display, we add a new output format to g-ir-doc-tool (--format=devdocs).
It works similarly to the Mallard output format, but generates very simple
HTML instead. We add a new set of Mako templates to generate this output.

19 files changed, 359 insertions, 5 deletions

diff --git a/Makefile-giscanner.am b/Makefile-giscanner.am
index 4f08934c..c51cbf23 100644
--- a/Makefile-giscanner.am
+++ b/Makefile-giscanner.am
@@ -47,6 +47,7 @@ pkgpyexec_PYTHON = 			\
 	giscanner/introspectablepass.py	\
 	giscanner/libtoolimporter.py	\
 	giscanner/maintransformer.py	\
+	giscanner/mdextensions.py	\
 	giscanner/message.py		\
 	giscanner/msvccompiler.py	\
 	giscanner/pkgconfig.py		\
@@ -105,7 +106,21 @@ nobase_dist_template_DATA =		\
 	giscanner/doctemplates/mallard/Gjs/property.tmpl	\
 	giscanner/doctemplates/mallard/Gjs/record.tmpl		\
 	giscanner/doctemplates/mallard/Gjs/signal.tmpl		\
-	giscanner/doctemplates/mallard/Gjs/vfunc.tmpl
+	giscanner/doctemplates/mallard/Gjs/vfunc.tmpl		\
+	giscanner/doctemplates/devdocs/Gjs/_method.tmpl		\
+	giscanner/doctemplates/devdocs/Gjs/_methods.tmpl	\
+	giscanner/doctemplates/devdocs/Gjs/_properties.tmpl	\
+	giscanner/doctemplates/devdocs/Gjs/_signals.tmpl	\
+	giscanner/doctemplates/devdocs/Gjs/_staticmethods.tmpl	\
+	giscanner/doctemplates/devdocs/Gjs/_vfuncs.tmpl		\
+	giscanner/doctemplates/devdocs/Gjs/base.tmpl		\
+	giscanner/doctemplates/devdocs/Gjs/callback.tmpl	\
+	giscanner/doctemplates/devdocs/Gjs/class.tmpl		\
+	giscanner/doctemplates/devdocs/Gjs/default.tmpl		\
+	giscanner/doctemplates/devdocs/Gjs/enum.tmpl		\
+	giscanner/doctemplates/devdocs/Gjs/function.tmpl	\
+	giscanner/doctemplates/devdocs/Gjs/interface.tmpl	\
+	giscanner/doctemplates/devdocs/Gjs/namespace.tmpl
 
 _giscanner_la_CFLAGS = \
 	$(PYTHON_INCLUDES) \
diff --git a/configure.ac b/configure.ac
index 29bddde3..e8c23523 100644
--- a/configure.ac
+++ b/configure.ac
@@ -276,13 +276,20 @@ dnl an external dependency
 AC_ARG_ENABLE(doctool,[  --disable-doctool           disable g-ir-doc-tool ],,enable_doctool=auto)
 AS_IF([ test x$enable_doctool != xno], [
    AM_CHECK_PYMOD(mako,,have_python_mako=yes,have_python_mako=no)
+   AM_CHECK_PYMOD(markdown,,have_python_markdown=yes,have_python_markdown=no)
 ])
-AS_IF([ test x$enable_doctool = xauto && test x$have_python_mako = xyes ],
+AS_IF([ test x$enable_doctool = xauto &&
+        test x$have_python_mako = xyes &&
+        test x$have_python_markdown = xyes ],
       [ enable_doctool=yes ],
-      [ test x$enable_doctool = xauto && test x$have_python_mako = xno ],
+      [ test x$enable_doctool = xauto &&
+        (test x$have_python_mako = xno ||
+         test x$have_python_markdown = xno) ],
       [ enable_doctool=no ],
       [ test x$enable_doctool = xyes && test x$have_python_mako = xno ],
-      [ AC_MSG_ERROR([Python mako module not found]) ])
+      [ AC_MSG_ERROR([Python mako module not found]) ],
+      [ test x$enable_doctool = xyes && test x$have_python_markdown = xno ],
+      [ AC_MSG_ERROR([Python markdown module not found]) ])
 AM_CONDITIONAL(BUILD_DOCTOOL, test x$enable_doctool != xno)
 
 # Glib documentation
diff --git a/giscanner/docmain.py b/giscanner/docmain.py
index c91cce95..0120022b 100644
--- a/giscanner/docmain.py
+++ b/giscanner/docmain.py
@@ -32,7 +32,7 @@ from .docwriter import DocWriter
 from .sectionparser import generate_sections_file, write_sections_file
 from .transformer import Transformer
 
-FORMATS = ('mallard', 'sections')
+FORMATS = ['devdocs', 'mallard', 'sections']
 
 
 def doc_main(args):
diff --git a/giscanner/doctemplates/devdocs/Gjs/_method.tmpl b/giscanner/doctemplates/devdocs/Gjs/_method.tmpl
new file mode 100644
index 00000000..0374ba40
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/_method.tmpl
@@ -0,0 +1,48 @@
+<%def name="describe_parameters(m, static=False, virtual=False)">
+  <dl>
+    % if static:
+      <dt>Type:</dt>
+      <dd>Static</dd>
+    % elif virtual:
+      <dt>Type:</dt>
+      <dd>Virtual</dd>
+    % endif
+    % if m.parameters:
+      <dt>Parameters:</dt>
+      <dd>
+        <ul>
+          % for p in m.parameters:
+            <li>
+              <strong>${p.argname}</strong>
+              (<code>${formatter.format_type(p.type)}</code>)
+              % if p.doc:
+                &mdash; ${formatter.format_inline(m, p.doc)}
+              % endif
+            </li>
+          % endfor
+        </ul>
+      </dd>
+    % endif
+    % if m.retval.type != ast.TYPE_NONE:
+      <dt>Returns:</dt>
+      <dd>
+        (<code>${formatter.format_type(m.retval.type)}</code>)
+        % if m.retval.doc:
+          ${formatter.format_inline(m, m.retval.doc)}
+        % endif
+      </dd>
+    % endif
+  </dl>
+</%def>
+
+<%def name="method(m, static=False, virtual=False)">
+  <% invocation = ", ".join(map(lambda p: p.argname, m.parameters)) %>
+  <h3>
+    <span class="entry ${get_node_kind(m)}" id="${formatter.make_anchor(m)}">
+    ${formatter.format_function_name(m)}<!-- no space
+    --></span><!-- no space
+    -->(${formatter.format_in_parameters(m)})
+  </h3>
+  ${describe_parameters(m, static, virtual)}
+  ${formatter.format(m, m.doc)}
+</%def>
diff --git a/giscanner/doctemplates/devdocs/Gjs/_methods.tmpl b/giscanner/doctemplates/devdocs/Gjs/_methods.tmpl
new file mode 100644
index 00000000..e876cd68
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/_methods.tmpl
@@ -0,0 +1,4 @@
+<%namespace name="method" file="_method.tmpl"/>
+% for m in getattr(node, 'methods', []):
+  ${method.method(m)}
+% endfor
diff --git a/giscanner/doctemplates/devdocs/Gjs/_properties.tmpl b/giscanner/doctemplates/devdocs/Gjs/_properties.tmpl
new file mode 100644
index 00000000..ef9913aa
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/_properties.tmpl
@@ -0,0 +1,34 @@
+<%!
+  def dash_to_underscore(string):
+    return '_'.join(string.split('-'))
+
+  def dash_to_camel(string):
+    words = string.split('-')
+    return ''.join([words[0]] + [word.title() for word in words[1:]])
+%>
+% if getattr(node, 'properties', []):
+  <h2>Property Details</h2>
+  % for p in node.properties:
+    <h3 class="entry property" id="${formatter.make_anchor(p)}">
+      ${p.name | dash_to_underscore}
+    </h3>
+    <dl>
+      % if p.name.lower() != p.name:
+        <dt>Names</dt>
+        <dd>
+          <code>${p.name}</code>, <code>${p.name | dash_to_underscore}</code>,
+          <code>${p.name | dash_to_camel}</code>
+        </dd>
+      % endif
+      <dt>Type</dt>
+      <dd><code>${formatter.format_type(p.type)}</code></dd>
+      ##<dt>Default value</dt>
+      ##<dd>Not currently stored in GIR</dd>
+      <dt>Flags</dt>
+      <dd>${formatter.format_property_flags(p)}</dd>
+    </dl>
+    % if p.doc:
+      ${formatter.format(node, p.doc)}
+    % endif
+  % endfor
+% endif
diff --git a/giscanner/doctemplates/devdocs/Gjs/_signals.tmpl b/giscanner/doctemplates/devdocs/Gjs/_signals.tmpl
new file mode 100644
index 00000000..4065df83
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/_signals.tmpl
@@ -0,0 +1,20 @@
+<%namespace name="method" file="_method.tmpl"/>
+% if getattr(node, 'signals', []):
+  <h2>Signal Details</h2>
+  % for s in node.signals:
+    <h3>
+      <span class="entry signal" id="${formatter.make_anchor(s)}">
+        ${s.name}<!-- no space
+      --></span><!--
+      -->(${formatter.format_in_parameters(s)})
+    </h3>
+    <dl>
+      <dt>Flags</dt>
+      <dd>${formatter.format_signal_flags(s)}</dd>
+      ${method.describe_parameters(s)}
+    </dl>
+    % if s.doc:
+      ${formatter.format(node, s.doc)}
+    % endif
+  % endfor
+% endif
diff --git a/giscanner/doctemplates/devdocs/Gjs/_staticmethods.tmpl b/giscanner/doctemplates/devdocs/Gjs/_staticmethods.tmpl
new file mode 100644
index 00000000..dcd542e1
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/_staticmethods.tmpl
@@ -0,0 +1,4 @@
+<%namespace name="method" file="_method.tmpl"/>
+% for m in getattr(node, 'static_methods', []) + getattr(node, 'constructors', []):
+  ${method.method(m, static=True)}
+% endfor
diff --git a/giscanner/doctemplates/devdocs/Gjs/_vfuncs.tmpl b/giscanner/doctemplates/devdocs/Gjs/_vfuncs.tmpl
new file mode 100644
index 00000000..2966ede4
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/_vfuncs.tmpl
@@ -0,0 +1,6 @@
+<%namespace name="method" file="_method.tmpl"/>
+% if getattr(node, 'virtual_methods', []):
+  % for m in node.virtual_methods:
+    ${method.method(m, virtual=True)}
+  % endfor
+% endif
diff --git a/giscanner/doctemplates/devdocs/Gjs/base.tmpl b/giscanner/doctemplates/devdocs/Gjs/base.tmpl
new file mode 100644
index 00000000..95800da8
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/base.tmpl
@@ -0,0 +1,15 @@
+<html>
+<body>
+  <section>
+    <h1 class="${page_kind}">${formatter.format_page_name(node)}</h1>
+    <h2>Details</h2>
+    ${formatter.format(node, node.doc)}
+    <%include file="_staticmethods.tmpl"/>
+    <%include file="_methods.tmpl"/>
+    <%include file="_vfuncs.tmpl"/>
+    <%include file="_signals.tmpl"/>
+    <%include file="_properties.tmpl"/>
+    ${self.body()}
+  </section>
+</body>
+</html>
diff --git a/giscanner/doctemplates/devdocs/Gjs/callback.tmpl b/giscanner/doctemplates/devdocs/Gjs/callback.tmpl
new file mode 100644
index 00000000..2795ee3c
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/callback.tmpl
@@ -0,0 +1,3 @@
+<%namespace name="method" file="_method.tmpl"/>
+<%inherit file="base.tmpl"/>
+${method.describe_parameters(node)}
diff --git a/giscanner/doctemplates/devdocs/Gjs/class.tmpl b/giscanner/doctemplates/devdocs/Gjs/class.tmpl
new file mode 100644
index 00000000..9d2b5238
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/class.tmpl
@@ -0,0 +1 @@
+<%inherit file="base.tmpl"/>
diff --git a/giscanner/doctemplates/devdocs/Gjs/default.tmpl b/giscanner/doctemplates/devdocs/Gjs/default.tmpl
new file mode 100644
index 00000000..5130fadc
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/default.tmpl
@@ -0,0 +1,6 @@
+<%inherit file="base.tmpl"/>
+% if isinstance(node, ast.Constant):
+  <code>
+    const ${formatter.format_page_name(node)} = ${node.value};
+  </code>
+% endif
diff --git a/giscanner/doctemplates/devdocs/Gjs/enum.tmpl b/giscanner/doctemplates/devdocs/Gjs/enum.tmpl
new file mode 100644
index 00000000..a66cbefa
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/enum.tmpl
@@ -0,0 +1,16 @@
+<%inherit file="base.tmpl"/>
+<ul>
+% for m in node.members:
+  <li>
+    <code>
+      <span class="entry" id="${formatter.make_anchor(m)}">
+        ${m.name.upper()}
+      </span>
+      = ${m.value}
+    </code>
+    % if m.doc:
+      &mdash; ${formatter.format_inline(node, m.doc)}
+    % endif
+  </li>
+% endfor
+</ul>
diff --git a/giscanner/doctemplates/devdocs/Gjs/function.tmpl b/giscanner/doctemplates/devdocs/Gjs/function.tmpl
new file mode 100644
index 00000000..9d2b5238
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/function.tmpl
@@ -0,0 +1 @@
+<%inherit file="base.tmpl"/>
diff --git a/giscanner/doctemplates/devdocs/Gjs/interface.tmpl b/giscanner/doctemplates/devdocs/Gjs/interface.tmpl
new file mode 100644
index 00000000..9d2b5238
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/interface.tmpl
@@ -0,0 +1 @@
+<%inherit file="base.tmpl"/>
diff --git a/giscanner/doctemplates/devdocs/Gjs/namespace.tmpl b/giscanner/doctemplates/devdocs/Gjs/namespace.tmpl
new file mode 100644
index 00000000..5a90634a
--- /dev/null
+++ b/giscanner/doctemplates/devdocs/Gjs/namespace.tmpl
@@ -0,0 +1,14 @@
+<html>
+<body>
+  <section>
+    <h1 class="namespace">${node.name}</h1>
+  </section>
+  <ul>
+  % for n in node.values():
+    % if formatter.should_render_node(n):
+      <li>${formatter.format_inline(n, formatter.format_xref(n))}</li>
+    % endif
+  % endfor
+  </ul>
+</body>
+</html>
diff --git a/giscanner/docwriter.py b/giscanner/docwriter.py
index 2c3920fe..f26ed5b1 100644
--- a/giscanner/docwriter.py
+++ b/giscanner/docwriter.py
@@ -32,9 +32,11 @@ import tempfile
 
 from xml.sax import saxutils
 from mako.lookup import TemplateLookup
+import markdown
 
 from . import ast, xmlwriter
 from .utils import to_underscores
+from .mdextensions import InlineMarkdown
 
 # Freely inspired from
 # https://github.com/GNOME/yelp-xsl/blob/master/js/syntax.html
@@ -468,6 +470,20 @@ class DocFormatter(object):
 
         return " / ".join(flags)
 
+    def format_signal_flags(self, signal):
+        flags = []
+        if signal.action:
+            flags.append("Action")
+        if signal.detailed:
+            flags.append("Detailed")
+        if signal.no_hooks:
+            flags.append("No Hooks")
+        if signal.no_recurse:
+            flags.append("No Recurse")
+        if signal.when:
+            flags.append("Run " + signal.when.capitalize())
+        return " / ".join(flags)
+
     def to_underscores(self, node):
         if isinstance(node, ast.Property):
             return node.name.replace('-', '_')
@@ -963,7 +979,135 @@ class DocFormatterGjs(DocFormatterIntrospectableBase):
             return ', '.join(('%s: %s' % (p.argname, self.format_type(p.type)))
                              for p in construct_params)
 
+
+class DevDocsFormatterGjs(DocFormatterGjs):
+    def _is_static_method(self, node):
+        if not hasattr(node.parent, "static_methods"):
+            return False
+        return node in node.parent.static_methods
+
+    def should_render_node(self, node):
+        # For DevDocs, we only want to render the top-level nodes.
+        if isinstance(node, (ast.Compound, ast.Boxed)):
+            self.resolve_gboxed_constructor(node)
+
+        if not super(DevDocsFormatterGjs, self).should_render_node(node):
+            return False
+
+        if isinstance(node, ast.Function) and not node.is_method and \
+           not node.is_constructor and not self._is_static_method(node):
+            return True  # module-level function
+        toplevel_types = [ast.Alias, ast.Bitfield, ast.Boxed, ast.Callback,
+            ast.Class, ast.Constant, ast.Enum, ast.Interface, ast.Namespace,
+            ast.Record, ast.Union]
+        for ast_type in toplevel_types:
+            if isinstance(node, ast_type):
+                return True
+
+        return False
+
+    def format_fundamental_type(self, name):
+        # Don't specify the C type after Number as the Mallard docs do; it's
+        # confusing to GJS newbies.
+        if name in ["gint8", "guint8", "gint16", "guint16", "gint32", "guint32",
+                    "gchar", "guchar", "gshort", "gint", "guint", "gfloat",
+                    "gdouble", "gsize", "gssize", "gintptr", "guintptr",
+                    "glong", "gulong", "gint64", "guint64", "long double",
+                    "long long", "unsigned long long"]:
+            return "Number"  # gsize and up cannot fully be represented in GJS
+        if name in ["none", "gpointer"]:
+            return "void"
+        if name in ["utf8", "gunichar", "filename"]:
+            return "String"
+        if name == "gboolean":
+            return "Boolean"
+        if name == "GType":
+            return "GObject.Type"
+        if name == "GVariant":
+            return "GLib.Variant"
+        return name
+
+    def format(self, node, doc):
+        if doc is None:
+            return ''
+
+        cleaned_up_gtkdoc = super(DevDocsFormatterGjs, self).format_inline(node, doc)
+        return markdown.markdown(cleaned_up_gtkdoc)
+
+    def format_function_name(self, func):
+        name = func.name
+        if func.shadows:
+            name = func.shadows
+
+        if isinstance(func, ast.VFunction):
+            return 'vfunc_' + name
+        return name
+
+    def format_page_name(self, node):
+        if isinstance(node, ast.Function) and node.parent is not None:
+            return node.parent.name + "." + self.format_function_name(node)
+        return super(DevDocsFormatterGjs, self).format_page_name(node)
+
+    def _write_xref_markdown(self, target, anchor=None, display_name=None, pluralize=False):
+        if display_name is None:
+            display_name = target
+        link = target + ".html"
+        if anchor is not None:
+            link += "#" + anchor
+        return "[{}]({}){}".format(display_name, link, 's' if pluralize else '')
+
+    def make_anchor(self, node):
+        style_class = get_node_kind(node)
+        return "{}-{}".format(style_class, self.to_underscores(node))
+
+    def _process_parameter(self, node, match, props):
+        # Display the instance parameter as "this" instead of whatever name it
+        # has in C.
+        if hasattr(node, 'instance_parameter') and \
+           node.instance_parameter is not None and \
+           props['param_name'] == node.instance_parameter.argname:
+            return '<code>this</code>'
+        return super(DevDocsFormatterGjs, self)._process_parameter(node, match, props)
+
+    def format_xref(self, node, pluralize=False, **attrdict):
+        if node is None or not hasattr(node, 'namespace'):
+            return self._write_xref_markdown('index')
+        if node.namespace is self._transformer.namespace:
+            return self.format_internal_xref(node, attrdict, pluralize=pluralize)
+        return self.format_external_xref(node, attrdict, pluralize=pluralize)
+
+    def format_internal_xref(self, node, attrdict, pluralize=False):
+        if not self.should_render_node(node):
+            # Non-toplevel nodes are linked to the main page.
+            page = make_page_id(node.parent)
+            return self._write_xref_markdown(page, self.make_anchor(node),
+                                             page + "." + node.name,
+                                             pluralize=pluralize)
+        return self._write_xref_markdown(make_page_id(node), pluralize=pluralize)
+
+    def format_external_xref(self, node, attrdict, pluralize=False):
+        ns = node.namespace
+        if not self.should_render_node(node):
+            target = '../%s-%s/%s' % (ns.name, str(ns.version), make_page_id(node.parent))
+            return self._write_xref_markdown(target, self.make_anchor(node),
+                                             self.format_page_name(node.parent),
+                                             pluralize=pluralize)
+        target = '../%s-%s/%s' % (ns.name, str(ns.version), make_page_id(node))
+        return self._write_xref_markdown(target, None,
+                                         self.format_page_name(node),
+                                         pluralize=pluralize)
+
+    def format_inline(self, node, para):
+        cleaned_up_gtkdoc = super(DevDocsFormatterGjs, self).format_inline(node, para)
+        return markdown.markdown(cleaned_up_gtkdoc, extensions=[InlineMarkdown()])
+
+    def format_in_parameters(self, node):
+        return ', '.join(p.argname for p in self.get_in_parameters(node))
+
 LANGUAGES = {
+    "devdocs": {
+        "gjs": DevDocsFormatterGjs,
+    },
     "mallard": {
         "c": DocFormatterC,
         "python": DocFormatterPython,
@@ -1043,6 +1187,7 @@ class DocWriter(object):
                                  node=node,
                                  page_id=page_id,
                                  page_kind=page_kind,
+                                 get_node_kind=get_node_kind,
                                  formatter=self._formatter,
                                  ast=ast)
 
diff --git a/giscanner/mdextensions.py b/giscanner/mdextensions.py
new file mode 100644
index 00000000..97be4ed1
--- /dev/null
+++ b/giscanner/mdextensions.py
@@ -0,0 +1,14 @@
+from markdown.extensions import Extension
+from markdown.treeprocessors import Treeprocessor
+from markdown.util import etree
+
+
+class RemoveOuterP(Treeprocessor):
+    def run(self, root):
+        if len(root) == 1 and root[0].tag == "p":
+            root[0].tag = "span"
+
+
+class InlineMarkdown(Extension):
+    def extendMarkdown(self, md, md_globals):
+        md.treeprocessors.add("remove_outer_p", RemoveOuterP(md), "_end")