website: Include manpagessam/wiki-to-website

The man pages are converted to XHTML using asciidoc and docbook,
then postprocessed in Python using BeautifulSoup to include them
in the mkdocs website output.

6 files changed, 109 insertions, 53 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aeb209be6..9072854b2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -86,7 +86,7 @@ pages:
   dependencies: []
   before_script:
     - dnf install -y python3-pip
-    - pip3 install mkdocs mkdocs-cinder
+    - pip3 install beautifulsoup4 mkdocs mkdocs-cinder
   script:
     # Build tracker and install.
     - su tracker -c 'mkdir build; cd build; meson .. --prefix=/tmp/tracker; ninja install'
@@ -97,10 +97,10 @@ pages:
     - |
       tracker_commit=$CI_COMMIT_SHA1
       tracker_miners_commit=$(git -C ./extra/tracker-miners rev-parse HEAD)
-      ./docs/website/build.py --api-docs=/tmp/tracker/share/gtk-doc/html --tracker-commit=${tracker_commit}
+      ./docs/website/build.py --api-docs=/tmp/tracker/share/gtk-doc/html --tracker-commit=${tracker_commit} --man-pages ./docs/manpages/*.txt ./extra/tracker-miners/docs/manpages/*.txt
   artifacts:
     paths:
       - public
   only:
     - master
-    - sam/website
+    - /^sam\/website.*$/
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 6757bb3bc..687403c3c 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -7,4 +7,11 @@ nav:
   - Overview: 'overview.md'
   - FAQ: 'faq.md'
   - Community: 'community.md'
-  - Documentation: 'docs.md'
+  - Documentation:
+      User documentation: 'docs/user.md'
+      Developer documentation: 'docs/developer.md'
+      Commandline documentation: 'docs/commandline.md'
+
+markdown_extensions:
+  - admonition
+  - markdown_include.include
diff --git a/docs/website/build.py b/docs/website/build.py
index 49c086529..596717400 100755
--- a/docs/website/build.py
+++ b/docs/website/build.py
@@ -19,7 +19,11 @@
 # Boston, MA  02110-1301, USA.
 
 
+import bs4
+
 import argparse
+import contextlib
+import html.parser
 import logging
 import pathlib
 import shutil
@@ -30,7 +34,12 @@ import tempfile
 log = logging.getLogger('build.py')
 
 output_path = pathlib.Path('public')
-mkdocs_root = pathlib.Path(__file__).parent.parent
+website_root = pathlib.Path(__file__).parent
+docs_root = website_root.parent
+source_root = docs_root.parent
+
+asciidoc = shutil.which('asciidoc')
+xmlto = shutil.which('xmlto')
 
 
 def argument_parser():
@@ -43,6 +52,8 @@ def argument_parser():
                              "$prefix/share/gtk-doc/html/")
     parser.add_argument('--tracker-commit', required=True, metavar='COMMIT',
                         help="Commit ID of tracker.git repo used to build")
+    parser.add_argument('--man-pages', nargs='+', required=True,
+                        help="List of Asciidoc manual page source")
     return parser
 
 
@@ -79,6 +90,54 @@ def add_apidocs_header(text, filename):
     shutil.move(f_out.name, filename)
 
 
+class Manpages():
+    def run(self, command):
+        command = [str(c) for c in command]
+        log.debug("Running: %s", ' '.join(command))
+        subprocess.run(command, check=True)
+
+    def generate_manpage_xml(self, in_path, out_path):
+        """Generate a docbook XML file for an Asciidoc manpage source file."""
+        self.run([asciidoc, '--backend', 'docbook', '--doctype', 'manpage',
+                 '--out-file', out_path, in_path])
+
+    def generate_manpage_html(self, in_path, out_path):
+        """Generate a HTML page from a docbook XML file"""
+        self.run([xmlto, 'xhtml-nochunks', '-o', out_path, in_path])
+
+    def generate_toplevel_markdown(self, in_path, out_path, html_files):
+        """Generate the master commandline.md page."""
+
+        includes = []
+        for path in sorted(html_files):
+            parser = bs4.BeautifulSoup(path.read_text(), 'html.parser')
+
+            title = parser.title.text
+            if title and title.startswith('tracker-'):
+                title = 'tracker ' + title[8:]
+
+            body = parser.body.contents[0]
+
+            includes.append("## %s\n\n%s\n---\n" % (title, body))
+
+        text = in_path.read_text()
+        text = text.format(
+            includes='\n'.join(includes)
+        )
+        out_path.write_text(text)
+
+
+@contextlib.contextmanager
+def tmpdir():
+    path = pathlib.Path(tempfile.mkdtemp())
+    log.debug("Created temporary directory %s", path)
+    try:
+        yield path
+    finally:
+        log.debug("Removed temporary directory %s", path)
+        shutil.rmtree(path)
+
+
 def main():
     args = argument_parser().parse_args()
 
@@ -90,8 +149,32 @@ def main():
     if output_path.exists():
         raise RuntimeError(f"Output path '{output_path}' already exists.")
 
+    log.info("Generating online man pages")
+    with tmpdir() as workdir:
+        manpages = Manpages()
+
+        htmldir = website_root.joinpath('docs')
+        htmlfiles = []
+
+        for man_page in args.man_pages:
+            asciidocpath = pathlib.Path(man_page)
+            xmlpath = workdir.joinpath(asciidocpath.stem + '.xml')
+
+            manpages.generate_manpage_xml(asciidocpath, xmlpath)
+            manpages.generate_manpage_html(xmlpath, htmldir)
+
+            htmlpath = htmldir.joinpath(xmlpath.with_suffix('.html').name)
+            htmlfiles.append(htmlpath)
+
+        template_in = website_root.joinpath('docs/commandline.md.in')
+        template_out = website_root.joinpath('docs/commandline.md')
+
+        manpages.generate_toplevel_markdown(template_in, template_out, htmlfiles)
+
+    #shutil.copy('/usr/share/asciidoc/stylesheets/docbook-xsl.css', manpage_output_path)
+
     log.info("Building website")
-    mkdocs_config = mkdocs_root.joinpath('mkdocs.yml')
+    mkdocs_config = docs_root.joinpath('mkdocs.yml')
     subprocess.run(['mkdocs', 'build', '--config-file', mkdocs_config,
                     '--site-dir', output_path.absolute()])
 
diff --git a/docs/website/docs.md b/docs/website/docs.md
deleted file mode 100644
index beed96a56..000000000
--- a/docs/website/docs.md
+++ /dev/null
@@ -1,45 +0,0 @@
-# Documentation
-
-## User Documentation
-
-Tracker is a system component and most users will not need to interact with
-it directly.
-
-GNOME has documentation on how to
-[search for files in the file manager](https://developer.gnome.org/libtracker-sparql/stable/).
-
-The `tracker` commandline tool provides direct access to Tracker. Use the
-`--help` option for documentation of this tool.
-
-## Developer Documentation
-    
-Application and platform developers using Tracker will interact with Tracker
-using one or more of the shared libraries it provides:
-
- * [libtracker-sparql](https://developer.gnome.org/libtracker-sparql/stable/) is
-   used to read and write data from the Tracker store using SPARQL.
- * [libtracker-control](https://developer.gnome.org/libtracker-control/stable/),
-   is used to manage Tracker daemons.
- * [libtracker-miner](https://developer.gnome.org/libtracker-miner/stable/) can
-   be used if you want to implement a new data provider while reusing existing
-   Tracker code.
-
-WARNING: The documention linked above is for out of date verions of Tracker 2.x.
-This is due to an [infrastructure
-issue](https://gitlab.gnome.org/Infrastructure/library-web/issues/50). See also
-[this bug](https://gitlab.gnome.org/GNOME/tracker/-/issues/100).
-
-The following documentation may be useful:
-
- * [Tracker ontology documentation](https://developer.gnome.org/ontology/stable/).
- * [Tracker documentation on wiki.gnome.org](https://wiki.gnome.org/Projects/Tracker).
-
-## Preview Documentation
-
-We provide an online version of the documentation for the latest in-development version
-of Tracker. You can browse it here:
-
-  * [libtracker-sparql](./api-preview/libtracker-sparql)
-  * [ontology](./api-preview/ontology)
-
-Be aware that some libraries from Tracker 2.0 will not be available for Tracker 3.0.
diff --git a/docs/website/docs/commandline.md.in b/docs/website/docs/commandline.md.in
new file mode 100644
index 000000000..becc0ab7a
--- /dev/null
+++ b/docs/website/docs/commandline.md.in
@@ -0,0 +1,11 @@
+# Commandline reference
+
+You can control tracker using the `tracker` commandline tool. The various
+commands are documented below.
+
+This documentation is also available on your computer using the `man` command.
+
+!!! note
+    This documentation is for the latest in-development version of Tracker.
+
+{includes}
diff --git a/docs/website/index.md b/docs/website/index.md
index 92becdd03..7e0f335d9 100644
--- a/docs/website/index.md
+++ b/docs/website/index.md
@@ -7,8 +7,8 @@ Are you having problems with Tracker in GNOME? Look at the [Frequently Asked
 Questions](faq).
 
 Are you interested in using or developing Tracker and want to find out more
-about it? Read the [overview](overview) first and then take a look at the
-[documentation](documentation).
+about it? Start with the [overview](overview), then see our documentation
+for [users](docs/user) and [developers](docs/developer).
 
 Do you want to contribute to Tracker? Join the [community](community)
 and look for some bugs marked 'Help wanted'