diff options
Diffstat (limited to 'hacking')
| -rwxr-xr-x | hacking/module_formatter.py | 450 | ||||
| -rw-r--r-- | hacking/templates/html.j2 | 7 | ||||
| -rw-r--r-- | hacking/templates/js.j2 | 5 | ||||
| -rw-r--r-- | hacking/templates/latex.j2 | 76 | ||||
| -rw-r--r-- | hacking/templates/markdown.j2 | 64 |
5 files changed, 161 insertions, 441 deletions
diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 24dc3bdce1..f6ab8d596a 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -24,16 +24,20 @@ import yaml import codecs import json import ast -from jinja2 import Environment, FileSystemLoader import re import optparse import time import datetime import subprocess import cgi +from jinja2 import Environment, FileSystemLoader + import ansible.utils import ansible.utils.module_docs as module_docs +##################################################################################### +# constants and paths + # if a module is added in a version of Ansible older than this, don't print the version added information # in the module documentation because everyone is assumed to be running something newer than this already. TO_OLD_TO_BE_NOTABLE = 1.0 @@ -48,65 +52,16 @@ EXAMPLE_YAML=os.path.abspath(os.path.join( os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml' )) -# There is a better way of doing this! -# TODO: somebody add U(text, http://foo.bar/) as described by Tim in #991 - _ITALIC = re.compile(r"I\(([^)]+)\)") _BOLD = re.compile(r"B\(([^)]+)\)") _MODULE = re.compile(r"M\(([^)]+)\)") _URL = re.compile(r"U\(([^)]+)\)") _CONST = re.compile(r"C\(([^)]+)\)") -def latex_ify(text): - - t = _ITALIC.sub("\\I{" + r"\1" + "}", text) - t = _BOLD.sub("\\B{" + r"\1" + "}", t) - t = _MODULE.sub("\\M{" + r"\1" + "}", t) - t = _URL.sub("\\url{" + r"\1" + "}", t) - t = _CONST.sub("\\C{" + r"\1" + "}", t) - - return t - -def html_ify(text): - - #print "DEBUG: text=%s" % text - - t = cgi.escape(text) - t = _ITALIC.sub("<em>" + r"\1" + "</em>", t) - t = _BOLD.sub("<b>" + r"\1" + "</b>", t) - t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t) - t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t) - t = _CONST.sub("<code>" + r"\1" + "</code>", t) - - return t - -def json_ify(text): - - t = _ITALIC.sub("<em>" + r"\1" + "</em>", text) - t = _BOLD.sub("<b>" + r"\1" + "</b>", t) - t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t) - t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t) - t = _CONST.sub("<code>" + r"\1" + "</code>", t) - - return t - - -def js_ify(text): - - return text - - -def man_ify(text): - - t = _ITALIC.sub(r'\\fI' + r"\1" + r"\\fR", text) - t = _BOLD.sub(r'\\fB' + r"\1" + r"\\fR", t) - t = _MODULE.sub(r'\\fI' + r"\1" + r"\\fR", t) - t = _URL.sub(r'\\fI' + r"\1" + r"\\fR", t) - t = _CONST.sub(r'\\fC' + r"\1" + r"\\fR", t) - - return t +##################################################################################### def rst_ify(text): + ''' convert symbols like I(this is in italics) to valid restructured text ''' t = _ITALIC.sub(r'*' + r"\1" + r"*", text) t = _BOLD.sub(r'**' + r"\1" + r"**", t) @@ -116,31 +71,40 @@ def rst_ify(text): return t -_MARKDOWN = re.compile(r"[*_`]") +##################################################################################### -def markdown_ify(text): +def html_ify(text): + ''' convert symbols like I(this is in italics) to valid HTML ''' t = cgi.escape(text) - t = _MARKDOWN.sub(r"\\\g<0>", t) - t = _ITALIC.sub("_" + r"\1" + "_", t) - t = _BOLD.sub("**" + r"\1" + "**", t) - t = _MODULE.sub("*" + r"\1" + "*", t) - t = _URL.sub("[" + r"\1" + "](" + r"\1" + ")", t) - t = _CONST.sub("`" + r"\1" + "`", t) + t = _ITALIC.sub("<em>" + r"\1" + "</em>", t) + t = _BOLD.sub("<b>" + r"\1" + "</b>", t) + t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t) + t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t) + t = _CONST.sub("<code>" + r"\1" + "</code>", t) return t -# Helper for Jinja2 (format() doesn't work here...) + +##################################################################################### + def rst_fmt(text, fmt): + ''' helper for Jinja2 to do format strings ''' + return fmt % (text) +##################################################################################### + def rst_xline(width, char="="): + ''' return a restructured text line of a given length ''' + return char * width -def load_examples_section(text): - return text.split('***BREAK***') +##################################################################################### def return_data(text, options, outputname, module): + ''' dumps module output to a file or the screen, as requested ''' + if options.output_dir is not None: f = open(os.path.join(options.output_dir, outputname % module), 'w') f.write(text.encode('utf-8')) @@ -148,15 +112,29 @@ def return_data(text, options, outputname, module): else: print text +##################################################################################### + def boilerplate(): + ''' prints the boilerplate for module docs ''' + if not os.path.exists(EXAMPLE_YAML): print >>sys.stderr, "Missing example boiler plate: %s" % EXAMPLE_YAML print "DOCUMENTATION = '''" print file(EXAMPLE_YAML).read() print "'''" print "" + print "" + print "EXAMPLES = '''" + print "# example of doing ___ from a playbook" + print "your_module: some_arg=1 other_arg=2" + print "'''" + print "" + +##################################################################################### def list_modules(module_dir): + ''' returns a hash of categories, each category being a hash of module names to file paths ''' + categories = {} files = glob.glob("%s/*" % module_dir) for d in files: @@ -171,271 +149,165 @@ def list_modules(module_dir): categories[category][module] = f return categories -def main(): +##################################################################################### + +def generate_parser(): + ''' generate an optparse parser ''' p = optparse.OptionParser( version='%prog 1.0', usage='usage: %prog [options] arg1 arg2', - description='Convert Ansible module DOCUMENTATION strings to other formats', + description='Generate module documentation from metadata', ) - p.add_option("-A", "--ansible-version", - action="store", - dest="ansible_version", - default="unknown", - help="Ansible version number") - p.add_option("-M", "--module-dir", - action="store", - dest="module_dir", - default=MODULEDIR, - help="Ansible modules/ directory") - p.add_option("-T", "--template-dir", - action="store", - dest="template_dir", - default="hacking/templates", - help="directory containing Jinja2 templates") - p.add_option("-t", "--type", - action='store', - dest='type', - choices=['html', 'latex', 'man', 'rst', 'json', 'markdown', 'js'], - default='latex', - help="Output type") - p.add_option("-m", "--module", - action='append', - default=[], - dest='module_list', - help="Add modules to process in module_dir") - p.add_option("-v", "--verbose", - action='store_true', - default=False, - help="Verbose") - p.add_option("-o", "--output-dir", - action="store", - dest="output_dir", - default=None, - help="Output directory for module files") - p.add_option("-I", "--includes-file", - action="store", - dest="includes_file", - default=None, - help="Create a file containing list of processed modules") - p.add_option("-G", "--generate", - action="store_true", - dest="do_boilerplate", - default=False, - help="generate boilerplate DOCUMENTATION to stdout") + p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number") + p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path") + p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates") + p.add_option("-t", "--type", action='store', dest='type', choices=['html', 'latex', 'man', 'rst', 'json', 'markdown', 'js'], default='latex', help="Document type") + p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose") + p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files") + p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules") + p.add_option("-G", "--generate", action="store_true", dest="do_boilerplate", default=False, help="generate boilerplate docs to stdout") p.add_option('-V', action='version', help='Show version number and exit') + return p - (options, args) = p.parse_args() - -# print "M: %s" % options.module_dir -# print "t: %s" % options.type -# print "m: %s" % options.module_list -# print "v: %s" % options.verbose - - if options.do_boilerplate: - boilerplate() - - print "" - print "EXAMPLES = '''" - print "# example of doing ___ from a playbook" - print "your_module: some_arg=1 other_arg=2" - print "'''" - print "" - - sys.exit(0) +##################################################################################### - if not options.module_dir: - print "Need module_dir" - sys.exit(1) - if not os.path.exists(options.module_dir): - print >>sys.stderr, "Module directory does not exist: %s" % options.module_dir - sys.exit(1) +def jinja2_environment(template_dir, typ): - - if not options.template_dir: - print "Need template_dir" - sys.exit(1) - - env = Environment(loader=FileSystemLoader(options.template_dir), + env = Environment(loader=FileSystemLoader(template_dir), variable_start_string="@{", variable_end_string="}@", trim_blocks=True, ) - env.globals['xline'] = rst_xline - if options.type == 'latex': - env.filters['convert_symbols_to_format'] = latex_ify - template = env.get_template('latex.j2') - outputname = "%s.tex" - includecmt = "" - includefmt = "%s\n" - if options.type == 'html': - env.filters['convert_symbols_to_format'] = html_ify - template = env.get_template('html.j2') - outputname = "%s.html" - includecmt = "" - includefmt = "" - if options.type == 'man': - env.filters['convert_symbols_to_format'] = man_ify - template = env.get_template('man.j2') - outputname = "ansible.%s.3" - includecmt = "" - includefmt = "" - if options.type == 'rst': + if typ == 'rst': env.filters['convert_symbols_to_format'] = rst_ify env.filters['html_ify'] = html_ify env.filters['fmt'] = rst_fmt env.filters['xline'] = rst_xline template = env.get_template('rst.j2') outputname = "%s.rst" - includecmt = ".. Generated by module_formatter\n" - includefmt = ".. include:: modules/%s.rst\n" - if options.type == 'json': - env.filters['convert_symbols_to_format'] = json_ify - outputname = "%s.json" - includecmt = "" - includefmt = "" - if options.type == 'js': - env.filters['convert_symbols_to_format'] = js_ify - template = env.get_template('js.j2') - outputname = "%s.js" - if options.type == 'markdown': - env.filters['convert_symbols_to_format'] = markdown_ify - env.filters['html_ify'] = html_ify - template = env.get_template('markdown.j2') - outputname = "%s.md" - includecmt = "" - includefmt = "" + else: + raise Exception("unknown module format type: %s" % typ) - if options.includes_file is not None and includefmt != "": - incfile = open(options.includes_file, "w") - incfile.write(includecmt) + return env, template, outputname - # Temporary variable required to genrate aggregated content in 'js' format. - js_data = [] +##################################################################################### - categories = list_modules(options.module_dir) - last_category = None - category_names = categories.keys() - category_names.sort() - - for category in category_names: - - module_map = categories[category] - - category = category.replace("_"," ") - category = category.title() +def process_module(module, options, env, template, outputname, module_map): + + print "rendering: %s" % module + + fname = module_map[module] + + # ignore files with extensions + if os.path.basename(fname).find(".") != -1: + return + + # use ansible core library to parse out doc metadata YAML and plaintext examples + doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose) + + # crash if module is missing documentation and not explicitly hidden from docs index + if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES: + sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) + sys.exit(1) + if doc is None: + return + + all_keys = [] - modules = module_map.keys() - modules.sort() + if not 'version_added' in doc: + sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module) + sys.exit(1) + + added = 0 + if doc['version_added'] == 'historical': + del doc['version_added'] + else: + added = doc['version_added'] + + # don't show version added information if it's too old to be called out + if added: + added_tokens = str(added).split(".") + added = added_tokens[0] + "." + added_tokens[1] + added_float = float(added) + if added and added_float < TO_OLD_TO_BE_NOTABLE: + del doc['version_added'] + + for (k,v) in doc['options'].iteritems(): + all_keys.append(k) + all_keys = sorted(all_keys) + doc['option_keys'] = all_keys + + doc['filename'] = fname + doc['docuri'] = doc['module'].replace('_', '-') + doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') + doc['ansible_version'] = options.ansible_version + doc['plainexamples'] = examples #plain text + + # here is where we build the table of contents... + + text = template.render(doc) + return_data(text, options, outputname, module) - for module in modules: +##################################################################################### - print "rendering: %s" % module +def process_category(category, categories, options, env, template, outputname): - fname = module_map[module] + module_map = categories[category] - if len(options.module_list): - if not module in options.module_list: - continue + # TODO: start a new category file - # fname = os.path.join(options.module_dir, module) + category = category.replace("_"," ") + category = category.title() + + modules = module_map.keys() + modules.sort() + + for module in modules: + process_module(module, options, env, template, outputname, module_map) + + # TODO: end a new category file + +##################################################################################### + +def validate_options(options): + ''' validate option parser options ''' + + if options.do_boilerplate: + boilerplate() + sys.exit(0) - extra = os.path.join("inc", "%s.tex" % module) + if not options.module_dir: + print >>sys.stderr, "--module-dir is required" + sys.exit(1) + if not os.path.exists(options.module_dir): + print >>sys.stderr, "--module-dir does not exist: %s" % options.module_dir + sys.exit(1) + if not options.template_dir: + print "--template-dir must be specified" + sys.exit(1) - # probably could just throw out everything with extensions - if fname.endswith(".swp") or fname.endswith(".orig") or fname.endswith(".rej"): - continue +##################################################################################### - # print " processing module source ---> %s" % fname +def main(): - if options.type == 'js': - if fname.endswith(".json"): - f = open(fname) - j = json.load(f) - f.close() - js_data.append(j) - continue + p = generate_parser() - doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose) + (options, args) = p.parse_args() + validate_options(options) - if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES: - print " while processing module source ---> %s" % fname - sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s ***\n" % module) - #sys.exit(1) + env, template, outputname = jinja2_environment(options.template_dir, options.type) - if not doc is None: + categories = list_modules(options.module_dir) + last_category = None + category_names = categories.keys() + category_names.sort() - all_keys = [] - - if not 'version_added' in doc: - sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module) - sys.exit(1) - - added = 0 - if doc['version_added'] == 'historical': - del doc['version_added'] - else: - added = doc['version_added'] - - # don't show version added information if it's too old to be called out - if added: - added_tokens = str(added).split(".") - added = added_tokens[0] + "." + added_tokens[1] - added_float = float(added) - if added and added_float < TO_OLD_TO_BE_NOTABLE: - del doc['version_added'] - - for (k,v) in doc['options'].iteritems(): - all_keys.append(k) - all_keys = sorted(all_keys) - doc['option_keys'] = all_keys - - doc['filename'] = fname - doc['docuri'] = doc['module'].replace('_', '-') - doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') - doc['ansible_version'] = options.ansible_version - doc['plainexamples'] = examples #plain text - - # BOOKMARK: here is where we build the table of contents... - - if options.includes_file is not None and includefmt != "": - - if last_category != category: - incfile.write("\n\n") - incfile.write(category) - incfile.write("\n") - incfile.write('`' * len(category)) - incfile.write("\n\n") - last_category = category - - incfile.write(includefmt % module) - - if options.verbose: - print json.dumps(doc, indent=4) - - - if options.type == 'latex': - if os.path.exists(extra): - f = open(extra) - extradata = f.read() - f.close() - doc['extradata'] = extradata - - if options.type == 'json': - text = json.dumps(doc, indent=2) - else: - text = template.render(doc) - - return_data(text, options, outputname, module) - - if options.type == 'js': - docs = {} - docs['json'] = json.dumps(js_data, indent=2) - text = template.render(docs) - return_data(text, options, outputname, 'modules') + for category in category_names: + process_category(category, categories, options, env, template, outputname) if __name__ == '__main__': main() diff --git a/hacking/templates/html.j2 b/hacking/templates/html.j2 deleted file mode 100644 index f80018bb6d..0000000000 --- a/hacking/templates/html.j2 +++ /dev/null @@ -1,7 +0,0 @@ -<!-- @{ module | upper }@ --> - -<h2>@{module}@</h2> - - {% for desc in description -%} - @{ desc | convert_symbols_to_format }@ - {% endfor %} diff --git a/hacking/templates/js.j2 b/hacking/templates/js.j2 deleted file mode 100644 index fa2d4f7f53..0000000000 --- a/hacking/templates/js.j2 +++ /dev/null @@ -1,5 +0,0 @@ -function AnsibleModules($scope) { - $scope.modules = @{ json }@; - - $scope.orderProp = "module"; -} diff --git a/hacking/templates/latex.j2 b/hacking/templates/latex.j2 deleted file mode 100644 index 5905477d60..0000000000 --- a/hacking/templates/latex.j2 +++ /dev/null @@ -1,76 +0,0 @@ -{# ------------------------------------------------------------------- - template for module_formatter.py for LaTeX output (Ansible Booklet) - by Jan-Piet Mens. - Note: nodes & code examples are omitted on purpose. - -------------------------------------------------------------------- #} -%--- @{ module | upper }@ ---- from @{ filename }@ --- - -%: -- module header -\mods{@{module}@}{@{docuri}@}{ - {% for desc in description -%} - @{ desc | convert_symbols_to_format }@ - {% endfor -%} - {% if version_added is defined -%} - (\I{* new in version @{ version_added }@}) - {% endif -%} - } - -%: -- module options - - - -{% if options %} -\begin{xlist}{abcdefghijklmno} - {% for (opt,v) in options.iteritems() %} - {% if v['required'] %} - \item[\man\,\C{@{ opt }@}] - {% else %} - \item[\opt\,\C{@{ opt }@}] - {% endif %} - - {# -------- option description ----------#} - {% for desc in v.description %} - @{ desc | convert_symbols_to_format }@ - {% endfor %} - {% if v['choices'] %} - \B{Choices}:\, - {% for choice in v['choices'] %}\C{@{ choice }@}{% if not loop.last %},{% else %}.{% endif %} - {% endfor %} - {% endif %} - {% if v['default'] %} - (default \C{@{ v['default'] }@}) - {% endif %} - {% if v['version_added'] is defined %} - (\I{* version @{ v['version_added'] }@}) - {% endif %} - {% endfor %} -\end{xlist} -{% endif %} - -{# --------------------------------------- -{% if notes %} - - {% for note in notes %} - \I{@{ note | convert_symbols_to_format }@} - {% endfor %} -{% endif %} - ----------------------------- #} - -{#------------------------------------------- - -{% if examples is defined -%} - {% for e in examples %} - \begin{extymeta} -@{ e['code'] }@ - \end{extymeta} - {% endfor %} -{% endif %} ------------------------------------ #} - -{% if extradata is defined %} -%--- BEGIN-EXTRADATA -\begin{extymeta} -@{ extradata }@ -\end{extymeta} -%----- END-EXTRADATA -{% endif %} diff --git a/hacking/templates/markdown.j2 b/hacking/templates/markdown.j2 deleted file mode 100644 index a734ece3ec..0000000000 --- a/hacking/templates/markdown.j2 +++ /dev/null @@ -1,64 +0,0 @@ -## @{ module | convert_symbols_to_format }@ - -{# ------------------------------------------ - # - # This is Github-flavored Markdown - # - --------------------------------------------#} - -{% if version_added is defined -%} -New in version @{ version_added }@. -{% endif %} - -{% for desc in description -%} -@{ desc | convert_symbols_to_format }@ - -{% endfor %} - -{% if options -%} -<table> -<tr> -<th class="head">parameter</th> -<th class="head">required</th> -<th class="head">default</th> -<th class="head">choices</th> -<th class="head">comments</th> -</tr> -{% for (k,v) in options.iteritems() %} -<tr> -<td>@{ k }@</td> -<td>{% if v.get('required', False) %}yes{% else %}no{% endif %}</td> -<td>{% if v['default'] %}@{ v['default'] }@{% endif %}</td> -<td><ul>{% for choice in v.get('choices',[]) -%}<li>@{ choice }@</li>{% endfor -%}</ul></td> -<td>{% for desc in v.description -%}@{ desc | html_ify }@{% endfor -%}{% if v['version_added'] %} (added in Ansible @{v['version_added']}@){% endif %}</td> -</tr> -{% endfor %} -</table> -{% endif %} - -{% if examples or plainexamples %} -#### Examples -{% endif %} - -{% for example in examples %} -{% if example['description'] %} -* @{ example['description'] | convert_symbols_to_format }@ -{% endif %} - -``` -@{ example['code'] }@ -``` -{% endfor %} -{% if plainexamples -%} -``` -@{ plainexamples }@ -``` -{% endif %} - -{% if notes %} -#### Notes -{% for note in notes %} -@{ note | convert_symbols_to_format }@ -{% endfor %} -{% endif %} - |