diff options
author | Anthon van der Neut <anthon@mnt.org> | 2016-02-29 15:03:18 +0100 |
---|---|---|
committer | Anthon van der Neut <anthon@mnt.org> | 2016-02-29 15:03:18 +0100 |
commit | d9cb9dea8e9dff1f0c91d1f2ad0ef1ad9c7ed569 (patch) | |
tree | 9854b0e33f72ef281665623ab49267d20821952a | |
parent | e600daa78ebdd7e5b4dd39344822699f1f6721c8 (diff) | |
download | ruamel.yaml-d9cb9dea8e9dff1f0c91d1f2ad0ef1ad9c7ed569.tar.gz |
guessing routine, RTD support
- guessing routine now returns block_seq_indent guess as well
as indent guess
- split README.rst up into _doc/*.rst entries
-rw-r--r-- | .hgignore | 1 | ||||
-rw-r--r-- | README.rst | 355 | ||||
-rw-r--r-- | __init__.py | 4 | ||||
-rw-r--r-- | _doc/Makefile | 216 | ||||
-rw-r--r-- | _doc/conf.py | 287 | ||||
-rw-r--r-- | _doc/detail.rst | 239 | ||||
-rw-r--r-- | _doc/example.rst | 73 | ||||
-rw-r--r-- | _doc/index.rst | 23 | ||||
-rw-r--r-- | _doc/install.rst | 31 | ||||
-rw-r--r-- | _doc/links.rst | 7 | ||||
-rw-r--r-- | _doc/overview.rst | 31 | ||||
-rw-r--r-- | _doc/pyyaml.rst | 68 | ||||
-rw-r--r-- | _test/test_indentation.py | 32 | ||||
-rw-r--r-- | main.py | 17 | ||||
-rw-r--r-- | util.py | 21 |
15 files changed, 1045 insertions, 360 deletions
@@ -24,3 +24,4 @@ venv convert cmd TODO.rst +_doc/_build @@ -2,361 +2,8 @@ ruamel.yaml =========== +``ruamel.yaml`` is a YAML 1.2 loader/dumper package for Python. -**Starting with 0.11.0 the RoundTripLoader differentiates -between YAML 1.2 and YAML 1.1. This may cause compatibility problems, -see the "Document version support" section below.** - ----- - -Starting with 0.10.7 the package has been reorganised and the -command line utility is in its own package ``ruamel.yaml.cmd`` (so -installing ``ruamel.yaml`` doesn't pull in possibly irrelevant modules -only used in the command line utility) - -``ruamel.yaml`` is a YAML package for Python. It is a derivative -of Kirill Simonov's `PyYAML 3.11 <https://bitbucket.org/xi/pyyaml>`_ -which supports YAML1.1 - -Major differences with PyYAML 3.11: - -- integrated Python 2 and 3 sources, running on Python 2.6, 2.7 (CPython, - PyPy), 3.3 and 3.4. -- round trip mode that **includes comments** (block mode, key ordering kept) -- support for simple lists as mapping keys by transforming these to tuples -- ``!!omap`` generates ordereddict (C) on Python 2, collections.OrderedDict - on Python 3, and ``!!omap`` is generated for these types. -- some `YAML 1.2 <http://yaml.org/spec/1.2/spec.html>`_ enhancements - (``0o`` octal prefix, ``\/`` escape) -- scalars in lists are indented -- pep8 compliance -- tox and py.test based testing -- Tests whether the C yaml library is installed as well as the header - files. That library doesn't generate CommentTokens, so it cannot be used to - do round trip editing on comments. It can be used to speed up normal - processing (so you don't need to install ``ruamel.yaml`` and ``PyYaml``). - See the section *Optional requirements*. -- Basic support for multiline strings with preserved newlines and - chomping ( '``|``', '``|+``', '``|-``' ). As this subclasses the string type - the information is lost on reassignment. (This might be changed - in the future so that the preservation/folding/chomping is part of the - parent container, like comments). -- RoundTrip preservation of flow style sequences ( 'a: b, c, d') (based - on request and test by Anthony Sottile) -- anchors names that are hand-crafted (not of the form``idNNN``) are preserved -- `merges <http://yaml.org/type/merge.html>`_ in dictionaries are preserved -- adding/replacing comments on block-style sequences and mappings - with smart column positioning -- collection objects (when read in via RoundTripParser) have an ``lc`` - property that contains line and column info ``lc.line`` and ``lc.col``. - Individual positions for mappings and sequences can also be retrieved - (``lc.key('a')``, ``lc.value('a')`` resp. ``lc.item(3)``) -- preservation of whitelines after block scalars. Contributed by Sam Thursfield. - -Indentation of block sequences -============================== - -Although ruamel.yaml doesn't preserve individual indentations of block sequence -items, it does properly dump:: - - x: - - b: 1 - - 2 - -back to:: - - x: - - b: 1 - - 2 - -if you specify ``indent=4``. - -PyYAML (and older versions of ruamel.yaml) gives you indented scalars:: - - x: - - b: 1 - - 2 - -The dump routine also has an additional ``block_seq_indent`` parameter that -can be used to push the dash inwards, *within the space defined by* ``indent``. - -The above example with the often seen ``indent=4, block_seq_indent=2`` -indentation:: - - x: - - b: 1 - - 2 - - -If the ``block_seq_indent`` is only one less than the indent, there is -not enough room to put the space that has to follow the dash. In that -case the element is pushed to the next line. If you specify ``block_seq_indent>=indent``, then the emitter adjusts the ``indent`` value to equal -``block_seq_indent + 1``. - - -Document version support. -========================= - -In YAML a document version can be explicitly set by using:: - - %YAML 1.x - -before the document start (at the top or before a -``---``). For ``ruamel.yaml`` x has to be 1 or 2. If no explicit -version is set `version 1.2 <http://www.yaml.org/spec/1.2/spec.html>`_ -is assumed (which has been released in 2009). - -The 1.2 version does **not** support: - -- sexagesimals like ``12:34:56`` -- octals that start with 0 only: like ``012`` for number 10 (``0o12`` **is** - supported by YAML 1.2) -- Unquoted Yes and On as alternatives for True and No and Off for False. - -If you cannot change your YAML files and you need them to load as 1.1 -you can load with: - - ruamel.yaml.load(some_str, Loader=ruamel.yaml.RoundTripLoader, version=(1, 1)) - -or the equivalent (version can be a tuple, list or string): - - ruamel.yaml.round_trip_load(some_str, version="1.1") - -this also works for ``load_all``/``round_trip_load_all``. - -*If you cannot change your code, stick with ruamel.yaml==0.10.23 and let -me know if it would help to be able to set an environment variable.* - -This does not affect dump as ruamel.yaml never emitted sexagesimals, nor -octal numbers, and emitted booleans always as true resp. false - -Round trip including comments -============================= - -The major motivation for this fork is the round-trip capability for -comments. The integration of the sources was just an initial step to -make this easier. - -adding/replacing comments -------------------------- - -Starting with version 0.8, you can add/replace comments on block style -collections (mappings/sequences resuting in Python dict/list). The basic -for for this is:: - - from __future__ import print_function - - import ruamel.yaml - - inp = """\ - abc: - - a # comment 1 - xyz: - a: 1 # comment 2 - b: 2 - c: 3 - d: 4 - e: 5 - f: 6 # comment 3 - """ - - data = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader) - data['abc'].append('b') - data['abc'].yaml_add_eol_comment('comment 4', 1) # takes column of comment 1 - data['xyz'].yaml_add_eol_comment('comment 5', 'c') # takes column of comment 2 - data['xyz'].yaml_add_eol_comment('comment 6', 'e') # takes column of comment 3 - data['xyz'].yaml_add_eol_comment('comment 7', 'd', column=20) - - print(ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper), end='') - -.. example code add_comment.py - -Resulting in:: - - abc: - - a # comment 1 - - b # comment 4 - xyz: - a: 1 # comment 2 - b: 2 - c: 3 # comment 5 - d: 4 # comment 7 - e: 5 # comment 6 - f: 6 # comment 3 - - -.. example output add_comment.py - - -If the comment doesn't start with '#', this will be added. The key is -the element index for list, the actual key for dictionaries. As can be seen -from the example, the column to choose for a comment is derived -from the previous, next or preceding comment column (picking the first one -found). - -Config file formats -=================== - -There are only a few configuration file formats that are easily -readable and editable: JSON, INI/ConfigParser, YAML (XML is to cluttered -to be called easily readable). - -Unfortunately `JSON <http://www.json.org/>`_ doesn't support comments, -and although there are some solutions with pre-processed filtering of -comments, there are no libraries that support round trip updating of -such commented files. - -INI files support comments, and the excellent `ConfigObj -<http://www.voidspace.org.uk/python/configobj.html>`_ library by Foord -and Larosa even supports round trip editing with comment preservation, -nesting of sections and limited lists (within a value). Retrieval of -particular value format is explicit (and extensible). - -YAML has basic mapping and sequence structures as well as support for -ordered mappings and sets. It supports scalars various types -including dates and datetimes (missing in JSON). -YAML has comments, but these are normally thrown away. - -Block structured YAML is a clean and very human readable -format. By extending the Python YAML parser to support round trip -preservation of comments, it makes YAML a very good choice for -configuration files that are human readable and editable while at -the same time interpretable and modifiable by a program. - -Extending -========= - -There are normally six files involved when extending the roundtrip -capabilities: the reader, parser, composer and constructor to go from YAML to -Python and the resolver, representer, serializer and emitter to go the other -way. - -Extending involves keeping extra data around for the next process step, -eventuallly resulting in a different Python object (subclass or alternative), -that should behave like the original, but on the way from Python to YAML -generates the original (or at least something much closer). - -Smartening -========== - -When you use round-tripping, then the complex data you get are -already subclasses of the built-in types. So you can patch -in extra methods or override existing ones. Some methods are already -included and you can do:: - - yaml_str = """\ - a: - - b: - c: 42 - - d: - f: 196 - e: - g: 3.14 - """ - - - data = yaml.load(yaml_str, Loader=yaml.RoundTripLoader) - - assert data.mlget(['a', 1, 'd', 'f'], list_ok=True) == 196 - - -Examples -======== - -Basic round trip of parsing YAML to Python objects, modifying -and generating YAML:: - - from __future__ import print_function - - import ruamel.yaml - - inp = """\ - # example - name: - # details - family: Smith # very common - given: Alice # one of the siblings - """ - - code = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader) - code['name']['given'] = 'Bob' - - print(ruamel.yaml.dump(code, Dumper=ruamel.yaml.RoundTripDumper), end='') - -.. example code small.py - -Resulting in :: - - # example - name: - # details - family: Smith # very common - given: Bob # one of the siblings - - -.. example output small.py - - -YAML handcrafted anchors and references as well as key merging -is preserved. The merged keys can transparently be accessed -using ``[]`` and ``.get()``:: - - import ruamel.yaml - - inp = """\ - - &CENTER {x: 1, y: 2} - - &LEFT {x: 0, y: 2} - - &BIG {r: 10} - - &SMALL {r: 1} - # All the following maps are equal: - # Explicit keys - - x: 1 - y: 2 - r: 10 - label: center/big - # Merge one map - - <<: *CENTER - r: 10 - label: center/big - # Merge multiple maps - - <<: [*CENTER, *BIG] - label: center/big - # Override - - <<: [*BIG, *LEFT, *SMALL] - x: 1 - label: center/big - """ - - data = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader) - assert data[7]['y'] == 2 - - -.. example code anchor_merge.py - - -Optional requirements -===================== - -If you have the C yaml library and headers installed, as well as -the header files for your Python executables then you can use the -non-roundtrip but faster C loader and emitter. - -On Debian systems you should use:: - - sudo apt-get install libyaml-dev python-dev python3-dev - -you can leave out ``python3-dev`` if you don't use python3 - -For CentOS (7) based systems you should do:: - - sudo yum install libyaml-devel python-devel - -Testing -======= - -Testing is done using `tox <https://pypi.python.org/pypi/tox>`_, which -uses `virtualenv <https://pypi.python.org/pypi/virtualenv>`_ and -`pytest <http://pytest.org/latest/>`_. ChangeLog ========= diff --git a/__init__.py b/__init__.py index a273f61..ba09dfe 100644 --- a/__init__.py +++ b/__init__.py @@ -9,13 +9,13 @@ from __future__ import absolute_import _package_data = dict( full_package_name="ruamel.yaml", - version_info=(0, 11, 3), + version_info=(0, 11, 4, "dev"), author="Anthon van der Neut", author_email="a.van.der.neut@ruamel.eu", description="ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order", # NOQA entry_points=None, install_requires=dict( - any=["ruamel.base>=1.0.0"], + any=[], py26=["ruamel.ordereddict"], py27=["ruamel.ordereddict"] ), diff --git a/_doc/Makefile b/_doc/Makefile new file mode 100644 index 0000000..630532d --- /dev/null +++ b/_doc/Makefile @@ -0,0 +1,216 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/yaml.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/yaml.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/yaml" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/yaml" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/_doc/conf.py b/_doc/conf.py new file mode 100644 index 0000000..a40a614 --- /dev/null +++ b/_doc/conf.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +# +# yaml documentation build configuration file, created by +# sphinx-quickstart on Mon Feb 29 12:03:00 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'yaml' +copyright = u'2016, Anthon van der Neut, Ruamel bvba' +author = u'Anthon van der Neut' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +try: + from ruamel.yaml import __version__, version_info + # The short X.Y version. + version = '.'.join(version_info[:2]) + # The full version, including alpha/beta/rc tags. + release = __version__ +except: + version = release = 'dev' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'yamldoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'yaml.tex', u'yaml Documentation', + u'Anthon van der Neut', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'yaml', u'yaml Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'yaml', u'yaml Documentation', + author, 'yaml', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/_doc/detail.rst b/_doc/detail.rst new file mode 100644 index 0000000..efcdac7 --- /dev/null +++ b/_doc/detail.rst @@ -0,0 +1,239 @@ +Details +======= + + + +- support for simple lists as mapping keys by transforming these to tuples +- ``!!omap`` generates ordereddict (C) on Python 2, collections.OrderedDict + on Python 3, and ``!!omap`` is generated for these types. +- Tests whether the C yaml library is installed as well as the header + files. That library doesn't generate CommentTokens, so it cannot be used to + do round trip editing on comments. It can be used to speed up normal + processing (so you don't need to install ``ruamel.yaml`` and ``PyYaml``). + See the section *Optional requirements*. +- Basic support for multiline strings with preserved newlines and + chomping ( '``|``', '``|+``', '``|-``' ). As this subclasses the string type + the information is lost on reassignment. (This might be changed + in the future so that the preservation/folding/chomping is part of the + parent container, like comments). +- anchors names that are hand-crafted (not of the form``idNNN``) are preserved +- `merges <http://yaml.org/type/merge.html>`_ in dictionaries are preserved +- adding/replacing comments on block-style sequences and mappings + with smart column positioning +- collection objects (when read in via RoundTripParser) have an ``lc`` + property that contains line and column info ``lc.line`` and ``lc.col``. + Individual positions for mappings and sequences can also be retrieved + (``lc.key('a')``, ``lc.value('a')`` resp. ``lc.item(3)``) +- preservation of whitelines after block scalars. Contributed by Sam Thursfield. + + + +Indentation of block sequences +------------------------------ + +Although ruamel.yaml doesn't preserve individual indentations of block sequence +items, it does properly dump:: + + x: + - b: 1 + - 2 + +back to:: + + x: + - b: 1 + - 2 + +if you specify ``indent=4``. + +PyYAML (and older versions of ruamel.yaml) gives you non-indented +scalars (when specifying default_flow_style=False):: + + x: + - b: 1 + - 2 + +The dump routine also has an additional ``block_seq_indent`` parameter that +can be used to push the dash inwards, *within the space defined by* ``indent``. + +The above example with the often seen ``indent=4, block_seq_indent=2`` +indentation:: + + x: + - b: 1 + - 2 + + +If the ``block_seq_indent`` is only one less than the indent, there is +not enough room to put the space that has to follow the dash. In that +case the element is pushed to the next line. If you specify ``block_seq_indent>=indent``, then the emitter adjusts the ``indent`` value to equal +``block_seq_indent + 1``. + + +Document version support. +------------------------- + +In YAML a document version can be explicitly set by using:: + + %YAML 1.x + +before the document start (at the top or before a +``---``). For ``ruamel.yaml`` x has to be 1 or 2. If no explicit +version is set `version 1.2 <http://www.yaml.org/spec/1.2/spec.html>`_ +is assumed (which has been released in 2009). + +The 1.2 version does **not** support: + +- sexagesimals like ``12:34:56`` +- octals that start with 0 only: like ``012`` for number 10 (``0o12`` **is** + supported by YAML 1.2) +- Unquoted Yes and On as alternatives for True and No and Off for False. + +If you cannot change your YAML files and you need them to load as 1.1 +you can load with: + + ruamel.yaml.load(some_str, Loader=ruamel.yaml.RoundTripLoader, version=(1, 1)) + +or the equivalent (version can be a tuple, list or string): + + ruamel.yaml.round_trip_load(some_str, version="1.1") + +this also works for ``load_all``/``round_trip_load_all``. + +*If you cannot change your code, stick with ruamel.yaml==0.10.23 and let +me know if it would help to be able to set an environment variable.* + +This does not affect dump as ruamel.yaml never emitted sexagesimals, nor +octal numbers, and emitted booleans always as true resp. false + +Round trip including comments +----------------------------- + +The major motivation for this fork is the round-trip capability for +comments. The integration of the sources was just an initial step to +make this easier. + +adding/replacing comments +......................... + +Starting with version 0.8, you can add/replace comments on block style +collections (mappings/sequences resuting in Python dict/list). The basic +for for this is:: + + from __future__ import print_function + + import ruamel.yaml + + inp = """\ + abc: + - a # comment 1 + xyz: + a: 1 # comment 2 + b: 2 + c: 3 + d: 4 + e: 5 + f: 6 # comment 3 + """ + + data = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader) + data['abc'].append('b') + data['abc'].yaml_add_eol_comment('comment 4', 1) # takes column of comment 1 + data['xyz'].yaml_add_eol_comment('comment 5', 'c') # takes column of comment 2 + data['xyz'].yaml_add_eol_comment('comment 6', 'e') # takes column of comment 3 + data['xyz'].yaml_add_eol_comment('comment 7', 'd', column=20) + + print(ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper), end='') + +.. example code add_comment.py + +Resulting in:: + + abc: + - a # comment 1 + - b # comment 4 + xyz: + a: 1 # comment 2 + b: 2 + c: 3 # comment 5 + d: 4 # comment 7 + e: 5 # comment 6 + f: 6 # comment 3 + + +.. example output add_comment.py + + +If the comment doesn't start with '#', this will be added. The key is +the element index for list, the actual key for dictionaries. As can be seen +from the example, the column to choose for a comment is derived +from the previous, next or preceding comment column (picking the first one +found). + +Config file formats +------------------- + +There are only a few configuration file formats that are easily +readable and editable: JSON, INI/ConfigParser, YAML (XML is to cluttered +to be called easily readable). + +Unfortunately `JSON <http://www.json.org/>`_ doesn't support comments, +and although there are some solutions with pre-processed filtering of +comments, there are no libraries that support round trip updating of +such commented files. + +INI files support comments, and the excellent `ConfigObj +<http://www.voidspace.org.uk/python/configobj.html>`_ library by Foord +and Larosa even supports round trip editing with comment preservation, +nesting of sections and limited lists (within a value). Retrieval of +particular value format is explicit (and extensible). + +YAML has basic mapping and sequence structures as well as support for +ordered mappings and sets. It supports scalars various types +including dates and datetimes (missing in JSON). +YAML has comments, but these are normally thrown away. + +Block structured YAML is a clean and very human readable +format. By extending the Python YAML parser to support round trip +preservation of comments, it makes YAML a very good choice for +configuration files that are human readable and editable while at +the same time interpretable and modifiable by a program. + +Extending +--------- + +There are normally six files involved when extending the roundtrip +capabilities: the reader, parser, composer and constructor to go from YAML to +Python and the resolver, representer, serializer and emitter to go the other +way. + +Extending involves keeping extra data around for the next process step, +eventuallly resulting in a different Python object (subclass or alternative), +that should behave like the original, but on the way from Python to YAML +generates the original (or at least something much closer). + +Smartening +---------- + +When you use round-tripping, then the complex data you get are +already subclasses of the built-in types. So you can patch +in extra methods or override existing ones. Some methods are already +included and you can do:: + + yaml_str = """\ + a: + - b: + c: 42 + - d: + f: 196 + e: + g: 3.14 + """ + + + data = yaml.load(yaml_str, Loader=yaml.RoundTripLoader) + + assert data.mlget(['a', 1, 'd', 'f'], list_ok=True) == 196 + + + diff --git a/_doc/example.rst b/_doc/example.rst new file mode 100644 index 0000000..8bf9c6a --- /dev/null +++ b/_doc/example.rst @@ -0,0 +1,73 @@ +Examples +======== + +Basic round trip of parsing YAML to Python objects, modifying +and generating YAML:: + + from __future__ import print_function + + import ruamel.yaml + + inp = """\ + # example + name: + # details + family: Smith # very common + given: Alice # one of the siblings + """ + + code = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader) + code['name']['given'] = 'Bob' + + print(ruamel.yaml.dump(code, Dumper=ruamel.yaml.RoundTripDumper), end='') + +.. example code small.py + +Resulting in :: + + # example + name: + # details + family: Smith # very common + given: Bob # one of the siblings + + +.. example output small.py + + +YAML handcrafted anchors and references as well as key merging +is preserved. The merged keys can transparently be accessed +using ``[]`` and ``.get()``:: + + import ruamel.yaml + + inp = """\ + - &CENTER {x: 1, y: 2} + - &LEFT {x: 0, y: 2} + - &BIG {r: 10} + - &SMALL {r: 1} + # All the following maps are equal: + # Explicit keys + - x: 1 + y: 2 + r: 10 + label: center/big + # Merge one map + - <<: *CENTER + r: 10 + label: center/big + # Merge multiple maps + - <<: [*CENTER, *BIG] + label: center/big + # Override + - <<: [*BIG, *LEFT, *SMALL] + x: 1 + label: center/big + """ + + data = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader) + assert data[7]['y'] == 2 + + +.. example code anchor_merge.py + diff --git a/_doc/index.rst b/_doc/index.rst new file mode 100644 index 0000000..6bfbdd3 --- /dev/null +++ b/_doc/index.rst @@ -0,0 +1,23 @@ + +ruamel.yaml +=========== + +`BitBucket <https://bitbucket.org/ruamel/yaml>`_ | +`PyPI <https://pypi.python.org/pypi/ruamel.yaml/>`_ + + +Contents: + +.. toctree:: + :maxdepth: 2 + + overview + install + detail + example + pyyaml + + + + + diff --git a/_doc/install.rst b/_doc/install.rst new file mode 100644 index 0000000..be3c4f4 --- /dev/null +++ b/_doc/install.rst @@ -0,0 +1,31 @@ +Installing +========== + +``ruamel.yaml`` can be installed from PyPI_ using:: + + pip install ruamel.yaml + +There is a a commandline utility ``yaml`` available after installing:: + + pip install ruamel.yaml.cmd + +that allows for round-trip testing/re-indenting and conversion of YAML +files (JSON,INI,HTML tables) + +Optional requirements +--------------------- + +If you have the C yaml library and headers installed, as well as +the header files for your Python executables then you can use the +non-roundtrip but faster C loader and emitter. + +On Debian systems you should use:: + + sudo apt-get install libyaml-dev python-dev python3-dev + +you can leave out ``python3-dev`` if you don't use python3 + +For CentOS (7) based systems you should do:: + + sudo yum install libyaml-devel python-devel + diff --git a/_doc/links.rst b/_doc/links.rst new file mode 100644 index 0000000..c3dbb1b --- /dev/null +++ b/_doc/links.rst @@ -0,0 +1,7 @@ +.. _tox: https://pypi.python.org/pypi/tox +.. _py.test: http://pytest.org/latest/ +.. _YAML 1.1: http://www.yaml.org/spec/1.1/spec.html +.. _YAML 1.2: http://www.yaml.org/spec/1.2/spec.html +.. _PyPI: https: https://pypi.python.org/pypi +.. _ruamel.yaml: https://pypi.python.org/pypi/ruamel.yaml + diff --git a/_doc/overview.rst b/_doc/overview.rst new file mode 100644 index 0000000..1c664e0 --- /dev/null +++ b/_doc/overview.rst @@ -0,0 +1,31 @@ +Overview +======== + +``ruamel.yaml`` is a YAML 1.2 loader/dumper package for Python. It is a +derivative of Kirill Simonov's `PyYAML 3.11 +<https://bitbucket.org/xi/pyyaml>`_ + +``ruamel.yaml`` supports `YAML 1.2`_ and has round-trip loaders and dumpers +that preserves, among others: + +- comments +- block style and key ordering are kept, so you can diff the round-tripped + source +- flow style sequences ( 'a: b, c, d') (based on request and test by + Anthony Sottile) +- anchors names that are hand-crafted (i.e. not of the form``idNNN``) +- `merges <http://yaml.org/type/merge.html>`_ in dictionaries are preserved + +This preservation is normally not broken unless you severely alter +the structure of a component (delete a key in a dict, remove list entries). +Reassigning values or replacing list items, etc., is fine. + +For the specific 1.2 differences see :ref:`yaml-1-2-support` + +Although individual indentation is not preserved, you can specify both +indentation and specific offset of block sequence dashes within that +indentation. There is a utility function that tries to determine the +correct values for ``indent`` and ``block_seq_indent`` that can +then be passed to the dumper. + +.. include:: links.rst diff --git a/_doc/pyyaml.rst b/_doc/pyyaml.rst new file mode 100644 index 0000000..7182021 --- /dev/null +++ b/_doc/pyyaml.rst @@ -0,0 +1,68 @@ +Differences with PyYAML +======================= + +.. parsed-literal:: + + *If I have seen further, it is by standing on the shoulders of giants*. + Isaac Newton (1676) + + + +``ruamel.yaml`` is a derivative of Kirill Simonov's `PyYAML 3.11 +<https://bitbucket.org/xi/pyyaml>`_ and would not exist without that +excellent base to start from. + +The following a summary of the major differences with PyYAML 3.11 + +.. _yaml-1-2-support: + +Defaulting to YAML 1.2 support +------------------------------ + +PyYAML supports the `YAML 1.1`_ standard, ``ruamel.yaml`` supports +`YAML 1.2`_ as released in 2009. + +- YAML 1.2 dropped support for several features unquoted ``Yes``, + ``No``, ``On``, ``Off`` +- YAML 1.2 no longer accepts strings that start with a ``0`` and solely + consist of number characters as octal, you need to specify such strings with + ``0o[0-7]+`` (zero + lower-case o for octal + one or more octal characters). +- YAML 1.2 no longer supports `sexagesimals + <https://en.wikipedia.org/wiki/Sexagesimal>`_, so the string scalar + ``12:34:56`` doesn't need quoting. +- ``\/`` escape for JSON compatibility +- correct parsing of floating point scalars with exponentials + +unless the YAML document is loaded with an explicit ``version==1.1`` or +the document starts with:: + + % YAML 1.1 + +, ``ruamel.yaml`` will load the document as version 1.2. + + +PY2/PY3 reintegration +--------------------- + +``ruamel.yaml`` re-integrates the Python 2 and 3 sources, running on +Python 2.6, 2.7 (CPython, PyPy), 3.3, 3.4, 3.5. It is more easy to +extend and maintain as only a miniscule part of the code is version +specific. + +Fixes +----- + +- ``ruamel.yaml`` follows the ``indent`` keyword argument on scalars + when dumping. + +Testing +------- + +``ruamel.yaml`` is tested using `tox`_ and `py.test`_. In addition to +new tests the original PyYAML +test framework is called from within ``tox`` runs. + +Before versions are pushed to PyPI, ``tox`` is invoked, and has to pass, on all +Python versions, PyPI as well as flake8/pep8 + +.. include:: links.rst diff --git a/_test/test_indentation.py b/_test/test_indentation.py index 72ef8dc..8764109 100644 --- a/_test/test_indentation.py +++ b/_test/test_indentation.py @@ -10,6 +10,8 @@ from textwrap import dedent import pytest # NOQA import ruamel.yaml +from ruamel.yaml.util import load_yaml_guess_indent + from roundtrip import round_trip @@ -148,5 +150,35 @@ class TestIndent: 2 """, indent=2, block_seq_indent=2) +def guess(s): + x, y, z = load_yaml_guess_indent(dedent(s)) + return y, z + +class TestGuessIndent: + def test_guess_20(self): + assert guess("""\ + a: + - 1 + """) == (2, 0) + + def test_guess_42(self): + assert guess("""\ + a: + - 1 + """) == (4, 2) + + def test_guess_42(self): + assert guess("""\ + b: + a: + - 1 + """) == (4, 2) + + def test_guess_3None(self): + assert guess("""\ + b: + a: 1 + """) == (3, None) + # ############ indentation @@ -267,6 +267,23 @@ def safe_dump(data, stream=None, **kwds): """ return dump_all([data], stream, Dumper=SafeDumper, **kwds) +def round_trip_dump(data, stream=None, Dumper=RoundTripDumper, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=enc, explicit_start=None, explicit_end=None, + version=None, tags=None, block_seq_indent=None): + allow_unicode = True if allow_unicode is None else allow_unicode + return dump_all([data], stream, Dumper=Dumper, + default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, + line_break=line_break, + encoding=encoding, explicit_start=explicit_start, + explicit_end=explicit_end, + version=version, tags=tags, block_seq_indent=block_seq_indent) def add_implicit_resolver(tag, regexp, first=None, Loader=Loader, Dumper=Dumper): @@ -16,8 +16,14 @@ from .main import round_trip_load # if you use this in your code, I suggest adding a test in your test suite # that check this routines output against a known piece of your YAML # before upgrades to this code break your round-tripped YAML -def load_yaml_guess_indent(stream): +def load_yaml_guess_indent(stream, **kw): # load a yaml file guess the indentation, if you use TABs ... + def leading_spaces(l): + idx = 0 + while idx < len(l) and l[idx] == ' ': + idx += 1 + return idx + if isinstance(stream, text_type): yaml_str = stream elif isinstance(stream, binary_type): @@ -25,18 +31,23 @@ def load_yaml_guess_indent(stream): else: yaml_str = stream.read() indent = None # default if not found for some reason + block_seq_indent = None prev_line_key_only = None + key_indent = 0 for line in yaml_str.splitlines(): rline = line.rstrip() - if rline.startswith('- '): + lline = rline.lstrip() + if lline.startswith('- '): + block_seq_indent = leading_spaces(line) - key_indent idx = 1 while line[idx] == ' ': # this will end as we rstripped idx += 1 if line[idx] == '#': # comment after - continue - indent = idx + indent = idx - key_indent break if rline.endswith(':'): + key_indent = leading_spaces(line) idx = 0 while line[idx] == ' ': # this will end on ':' idx += 1 @@ -52,7 +63,9 @@ def load_yaml_guess_indent(stream): else: indent = 2 prev_line_key_only = None - return round_trip_load(yaml_str), indent + if block_seq_indent: + indent += block_seq_indent + return round_trip_load(yaml_str, **kw), indent, block_seq_indent def configobj_walker(cfg): |