Refactor tox & update docs

- Updated tox envlist, so just running `tox` from the CLI will now run all
voting gate tests

- Reduce duplicated definitions and commands

- Remove any reliance on run_tests within tox

- Removes all doc references to run_tests.sh, and replaces them
with their tox equivalent. Where necessary, language around the tox
commands has been altered or extended so that it makes sense and is
consistent with other parts of the docs. Also adds a new "Test Environment"
list to the docs, so that newcomers do not have to piece together CLI
commands and their cryptic extensions from tox.ini

- Move the inline shell scripting to its own file. Also fixes a bug when
passing args, since the logic assumed you were attempting a subset test
run (try `tox -e py27 -- --pdb` on master to compare)

- Moved translation tooling from run_tests to manage.py, w/ help text
and arg restrictions. This is much more flexible so that plugins can use
it without having to copy commands, but still defaults to exactly the
same parameters/behaviour from run_tests. Docs updated appropriately.

- Removed npm/karma strange reliance on either .venv or tox/py27. Now
it only uses tox/npm.

Change-Id: I883f885bd424955d39ddcfde5ba396a88cfc041e
Implements: blueprint enhance-tox
Closes-Bug: 1638672

2 files changed, 179 insertions, 0 deletions

diff --git a/openstack_dashboard/management/commands/extract_messages.py b/openstack_dashboard/management/commands/extract_messages.py
new file mode 100644
index 000000000..f015f26b3
--- /dev/null
+++ b/openstack_dashboard/management/commands/extract_messages.py
@@ -0,0 +1,57 @@
+# Copyright 2016 Cisco Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distutils.dist import Distribution
+import os
+from subprocess import call
+
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = ('Extract strings that have been marked for translation into .POT '
+            'files.')
+
+    def add_arguments(self, parser):
+        parser.add_argument('-m', '--module', type=str, nargs='+',
+                            default=['openstack_dashboard', 'horizon'],
+                            help=("The target python module(s) to extract "
+                                  "strings from"))
+        parser.add_argument('-d', '--domain', choices=['django', 'djangojs'],
+                            nargs='+', default=['django', 'djangojs'],
+                            help="Domain(s) of the .pot file")
+        parser.add_argument('--check-only', action='store_true',
+                            help=("Checks that extraction works correctly, "
+                                  "then deletes the .pot file to avoid "
+                                  "polluting the source code"))
+
+    def handle(self, *args, **options):
+        cmd = ('python setup.py extract_messages -F babel-{domain}.cfg '
+               '-o {module}/locale/{domain}.pot')
+        distribution = Distribution()
+        distribution.parse_config_files(distribution.find_config_files())
+
+        if options['check_only']:
+            cmd += " ; rm {module}/locale/{domain}.pot"
+
+        for module in options['module']:
+            for domain in options['domain']:
+                potfile = '{module}/locale/{domain}.pot'.format(module=module,
+                                                                domain=domain)
+                if not os.path.exists(potfile):
+                    with open(potfile, 'wb') as f:
+                        f.write(b'')
+
+                call(cmd.format(module=module, domain=domain, potfile=potfile),
+                     shell=True)
diff --git a/openstack_dashboard/management/commands/update_catalog.py b/openstack_dashboard/management/commands/update_catalog.py
new file mode 100644
index 000000000..522dd7f28
--- /dev/null
+++ b/openstack_dashboard/management/commands/update_catalog.py
@@ -0,0 +1,122 @@
+# coding: utf-8
+
+# Copyright 2016 Cisco Systems, Inc.
+# Copyright 2015 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import babel.messages.catalog as catalog
+import os
+from subprocess import call
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.utils import translation
+
+LANGUAGE_CODES = [language[0] for language in settings.LANGUAGES
+                  if language[0] != 'en']
+POTFILE = "{module}/locale/{domain}.pot"
+POFILE = "{module}/locale/{locale}/LC_MESSAGES/{domain}.po"
+
+
+def translate(segment):
+    prefix = u""
+    # When the id starts with a newline the mo compiler enforces that
+    # the translated message must also start with a newline.  Make
+    # sure that doesn't get broken when prepending the bracket.
+    if segment.startswith('\n'):
+        prefix = u"\n"
+    orig_size = len(segment)
+    # Add extra expansion space based on recommendation from
+    # http://www-01.ibm.com/software/globalization/guidelines/a3.html
+    if orig_size < 20:
+        multiplier = 1
+    elif orig_size < 30:
+        multiplier = 0.8
+    elif orig_size < 50:
+        multiplier = 0.6
+    elif orig_size < 70:
+        multiplier = 0.4
+    else:
+        multiplier = 0.3
+    extra_length = int(max(0, (orig_size * multiplier) - 10))
+    extra_chars = "~" * extra_length
+    return u"{0}[~{1}~您好яшçあ{2}]".format(prefix, segment, extra_chars)
+
+
+class Command(BaseCommand):
+    help = 'Update a translation catalog for a specified language'
+
+    def add_arguments(self, parser):
+        parser.add_argument('-l', '--language', choices=LANGUAGE_CODES,
+                            default=LANGUAGE_CODES, nargs='+',
+                            help=("The language code(s) to pseudo translate"))
+        parser.add_argument('-m', '--module', type=str, nargs='+',
+                            default=['openstack_dashboard', 'horizon'],
+                            help=("The target python module(s) to extract "
+                                  "strings from"))
+        parser.add_argument('-d', '--domain', choices=['django', 'djangojs'],
+                            nargs='+', default=['django', 'djangojs'],
+                            help="Domain(s) of the .POT file")
+        parser.add_argument('--pseudo', action='store_true',
+                            help=("Creates a pseudo translation for the "
+                                  "specified locale, to check for "
+                                  "translatable string coverage"))
+
+    def handle(self, *args, **options):
+        for module in options['module']:
+            for domain in options['domain']:
+                potfile = POTFILE.format(module=module, domain=domain)
+
+                for language in options['language']:
+                    # Get the locale code for the language code given and
+                    # work around broken django conversion function
+                    locales = {'ko': 'ko_KR', 'pl': 'pl_PL', 'tr': 'tr_TR'}
+                    locale = locales.get(language,
+                                         translation.to_locale(language))
+                    pofile = POFILE.format(module=module, locale=locale,
+                                           domain=domain)
+
+                    # If this isn't a pseudo translation, execute pybabel
+                    if not options['pseudo']:
+                        if not os.path.exists(pofile):
+                            with open(pofile, 'wb') as fobj:
+                                fobj.write(b'')
+
+                        cmd = ('pybabel update -l {locale} -i {potfile} '
+                               '-o {pofile}').format(locale=locale,
+                                                     potfile=potfile,
+                                                     pofile=pofile)
+                        call(cmd, shell=True)
+                        continue
+
+                    # Pseudo translation logic
+                    with open(potfile, 'r') as f:
+                        pot_cat = pofile.read_po(f, ignore_obsolete=True)
+
+                    new_cat = catalog.Catalog(locale=locale,
+                                              last_translator="pseudo.py",
+                                              charset="utf-8")
+                    num_plurals = new_cat.num_plurals
+
+                    for msg in pot_cat:
+                        if msg.pluralizable:
+                            msg.string = [
+                                translate(u"{}:{}".format(i, msg.id[0]))
+                                for i in range(num_plurals)]
+                        else:
+                            msg.string = translate(msg.id)
+                        new_cat[msg.id] = msg
+
+                    with open(pofile, 'w') as f:
+                        pofile.write_po(f, new_cat, ignore_obsolete=True)