Retain vault password as bytes in 2.2 (#22378)

* Retain vault password as bytes in 2.2

Prior to 2.2.1, the vault password was read in as byes and then remained
bytes all the way through the code.  A bug existed where bytes and text
were mixed, leading to a traceback with non-ascii passwords.  In devel,
this was fixed by changing the read in password to text type to match
with our overall strategy of converting at the borders.  This was
backported to stable-2.2 for the 2.2.1 release.

On reflection, this should not have been backported as it causes
passwords which were originally non-utf-8 to become utf-8.  People will
then have their working 2.2.x vault files become in-accessible.

this commit pipes bytes all the way through the system for vault
password.  That way if a password is read in as a non-utf-8 character
sequence, it will continue to work in 2.2.2+.  This change is only for
the 2.2 branch, not for 2.3 and beyond.

Why not everywhere?  The reason is that non-utf-8 passwords will cause
problems when vault files are shared between systems or users.  If the
password is read from the prompt and one user/machine has a latin1
encoded locale while a second one has utf-8, the non-ascii password
typed in won't match between machines.  Deal with this by making sure
that when we encrypt the data, we always use valid utf-8.

Fixes #20398

(cherry picked from commit 5dcce0666a81917c68b76286685642fd72d84327)

4 files changed, 25 insertions, 27 deletions

diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py
index 7b50e9bbbb..a271043133 100644
--- a/lib/ansible/cli/__init__.py
+++ b/lib/ansible/cli/__init__.py
@@ -174,7 +174,7 @@ class CLI(with_metaclass(ABCMeta, object)):
 
         # enforce no newline chars at the end of passwords
         if vault_pass:
-            vault_pass = to_text(vault_pass, errors='surrogate_or_strict', nonstring='simplerepr').strip()
+            vault_pass = to_bytes(vault_pass, errors='surrogate_or_strict', nonstring='simplerepr').strip()
 
         return vault_pass
 
@@ -190,7 +190,7 @@ class CLI(with_metaclass(ABCMeta, object)):
             pass
 
         if new_vault_pass:
-            new_vault_pass = to_text(new_vault_pass, errors='surrogate_or_strict', nonstring='simplerepr').strip()
+            new_vault_pass = to_bytes(new_vault_pass, errors='surrogate_or_strict', nonstring='simplerepr').strip()
 
         return new_vault_pass
 
@@ -641,7 +641,7 @@ class CLI(with_metaclass(ABCMeta, object)):
             except (OSError, IOError) as e:
                 raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e))
 
-        return to_text(vault_pass, errors='surrogate_or_strict')
+        return vault_pass
 
     def get_opt(self, k, defval=""):
         """
diff --git a/lib/ansible/cli/adhoc.py b/lib/ansible/cli/adhoc.py
index df855a32b6..d5d9ed1b4d 100644
--- a/lib/ansible/cli/adhoc.py
+++ b/lib/ansible/cli/adhoc.py
@@ -105,7 +105,7 @@ class AdHocCLI(CLI):
 
         sshpass    = None
         becomepass = None
-        vault_pass = None
+        b_vault_pass = None
 
         self.normalize_become_options()
         (sshpass, becomepass) = self.ask_passwords()
@@ -115,11 +115,11 @@ class AdHocCLI(CLI):
 
         if self.options.vault_password_file:
             # read vault_pass from a file
-            vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
-            loader.set_vault_password(vault_pass)
+            b_vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
+            loader.set_vault_password(b_vault_pass)
         elif self.options.ask_vault_pass:
-            vault_pass = self.ask_vault_passwords()
-            loader.set_vault_password(vault_pass)
+            b_vault_pass = self.ask_vault_passwords()
+            loader.set_vault_password(b_vault_pass)
 
         variable_manager = VariableManager()
         variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options)
diff --git a/lib/ansible/cli/playbook.py b/lib/ansible/cli/playbook.py
index 274d42400d..9b50754e70 100644
--- a/lib/ansible/cli/playbook.py
+++ b/lib/ansible/cli/playbook.py
@@ -90,7 +90,7 @@ class PlaybookCLI(CLI):
         # Manage passwords
         sshpass    = None
         becomepass    = None
-        vault_pass = None
+        b_vault_pass = None
         passwords = {}
 
         # initial error check, to make sure all specified playbooks are accessible
@@ -111,11 +111,11 @@ class PlaybookCLI(CLI):
 
         if self.options.vault_password_file:
             # read vault_pass from a file
-            vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
-            loader.set_vault_password(vault_pass)
+            b_vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
+            loader.set_vault_password(b_vault_pass)
         elif self.options.ask_vault_pass:
-            vault_pass = self.ask_vault_passwords()
-            loader.set_vault_password(vault_pass)
+            b_vault_pass = self.ask_vault_passwords()
+            loader.set_vault_password(b_vault_pass)
 
         # create the variable manager, which will be shared throughout
         # the code, ensuring a consistent view of global variables
diff --git a/lib/ansible/cli/vault.py b/lib/ansible/cli/vault.py
index b95db2c240..31579707c6 100644
--- a/lib/ansible/cli/vault.py
+++ b/lib/ansible/cli/vault.py
@@ -46,11 +46,9 @@ class VaultCLI(CLI):
 
     def __init__(self, args):
 
-        self.vault_pass = None
-        self.new_vault_pass = None
-
+        self.b_vault_pass = None
+        self.b_new_vault_pass = None
         self.encrypt_string_read_stdin = False
-
         super(VaultCLI, self).__init__(args)
 
     def parse(self):
@@ -128,29 +126,29 @@ class VaultCLI(CLI):
 
         if self.options.vault_password_file:
             # read vault_pass from a file
-            self.vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader)
+            self.b_vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader)
 
         if self.options.new_vault_password_file:
             # for rekey only
-            self.new_vault_pass = CLI.read_vault_password_file(self.options.new_vault_password_file, loader)
+            self.b_new_vault_pass = CLI.read_vault_password_file(self.options.new_vault_password_file, loader)
 
-        if not self.vault_pass or self.options.ask_vault_pass:
-            self.vault_pass = self.ask_vault_passwords()
+        if not self.b_vault_pass or self.options.ask_vault_pass:
+            self.b_vault_pass = self.ask_vault_passwords()
 
-        if not self.vault_pass:
+        if not self.b_vault_pass:
             raise AnsibleOptionsError("A password is required to use Ansible's Vault")
 
         if self.action == 'rekey':
-            if not self.new_vault_pass:
-                self.new_vault_pass = self.ask_new_vault_passwords()
-            if not self.new_vault_pass:
+            if not self.b_new_vault_pass:
+                self.b_new_vault_pass = self.ask_new_vault_passwords()
+            if not self.b_new_vault_pass:
                 raise AnsibleOptionsError("A password is required to rekey Ansible's Vault")
 
         if self.action == 'encrypt_string':
             if self.options.encrypt_string_prompt:
                 self.encrypt_string_prompt = True
 
-        self.editor = VaultEditor(self.vault_pass)
+        self.editor = VaultEditor(self.b_vault_pass)
 
         self.execute()
 
@@ -347,6 +345,6 @@ class VaultCLI(CLI):
                 raise AnsibleError(f + " does not exist")
 
         for f in self.args:
-            self.editor.rekey_file(f, self.new_vault_pass)
+            self.editor.rekey_file(f, self.b_new_vault_pass)
 
         display.display("Rekey successful", stderr=True)