[stable-2.10] Changes required for macOS 10.15 (#71841) (#71954)

* Use default group of staff on macos to prevent sudo issues
* Install gnu-tar for macos in git and unarchive tests
* Enable timezone module to support py3 on macos
* If the virtualenv command is missing, try python -m virtualenv
* Install passlib for filter_core on macos
* Install paramiko via pip on macos for paramiko tests
* Normalize discovered python interpreter on macos
* Get pip tests passing, by ensuring we have wheel installed
* Create /etc/ansible for ca certs on mac, list lookup_url as destructive
* Fixups for CA certs
* Include macos
* Dynamically get cafile instead of hardcoding the path
(cherry picked from commit 35b0fef536)

Co-authored-by: Matt Martz <matt@sivel.net>

17 files changed, 93 insertions, 20 deletions

diff --git a/shippable.yml b/shippable.yml
index 701e83c84f..9f19d381c1 100644
--- a/shippable.yml
+++ b/shippable.yml
@@ -27,7 +27,7 @@ matrix:
     - env: T=windows/2016/1
     - env: T=windows/2019/1
 
-    - env: T=osx/10.11/1
+    - env: T=macos/10.15/1
     - env: T=rhel/7.8/1
     - env: T=rhel/8.2/1
     - env: T=freebsd/11.1/1
@@ -42,7 +42,7 @@ matrix:
     - env: T=linux/ubuntu1604/1
     - env: T=linux/ubuntu1804/1
 
-    - env: T=osx/10.11/2
+    - env: T=macos/10.15/2
     - env: T=rhel/7.8/2
     - env: T=rhel/8.2/2
     - env: T=freebsd/11.1/2
@@ -57,7 +57,7 @@ matrix:
     - env: T=linux/ubuntu1604/2
     - env: T=linux/ubuntu1804/2
 
-    - env: T=osx/10.11/3
+    - env: T=macos/10.15/3
     - env: T=rhel/7.8/3
     - env: T=rhel/8.2/3
     - env: T=freebsd/11.1/3
@@ -72,7 +72,7 @@ matrix:
     - env: T=linux/ubuntu1604/3
     - env: T=linux/ubuntu1804/3
 
-    - env: T=osx/10.11/4
+    - env: T=macos/10.15/4
     - env: T=rhel/7.8/4
     - env: T=rhel/8.2/4
     - env: T=freebsd/11.1/4
@@ -87,7 +87,7 @@ matrix:
     - env: T=linux/ubuntu1604/4
     - env: T=linux/ubuntu1804/4
 
-    - env: T=osx/10.11/5
+    - env: T=macos/10.15/5
     - env: T=rhel/7.8/5
     - env: T=rhel/8.2/5
     - env: T=freebsd/11.1/5
diff --git a/test/integration/targets/become/tasks/default.yml b/test/integration/targets/become/tasks/default.yml
index 59568ed662..4ba10170dd 100644
--- a/test/integration/targets/become/tasks/default.yml
+++ b/test/integration/targets/become/tasks/default.yml
@@ -3,6 +3,7 @@
   become_user: root
   user:
     name: "{{ become_test_user }}"
+    group: '{{ "staff" if ansible_facts.distribution == "MacOSX" else omit }}'
 
 - name: test becoming user (become_method=default)
   shell: whoami
diff --git a/test/integration/targets/become/tasks/su.yml b/test/integration/targets/become/tasks/su.yml
index 01e40b6aa7..d314b0a87e 100644
--- a/test/integration/targets/become/tasks/su.yml
+++ b/test/integration/targets/become/tasks/su.yml
@@ -4,6 +4,7 @@
   become_method: su
   user:
     name: "{{ become_test_user }}"
+    group: '{{ "staff" if ansible_facts.distribution == "MacOSX" else omit }}'
 
 - name: test becoming user (become_method=su)
   shell: whoami
diff --git a/test/integration/targets/become/tasks/sudo.yml b/test/integration/targets/become/tasks/sudo.yml
index 8b0cce5a76..636ec378be 100644
--- a/test/integration/targets/become/tasks/sudo.yml
+++ b/test/integration/targets/become/tasks/sudo.yml
@@ -4,6 +4,7 @@
   become_method: sudo
   user:
     name: "{{ become_test_user }}"
+    group: '{{ "staff" if ansible_facts.distribution == "MacOSX" else omit }}'
 
 - name: test becoming user (become_method=sudo)
   shell: whoami
diff --git a/test/integration/targets/filter_core/meta/main.yml b/test/integration/targets/filter_core/meta/main.yml
new file mode 100644
index 0000000000..e430ea6f9a
--- /dev/null
+++ b/test/integration/targets/filter_core/meta/main.yml
@@ -0,0 +1,3 @@
+dependencies:
+  - role: setup_passlib
+    when: ansible_facts.distribution == 'MacOSX'
diff --git a/test/integration/targets/git/meta/main.yml b/test/integration/targets/git/meta/main.yml
index 07faa21776..34a77cb7e4 100644
--- a/test/integration/targets/git/meta/main.yml
+++ b/test/integration/targets/git/meta/main.yml
@@ -1,2 +1,3 @@
 dependencies:
   - prepare_tests
+  - setup_gnutar
diff --git a/test/integration/targets/interpreter_discovery_python/tasks/main.yml b/test/integration/targets/interpreter_discovery_python/tasks/main.yml
index efba1cd7b8..50c44dac8e 100644
--- a/test/integration/targets/interpreter_discovery_python/tasks/main.yml
+++ b/test/integration/targets/interpreter_discovery_python/tasks/main.yml
@@ -31,10 +31,23 @@
     test_echo_module:
     register: echoout
 
+  - when: distro == 'macosx'
+    block:
+      - name: Get the sys.executable for the macos discovered interpreter, as it may be different than the actual path
+        raw: '{{ auto_out.ansible_facts.discovered_interpreter_python }} -c "import sys; print(sys.executable)"'
+        register: discovered_sys_executable
+
+      - set_fact:
+          normalized_discovered_interpreter: '{{ discovered_sys_executable.stdout_lines[0] }}'
+
+  - set_fact:
+      normalized_discovered_interpreter: '{{ auto_out.ansible_facts.discovered_interpreter_python }}'
+    when: distro != 'macosx'
+
   - assert:
       that:
       - auto_out.ansible_facts.discovered_interpreter_python is defined
-      - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python
+      - echoout.running_python_interpreter == normalized_discovered_interpreter
       # verify that discovery didn't run again (if it did, we'd have the fact in the result)
       - echoout.ansible_facts is not defined or echoout.ansible_facts.discovered_interpreter_python is not defined
 
diff --git a/test/integration/targets/lookup_url/aliases b/test/integration/targets/lookup_url/aliases
index 9419339aa7..28990148bb 100644
--- a/test/integration/targets/lookup_url/aliases
+++ b/test/integration/targets/lookup_url/aliases
@@ -1,3 +1,4 @@
+destructive
 shippable/posix/group1
 needs/httptester
 skip/aix
diff --git a/test/integration/targets/pip/tasks/main.yml b/test/integration/targets/pip/tasks/main.yml
index 20e84d7943..05879c18a9 100644
--- a/test/integration/targets/pip/tasks/main.yml
+++ b/test/integration/targets/pip/tasks/main.yml
@@ -4,11 +4,17 @@
 - name: find virtualenv command
   command: "which virtualenv virtualenv-{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
   register: command
-  failed_when: not command.stdout_lines
+  ignore_errors: true
+
+- name: is virtualenv available to python -m
+  command: '{{ ansible_python_interpreter }} -m virtualenv'
+  register: python_m
+  when: not command.stdout_lines
+  failed_when: python_m.rc != 2
 
 - name: remember selected virtualenv command
   set_fact:
-    virtualenv: "{{ command.stdout_lines[0] }}"
+    virtualenv: "{{ command.stdout_lines[0] if command is successful else ansible_python_interpreter ~ ' -m virtualenv' }}"
 
 - block:
     - name: install git, needed for repo installs
@@ -18,6 +24,10 @@
       when: ansible_distribution != "MacOSX"
       register: git_install
 
+    - name: ensure wheel is installed
+      pip:
+        name: wheel
+
     - include_tasks: pip.yml
   always:
     - name: platform specific cleanup
diff --git a/test/integration/targets/prepare_http_tests/tasks/default.yml b/test/integration/targets/prepare_http_tests/tasks/default.yml
index 5f098e7206..bff903501b 100644
--- a/test/integration/targets/prepare_http_tests/tasks/default.yml
+++ b/test/integration/targets/prepare_http_tests/tasks/default.yml
@@ -49,11 +49,16 @@
   when: ansible_os_family == 'FreeBSD'
 
 - name: MacOS - Retrieve test cacert
-  get_url:
-    url: "http://ansible.http.tests/cacert.pem"
-    dest: "/usr/local/etc/openssl/certs/ansible.pem"
   when: ansible_os_family == 'Darwin'
+  block:
+    - uri:
+        url: "http://ansible.http.tests/cacert.pem"
+        return_content: true
+      register: cacert_pem
 
-- name: MacOS - Update ca certificates
-  command: /usr/local/opt/openssl/bin/c_rehash
-  when: ansible_os_family == 'Darwin'
+    - raw: '{{ ansible_python_interpreter }} -c "import ssl; print(ssl.get_default_verify_paths().cafile)"'
+      register: macos_cafile
+
+    - blockinfile:
+        path: "{{ macos_cafile.stdout_lines|first }}"
+        block: "{{ cacert_pem.content }}"
diff --git a/test/integration/targets/remote_tmp/playbook.yml b/test/integration/targets/remote_tmp/playbook.yml
index cfd44b75e6..43f99ca5f2 100644
--- a/test/integration/targets/remote_tmp/playbook.yml
+++ b/test/integration/targets/remote_tmp/playbook.yml
@@ -1,10 +1,12 @@
 - name: Test temp dir on de escalation
   hosts: testhost
-  gather_facts: false
   become: yes
   tasks:
     - name: create test user
-      user: name=tmptest state=present
+      user:
+        name: tmptest
+        state: present
+        group: '{{ "staff" if ansible_facts.distribution == "MacOSX" else omit }}'
 
     - name: execute test case
       become_user: tmptest
@@ -31,11 +33,11 @@
   tasks:
     - file:
         state: touch
-        path: "/{{ output_dir }}/65393"
+        path: "{{ output_dir }}/65393"
 
     - copy:
-        src: "/{{ output_dir }}/65393"
-        dest: "/{{ output_dir }}/65393.2"
+        src: "{{ output_dir }}/65393"
+        dest: "{{ output_dir }}/65393.2"
         remote_src: true
 
     - find:
diff --git a/test/integration/targets/setup_gnutar/handlers/main.yml b/test/integration/targets/setup_gnutar/handlers/main.yml
new file mode 100644
index 0000000000..d3fa7c27cd
--- /dev/null
+++ b/test/integration/targets/setup_gnutar/handlers/main.yml
@@ -0,0 +1,6 @@
+- name: uninstall gnu-tar
+  command: brew uninstall gnu-tar
+  become: yes
+  become_user: "{{ brew_stat.stat.pw_name }}"
+  environment:
+    HOMEBREW_NO_AUTO_UPDATE: True
diff --git a/test/integration/targets/setup_gnutar/tasks/main.yml b/test/integration/targets/setup_gnutar/tasks/main.yml
new file mode 100644
index 0000000000..b7d841cde6
--- /dev/null
+++ b/test/integration/targets/setup_gnutar/tasks/main.yml
@@ -0,0 +1,18 @@
+- when: ansible_facts.distribution == 'MacOSX'
+  block:
+    - name: MACOS | Find brew binary
+      command: which brew
+      register: brew_which
+
+    - name: MACOS | Get owner of brew binary
+      stat:
+        path: "{{ brew_which.stdout }}"
+      register: brew_stat
+
+    - command: brew install gnu-tar
+      become: yes
+      become_user: "{{ brew_stat.stat.pw_name }}"
+      environment:
+        HOMEBREW_NO_AUTO_UPDATE: True
+      notify:
+        - uninstall gnu-tar
diff --git a/test/integration/targets/setup_paramiko/install-MacOSX-10-python-3.yml b/test/integration/targets/setup_paramiko/install-MacOSX-10-python-3.yml
new file mode 100644
index 0000000000..a156f8066a
--- /dev/null
+++ b/test/integration/targets/setup_paramiko/install-MacOSX-10-python-3.yml
@@ -0,0 +1,6 @@
+- name: Install Paramiko for Python 3 on MacOS
+  pip: # no homebrew package manager in core, just use pip
+    name: paramiko
+  environment:
+    # Not sure why this fixes the test, but it does.
+    SETUPTOOLS_USE_DISTUTILS: stdlib
diff --git a/test/integration/targets/setup_paramiko/uninstall-MacOSX-10-python-3.yml b/test/integration/targets/setup_paramiko/uninstall-MacOSX-10-python-3.yml
new file mode 100644
index 0000000000..69a68e4209
--- /dev/null
+++ b/test/integration/targets/setup_paramiko/uninstall-MacOSX-10-python-3.yml
@@ -0,0 +1,4 @@
+- name: Uninstall Paramiko for Python 3 on MacOS
+  pip:
+    name: paramiko
+    state: absent
diff --git a/test/integration/targets/unarchive/meta/main.yml b/test/integration/targets/unarchive/meta/main.yml
index cb6005d042..56245b3d2d 100644
--- a/test/integration/targets/unarchive/meta/main.yml
+++ b/test/integration/targets/unarchive/meta/main.yml
@@ -1,3 +1,4 @@
 dependencies:
   - prepare_tests
   - setup_remote_tmp_dir
+  - setup_gnutar
diff --git a/test/support/integration/plugins/modules/timezone.py b/test/support/integration/plugins/modules/timezone.py
index d6fc1c76e7..b7439a1272 100644
--- a/test/support/integration/plugins/modules/timezone.py
+++ b/test/support/integration/plugins/modules/timezone.py
@@ -126,7 +126,7 @@ class Timezone(object):
                     module.fail_json(msg='Adjusting timezone is not supported in Global Zone')
 
             return super(Timezone, SmartOSTimezone).__new__(SmartOSTimezone)
-        elif re.match('^Darwin', platform.platform()):
+        elif platform.system() == 'Darwin':
             return super(Timezone, DarwinTimezone).__new__(DarwinTimezone)
         elif re.match('^(Free|Net|Open)BSD', platform.platform()):
             return super(Timezone, BSDTimezone).__new__(BSDTimezone)