* adds commit replace with config file for iosxr (#35564)

* * adds commit replace with config file for iosxr
* fixes dci failure in iosxr_logging

* * review comment changes

7 files changed, 122 insertions, 48 deletions

diff --git a/lib/ansible/module_utils/network/iosxr/iosxr.py b/lib/ansible/module_utils/network/iosxr/iosxr.py
index 114f848132..e37a0d8485 100644
--- a/lib/ansible/module_utils/network/iosxr/iosxr.py
+++ b/lib/ansible/module_utils/network/iosxr/iosxr.py
@@ -29,6 +29,7 @@
 import json
 from difflib import Differ
 from copy import deepcopy
+from time import sleep
 
 from ansible.module_utils._text import to_text, to_bytes
 from ansible.module_utils.basic import env_fallback
@@ -415,7 +416,14 @@ def load_config(module, command_filter, commit=False, replace=False,
         if module._diff:
             diff = get_config_diff(module)
 
-        if commit:
+        if replace:
+            cmd = list()
+            cmd.append({'command': 'commit replace',
+                        'prompt': 'This commit will replace or remove the entire running configuration',
+                        'answer': 'yes'})
+            cmd.append('end')
+            conn.edit_config(cmd)
+        elif commit:
             commit_config(module, comment=comment)
             conn.edit_config('end')
         else:
@@ -428,20 +436,36 @@ def run_command(module, commands):
     conn = get_connection(module)
     responses = list()
     for cmd in to_list(commands):
+
         try:
-            cmd = json.loads(cmd)
-            command = cmd['command']
-            prompt = cmd['prompt']
-            answer = cmd['answer']
+            if isinstance(cmd, str):
+                cmd = json.loads(cmd)
+            command = cmd.get('command', None)
+            prompt = cmd.get('prompt', None)
+            answer = cmd.get('answer', None)
+            sendonly = cmd.get('sendonly', False)
+            newline = cmd.get('newline', True)
         except:
             command = cmd
             prompt = None
             answer = None
+            sendonly = False
+            newline = True
 
-        out = conn.get(command, prompt, answer)
+        out = conn.get(command, prompt=prompt, answer=answer, sendonly=sendonly, newline=newline)
 
         try:
             responses.append(to_text(out, errors='surrogate_or_strict'))
         except UnicodeError:
             module.fail_json(msg=u'failed to decode output from {0}:{1}'.format(cmd, to_text(out)))
     return responses
+
+
+def copy_file(module, src, dst, proto='scp'):
+    conn = get_connection(module)
+    conn.copy_file(source=src, destination=dst, proto=proto)
+
+
+def get_file(module, src, dst, proto='scp'):
+    conn = get_connection(module)
+    conn.get_file(source=src, destination=dst, proto=proto)
diff --git a/lib/ansible/modules/network/iosxr/iosxr_command.py b/lib/ansible/modules/network/iosxr/iosxr_command.py
index 5fca0fdb26..fea39f9ef7 100644
--- a/lib/ansible/modules/network/iosxr/iosxr_command.py
+++ b/lib/ansible/modules/network/iosxr/iosxr_command.py
@@ -94,7 +94,7 @@ tasks:
       commands:
         - show version
         - show interfaces
-        - [{ command: example command that prompts, prompt: expected prompt, answer: yes}]
+        - { command: example command that prompts, prompt: expected prompt, answer: yes}
 
   - name: run multiple commands and evaluate the output
     iosxr_command:
diff --git a/lib/ansible/modules/network/iosxr/iosxr_config.py b/lib/ansible/modules/network/iosxr/iosxr_config.py
index 7dc747e661..057de6ae3e 100644
--- a/lib/ansible/modules/network/iosxr/iosxr_config.py
+++ b/lib/ansible/modules/network/iosxr/iosxr_config.py
@@ -25,7 +25,9 @@ description:
     a deterministic way.
 extends_documentation_fragment: iosxr
 notes:
-  - Tested against IOS XR 6.1.2
+  - Tested against IOS XRv 6.1.2
+  - Avoid service disrupting changes (viz. Management IP) from config replace.
+  - Do not use C(end) in the replace config file.
 options:
   lines:
     description:
@@ -164,6 +166,7 @@ EXAMPLES = """
 - name: load a config from disk and replace the current config
   iosxr_config:
     src: config.cfg
+    replace: config
     backup: yes
 """
 
@@ -181,13 +184,26 @@ backup_path:
 """
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.iosxr.iosxr import load_config, get_config
-from ansible.module_utils.network.iosxr.iosxr import iosxr_argument_spec
+from ansible.module_utils.network.iosxr.iosxr import iosxr_argument_spec, copy_file
 from ansible.module_utils.network.common.config import NetworkConfig, dumps
 
-
 DEFAULT_COMMIT_COMMENT = 'configured by iosxr_config'
 
 
+def copy_file_to_node(module):
+    """ Copy config file to IOS-XR node. We use SFTP because older IOS-XR versions don't handle SCP very well.
+    """
+    src = '/tmp/ansible_config.txt'
+    file = open(src, 'wb')
+    file.write(module.params['src'])
+    file.close()
+
+    dst = '/harddisk:/ansible_config.txt'
+    copy_file(module, src, dst, 'sftp')
+
+    return True
+
+
 def check_args(module, warnings):
     if module.params['comment']:
         if len(module.params['comment']) > 60:
@@ -224,18 +240,30 @@ def run(module, result):
     admin = module.params['admin']
     check_mode = module.check_mode
 
-    candidate = get_candidate(module)
+    candidate_config = get_candidate(module)
+    running_config = get_running_config(module)
 
+    commands = None
     if match != 'none' and replace != 'config':
-        contents = get_running_config(module)
-        configobj = NetworkConfig(contents=contents, indent=1)
-        commands = candidate.difference(configobj, path=path, match=match,
-                                        replace=replace)
+        commands = candidate_config.difference(running_config, path=path, match=match, replace=replace)
+    elif replace_config:
+        can_config = candidate_config.difference(running_config, path=path, match=match, replace=replace)
+        candidate = dumps(can_config, 'commands').split('\n')
+        run_config = running_config.difference(candidate_config, path=path, match=match, replace=replace)
+        running = dumps(run_config, 'commands').split('\n')
+
+        if len(candidate) > 1 or len(running) > 1:
+            ret = copy_file_to_node(module)
+            if not ret:
+                module.fail_json(msg='Copy of config file to the node failed')
+
+            commands = ['load harddisk:/ansible_config.txt']
     else:
-        commands = candidate.items
+        commands = candidate_config.items
 
     if commands:
-        commands = dumps(commands, 'commands').split('\n')
+        if not replace_config:
+            commands = dumps(commands, 'commands').split('\n')
 
         if any((module.params['lines'], module.params['src'])):
             if module.params['before']:
@@ -247,10 +275,10 @@ def run(module, result):
             result['commands'] = commands
 
         commit = not check_mode
-        diff = load_config(module, commands, commit=commit, replace=replace_config,
-                           comment=comment, admin=admin)
+        diff = load_config(module, commands, commit=commit, replace=replace_config, comment=comment, admin=admin)
         if diff:
             result['diff'] = dict(prepared=diff)
+
         result['changed'] = True
 
 
diff --git a/lib/ansible/modules/network/iosxr/iosxr_logging.py b/lib/ansible/modules/network/iosxr/iosxr_logging.py
index 039fa163e5..e1645a9129 100644
--- a/lib/ansible/modules/network/iosxr/iosxr_logging.py
+++ b/lib/ansible/modules/network/iosxr/iosxr_logging.py
@@ -586,7 +586,7 @@ class NCConfiguration(ConfigBase):
                 elif item['dest'] == 'host' and item['name'] in host_list:
                     item['level'] = severity_level[item['level']]
                     host_params.append(item)
-                elif item['dest'] == 'console' and have_console and have_console_enable:
+                elif item['dest'] == 'console' and have_console:
                     console_params.update({'console-level': item['level']})
                 elif item['dest'] == 'monitor' and have_monitor:
                     monitor_params.update({'monitor-level': item['level']})
diff --git a/lib/ansible/plugins/cliconf/__init__.py b/lib/ansible/plugins/cliconf/__init__.py
index 74cac5d54e..8cb52f9f78 100644
--- a/lib/ansible/plugins/cliconf/__init__.py
+++ b/lib/ansible/plugins/cliconf/__init__.py
@@ -34,7 +34,6 @@ try:
 except ImportError:
     HAS_SCP = False
 
-
 try:
     from __main__ import display
 except ImportError:
@@ -135,7 +134,7 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
         pass
 
     @abstractmethod
-    def edit_config(self, commands):
+    def edit_config(self, commands=None):
         """Loads the specified commands into the remote device
         This method will load the commands into the remote device.  This
         method will make sure the device is in the proper context before
@@ -150,7 +149,7 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
         pass
 
     @abstractmethod
-    def get(self, command, prompt=None, answer=None, sendonly=False):
+    def get(self, command=None, prompt=None, answer=None, sendonly=False, newline=True):
         """Execute specified command on remote device
         This method will retrieve the specified data and
         return it to the caller as a string.
@@ -181,18 +180,26 @@ class CliconfBase(with_metaclass(ABCMeta, object)):
         "Discard changes in candidate datastore"
         return self._connection.method_not_found("discard_changes is not supported by network_os %s" % self._play_context.network_os)
 
-    def put_file(self, source, destination):
-        """Copies file over scp to remote device"""
-        if not HAS_SCP:
-            self._connection.internal_error("Required library scp is not installed.  Please install it using `pip install scp`")
-        ssh = self._connection._connect_uncached()
-        with SCPClient(ssh.get_transport()) as scp:
-            scp.put(source, destination)
-
-    def fetch_file(self, source, destination):
-        """Fetch file over scp from remote device"""
-        if not HAS_SCP:
-            self._connection.internal_error("Required library scp is not installed.  Please install it using `pip install scp`")
-        ssh = self._connection._connect_uncached()
-        with SCPClient(ssh.get_transport()) as scp:
-            scp.get(source, destination)
+    def copy_file(self, source=None, destination=None, proto='scp'):
+        """Copies file over scp/sftp to remote device"""
+        ssh = self._connection.paramiko_conn._connect_uncached()
+        if proto == 'scp':
+            if not HAS_SCP:
+                self._connection.internal_error("Required library scp is not installed.  Please install it using `pip install scp`")
+            with SCPClient(ssh.get_transport()) as scp:
+                scp.put(source, destination)
+        elif proto == 'sftp':
+            with ssh.open_sftp() as sftp:
+                sftp.put(source, destination)
+
+    def get_file(self, source=None, destination=None, proto='scp'):
+        """Fetch file over scp/sftp from remote device"""
+        ssh = self._connection.paramiko_conn._connect_uncached()
+        if proto == 'scp':
+            if not HAS_SCP:
+                self._connection.internal_error("Required library scp is not installed.  Please install it using `pip install scp`")
+            with SCPClient(ssh.get_transport()) as scp:
+                scp.get(source, destination)
+        elif proto == 'sftp':
+            with ssh.open_sftp() as sftp:
+                sftp.get(source, destination)
diff --git a/lib/ansible/plugins/cliconf/iosxr.py b/lib/ansible/plugins/cliconf/iosxr.py
index 42aa864487..37afb3f21c 100644
--- a/lib/ansible/plugins/cliconf/iosxr.py
+++ b/lib/ansible/plugins/cliconf/iosxr.py
@@ -67,12 +67,27 @@ class Cliconf(CliconfBase):
 
         return self.send_command(cmd)
 
-    def edit_config(self, command):
-        for cmd in chain(to_list(command)):
-            self.send_command(cmd)
-
-    def get(self, command, prompt=None, answer=None, sendonly=False):
-        return self.send_command(command, prompt=prompt, answer=answer, sendonly=sendonly)
+    def edit_config(self, commands=None):
+        for cmd in chain(to_list(commands)):
+            try:
+                if isinstance(cmd, str):
+                    cmd = json.loads(cmd)
+                command = cmd.get('command', None)
+                prompt = cmd.get('prompt', None)
+                answer = cmd.get('answer', None)
+                sendonly = cmd.get('sendonly', False)
+                newline = cmd.get('newline', True)
+            except:
+                command = cmd
+                prompt = None
+                answer = None
+                sendonly = None
+                newline = None
+
+            self.send_command(command=command, prompt=prompt, answer=answer, sendonly=sendonly, newline=newline)
+
+    def get(self, command=None, prompt=None, answer=None, sendonly=False, newline=True):
+        return self.send_command(command=command, prompt=prompt, answer=answer, sendonly=sendonly, newline=newline)
 
     def commit(self, comment=None):
         if comment:
diff --git a/lib/ansible/plugins/connection/network_cli.py b/lib/ansible/plugins/connection/network_cli.py
index 73a2974b4b..fedbadafa5 100644
--- a/lib/ansible/plugins/connection/network_cli.py
+++ b/lib/ansible/plugins/connection/network_cli.py
@@ -283,10 +283,10 @@ class Connection(ConnectionBase):
         if self.connected:
             return
 
-        p = connection_loader.get('paramiko', self._play_context, '/dev/null')
-        p.set_options(direct={'look_for_keys': not bool(self._play_context.password and not self._play_context.private_key_file)})
-        p.force_persistence = self.force_persistence
-        ssh = p._connect()
+        self.paramiko_conn = connection_loader.get('paramiko', self._play_context, '/dev/null')
+        self.paramiko_conn.set_options(direct={'look_for_keys': not bool(self._play_context.password and not self._play_context.private_key_file)})
+        self.paramiko_conn.force_persistence = self.force_persistence
+        ssh = self.paramiko_conn._connect()
 
         display.vvvv('ssh connection done, setting terminal', host=self._play_context.remote_addr)