Improve integration tests logging

* Add fetch-video-output role based on fetch-tox-output
* Fix test log collection to work with Python3
* Fix page HTML source log

Change-Id: I2e865cc5544d6e77bc3af5e413cbaedceb6df1c1

3 files changed, 65 insertions, 0 deletions

diff --git a/roles/fetch-video-output/README.rst b/roles/fetch-video-output/README.rst
new file mode 100644
index 000000000..48fd1ef31
--- /dev/null
+++ b/roles/fetch-video-output/README.rst
@@ -0,0 +1,16 @@
+Collect log output from a tox build
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory tox was run in.
+
+.. zuul:rolevar:: zuul_use_fetch_output
+   :default: false
+
+   Whether to synchronize files to the executor work dir, or to copy them
+   on the test instance.
+   When set to false, the role synchronizes the file to the executor.
+   When set to true, the job needs to use the fetch-output role later.
diff --git a/roles/fetch-video-output/defaults/main.yaml b/roles/fetch-video-output/defaults/main.yaml
new file mode 100644
index 000000000..a305de523
--- /dev/null
+++ b/roles/fetch-video-output/defaults/main.yaml
@@ -0,0 +1,5 @@
+---
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+
+zuul_output_dir: "{{ ansible_user_dir }}/zuul-output"
+zuul_use_fetch_output: "{{ zuul_site_use_fetch_output|default(false) }}"
diff --git a/roles/fetch-video-output/tasks/main.yaml b/roles/fetch-video-output/tasks/main.yaml
new file mode 100644
index 000000000..323e4aade
--- /dev/null
+++ b/roles/fetch-video-output/tasks/main.yaml
@@ -0,0 +1,44 @@
+- name: Set tox log path for multiple nodes
+  set_fact:
+    log_path: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}/tox"
+  when: groups['all'] | length > 1
+
+- name: Set tox log path for single node
+  set_fact:
+    log_path: "{{ zuul.executor.log_root }}/tox"
+  when: log_path is not defined
+
+- name: Ensure local tox dir
+  file:
+    path: "{{ log_path }}"
+    state: directory
+  delegate_to: localhost
+
+- name: Copy tox logs
+  copy:
+    dest: "{{ zuul_output_dir }}/logs/tox/"
+    src: "{{ zuul_work_dir }}/openstack_dashboard/test/integration_tests/test_reports"
+    remote_src: true
+  loop: "{{ envlist }}"
+  loop_control:
+    loop_var: zj_testenv
+  # some tox runs may not create a virtualenv and thus have
+  # no ./tox/env directory
+  failed_when: false
+  when: zuul_use_fetch_output
+
+- name: Collect tox logs
+  synchronize:
+    dest: "{{ log_path }}"
+    mode: pull
+    src: "{{ zuul_work_dir }}/openstack_dashboard/test/integration_tests/test_reports"
+    verify_host: true
+    owner: no
+    group: no
+  loop: "{{ envlist }}"
+  loop_control:
+    loop_var: zj_testenv
+  # some tox runs may not create a virtualenv and thus have
+  # no ./tox/env directory
+  failed_when: false
+  when: not zuul_use_fetch_output