Merge "Add tracing tutorial"

7 files changed, 302 insertions, 1 deletions

diff --git a/doc/source/examples/tracing/docker-compose.yaml b/doc/source/examples/tracing/docker-compose.yaml
new file mode 100644
index 000000000..d732dff90
--- /dev/null
+++ b/doc/source/examples/tracing/docker-compose.yaml
@@ -0,0 +1,32 @@
+# Start the quickstart tutorial with `docker-compose -p zuul-tutorial
+# up` (as directed in the instructions) in order for the network to
+# have the expected name so that it can be shared with Jaeger.
+
+# Version 2.1 is required to specify the network name
+version: '2.1'
+
+services:
+  jaeger:
+    image: docker.io/jaegertracing/all-in-one:latest
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
+    ports:
+      - "16686:16686"
+      - "4317:4317"
+      - "4318:4318"
+    volumes:
+      - "badger:/badger"
+    networks:
+      - zuul
+
+volumes:
+  badger:
+
+networks:
+  zuul:
+    external: true
+    name: zuul-tutorial_zuul
diff --git a/doc/source/examples/tracing/etc_zuul/main.yaml b/doc/source/examples/tracing/etc_zuul/main.yaml
new file mode 100644
index 000000000..b3674844e
--- /dev/null
+++ b/doc/source/examples/tracing/etc_zuul/main.yaml
@@ -0,0 +1,14 @@
+- tenant:
+    name: example-tenant
+    source:
+      gerrit:
+        config-projects:
+          - zuul-config
+        untrusted-projects:
+          - test1
+          - test2
+      opendev.org:
+        untrusted-projects:
+          - zuul/zuul-jobs:
+              include:
+                - job
diff --git a/doc/source/examples/tracing/etc_zuul/zuul.conf b/doc/source/examples/tracing/etc_zuul/zuul.conf
new file mode 100644
index 000000000..76f4efe56
--- /dev/null
+++ b/doc/source/examples/tracing/etc_zuul/zuul.conf
@@ -0,0 +1,59 @@
+[zookeeper]
+hosts=zk:2281
+tls_cert=/var/certs/certs/client.pem
+tls_key=/var/certs/keys/clientkey.pem
+tls_ca=/var/certs/certs/cacert.pem
+
+[keystore]
+password=secret
+
+[scheduler]
+tenant_config=/etc/zuul/main.yaml
+
+[connection "gerrit"]
+name=gerrit
+driver=gerrit
+server=gerrit
+sshkey=/var/ssh/zuul
+user=zuul
+password=secret
+baseurl=http://gerrit:8080
+auth_type=basic
+
+[connection "opendev.org"]
+name=opendev
+driver=git
+baseurl=https://opendev.org
+
+[database]
+# Use variable interpolation to supply the password from the
+# docker-compose file.
+# https://zuul-ci.org/docs/zuul/latest/configuration.html
+dburi=mysql+pymysql://zuul:%(ZUUL_MYSQL_PASSWORD)s@mysql/zuul
+
+[web]
+listen_address=0.0.0.0
+port=9000
+root=http://localhost:9000
+
+[executor]
+private_key_file=/var/ssh/nodepool
+default_username=root
+trusted_rw_paths=/srv/static/logs
+
+[auth zuul_operator]
+driver=HS256
+allow_authz_override=true
+realm=zuul.example.com
+client_id=zuul.example.com
+issuer_id=zuul_operator
+secret=exampleSecret
+
+[webclient]
+url=http://localhost:9000
+verify_ssl=false
+
+[tracing]
+enabled=true
+endpoint=jaeger:4317
+insecure=true
diff --git a/doc/source/tracing.rst b/doc/source/tracing.rst
index e973a77dd..267ce4226 100644
--- a/doc/source/tracing.rst
+++ b/doc/source/tracing.rst
@@ -5,7 +5,7 @@
 Tracing
 =======
 
-Zuul includes support for distributed `tracing`_ as described by the
+Zuul includes support for `distributed tracing`_ as described by the
 OpenTelemetry project.  This allows operators (and potentially users)
 to visualize the progress of events and queue items through the
 various Zuul components as an aid to debugging.
@@ -18,6 +18,19 @@ Zuul supports the OpenTelemetry Protocol (OTLP) for exporting traces.
 Many observability systems support receiving traces via OTLP
 (including Jaeger tracing).
 
+Configuration
+-------------
+
 Related configuration is in the :attr:`tracing` section of ``zuul.conf``.
 
+Tutorial
+--------
+
+Here is a tutorial that shows how to enable tracing with Zuul and Jaeger.
+
+.. toctree::
+   :maxdepth: 1
+
+   tutorials/tracing
+
 _`distributed tracing`: https://opentelemetry.io/docs/concepts/observability-primer/#distributed-traces
diff --git a/doc/source/tutorials/tracing.rst b/doc/source/tutorials/tracing.rst
new file mode 100644
index 000000000..bc0ca0ea6
--- /dev/null
+++ b/doc/source/tutorials/tracing.rst
@@ -0,0 +1,80 @@
+Jaeger Tracing Tutorial
+=======================
+
+Zuul includes support for `distributed tracing`_ as described by the
+OpenTelemetry project.  This allows operators (and potentially users)
+to visualize the progress of events and queue items through the
+various Zuul components as an aid to debugging.
+
+Zuul supports the OpenTelemetry Protocol (OTLP) for exporting traces.
+Many observability systems support receiving traces via OTLP.  One of
+these is Jaeger.  Because it can be run as a standalone service with
+local storage, this tutorial describes how to set up a Jaeger server
+and configure Zuul to export data to it.
+
+For more information about tracing in Zuul, see :ref:`tracing`.
+
+To get started, first run the :ref:`quick-start` and then follow the
+steps in this tutorial to add a Jaeger server.
+
+Restart Zuul Containers
+-----------------------
+
+After completing the initial tutorial, stop the Zuul containers so
+that we can update Zuul's configuration to enable tracing.
+
+.. code-block:: shell
+
+   cd zuul/doc/source/examples
+   sudo -E docker-compose -p zuul-tutorial stop
+
+Restart the containers with a new Zuul configuration.
+
+.. code-block:: shell
+
+   cd zuul/doc/source/examples
+   ZUUL_TUTORIAL_CONFIG="./tracing/etc_zuul/" sudo -E docker-compose -p zuul-tutorial up -d
+
+This tells docker-compose to use these Zuul `config files
+<https://opendev.org/zuul/zuul/src/branch/master/doc/source/examples/tracing>`_.
+The only change compared to the quick-start is to add a
+:attr:`tracing` section to ``zuul.conf``:
+
+.. code-block:: ini
+
+   [tracing]
+   enabled=true
+   endpoint=jaeger:4317
+   insecure=true
+
+This instructs Zuul to send tracing information to the Jaeger server
+we will start below.
+
+Start Jaeger
+------------
+
+A separate docker-compose file is provided to run Jaeger.  Start it
+with this command:
+
+.. code-block:: shell
+
+   cd zuul/doc/source/examples/tracing
+   sudo -E docker-compose -p zuul-tutorial-tracing up -d
+
+You can visit http://localhost:16686/search to verify it is running.
+
+Recheck a change
+----------------
+
+Visit Gerrit at http://localhost:8080/dashboard/self and return to the
+``test1`` change you uploaded earlier.  Click `Reply` then type
+`recheck` into the text field and click `Send`.  This will tell Zuul
+to run the test job once again.  When the job is complete, you should
+have a trace available in Jaeger.
+
+To see the trace, visit http://localhost:16686/search and select the
+`zuul` service (reload the page if it doesn't show up at first).
+Press `Find Traces` and you should see the trace for your build
+appear.
+
+_`distributed tracing`: https://opentelemetry.io/docs/concepts/observability-primer/#distributed-traces
diff --git a/playbooks/tutorial/run-tutorial.yaml b/playbooks/tutorial/run-tutorial.yaml
index abcbbba72..1d065aa82 100644
--- a/playbooks/tutorial/run-tutorial.yaml
+++ b/playbooks/tutorial/run-tutorial.yaml
@@ -13,3 +13,6 @@
 
 - name: Run admin tutorial
   include_tasks: admin.yaml
+
+- name: Run tracing tutorial
+  include_tasks: tracing.yaml
diff --git a/playbooks/tutorial/tracing.yaml b/playbooks/tutorial/tracing.yaml
new file mode 100644
index 000000000..3b01f1047
--- /dev/null
+++ b/playbooks/tutorial/tracing.yaml
@@ -0,0 +1,100 @@
+# Stop the basic tutorial
+- name: Run docker-compose down
+  when: not local
+  shell:
+    cmd: docker-compose -p zuul-tutorial stop
+    chdir: src/opendev.org/zuul/zuul/doc/source/examples
+
+- name: Run docker-compose down
+  when: local
+  shell:
+    cmd: docker-compose -p zuul-tutorial stop
+    chdir: ../../doc/source/examples
+
+# Restart with the new config
+- name: Run docker-compose up
+  when: not local
+  shell:
+    cmd: docker-compose -p zuul-tutorial up -d
+    chdir: src/opendev.org/zuul/zuul/doc/source/examples
+  environment:
+    ZUUL_TUTORIAL_CONFIG: "./tracing/etc_zuul/"
+
+- name: Run docker-compose up
+  when: local
+  shell:
+    cmd: docker-compose -p zuul-tutorial up -d
+    chdir: ../../doc/source/examples
+  environment:
+    ZUUL_TUTORIAL_CONFIG: "./tracing/etc_zuul/"
+
+# Start Jaeger
+- name: Run docker-compose up
+  when: not local
+  shell:
+    cmd: docker-compose -p zuul-tutorial-tracing up -d
+    chdir: src/opendev.org/zuul/zuul/doc/source/examples/tracing
+
+- name: Run docker-compose up
+  when: local
+  shell:
+    cmd: docker-compose -p zuul-tutorial-tracing up -d
+    chdir: ../../doc/source/examples/tracing
+
+# Verify that Zuul runs with the new config
+- name: Wait for Zuul
+  uri:
+    url: http://localhost:9000/api/tenant/example-tenant/status
+    method: GET
+    return_content: true
+    status_code: 200
+    body_format: json
+  register: result
+  retries: 30
+  delay: 10
+  until: result.status == 200 and result.json["zuul_version"] is defined
+  changed_when: false
+
+- name: Verify that old builds are available
+  uri:
+    url: "http://localhost:9000/api/tenant/example-tenant/builds"
+    method: GET
+    return_content: true
+    status_code: 200
+    body_format: json
+  register: result
+  failed_when: "result.json | length < 4"
+  changed_when: false
+
+# Remove the label so Zuul will post again Verified+1 which is what
+# check-pipeline is looking for
+- include_role:
+    name: remove-verified
+  vars:
+    change_id: "{{ changetest1.id }}"
+
+# Recheck the change to issue a trace
+- include_role:
+    name: recheck-change
+  vars:
+    change_id: "{{ changetest1.id }}"
+
+- include_role:
+    name: check-pipeline
+  vars:
+    title: "test job test1"
+    projectname: test1
+    check_number: 3
+
+- name: Verify tracing information is available
+  uri:
+    url: http://localhost:16686/api/traces?lookback=1h&service=zuul
+    method: GET
+    return_content: true
+    status_code: 200
+    body_format: json
+  register: result
+  retries: 30
+  delay: 10
+  until: result.status == 200 and result.json["data"] | length > 0
+  changed_when: false