baserock_storyboard: Upgrade to latest and use Ansible for deployment

Change-Id: If4578c0d97aa2aee1a1a7e57bb7e2c42917ba077

9 files changed, 162 insertions, 17 deletions

diff --git a/README.mdwn b/README.mdwn
index c39a3b61..0f50546a 100644
--- a/README.mdwn
+++ b/README.mdwn
@@ -483,27 +483,31 @@ the final SSH command showing any errors.
     ssh $GERRIT_ADMIN_USERNAME@gerrit.baserock.org -p 29418 gerrit plugin ls
     ssh $GERRIT_ADMIN_USERNAME@gerrit.baserock.org -p 29418 replication start --all --wait
 
-### Storyboard
+### StoryBoard
 
-We use a slightly adapted version of
-<https://github.com/openstack-infra/puppet-storyboard> to deploy Storyboard.
+    ansible-galaxy install palvarez89.storyboard -p `pwd`/baserock_storyboard/roles
+    ansible-galaxy install Mayeu.RabbitMQ,1.4.0 -p `pwd`/baserock_storyboard/roles
+    ansible-galaxy install geerlingguy.mysql,1.5.0 -p `pwd`/baserock_storyboard/roles
 
-There's no development deployment for Storyboard at this time: the Puppet
-script expects to start services using systemd, and that doesn't work by
-default in a Docker container.
-
-To deploy the production version:
+    nova volume-create \
+        --display-name storyboard-volume \
+        --display-description 'Storyboard volume' \
+        --volume-type Ceph \
+        100
 
-    packer build -only=production baserock_storyboard/packer_template.json
-    nova boot openid_provider
-        --flavor dc1.1x1 --image 'baserock_storyboard' \
-        --key-name=$keyname storyboard.baserock.org \
-        --nic="net-id=$network_id"
-        --security-groups default,web-server
+    nova boot storyboard.baserock.org \
+        --key-name $keyname \
+        --flavor 'dc1.1x1.20' \
+        --image $ubuntu_image_id \
+        --nic "net-id=$network_id,v4-fixed-ip=192.168.222.131" \
+        --security-groups default,web-server \
         --user-data baserock-ops-team.cloud-config
 
-Storyboard deployment does not yet work fully (you can manually kludge it into
-working after deploying it, though).
+    nova volume-attach storyboard.baserock.org <volume-id> /dev/vdb
+
+    ansible-playbook -i hosts baserock_storyboard/instance-config.yml
+    ansible-playbook -i hosts baserock_storyboard/instance-backup-config.yml
+    ansible-playbook -i hosts baserock_storyboard/instance-storyboard-config.yml
 
 ### Masons
 
diff --git a/baserock_backup/backup.sh b/baserock_backup/backup.sh
index f16ba447..3fded6bd 100755
--- a/baserock_backup/backup.sh
+++ b/baserock_backup/backup.sh
@@ -23,3 +23,8 @@ date > /srv/backup/database.timestamp
     root@192.168.222.69: /srv/backup/gerrit
 date > /srv/backup/gerrit.timestamp
 
+# Storyboard Database
+/usr/bin/rsync --archive --delete-before --delete-excluded \
+    --hard-links --human-readable --progress --sparse \
+    root@192.168.222.30: /srv/backup/storyboard-database
+date > /srv/backup/storyboard-database.timestamp
diff --git a/baserock_hosts b/baserock_hosts
index bcde97d2..e397db55 100644
--- a/baserock_hosts
+++ b/baserock_hosts
@@ -23,11 +23,11 @@ frontend-haproxy ansible_ssh_host=185.43.218.170
 database-mariadb ansible_ssh_host=192.168.222.30
 mail ansible_ssh_host=192.168.222.111
 openid ansible_ssh_host=192.168.222.67
-storyboard ansible_ssh_host=192.168.222.40
 webserver ansible_ssh_host=192.168.222.127
 
 [ubuntu]
 paste ansible_ssh_host=192.168.222.6
+storyboard ansible_ssh_host=192.168.222.131
 #testgerrit ansible_ssh_host=192.168.222.46
 
 
diff --git a/baserock_storyboard/backup-snapshot.conf b/baserock_storyboard/backup-snapshot.conf
new file mode 100644
index 00000000..8a5dd8d3
--- /dev/null
+++ b/baserock_storyboard/backup-snapshot.conf
@@ -0,0 +1,4 @@
+services:
+  - mysql.service
+
+volume: /dev/vg0/database-storyboard
diff --git a/baserock_storyboard/instance-backup-config.yml b/baserock_storyboard/instance-backup-config.yml
new file mode 100644
index 00000000..124dabc9
--- /dev/null
+++ b/baserock_storyboard/instance-backup-config.yml
@@ -0,0 +1,26 @@
+# Instance backup configuration for the baserock.org database.
+---
+- hosts: storyboard
+  gather_facts: false
+  sudo: yes
+  vars:
+    FRONTEND_IP: 192.168.222.116
+  tasks:
+    - name: backup-snapshot script
+      copy: src=../backup-snapshot dest=/usr/bin/backup-snapshot mode=755
+
+    - name: backup-snapshot config
+      copy: src=backup-snapshot.conf dest=/etc/backup-snapshot.conf
+
+    # We need to give the backup automation 'root' access, because it needs to
+    # manage system services, LVM volumes, and mounts, and because it needs to
+    # be able to read private data. The risk of having the backup key
+    # compromised is mitigated by only allowing it to execute the
+    # 'backup-snapshot' script, and limiting the hosts it can be used from.
+    - name: access for backup SSH key
+      authorized_key:
+        user: root
+        key: "{{ lookup('file', '../keys/backup.key.pub') }}"
+        # Quotes are important in this options, the OpenSSH server will reject
+        # the entry if the 'from' or 'command' values are not quoted.
+        key_options: 'from="{{FRONTEND_IP}}",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,command="/usr/bin/backup-snapshot"'
diff --git a/baserock_storyboard/instance-config.yml b/baserock_storyboard/instance-config.yml
new file mode 100644
index 00000000..6a1b2cf5
--- /dev/null
+++ b/baserock_storyboard/instance-config.yml
@@ -0,0 +1,35 @@
+# Instance configuration for Baserock MySQL on for StoryBoard host
+#
+# This script expects a volume to be available at /dev/vdb.
+---
+- hosts: storyboard
+  gather_facts: False
+  sudo: yes
+  vars:
+  - lv_size: 25g
+  - mountpoint: /var/lib/mysql
+  - lv_name: database-storyboard
+  tasks:
+  - name: install lvm2 tools
+    apt: name=lvm2 state=latest
+
+  - name: LVM logical volume group on /dev/vdb
+    lvg: vg=vg0 pvs=/dev/vdb
+
+# Duplicated from:
+#- include: ../tasks/create-data-volume.yml lv_name=database-storyboard lv_size=25g mountpoint=/var/lib/mysql
+# given that is not  ubuntu compatible
+
+  - name: logical volume for {{ lv_name }}
+    lvol: vg=vg0 lv={{ lv_name }} size={{ lv_size }}
+
+# This will NEVER overwrite an existing filesystem. Unless you add
+# 'force=yes' to the arguments. So don't do that. See:
+# http://docs.ansible.com/filesystem_module.html.
+#
+  - name: ext4 filesystem on /dev/vg0/{{ lv_name }}
+    filesystem: fstype=ext4 dev=/dev/vg0/{{ lv_name }}
+
+  - name: mount {{ lv_name }} logical volume
+    mount: src=/dev/vg0/{{ lv_name }} name={{ mountpoint }} fstype=ext4 state=mounted
+# End of duplication
diff --git a/baserock_storyboard/instance-storyboard-config.yml b/baserock_storyboard/instance-storyboard-config.yml
new file mode 100644
index 00000000..cf74f551
--- /dev/null
+++ b/baserock_storyboard/instance-storyboard-config.yml
@@ -0,0 +1,14 @@
+# Instance-specific configuration for the baserock.org StoryBoard instance.
+---
+- hosts: storyboard
+  vars_files:
+  - ../baserock_database/baserock_storyboard.database_password.yml
+  - ../baserock_database/root.database_password.yml
+  - storyboard-vars.yml
+  sudo: yes
+  roles:
+  # We are using a new database here because StoryBoard is not yet compatible
+  # with MariaDB
+  - { role: geerlingguy.mysql }
+  - { role: Mayeu.RabbitMQ }
+  - { role: palvarez89.storyboard }
diff --git a/baserock_storyboard/storyboard-vars.yml b/baserock_storyboard/storyboard-vars.yml
new file mode 100644
index 00000000..5b404bd4
--- /dev/null
+++ b/baserock_storyboard/storyboard-vars.yml
@@ -0,0 +1,53 @@
+# For rabbitmq role
+rabbitmq_host: localhost
+rabbitmq_port: 5672
+rabbitmq_vhost: '/'
+rabbitmq_user: storyboard
+rabbitmq_user_password: storyboard
+rabbitmq_ssl: false
+rabbitmq_vhost_definitions:
+  - name:    "{{ rabbitmq_vhost }}"
+rabbitmq_users_definitions:
+  - vhost:    "{{ rabbitmq_vhost }}"
+    user:     "{{ rabbitmq_user }}"
+    password: "{{ rabbitmq_user_password }}"
+rabbitmq_conf_tcp_listeners_address: '127.0.0.1'
+
+# For mysql role
+mysql_host: localhost
+mysql_port: 3306
+mysql_database: storyboard
+mysql_user: storyboard
+mysql_user_password: "{{ baserock_storyboard_password }}"
+mysql_root_password: "{{ root_password }}"
+mysql_databases:
+  - name: "{{ mysql_database }}"
+mysql_users:
+  - name: "{{ mysql_user }}"
+    host: "{{ mysql_host }}"
+    password: "{{ mysql_user_password }}"
+    priv: "{{ mysql_database }}.*:ALL"
+mysql_packages:
+  - mysql-server-5.6
+  - python-mysqldb
+
+storyboard_enable_email: 'True'
+storyboard_email_sender: StoryBoard (Do Not Reply) <do_not_reply@baserock.org>
+storyboard_email_smtp_host: 192.168.222.111
+storyboard_email_smtp_timeout: 10
+
+# Install from this branch that includes a patch to enable
+# email notifications. This patch is about to be merged upstream. 
+storyboard_webclient_repo: https://github.com/palvarez89/storyboard-webclient.git
+storyboard_webclient_version: email-notifications
+
+storyboard_fqdn: storyboard.baserock.org
+storyboard_openid_url: https://openid.baserock.org/openid/
+
+storyboard_projects: projects.yaml
+storyboard_superusers: users.yaml
+storyboard_mysql_user_password: "{{ baserock_storyboard_password }}"
+
+storyboard_ssl_cert: ../certs/baserock.org-ssl-certificate-temporary-dsilverstone.full.cert
+storyboard_ssl_key: ../private/baserock.org-ssl-certificate-temporary-dsilverstone.pem
+storyboard_resolved_ssl_ca: ../certs/startcom-class2-ca-chain-certificate.cert
diff --git a/baserock_storyboard/users.yaml b/baserock_storyboard/users.yaml
new file mode 100644
index 00000000..b42efca9
--- /dev/null
+++ b/baserock_storyboard/users.yaml
@@ -0,0 +1,4 @@
+- openid: https://openid.baserock.org/openid/pedroalvarez/
+  email: pedro.alvarez@codethink.co.uk
+- openid: https://openid.baserock.org/openid/samthursfield/
+  email: sam.thursfield@codethink.co.uk