Break up Turbo-Hipster configuration

Seeing as the power of Turbo-Hipster is in its pluggable design,
this change allows the configuration to be spit up allowing
each plug in to maintain it's own part of the configuration
including overwriting any default configuration.

There is a new configuration parameter 'conf_d', which is
mapped to a directory:

    "conf_d": "/etc/turbo-hipster/conf.d"

worker_server.py grabs all files inside this directory and
attempts to load them. If it fails, the error is logged.

The motivating factor for this change for me, is that it allows
the deployment of turbo-hipster via puppet much simpler.
The base TH puppet class will create the config.json, whereas
there puppet TH plug in classes can pop extra configuration into
the conf_d directory.

Change-Id: Ied20b46d4caa642d130097f3fe019df9c0ec5851

8 files changed, 106 insertions, 27 deletions

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
index 101a8fa..eb01021 100644
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -77,6 +77,10 @@ for your environment::
            to use a script to authenticate against a swift
            account or to use *laughing_spice* to format the logs
            etc.
+  **conf_d**
+    A path of a directory containing pieces of json confiuration.
+    This is helpful when you want different plugins to add extra
+    or even modify the default configuration.
 
 4. Create a turbo-hipster user:
 
diff --git a/requirements.txt b/requirements.txt
index 7c53f67..d64b484 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,3 +13,4 @@ sphinxcontrib-seqdiag
 mysql-python
 
 requests
+PyYAML
diff --git a/tests/etc/conf.d/extra.yaml b/tests/etc/conf.d/extra.yaml
new file mode 100644
index 0000000..44fe974
--- /dev/null
+++ b/tests/etc/conf.d/extra.yaml
@@ -0,0 +1 @@
+extra_configuration: testing123
diff --git a/tests/etc/config.json b/tests/etc/config.json
deleted file mode 100644
index 587b110..0000000
--- a/tests/etc/config.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-    "zuul_server": {
-        "git_url": "/home/josh/var/lib/zuul/git/",
-        "gearman_host": "localhost",
-        "gearman_port": 0
-    },
-    "debug_log": "/home/josh/var/log/turbo-hipster/debug.log",
-    "jobs_working_dir": "/home/josh/var/lib/turbo-hipster/jobs",
-    "git_working_dir": "/home/josh/var/lib/turbo-hipster/git",
-    "pip_download_cache": "/home/josh/var/cache/pip",
-    "plugins": [
-        {
-            "name": "gate_real_db_upgrade",
-            "datasets_dir": "/home/josh/var/lib/turbo-hipster/datasets",
-            "job": "gate-real-db-upgrade_nova_mysql"
-        }
-    ],
-    "publish_logs":
-    {
-        "type": "local",
-        "path": "/home/josh/var/www/results/",
-        "prepend_url": "http://localhost/results/"
-    }
-}
\ No newline at end of file
diff --git a/tests/etc/config.yaml b/tests/etc/config.yaml
new file mode 100644
index 0000000..0e332ee
--- /dev/null
+++ b/tests/etc/config.yaml
@@ -0,0 +1,18 @@
+zuul_server:
+    git_url: /home/josh/var/lib/zuul/git/
+    gearman_host: localhost
+    gearman_port: 0
+
+debug_log: /home/josh/var/log/turbo-hipster/debug.log
+jobs_working_dir: /home/josh/var/lib/turbo-hipster/jobs
+git_working_dir: /home/josh/var/lib/turbo-hipster/git
+pip_download_cache: /home/josh/var/cache/pip
+plugins:
+    - name: gate_real_db_upgrade
+      datasets_dir": /home/josh/var/lib/turbo-hipster/datasets
+      job: gate-real-db-upgrade_nova_mysql
+
+publish_logs:
+    type: local
+    path: /home/josh/var/www/results/
+    prepend_url: http://localhost/results/
diff --git a/tests/test_worker_manager.py b/tests/test_worker_manager.py
index eefbcbe..ae2d5b5 100644
--- a/tests/test_worker_manager.py
+++ b/tests/test_worker_manager.py
@@ -15,16 +15,16 @@
 # under the License.
 
 
-import json
 import os
 import testtools
 import time
+import yaml
 from fakes import FakeZuulManager, FakeGearmanServer,\
     FakeRealDbUpgradeRunner
 
 CONFIG_DIR = os.path.join(os.path.dirname(__file__), 'etc')
-with open(os.path.join(CONFIG_DIR, 'config.json'), 'r') as config_stream:
-    CONFIG = json.load(config_stream)
+with open(os.path.join(CONFIG_DIR, 'config.yaml'), 'r') as config_stream:
+    CONFIG = yaml.safe_load(config_stream)
 
 
 class TestZuulManager(testtools.TestCase):
diff --git a/tests/test_worker_server.py b/tests/test_worker_server.py
new file mode 100644
index 0000000..200a7b3
--- /dev/null
+++ b/tests/test_worker_server.py
@@ -0,0 +1,52 @@
+#!/usr/bin/python2
+#
+# Copyright 2014 Rackspace Australia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import os
+import testtools
+import yaml
+
+from turbo_hipster import worker_server
+
+CONFIG_DIR = os.path.join(os.path.dirname(__file__), 'etc')
+with open(os.path.join(CONFIG_DIR, 'config.yaml'), 'r') as config_stream:
+    CONFIG = yaml.safe_load(config_stream)
+
+CONF_D_DIR = os.path.join(CONFIG_DIR, "conf.d")
+
+
+class TestServerManager(testtools.TestCase):
+    def setUp(self):
+        super(TestServerManager, self).setUp()
+        self.config = CONFIG
+
+    def tearDown(self):
+        super(TestServerManager, self).tearDown()
+
+    def test_confd_configuration(self):
+        """ Check that the server can load in other configuration from a
+        conf.d directory """
+
+        def pass_function(*args, **kargs):
+            pass
+
+        self.config["conf_d"] = CONF_D_DIR
+
+        worker_server.Server.setup_logging = pass_function
+        serv = worker_server.Server(self.config)
+        serv_config = serv.config
+        self.assertIn("extra_configuration", serv_config)
+        self.assertEquals("testing123", serv_config["extra_configuration"])
diff --git a/turbo_hipster/worker_server.py b/turbo_hipster/worker_server.py
index d2edf18..a0f3cf2 100755
--- a/turbo_hipster/worker_server.py
+++ b/turbo_hipster/worker_server.py
@@ -21,8 +21,10 @@ task_plugins. """
 import logging
 import os
 import threading
+import yaml
 
 import worker_manager
+from os.path import join, isdir, isfile
 
 
 class Server(threading.Thread):
@@ -34,6 +36,13 @@ class Server(threading.Thread):
         super(Server, self).__init__()
         self._stop = threading.Event()
         self.config = config
+
+        # Load extra configuration first
+        # NOTE(Mattoliverau): debug_log might be specified in
+        # a conf.d snippet.
+        if 'conf_d' in self.config:
+            self.load_extra_configuration()
+
         # Python logging output file.
         self.debug_log = self.config['debug_log']
         self.setup_logging()
@@ -51,6 +60,24 @@ class Server(threading.Thread):
         self.tasks = {}
         self.load_plugins()
 
+    def load_extra_configuration(self):
+        if isdir(self.config["conf_d"]):
+            extra_configs = (join(self.config["conf_d"], item)
+                             for item in os.listdir(self.config["conf_d"])
+                             if isfile(join(self.config["conf_d"], item)))
+            for conf in extra_configs:
+                try:
+                    with open(conf, 'r') as config_stream:
+                        extra_config = yaml.safe_load(config_stream)
+                        self.config.update(extra_config)
+                except:
+                    self.log.warn("Failed to load extra configuration: '%s'" %
+                                  (conf))
+                    continue
+        else:
+            self.log.warn("conf_d parameter '%s' isn't a directory" %
+                          (self.config["conf_d"]))
+
     def setup_logging(self):
         if not self.debug_log:
             raise Exception('Debug log not configured')