Merge "secure oslo_messaging.rpc"

39 files changed, 1586 insertions, 81 deletions

diff --git a/api-ref/source/samples/db-mgmt-get-instance-details-response-json-http.txt b/api-ref/source/samples/db-mgmt-get-instance-details-response-json-http.txt
index feb89a88..6580c3b8 100644
--- a/api-ref/source/samples/db-mgmt-get-instance-details-response-json-http.txt
+++ b/api-ref/source/samples/db-mgmt-get-instance-details-response-json-http.txt
@@ -1,5 +1,5 @@
 HTTP/1.1 200 OK
 Content-Type: application/json
-Content-Length: 1676
+Content-Length: 1709
 Date: Mon, 18 Mar 2013 19:09:17 GMT
 
diff --git a/api-ref/source/samples/db-mgmt-get-instance-details-response.json b/api-ref/source/samples/db-mgmt-get-instance-details-response.json
index 203159d6..ef8b7efc 100644
--- a/api-ref/source/samples/db-mgmt-get-instance-details-response.json
+++ b/api-ref/source/samples/db-mgmt-get-instance-details-response.json
@@ -7,6 +7,7 @@
         },
         "deleted": false,
         "deleted_at": null,
+        "encrypted_rpc_messaging": true,
         "flavor": {
             "id": "3",
             "links": [
@@ -80,3 +81,4 @@
         "volume_id": "VOL_44b277eb-39be-4921-be31-3d61b43651d7"
     }
 }
+
diff --git a/api-ref/source/samples/db-mgmt-instance-index-response-json-http.txt b/api-ref/source/samples/db-mgmt-instance-index-response-json-http.txt
index 875f0f20..3994d592 100644
--- a/api-ref/source/samples/db-mgmt-instance-index-response-json-http.txt
+++ b/api-ref/source/samples/db-mgmt-instance-index-response-json-http.txt
@@ -1,5 +1,5 @@
 HTTP/1.1 200 OK
 Content-Type: application/json
-Content-Length: 1225
+Content-Length: 1258
 Date: Mon, 18 Mar 2013 19:09:17 GMT
 
diff --git a/api-ref/source/samples/db-mgmt-instance-index-response.json b/api-ref/source/samples/db-mgmt-instance-index-response.json
index 5736bb17..6b26254a 100644
--- a/api-ref/source/samples/db-mgmt-instance-index-response.json
+++ b/api-ref/source/samples/db-mgmt-instance-index-response.json
@@ -8,6 +8,7 @@
             },
             "deleted": false,
             "deleted_at": null,
+            "encrypted_rpc_messaging": true,
             "flavor": {
                 "id": "3",
                 "links": [
@@ -58,3 +59,4 @@
         }
     ]
 }
+
diff --git a/doc/source/dev/secure_oslo_messaging.rst b/doc/source/dev/secure_oslo_messaging.rst
new file mode 100644
index 00000000..beabd339
--- /dev/null
+++ b/doc/source/dev/secure_oslo_messaging.rst
@@ -0,0 +1,655 @@
+.. _secure_rpc_messaging:
+
+======================
+ Secure RPC messaging
+======================
+
+Background
+----------
+
+Trove uses oslo_messaging.rpc for communication amongst the various
+control plane components and the guest agents. For secure operation of
+the system, these RPC calls can be fully encrypted. A control plane
+encryption key is used for communications between the API service and
+the taskmanager, and system generated per-instance keys are used for
+communication between the control plane and guest instances.
+
+This document provides some useful tips on how to use this mechanism.
+
+The default system behavior
+---------------------------
+
+By default, the system will attempt to encrypt all RPC
+communication. This behavior is controlled by the following
+configuration parameters:
+
+- enable_secure_rpc_messaging
+
+  boolean that determines whether rpc messages will be secured by
+  encryption. The default value is True.
+
+- taskmanager_rpc_encr_key
+
+  the key used for encrypting messages sent to the taskmanager. A
+  default value is provided for this and it is important that
+  deployers change this.
+
+- inst_rpc_key_encr_key
+
+  the key used for encrypting the per-instance keys when they are
+  stored in the trove infrastructure database (catalog). A default is
+  provided for this and it is important that deployers change this.
+
+
+Interoperability and Upgrade
+----------------------------
+
+Consider the system as shown below which runs a version of code prior
+to the introduciton of this oslo_messaging.rpc security. Observe, for
+example that the instances table in the system catalog does not
+include the per-instance encrypted key column.
+
+mysql> describe instances;
++----------------------+--------------+------+-----+---------+-------+
+| Field                | Type         | Null | Key | Default | Extra |
++----------------------+--------------+------+-----+---------+-------+
+| id                   | varchar(36)  | NO   | PRI | NULL    |       |
+| created              | datetime     | YES  |     | NULL    |       |
+| updated              | datetime     | YES  |     | NULL    |       |
+| name                 | varchar(255) | YES  |     | NULL    |       |
+| hostname             | varchar(255) | YES  |     | NULL    |       |
+| compute_instance_id  | varchar(36)  | YES  |     | NULL    |       |
+| task_id              | int(11)      | YES  |     | NULL    |       |
+| task_description     | varchar(255) | YES  |     | NULL    |       |
+| task_start_time      | datetime     | YES  |     | NULL    |       |
+| volume_id            | varchar(36)  | YES  |     | NULL    |       |
+| flavor_id            | varchar(255) | YES  |     | NULL    |       |
+| volume_size          | int(11)      | YES  |     | NULL    |       |
+| tenant_id            | varchar(36)  | YES  | MUL | NULL    |       |
+| server_status        | varchar(64)  | YES  |     | NULL    |       |
+| deleted              | tinyint(1)   | YES  | MUL | NULL    |       |
+| deleted_at           | datetime     | YES  |     | NULL    |       |
+| datastore_version_id | varchar(36)  | NO   | MUL | NULL    |       |
+| configuration_id     | varchar(36)  | YES  | MUL | NULL    |       |
+| slave_of_id          | varchar(36)  | YES  | MUL | NULL    |       |
+| cluster_id           | varchar(36)  | YES  | MUL | NULL    |       |
+| shard_id             | varchar(36)  | YES  |     | NULL    |       |
+| type                 | varchar(64)  | YES  |     | NULL    |       |
+| region_id            | varchar(255) | YES  |     | NULL    |       |
++----------------------+--------------+------+-----+---------+-------+
+23 rows in set (0.00 sec)
+
+We launch an instance of MySQL using this version of the software.
+
+amrith@amrith-work:/opt/stack/trove/integration/scripts$ openstack network list
++--------------------------------------+-------------+--------------------------------------+
+| ID                                   | Name        | Subnets                              |
++--------------------------------------+-------------+--------------------------------------+
+[...]
+| 4bab02e7-87bb-4cc0-8c07-2f282c777c85 | public      | e620c4f5-749c-4212-b1d1-4a6e2c0a3f16 |
+[...]
++--------------------------------------+-------------+--------------------------------------+
+
+amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove create m2 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85
++-------------------+--------------------------------------+
+| Property          | Value                                |
++-------------------+--------------------------------------+
+| created           | 2017-01-09T18:17:13                  |
+| datastore         | mysql                                |
+| datastore_version | 5.6                                  |
+| flavor            | 25                                   |
+| id                | bb0c9213-31f8-4427-8898-c644254b3642 |
+| name              | m2                                   |
+| region            | RegionOne                            |
+| server_id         | None                                 |
+| status            | BUILD                                |
+| updated           | 2017-01-09T18:17:13                  |
+| volume            | 3                                    |
+| volume_id         | None                                 |
++-------------------+--------------------------------------+
+
+amrith@amrith-work:/opt/stack/trove/integration/scripts$ nova list
++--------------------------------------+------+--------+------------+-------------+-------------------+
+| ID                                   | Name | Status | Task State | Power State | Networks          |
++--------------------------------------+------+--------+------------+-------------+-------------------+
+| a4769ce2-4e22-4134-b958-6db6c23cb221 | m2   | BUILD  | spawning   | NOSTATE     | public=172.24.4.4 |
++--------------------------------------+------+--------+------------+-------------+-------------------+
+
+And on that machine, the configuration file looks like this:
+
+amrith@m2:~$ cat /etc/trove/conf.d/guest_info.conf
+[DEFAULT]
+guest_id=bb0c9213-31f8-4427-8898-c644254b3642
+datastore_manager=mysql
+tenant_id=56cca8484d3e48869126ada4f355c284
+
+The instance goes online
+
+amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove show m2
++-------------------+--------------------------------------+
+| Property          | Value                                |
++-------------------+--------------------------------------+
+| created           | 2017-01-09T18:17:13                  |
+| datastore         | mysql                                |
+| datastore_version | 5.6                                  |
+| flavor            | 25                                   |
+| id                | bb0c9213-31f8-4427-8898-c644254b3642 |
+| name              | m2                                   |
+| region            | RegionOne                            |
+| server_id         | a4769ce2-4e22-4134-b958-6db6c23cb221 |
+| status            | ACTIVE                               |
+| updated           | 2017-01-09T18:17:17                  |
+| volume            | 3                                    |
+| volume_id         | 16e57e3f-b462-4db2-968b-3c284aa2751c |
+| volume_used       | 0.11                                 |
++-------------------+--------------------------------------+
+
+For testing later, we launch a few more instances.
+
+amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove create m3 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85
+amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove create m4 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85
+
+amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove list
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| ID                                   | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region    |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| 9ceebd62-e13d-43c5-953a-c0f24f08757e | m3   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| bb0c9213-31f8-4427-8898-c644254b3642 | m2   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+
+In this condition, we take down the control plane and upgrade the
+software running on it. This will result in a catalog upgrade. Since
+this system is based on devstack, here's what that looks like.
+
+amrith@amrith-work:/opt/stack/trove$ git branch
+* master
+  review/amrith/bp/secure-oslo-messaging-messages
+amrith@amrith-work:/opt/stack/trove$ git checkout review/amrith/bp/secure-oslo-messaging-messages
+Switched to branch 'review/amrith/bp/secure-oslo-messaging-messages'
+Your branch is ahead of 'gerrit/master' by 1 commit.
+  (use "git push" to publish your local commits)
+amrith@amrith-work:/opt/stack/trove$ find . -name '*.pyc' -delete
+amrith@amrith-work:/opt/stack/trove$
+
+amrith@amrith-work:/opt/stack/trove$ trove-manage db_sync
+[...]
+2017-01-09 13:24:25.251 DEBUG migrate.versioning.repository [-] Config: OrderedDict([('db_settings', OrderedDict([('__name__', 'db_settings'), ('repository_id', 'Trove Migrations'), ('version_table', 'migrate_version'), ('required_dbs', "['mysql','postgres','sqlite']")]))]) from (pid=96180) __init__ /usr/local/lib/python2.7/dist-packages/migrate/versioning/repository.py:83
+2017-01-09 13:24:25.260 INFO migrate.versioning.api [-] 40 -> 41...
+2017-01-09 13:24:25.328 INFO migrate.versioning.api [-] done
+2017-01-09 13:24:25.329 DEBUG migrate.versioning.util [-] Disposing SQLAlchemy engine Engine(mysql+pymysql://root:***@127.0.0.1/trove?charset=utf8) from (pid=96180) with_engine /usr/local/lib/python2.7/dist-packages/migrate/versioning/util/__init__.py:163
+[...]
+
+We observe that the new table in the system has the encrypted_key column
+
+mysql> describe instances;
++----------------------+--------------+------+-----+---------+-------+
+| Field                | Type         | Null | Key | Default | Extra |
++----------------------+--------------+------+-----+---------+-------+
+| id                   | varchar(36)  | NO   | PRI | NULL    |       |
+| created              | datetime     | YES  |     | NULL    |       |
+| updated              | datetime     | YES  |     | NULL    |       |
+| name                 | varchar(255) | YES  |     | NULL    |       |
+| hostname             | varchar(255) | YES  |     | NULL    |       |
+| compute_instance_id  | varchar(36)  | YES  |     | NULL    |       |
+| task_id              | int(11)      | YES  |     | NULL    |       |
+| task_description     | varchar(255) | YES  |     | NULL    |       |
+| task_start_time      | datetime     | YES  |     | NULL    |       |
+| volume_id            | varchar(36)  | YES  |     | NULL    |       |
+| flavor_id            | varchar(255) | YES  |     | NULL    |       |
+| volume_size          | int(11)      | YES  |     | NULL    |       |
+| tenant_id            | varchar(36)  | YES  | MUL | NULL    |       |
+| server_status        | varchar(64)  | YES  |     | NULL    |       |
+| deleted              | tinyint(1)   | YES  | MUL | NULL    |       |
+| deleted_at           | datetime     | YES  |     | NULL    |       |
+| datastore_version_id | varchar(36)  | NO   | MUL | NULL    |       |
+| configuration_id     | varchar(36)  | YES  | MUL | NULL    |       |
+| slave_of_id          | varchar(36)  | YES  | MUL | NULL    |       |
+| cluster_id           | varchar(36)  | YES  | MUL | NULL    |       |
+| shard_id             | varchar(36)  | YES  |     | NULL    |       |
+| type                 | varchar(64)  | YES  |     | NULL    |       |
+| region_id            | varchar(255) | YES  |     | NULL    |       |
+| encrypted_key        | varchar(255) | YES  |     | NULL    |       |
++----------------------+--------------+------+-----+---------+-------+
+
+
+mysql> select id, encrypted_key from instances;
++--------------------------------------+---------------+
+| id                                   | encrypted_key |
++--------------------------------------+---------------+
+| 13a787f2-b699-4867-a727-b3f4d8040a12 | NULL          |
++--------------------------------------+---------------+
+1 row in set (0.00 sec)
+
+amrith@amrith-work:/opt/stack/trove$ sudo python setup.py install -f
+[...]
+
+We can now relaunch the control plane software but before we do that,
+we inspect the configuration parameters and disable secure RPC
+messaging by adding this line into the configuration files.
+
+amrith@amrith-work:/etc/trove$ grep enable_secure_rpc_messaging *.conf
+trove-conductor.conf:enable_secure_rpc_messaging = False
+trove.conf:enable_secure_rpc_messaging = False
+trove-taskmanager.conf:enable_secure_rpc_messaging = False
+
+The first thing we observe is that heartbeat messages from the
+existing instance are still properly handled by the conductor and the
+instance remains active.
+
+2017-01-09 13:26:57.742 DEBUG oslo_messaging._drivers.amqpdriver [-] received message with unique_id: eafe22c08bae485e9346ce0fbdaa4d6c from (pid=96551) __call__ /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:196
+2017-01-09 13:26:57.744 DEBUG trove.conductor.manager [-] Instance ID: bb0c9213-31f8-4427-8898-c644254b3642, Payload: {u'service_status': u'running'} from (pid=96551) heartbeat /opt/stack/trove/trove/conductor/manager.py:88
+2017-01-09 13:26:57.748 DEBUG trove.conductor.manager [-] Instance bb0c9213-31f8-4427-8898-c644254b3642 sent heartbeat at 1483986416.52  from (pid=96551) _message_too_old /opt/stack/trove/trove/conductor/manager.py:54
+2017-01-09 13:26:57.750 DEBUG trove.conductor.manager [-] [Instance bb0c9213-31f8-4427-8898-c644254b3642] Rec'd message is younger than last seen. Updating. from (pid=96551) _message_too_old /opt/stack/trove/trove/conductor/manager.py:76
+2017-01-09 13:27:01.197 DEBUG oslo_messaging._drivers.amqpdriver [-] received message with unique_id: df62b76523004338876bc7b08f8b7711 from (pid=96552) __call__ /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:196
+2017-01-09 13:27:01.200 DEBUG trove.conductor.manager [-] Instance ID: 9ceebd62-e13d-43c5-953a-c0f24f08757e, Payload: {u'service_status': u'running'} from (pid=96552) heartbeat /opt/stack/trove/trove/conductor/manager.py:88
+2017-01-09 13:27:01.219 DEBUG oslo_db.sqlalchemy.engines [-] Parent process 96542 forked (96552) with an open database connection, which is being discarded and recreated. from (pid=96552) checkout /usr/local/lib/python2.7/dist-packages/oslo_db/sqlalchemy/engines.py:362
+2017-01-09 13:27:01.225 DEBUG trove.conductor.manager [-] Instance 9ceebd62-e13d-43c5-953a-c0f24f08757e sent heartbeat at 1483986419.99  from (pid=96552) _message_too_old /opt/stack/trove/trove/conductor/manager.py:54
+2017-01-09 13:27:01.231 DEBUG trove.conductor.manager [-] [Instance 9ceebd62-e13d-43c5-953a-c0f24f08757e] Rec'd message is younger than last seen. Updating. from (pid=96552) _message_too_old /opt/stack/trove/trove/conductor/manager.py:76
+
+amrith@amrith-work:/etc/trove$ trove list
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| ID                                   | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region    |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| 9ceebd62-e13d-43c5-953a-c0f24f08757e | m3   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| bb0c9213-31f8-4427-8898-c644254b3642 | m2   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+
+amrith@amrith-work:/etc/trove$ trove show m2
++-------------------+--------------------------------------+
+| Property          | Value                                |
++-------------------+--------------------------------------+
+| created           | 2017-01-09T18:17:13                  |
+| datastore         | mysql                                |
+| datastore_version | 5.6                                  |
+| flavor            | 25                                   |
+| id                | bb0c9213-31f8-4427-8898-c644254b3642 |
+| name              | m2                                   |
+| region            | RegionOne                            |
+| server_id         | a4769ce2-4e22-4134-b958-6db6c23cb221 |
+| status            | ACTIVE                               |
+| updated           | 2017-01-09T18:17:17                  |
+| volume            | 3                                    |
+| volume_id         | 16e57e3f-b462-4db2-968b-3c284aa2751c |
+| volume_used       | 0.11                                 |
++-------------------+--------------------------------------+
+
+We now launch a new instance, recall that secure_rpc_messaging is disabled.
+
+amrith@amrith-work:/etc/trove$ trove create m10 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85
++-------------------+--------------------------------------+
+| Property          | Value                                |
++-------------------+--------------------------------------+
+| created           | 2017-01-09T18:28:56                  |
+| datastore         | mysql                                |
+| datastore_version | 5.6                                  |
+| flavor            | 25                                   |
+| id                | 514ef051-0bf7-48a5-adcf-071d4a6625fb |
+| name              | m10                                  |
+| region            | RegionOne                            |
+| server_id         | None                                 |
+| status            | BUILD                                |
+| updated           | 2017-01-09T18:28:56                  |
+| volume            | 3                                    |
+| volume_id         | None                                 |
++-------------------+--------------------------------------+
+
+Observe that the task manager does not create a password for the instance.
+
+2017-01-09 13:29:00.111 INFO trove.instance.models [-] Resetting task status to NONE on instance 514ef051-0bf7-48a5-adcf-071d4a6625fb.
+2017-01-09 13:29:00.115 DEBUG trove.db.models [-] Saving DBInstance: {u'region_id': u'RegionOne', u'cluster_id': None, u'shard_id': None, u'deleted_at': None, u'id': u'514ef051-0bf7-48a5-adcf-071d4a6625fb', u'datastore_version_id': u'4a881cb5-9e48-4cb2-a209-4283ed44eb01', 'errors': {}, u'hostname': None, u'server_status': None, u'task_description': u'No tasks for the instance.', u'volume_size': 3, u'type': None, u'updated': datetime.datetime(2017, 1, 9, 18, 29, 0, 114971), '_sa_instance_state': <sqlalchemy.orm.state.InstanceState object at 0x7f460dbca410>, u'encrypted_key': None, u'deleted': 0, u'configuration_id': None, u'volume_id': u'cee2e17b-80fa-48e5-a488-da8b7809373a', u'slave_of_id': None, u'task_start_time': None, u'name': u'm10', u'task_id': 1, u'created': datetime.datetime(2017, 1, 9, 18, 28, 56), u'tenant_id': u'56cca8484d3e48869126ada4f355c284', u'compute_instance_id': u'2452263e-3d33-48ec-8f24-2851fe74db28', u'flavor_id': u'25'} from (pid=96635) save /opt/stack/trove/trove/db/models.py:64
+
+
+the configuration file for this instance is:
+
+amrith@m10:~$ cat /etc/trove/conf.d/guest_info.conf
+[DEFAULT]
+guest_id=514ef051-0bf7-48a5-adcf-071d4a6625fb
+datastore_manager=mysql
+tenant_id=56cca8484d3e48869126ada4f355c284
+
+We can now shutdown the control plane again and enable the secure RPC
+capability. Observe that we've just commented out the lines (below).
+
+trove-conductor.conf:# enable_secure_rpc_messaging = False
+trove.conf:# enable_secure_rpc_messaging = False
+trove-taskmanager.conf:# enable_secure_rpc_messaging = False
+
+And create another database instance
+
+amrith@amrith-work:/etc/trove$ trove create m20 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85
++-------------------+--------------------------------------+
+| Property          | Value                                |
++-------------------+--------------------------------------+
+| created           | 2017-01-09T18:31:48                  |
+| datastore         | mysql                                |
+| datastore_version | 5.6                                  |
+| flavor            | 25                                   |
+| id                | 792fa220-2a40-4831-85af-cfb0ded8033c |
+| name              | m20                                  |
+| region            | RegionOne                            |
+| server_id         | None                                 |
+| status            | BUILD                                |
+| updated           | 2017-01-09T18:31:48                  |
+| volume            | 3                                    |
+| volume_id         | None                                 |
++-------------------+--------------------------------------+
+
+Observe that a unique per-instance encryption key was created for this instance.
+
+2017-01-09 13:31:52.474 DEBUG trove.db.models [-] Saving DBInstance: {u'region_id': u'RegionOne', u'cluster_id': None, u'shard_id': None, u'deleted_at': None, u'id': u'792fa220-2a40-4831-85af-cfb0ded8033c', u'datastore_version_id': u'4a881cb5-9e48-4cb2-a209-4283ed44eb01', 'errors': {}, u'hostname': None, u'server_status': None, u'task_description': u'No tasks for the instance.', u'volume_size': 3, u'type': None, u'updated': datetime.datetime(2017, 1, 9, 18, 31, 52, 473552), '_sa_instance_state': <sqlalchemy.orm.state.InstanceState object at 0x7fdb14d44550>, u'encrypted_key': u'fVpHrkUIjVsXe7Fj7Lm4u2xnJUsWX2rMC9GL0AppILJINBZxLvkowY8FOa+asKS+8pWb4iNyukQQ4AQoLEUHUQ==', u'deleted': 0, u'configuration_id': None, u'volume_id': u'4cd563dc-fe08-477b-828f-120facf4351b', u'slave_of_id': None, u'task_start_time': None, u'name': u'm20', u'task_id': 1, u'created': datetime.datetime(2017, 1, 9, 18, 31, 49), u'tenant_id': u'56cca8484d3e48869126ada4f355c284', u'compute_instance_id': u'1e62a192-83d3-43fd-b32e-b5ee2fa4e24b', u'flavor_id': u'25'} from (pid=97562) save /opt/stack/trove/trove/db/models.py:64
+
+And the configuration file on that instance includes an encryption key.
+
+amrith@m20:~$ cat /etc/trove/conf.d/guest_info.conf
+[DEFAULT]
+guest_id=792fa220-2a40-4831-85af-cfb0ded8033c
+datastore_manager=mysql
+tenant_id=56cca8484d3e48869126ada4f355c284
+instance_rpc_encr_key=eRz43LwE6eaxIbBlA2pNukzPjSdcQkVi
+
+amrith@amrith-work:/etc/trove$ trove list
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| ID                                   | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region    |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| 514ef051-0bf7-48a5-adcf-071d4a6625fb | m10  | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| 792fa220-2a40-4831-85af-cfb0ded8033c | m20  | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| 9ceebd62-e13d-43c5-953a-c0f24f08757e | m3   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| bb0c9213-31f8-4427-8898-c644254b3642 | m2   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+
+At this point communication between API service and Task Manager, and
+between the control plane and instance m20 is encrypted but
+communication between control plane and all other instances is not
+encrypted.
+
+In this condition we can attempt some operations on the various
+instances. First with the legacy instances created on software that
+predated the secure RPC mechanism.
+
+amrith@amrith-work:/etc/trove$ trove database-list m2
++------+
+| Name |
++------+
++------+
+amrith@amrith-work:/etc/trove$ trove database-create m2 foo2
+amrith@amrith-work:/etc/trove$ trove database-list m2
++------+
+| Name |
++------+
+| foo2 |
++------+
+
+And at the same time with the instance m10 which is created with the
+current software but without RPC encryption.
+
+amrith@amrith-work:/etc/trove$ trove database-list m10
++------+
+| Name |
++------+
++------+
+amrith@amrith-work:/etc/trove$ trove database-create m10 foo10
+amrith@amrith-work:/etc/trove$ trove database-list m10
++-------+
+| Name  |
++-------+
+| foo10 |
++-------+
+amrith@amrith-work:/etc/trove$
+
+And finally with an instance that uses encrypted RPC communications.
+
+amrith@amrith-work:/etc/trove$ trove database-list m20
++------+
+| Name |
++------+
++------+
+amrith@amrith-work:/etc/trove$ trove database-create m20 foo20
+amrith@amrith-work:/etc/trove$ trove database-list m20
++-------+
+| Name  |
++-------+
+| foo20 |
++-------+
+
+Finally, we can upgrade an instance that has no encryption to have rpc
+encryption.
+
+amrith@amrith-work:/etc/trove$ trove datastore-list
++--------------------------------------+------------------+
+| ID                                   | Name             |
++--------------------------------------+------------------+
+| 8e052edb-5f14-4aec-9149-0a80a30cf5e4 | mysql            |
++--------------------------------------+------------------+
+amrith@amrith-work:/etc/trove$ trove datastore-version-list mysql
++--------------------------------------+------------------+
+| ID                                   | Name             |
++--------------------------------------+------------------+
+| 4a881cb5-9e48-4cb2-a209-4283ed44eb01 | 5.6              |
++--------------------------------------+------------------+
+
+Let's look at instance m2.
+
+mysql> select id, name, encrypted_key from instances where id = 'bb0c9213-31f8-4427-8898-c644254b3642';
++--------------------------------------+------+---------------+
+| id                                   | name | encrypted_key |
++--------------------------------------+------+---------------+
+| bb0c9213-31f8-4427-8898-c644254b3642 | m2   | NULL          |
++--------------------------------------+------+---------------+
+1 row in set (0.00 sec)
+
+amrith@amrith-work:/etc/trove$ trove upgrade m2 4a881cb5-9e48-4cb2-a209-4283ed44eb01
+
+amrith@amrith-work:/etc/trove$ trove list
++--------------------------------------+------+-----------+-------------------+---------+-----------+------+-----------+
+| ID                                   | Name | Datastore | Datastore Version | Status  | Flavor ID | Size | Region    |
++--------------------------------------+------+-----------+-------------------+---------+-----------+------+-----------+
+| 514ef051-0bf7-48a5-adcf-071d4a6625fb | m10  | mysql     | 5.6               | ACTIVE  | 25        |    3 | RegionOne |
+| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4   | mysql     | 5.6               | ACTIVE  | 25        |    3 | RegionOne |
+| 792fa220-2a40-4831-85af-cfb0ded8033c | m20  | mysql     | 5.6               | ACTIVE  | 25        |    3 | RegionOne |
+| 9ceebd62-e13d-43c5-953a-c0f24f08757e | m3   | mysql     | 5.6               | ACTIVE  | 25        |    3 | RegionOne |
+| bb0c9213-31f8-4427-8898-c644254b3642 | m2   | mysql     | 5.6               | UPGRADE | 25        |    3 | RegionOne |
++--------------------------------------+------+-----------+-------------------+---------+-----------+------+-----------+
+
+amrith@amrith-work:/etc/trove$ nova list
++--------------------------------------+------+---------+------------+-------------+--------------------+
+| ID                                   | Name | Status  | Task State | Power State | Networks           |
++--------------------------------------+------+---------+------------+-------------+--------------------+
+[...]
+| a4769ce2-4e22-4134-b958-6db6c23cb221 | m2   | REBUILD | rebuilding | Running     | public=172.24.4.4  |
+[...]
++--------------------------------------+------+---------+------------+-------------+--------------------+
+
+
+2017-01-09 13:47:24.337 DEBUG trove.db.models [-] Saving DBInstance: {u'region_id': u'RegionOne', u'cluster_id': None, u'shard_id': None, u'deleted_at': None, u'id': u'bb0c9213-31f8-4427-8898-c644254b3642', u'datastore_version_id': u'4a881cb5-9e48-4cb2-a209-4283ed44eb01', 'errors': {}, u'hostname': None, u'server_status': None, u'task_description': u'Upgrading the instance.', u'volume_size': 3, u'type': None, u'updated': datetime.datetime(2017, 1, 9, 18, 47, 24, 337400), '_sa_instance_state': <sqlalchemy.orm.state.InstanceState object at 0x7fdb14d44150>, u'encrypted_key': u'gMrlHkEVxKgEFMTabzZr2TLJ6r5+wgfJfhohs7K/BzutWxs1wXfBswyV5Bgw4qeD212msmgSdOUCFov5otgzyg==', u'deleted': 0, u'configuration_id': None, u'volume_id': u'16e57e3f-b462-4db2-968b-3c284aa2751c', u'slave_of_id': None, u'task_start_time': None, u'name': u'm2', u'task_id': 89, u'created': datetime.datetime(2017, 1, 9, 18, 17, 13), u'tenant_id': u'56cca8484d3e48869126ada4f355c284', u'compute_instance_id': u'a4769ce2-4e22-4134-b958-6db6c23cb221', u'flavor_id': u'25'} from (pid=97562) save /opt/stack/trove/trove/db/models.py:64
+2017-01-09 13:47:24.347 DEBUG trove.taskmanager.models [-] Generated unique RPC encryption key for instance = bb0c9213-31f8-4427-8898-c644254b3642, key = gMrlHkEVxKgEFMTabzZr2TLJ6r5+wgfJfhohs7K/BzutWxs1wXfBswyV5Bgw4qeD212msmgSdOUCFov5otgzyg== from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1440
+2017-01-09 13:47:24.350 DEBUG trove.taskmanager.models [-] Rebuilding instance m2(bb0c9213-31f8-4427-8898-c644254b3642) with image ea05cba7-2f70-4745-abea-136d7bcc16c7. from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1445
+
+The instance now has an encryption key in its configuration
+
+amrith@m2:~$ cat /etc/trove/conf.d/guest_info.conf
+[DEFAULT]
+guest_id=bb0c9213-31f8-4427-8898-c644254b3642
+datastore_manager=mysql
+tenant_id=56cca8484d3e48869126ada4f355c284
+instance_rpc_encr_key=pN2hHEl171ngyD0mPvyV1xKJF2im01Gv
+
+amrith@amrith-work:/etc/trove$ trove list
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| ID                                   | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region    |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+[...]
+| bb0c9213-31f8-4427-8898-c644254b3642 | m2   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+[...]
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+
+amrith@amrith-work:/etc/trove$ trove show m2
++-------------------+--------------------------------------+
+| Property          | Value                                |
++-------------------+--------------------------------------+
+| created           | 2017-01-09T18:17:13                  |
+| datastore         | mysql                                |
+| datastore_version | 5.6                                  |
+| flavor            | 25                                   |
+| id                | bb0c9213-31f8-4427-8898-c644254b3642 |
+| name              | m2                                   |
+| region            | RegionOne                            |
+| server_id         | a4769ce2-4e22-4134-b958-6db6c23cb221 |
+| status            | ACTIVE                               |
+| updated           | 2017-01-09T18:50:07                  |
+| volume            | 3                                    |
+| volume_id         | 16e57e3f-b462-4db2-968b-3c284aa2751c |
+| volume_used       | 0.13                                 |
++-------------------+--------------------------------------+
+
+amrith@amrith-work:/etc/trove$ trove database-list m2
++------+
+| Name |
++------+
+| foo2 |
++------+
+
+We can similarly upgrade m4.
+
+2017-01-09 13:51:43.078 DEBUG trove.instance.models [-] Instance 6d55ab3a-267f-4b95-8ada-33fc98fd1767 service status is running. from (pid=97562) load_instance /opt/stack/trove/trove/instance/models.py:534
+2017-01-09 13:51:43.083 DEBUG trove.taskmanager.models [-] Upgrading instance m4(6d55ab3a-267f-4b95-8ada-33fc98fd1767) to new datastore version 5.6(4a881cb5-9e48-4cb2-a209-4283ed44eb01) from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1410
+2017-01-09 13:51:43.087 DEBUG trove.guestagent.api [-] Sending the call to prepare the guest for upgrade. from (pid=97562) pre_upgrade /opt/stack/trove/trove/guestagent/api.py:351
+2017-01-09 13:51:43.087 DEBUG trove.guestagent.api [-] Calling pre_upgrade with timeout 600 from (pid=97562) _call /opt/stack/trove/trove/guestagent/api.py:86
+2017-01-09 13:51:43.088 DEBUG oslo_messaging._drivers.amqpdriver [-] CALL msg_id: 41dbb7fff3dc4f8fa69d8b5f219809e0 exchange 'trove' topic 'guestagent.6d55ab3a-267f-4b95-8ada-33fc98fd1767' from (pid=97562) _send /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:442
+2017-01-09 13:51:45.452 DEBUG oslo_messaging._drivers.amqpdriver [-] received reply msg_id: 41dbb7fff3dc4f8fa69d8b5f219809e0 from (pid=97562) __call__ /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:299
+2017-01-09 13:51:45.452 DEBUG trove.guestagent.api [-] Result is {u'mount_point': u'/var/lib/mysql', u'save_etc_dir': u'/var/lib/mysql/etc', u'home_save': u'/var/lib/mysql/trove_user', u'save_dir': u'/var/lib/mysql/etc_mysql'}. from (pid=97562) _call /opt/stack/trove/trove/guestagent/api.py:91
+2017-01-09 13:51:45.544 DEBUG trove.db.models [-] Saving DBInstance: {u'region_id': u'RegionOne', u'cluster_id': None, u'shard_id': None, u'deleted_at': None, u'id': u'6d55ab3a-267f-4b95-8ada-33fc98fd1767', u'datastore_version_id': u'4a881cb5-9e48-4cb2-a209-4283ed44eb01', 'errors': {}, u'hostname': None, u'server_status': None, u'task_description': u'Upgrading the instance.', u'volume_size': 3, u'type': None, u'updated': datetime.datetime(2017, 1, 9, 18, 51, 45, 544496), '_sa_instance_state': <sqlalchemy.orm.state.InstanceState object at 0x7fdb14972c10>, u'encrypted_key': u'0gBkJl5Aqb4kFIPeJDMTNIymEUuUUB8NBksecTiYyQl+Ibrfi7ME8Bi58q2n61AxbG2coOqp97ETjHRyN7mYTg==', u'deleted': 0, u'configuration_id': None, u'volume_id': u'b7dc17b5-d0a8-47bb-aef4-ef9432c269e9', u'slave_of_id': None, u'task_start_time': None, u'name': u'm4', u'task_id': 89, u'created': datetime.datetime(2017, 1, 9, 18, 20, 58), u'tenant_id': u'56cca8484d3e48869126ada4f355c284', u'compute_instance_id': u'f43bba63-3be6-4993-b2d0-4ddfb7818d27', u'flavor_id': u'25'} from (pid=97562) save /opt/stack/trove/trove/db/models.py:64
+2017-01-09 13:51:45.557 DEBUG trove.taskmanager.models [-] Generated unique RPC encryption key for instance = 6d55ab3a-267f-4b95-8ada-33fc98fd1767, key = 0gBkJl5Aqb4kFIPeJDMTNIymEUuUUB8NBksecTiYyQl+Ibrfi7ME8Bi58q2n61AxbG2coOqp97ETjHRyN7mYTg== from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1440
+2017-01-09 13:51:45.560 DEBUG trove.taskmanager.models [-] Rebuilding instance m4(6d55ab3a-267f-4b95-8ada-33fc98fd1767) with image ea05cba7-2f70-4745-abea-136d7bcc16c7. from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1445
+
+amrith@amrith-work:/etc/trove$ nova list
++--------------------------------------+------+---------+------------+-------------+--------------------+
+| ID                                   | Name | Status  | Task State | Power State | Networks           |
++--------------------------------------+------+---------+------------+-------------+--------------------+
+[...]
+| f43bba63-3be6-4993-b2d0-4ddfb7818d27 | m4   | REBUILD | rebuilding | Running     | public=172.24.4.11 |
+[...]
++--------------------------------------+------+---------+------------+-------------+--------------------+
+
+2017-01-09 13:53:26.581 DEBUG trove.guestagent.api [-] Recover the guest after upgrading the guest's image. from (pid=97562) post_upgrade /opt/stack/trove/trove/guestagent/api.py:359
+2017-01-09 13:53:26.581 DEBUG trove.guestagent.api [-] Recycling the client ... from (pid=97562) post_upgrade /opt/stack/trove/trove/guestagent/api.py:361
+2017-01-09 13:53:26.581 DEBUG trove.guestagent.api [-] Calling post_upgrade with timeout 600 from (pid=97562) _call /opt/stack/trove/trove/guestagent/api.py:86
+2017-01-09 13:53:26.583 DEBUG oslo_messaging._drivers.amqpdriver [-] CALL msg_id: 2e9ccc88715b4b98848a017e19b2938d exchange 'trove' topic 'guestagent.6d55ab3a-267f-4b95-8ada-33fc98fd1767' from (pid=97562) _send /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:442
+
+mysql> select id, name, encrypted_key from instances where name in ('m2', 'm4', 'm10', 'm20');
++--------------------------------------+------+------------------------------------------------------------------------------------------+
+| id                                   | name | encrypted_key                                                                            |
++--------------------------------------+------+------------------------------------------------------------------------------------------+
+| 514ef051-0bf7-48a5-adcf-071d4a6625fb | m10  | NULL                                                                                     |
+| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4   | 0gBkJl5Aqb4kFIPeJDMTNIymEUuUUB8NBksecTiYyQl+Ibrfi7ME8Bi58q2n61AxbG2coOqp97ETjHRyN7mYTg== |
+| 792fa220-2a40-4831-85af-cfb0ded8033c | m20  | fVpHrkUIjVsXe7Fj7Lm4u2xnJUsWX2rMC9GL0AppILJINBZxLvkowY8FOa+asKS+8pWb4iNyukQQ4AQoLEUHUQ== |
+| bb0c9213-31f8-4427-8898-c644254b3642 | m2   | gMrlHkEVxKgEFMTabzZr2TLJ6r5+wgfJfhohs7K/BzutWxs1wXfBswyV5Bgw4qeD212msmgSdOUCFov5otgzyg== |
++--------------------------------------+------+------------------------------------------------------------------------------------------+
+
+amrith@amrith-work:/etc/trove$ trove list
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| ID                                   | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region    |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+| 514ef051-0bf7-48a5-adcf-071d4a6625fb | m10  | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| 792fa220-2a40-4831-85af-cfb0ded8033c | m20  | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
+| bb0c9213-31f8-4427-8898-c644254b3642 | m2   | mysql     | 5.6               | ACTIVE | 25        |    3 | RegionOne |
++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+
+
+Inspecting which instances are using secure RPC communications
+--------------------------------------------------------------
+
+An additional field is returned in the trove show command output to
+indicate whether any given instance is using secure RPC communication
+or not.
+
+NOTE: This field is only returned if the user is an 'admin'. Non admin
+users do not see the field.
+
+amrith@amrith-work:/opt/stack/trove$ trove show m20
++-------------------------+--------------------------------------+
+| Property                | Value                                |
++-------------------------+--------------------------------------+
+| created                 | 2017-01-09T18:31:49                  |
+| datastore               | mysql                                |
+| datastore_version       | 5.6                                  |
+| encrypted_rpc_messaging | True                                 |
+| flavor                  | 25                                   |
+| id                      | 792fa220-2a40-4831-85af-cfb0ded8033c |
+| name                    | m20                                  |
+| region                  | RegionOne                            |
+| server_id               | 1e62a192-83d3-43fd-b32e-b5ee2fa4e24b |
+| status                  | ACTIVE                               |
+| updated                 | 2017-01-09T18:31:52                  |
+| volume                  | 3                                    |
+| volume_id               | 4cd563dc-fe08-477b-828f-120facf4351b |
+| volume_used             | 0.11                                 |
++-------------------------+--------------------------------------+
+amrith@amrith-work:/opt/stack/trove$ trove show m10
++-------------------------+--------------------------------------+
+| Property                | Value                                |
++-------------------------+--------------------------------------+
+| created                 | 2017-01-09T18:28:56                  |
+| datastore               | mysql                                |
+| datastore_version       | 5.6                                  |
+| encrypted_rpc_messaging | False                                |
+| flavor                  | 25                                   |
+| id                      | 514ef051-0bf7-48a5-adcf-071d4a6625fb |
+| name                    | m10                                  |
+| region                  | RegionOne                            |
+| server_id               | 2452263e-3d33-48ec-8f24-2851fe74db28 |
+| status                  | ACTIVE                               |
+| updated                 | 2017-01-09T18:29:00                  |
+| volume                  | 3                                    |
+| volume_id               | cee2e17b-80fa-48e5-a488-da8b7809373a |
+| volume_used             | 0.11                                 |
++-------------------------+--------------------------------------+
+amrith@amrith-work:/opt/stack/trove$ trove show m2
++-------------------------+--------------------------------------+
+| Property                | Value                                |
++-------------------------+--------------------------------------+
+| created                 | 2017-01-09T18:17:13                  |
+| datastore               | mysql                                |
+| datastore_version       | 5.6                                  |
+| encrypted_rpc_messaging | True                                 |
+| flavor                  | 25                                   |
+| id                      | bb0c9213-31f8-4427-8898-c644254b3642 |
+| name                    | m2                                   |
+| region                  | RegionOne                            |
+| server_id               | a4769ce2-4e22-4134-b958-6db6c23cb221 |
+| status                  | ACTIVE                               |
+| updated                 | 2017-01-09T18:50:07                  |
+| volume                  | 3                                    |
+| volume_id               | 16e57e3f-b462-4db2-968b-3c284aa2751c |
+| volume_used             | 0.13                                 |
++-------------------------+--------------------------------------+
+amrith@amrith-work:/opt/stack/trove$ trove show m4
++-------------------------+--------------------------------------+
+| Property                | Value                                |
++-------------------------+--------------------------------------+
+| created                 | 2017-01-09T18:20:58                  |
+| datastore               | mysql                                |
+| datastore_version       | 5.6                                  |
+| encrypted_rpc_messaging | True                                 |
+| flavor                  | 25                                   |
+| id                      | 6d55ab3a-267f-4b95-8ada-33fc98fd1767 |
+| name                    | m4                                   |
+| region                  | RegionOne                            |
+| server_id               | f43bba63-3be6-4993-b2d0-4ddfb7818d27 |
+| status                  | ACTIVE                               |
+| updated                 | 2017-01-09T18:54:30                  |
+| volume                  | 3                                    |
+| volume_id               | b7dc17b5-d0a8-47bb-aef4-ef9432c269e9 |
+| volume_used             | 0.13                                 |
++-------------------------+--------------------------------------+
+amrith@amrith-work:/opt/stack/trove$
+
+In the API response, note that the additional key
+"encrypted_rpc_messaging" has been added (as below).
+
+NOTE: This field is only returned if the user is an 'admin'. Non admin
+users do not see the field.
+
+RESP BODY: {"instance": {"status": "ACTIVE", "updated": "2017-01-09T18:29:00", "name": "m10", "links": [{"href": "https://192.168.126.130:8779/v1.0/56cca8484d3e48869126ada4f355c284/instances/514ef051-0bf7-48a5-adcf-071d4a6625fb", "rel": "self"}, {"href": "https://192.168.126.130:8779/instances/514ef051-0bf7-48a5-adcf-071d4a6625fb", "rel": "bookmark"}], "created": "2017-01-09T18:28:56", "region": "RegionOne", "server_id": "2452263e-3d33-48ec-8f24-2851fe74db28", "id": "514ef051-0bf7-48a5-adcf-071d4a6625fb", "volume": {"used": 0.11, "size": 3}, "volume_id": "cee2e17b-80fa-48e5-a488-da8b7809373a", "flavor": {"id": "25", "links": [{"href": "https://192.168.126.130:8779/v1.0/56cca8484d3e48869126ada4f355c284/flavors/25", "rel": "self"}, {"href": "https://192.168.126.130:8779/flavors/25", "rel": "bookmark"}]}, "datastore": {"version": "5.6", "type": "mysql"}, "encrypted_rpc_messaging": false}}
diff --git a/doc/source/index.rst b/doc/source/index.rst
index f803a376..ed1511d1 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -51,6 +51,7 @@ functionality, the following resources are provided.
   dev/guest_cloud_init.rst
   dev/notifier.rst
   dev/trove_api_extensions.rst
+  dev/secure_oslo_messaging.rst
 
 * Source Code Repositories
 
diff --git a/run_tests.py b/run_tests.py
index 5f4c98ee..eb00e032 100644
--- a/run_tests.py
+++ b/run_tests.py
@@ -76,7 +76,8 @@ def initialize_trove(config_file):
     rpc.init(CONF)
 
     taskman_service = rpc_service.RpcService(
-        None, topic=topic, rpc_api_version=rpc_version.RPC_API_VERSION,
+        CONF.taskmanager_rpc_encr_key, topic=topic,
+        rpc_api_version=rpc_version.RPC_API_VERSION,
         manager='trove.taskmanager.manager.Manager')
     taskman_service.start()
 
diff --git a/tools/trove-pylint.config b/tools/trove-pylint.config
index b67f5fa1..fad15049 100644
--- a/tools/trove-pylint.config
+++ b/tools/trove-pylint.config
@@ -730,6 +730,18 @@
       "upgrade"
     ],
     [
+      "trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py",
+      "E1101",
+      "Instance of 'Table' has no 'create_column' member",
+      "upgrade"
+    ],
+    [
+      "trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py",
+      "no-member",
+      "Instance of 'Table' has no 'create_column' member",
+      "upgrade"
+    ],
+    [
       "trove/db/sqlalchemy/migration.py",
       "E0611",
       "No name 'exceptions' in module 'migrate.versioning'",
@@ -1109,11 +1121,23 @@
     ],
     [
       "trove/instance/models.py",
+      "E1101",
+      "Instance of 'DBInstance' has no 'encrypted_key' member",
+      "DBInstance.key"
+    ],
+    [
+      "trove/instance/models.py",
       "no-member",
       "Class 'InstanceStatus' has no 'LOGGING' member",
       "SimpleInstance.status"
     ],
     [
+      "trove/instance/models.py",
+      "no-member",
+      "Instance of 'DBInstance' has no 'encrypted_key' member",
+      "DBInstance.key"
+    ],
+    [
       "trove/instance/service.py",
       "E1101",
       "Instance of 'BuiltInstance' has no 'get_default_configuration_template' member",
diff --git a/trove/cmd/conductor.py b/trove/cmd/conductor.py
index daff5df4..793ad6b6 100644
--- a/trove/cmd/conductor.py
+++ b/trove/cmd/conductor.py
@@ -22,6 +22,7 @@ from trove.conductor import api as conductor_api
 @with_initialize
 def main(conf):
     from trove.common import notification
+    from trove.common.rpc import conductor_host_serializer as sz
     from trove.common.rpc import service as rpc_service
     from trove.instance import models as inst_models
 
@@ -29,8 +30,9 @@ def main(conf):
         inst_models.persist_instance_fault)
     topic = conf.conductor_queue
     server = rpc_service.RpcService(
-        manager=conf.conductor_manager, topic=topic,
-        rpc_api_version=conductor_api.API.API_LATEST_VERSION)
+        key=None, manager=conf.conductor_manager, topic=topic,
+        rpc_api_version=conductor_api.API.API_LATEST_VERSION,
+        secure_serializer=sz.ConductorHostSerializer)
     workers = conf.trove_conductor_workers or processutils.get_worker_count()
     launcher = openstack_service.launch(conf, server, workers=workers)
     launcher.wait()
diff --git a/trove/cmd/fakemode.py b/trove/cmd/fakemode.py
index 66e5b3cd..e66431fd 100644
--- a/trove/cmd/fakemode.py
+++ b/trove/cmd/fakemode.py
@@ -54,7 +54,7 @@ def start_fake_taskmanager(conf):
     from trove.common.rpc import service as rpc_service
     from trove.common.rpc import version as rpc_version
     taskman_service = rpc_service.RpcService(
-        topic=topic, rpc_api_version=rpc_version.RPC_API_VERSION,
+        key='', topic=topic, rpc_api_version=rpc_version.RPC_API_VERSION,
         manager='trove.taskmanager.manager.Manager')
     taskman_service.start()
 
diff --git a/trove/cmd/guest.py b/trove/cmd/guest.py
index ccb33563..19692d14 100644
--- a/trove/cmd/guest.py
+++ b/trove/cmd/guest.py
@@ -30,13 +30,15 @@ from trove.guestagent import api as guest_api
 CONF = cfg.CONF
 # The guest_id opt definition must match the one in common/cfg.py
 CONF.register_opts([openstack_cfg.StrOpt('guest_id', default=None,
-                                         help="ID of the Guest Instance.")])
+                                         help="ID of the Guest Instance."),
+                    openstack_cfg.StrOpt('instance_rpc_encr_key',
+                                         help=('Key (OpenSSL aes_cbc) for '
+                                               'instance RPC encryption.'))])
 
 
 def main():
     cfg.parse_args(sys.argv)
     logging.setup(CONF, None)
-
     debug_utils.setup()
 
     from trove.guestagent import dbaas
@@ -51,6 +53,9 @@ def main():
                "was not injected into the guest or not read by guestagent"))
         raise RuntimeError(msg)
 
+    # BUG(1650518): Cleanup in the Pike release
+    # make it fatal if CONF.instance_rpc_encr_key is None
+
     # rpc module must be loaded after decision about thread monkeypatching
     # because if thread module is not monkeypatched we can't use eventlet
     # executor from oslo_messaging library.
@@ -59,6 +64,7 @@ def main():
 
     from trove.common.rpc import service as rpc_service
     server = rpc_service.RpcService(
+        key=CONF.instance_rpc_encr_key,
         topic="guestagent.%s" % CONF.guest_id,
         manager=manager, host=CONF.guest_id,
         rpc_api_version=guest_api.API.API_LATEST_VERSION)
diff --git a/trove/cmd/taskmanager.py b/trove/cmd/taskmanager.py
index aaef017c..549e14b2 100644
--- a/trove/cmd/taskmanager.py
+++ b/trove/cmd/taskmanager.py
@@ -29,8 +29,14 @@ def startup(conf, topic):
 
     notification.DBaaSAPINotification.register_notify_callback(
         inst_models.persist_instance_fault)
+
+    if conf.enable_secure_rpc_messaging:
+        key = conf.taskmanager_rpc_encr_key
+    else:
+        key = None
+
     server = rpc_service.RpcService(
-        manager=conf.taskmanager_manager, topic=topic,
+        key=key, manager=conf.taskmanager_manager, topic=topic,
         rpc_api_version=task_api.API.API_LATEST_VERSION)
     launcher = openstack_service.launch(conf, server)
     launcher.wait()
diff --git a/trove/common/cfg.py b/trove/common/cfg.py
index b4b0c655..20051650 100644
--- a/trove/common/cfg.py
+++ b/trove/common/cfg.py
@@ -444,6 +444,16 @@ common_opts = [
                help='Maximum size of a chunk saved in guest log container.'),
     cfg.IntOpt('guest_log_expiry', default=2592000,
                help='Expiry (in seconds) of objects in guest log container.'),
+    cfg.BoolOpt('enable_secure_rpc_messaging', default=True,
+                help='Should RPC messaging traffic be secured by encryption.'),
+    cfg.StrOpt('taskmanager_rpc_encr_key',
+               default='bzH6y0SGmjuoY0FNSTptrhgieGXNDX6PIhvz',
+               help='Key (OpenSSL aes_cbc) for taskmanager RPC encryption.'),
+    cfg.StrOpt('inst_rpc_key_encr_key',
+               default='emYjgHFqfXNB1NGehAFIUeoyw4V4XwWHEaKP',
+               help='Key (OpenSSL aes_cbc) to encrypt instance keys in DB.'),
+    cfg.StrOpt('instance_rpc_encr_key',
+               help='Key (OpenSSL aes_cbc) for instance RPC encryption.'),
 ]
 
 
diff --git a/trove/common/context.py b/trove/common/context.py
index 254993d8..73d8a858 100644
--- a/trove/common/context.py
+++ b/trove/common/context.py
@@ -39,6 +39,7 @@ class TroveContext(context.RequestContext):
         self.marker = kwargs.pop('marker', None)
         self.service_catalog = kwargs.pop('service_catalog', None)
         self.user_identity = kwargs.pop('user_identity', None)
+        self.instance_id = kwargs.pop('instance_id', None)
 
         # TODO(esp): not sure we need this
         self.timeout = kwargs.pop('timeout', None)
diff --git a/trove/common/crypto_utils.py b/trove/common/crypto_utils.py
index bd8e3fb0..9e3d5613 100644
--- a/trove/common/crypto_utils.py
+++ b/trove/common/crypto_utils.py
@@ -20,7 +20,9 @@ from Crypto.Cipher import AES
 from Crypto import Random
 import hashlib
 from oslo_utils import encodeutils
+import random
 import six
+import string
 
 from trove.common import stream_codecs
 
@@ -68,3 +70,9 @@ def decrypt_data(data, key, iv_bit_count=IV_BIT_COUNT):
     aes = AES.new(md5_key, AES.MODE_CBC, bytes(iv))
     decrypted = aes.decrypt(bytes(data[iv_bit_count:]))
     return unpad_after_decryption(decrypted)
+
+
+def generate_random_key(length=32, chars=None):
+    chars = chars if chars else (string.ascii_uppercase +
+                                 string.ascii_lowercase + string.digits)
+    return ''.join(random.choice(chars) for _ in range(length))
diff --git a/trove/common/rpc/conductor_guest_serializer.py b/trove/common/rpc/conductor_guest_serializer.py
new file mode 100644
index 00000000..e3b8afa3
--- /dev/null
+++ b/trove/common/rpc/conductor_guest_serializer.py
@@ -0,0 +1,60 @@
+# Copyright 2016 Tesora, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_serialization import jsonutils
+
+from trove.common import crypto_utils as crypto
+from trove.common.i18n import _
+from trove.common.rpc import serializer
+
+CONF = cfg.CONF
+
+
+# BUG(1650518): Cleanup in the Pike release
+class ConductorGuestSerializer(serializer.TroveSerializer):
+    def __init__(self, base, key):
+        self._key = key
+        super(ConductorGuestSerializer, self).__init__(base)
+
+    def _serialize_entity(self, ctxt, entity):
+        if self._key is None:
+            return entity
+
+        value = crypto.encode_data(
+            crypto.encrypt_data(
+                jsonutils.dumps(entity), self._key))
+
+        return jsonutils.dumps({'entity': value, 'csz-instance-id':
+                                CONF.guest_id})
+
+    def _deserialize_entity(self, ctxt, entity):
+        msg = (_("_deserialize_entity not implemented in "
+                 "ConductorGuestSerializer."))
+        raise Exception(msg)
+
+    def _serialize_context(self, ctxt):
+        if self._key is None:
+            return ctxt
+
+        cstr = jsonutils.dumps(ctxt)
+
+        return {'context':
+                crypto.encode_data(
+                    crypto.encrypt_data(cstr, self._key)),
+                'csz-instance-id': CONF.guest_id}
+
+    def _deserialize_context(self, ctxt):
+        msg = (_("_deserialize_context not implemented in "
+                 "ConductorGuestSerializer."))
+        raise Exception(msg)
diff --git a/trove/common/rpc/conductor_host_serializer.py b/trove/common/rpc/conductor_host_serializer.py
new file mode 100644
index 00000000..0e17efd2
--- /dev/null
+++ b/trove/common/rpc/conductor_host_serializer.py
@@ -0,0 +1,83 @@
+# Copyright 2016 Tesora, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_serialization import jsonutils
+
+from trove.common import crypto_utils as cu
+from trove.common.rpc import serializer
+from trove.instance.models import get_instance_encryption_key
+
+CONF = cfg.CONF
+
+
+# BUG(1650518): Cleanup in the Pike release
+class ConductorHostSerializer(serializer.TroveSerializer):
+    def __init__(self, base, *_):
+        super(ConductorHostSerializer, self).__init__(base)
+
+    def _serialize_entity(self, ctxt, entity):
+        try:
+            if ctxt.instance_id is None:
+                return entity
+        except (ValueError, TypeError):
+            return entity
+
+        instance_key = get_instance_encryption_key(ctxt.instance_id)
+
+        estr = jsonutils.dumps(entity)
+        return cu.encode_data(cu.encrypt_data(estr, instance_key))
+
+    def _deserialize_entity(self, ctxt, entity):
+        try:
+            entity = jsonutils.loads(entity)
+            instance_id = entity['csz-instance-id']
+        except (ValueError, TypeError):
+            return entity
+
+        instance_key = get_instance_encryption_key(instance_id)
+
+        estr = cu.decrypt_data(cu.decode_data(entity['entity']),
+                               instance_key)
+        entity = jsonutils.loads(estr)
+
+        return entity
+
+    def _serialize_context(self, ctxt):
+        try:
+            if ctxt.instance_id is None:
+                return ctxt
+        except (ValueError, TypeError):
+            return ctxt
+
+        instance_key = get_instance_encryption_key(ctxt.instance_id)
+
+        cstr = jsonutils.dumps(ctxt)
+        return {'context': cu.encode_data(cu.encrypt_data(cstr,
+                                                          instance_key))}
+
+    def _deserialize_context(self, ctxt):
+        try:
+            instance_id = ctxt.get('csz-instance-id', None)
+
+            if instance_id is not None:
+                instance_key = get_instance_encryption_key(instance_id)
+
+                cstr = cu.decrypt_data(cu.decode_data(ctxt['context']),
+                                       instance_key)
+                ctxt = jsonutils.loads(cstr)
+        except (ValueError, TypeError):
+            return ctxt
+
+        ctxt['instance_id'] = instance_id
+        return ctxt
diff --git a/trove/common/rpc/secure_serializer.py b/trove/common/rpc/secure_serializer.py
new file mode 100644
index 00000000..3430b939
--- /dev/null
+++ b/trove/common/rpc/secure_serializer.py
@@ -0,0 +1,59 @@
+# Copyright 2016 Tesora, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_serialization import jsonutils
+
+from trove.common import crypto_utils as cu
+from trove.common.rpc import serializer
+
+
+# BUG(1650518): Cleanup in the Pike release
+class SecureSerializer(serializer.TroveSerializer):
+    def __init__(self, base, key):
+        self._key = key
+        super(SecureSerializer, self).__init__(base)
+
+    def _serialize_entity(self, ctxt, entity):
+        if self._key is None:
+            return entity
+
+        estr = jsonutils.dumps(entity)
+        return cu.encode_data(cu.encrypt_data(estr, self._key))
+
+    def _deserialize_entity(self, ctxt, entity):
+        try:
+            if self._key is not None:
+                estr = cu.decrypt_data(cu.decode_data(entity), self._key)
+                entity = jsonutils.loads(estr)
+        except (ValueError, TypeError):
+            return entity
+
+        return entity
+
+    def _serialize_context(self, ctxt):
+        if self._key is None:
+            return ctxt
+
+        cstr = jsonutils.dumps(ctxt)
+        return {'context': cu.encode_data(cu.encrypt_data(cstr, self._key))}
+
+    def _deserialize_context(self, ctxt):
+        try:
+            if self._key is not None:
+                cstr = cu.decrypt_data(cu.decode_data(ctxt['context']),
+                                       self._key)
+                ctxt = jsonutils.loads(cstr)
+        except (ValueError, TypeError):
+            return ctxt
+
+        return ctxt
diff --git a/trove/common/rpc/serializer.py b/trove/common/rpc/serializer.py
new file mode 100644
index 00000000..0073f293
--- /dev/null
+++ b/trove/common/rpc/serializer.py
@@ -0,0 +1,86 @@
+# Copyright 2016 Tesora, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import oslo_messaging as messaging
+from osprofiler import profiler
+
+from trove.common.context import TroveContext
+
+
+class TroveSerializer(messaging.Serializer):
+    """The Trove serializer class that handles class inheritence and base
+       serializers.
+    """
+
+    def __init__(self, base):
+        self._base = base
+
+    def _serialize_entity(self, context, entity):
+        return entity
+
+    def serialize_entity(self, context, entity):
+        if self._base:
+            entity = self._base.serialize_entity(context, entity)
+
+        return self._serialize_entity(context, entity)
+
+    def _deserialize_entity(self, context, entity):
+        return entity
+
+    def deserialize_entity(self, context, entity):
+        entity = self._deserialize_entity(context, entity)
+
+        if self._base:
+            entity = self._base.deserialize_entity(context, entity)
+
+        return entity
+
+    def _serialize_context(self, context):
+        return context
+
+    def serialize_context(self, context):
+        if self._base:
+            context = self._base.serialize_context(context)
+
+        return self._serialize_context(context)
+
+    def _deserialize_context(self, context):
+        return context
+
+    def deserialize_context(self, context):
+        context = self._deserialize_context(context)
+
+        if self._base:
+            context = self._base.deserialize_context(context)
+
+        return context
+
+
+class TroveRequestContextSerializer(TroveSerializer):
+    def _serialize_context(self, context):
+        _context = context.to_dict()
+        prof = profiler.get()
+        if prof:
+            trace_info = {
+                "hmac_key": prof.hmac_key,
+                "base_id": prof.get_base_id(),
+                "parent_id": prof.get_id()
+            }
+            _context.update({"trace_info": trace_info})
+        return _context
+
+    def _deserialize_context(self, context):
+        trace_info = context.pop("trace_info", None)
+        if trace_info:
+            profiler.init(**trace_info)
+        return TroveContext.from_dict(context)
diff --git a/trove/common/rpc/service.py b/trove/common/rpc/service.py
index f5ff2af8..ed3924c0 100644
--- a/trove/common/rpc/service.py
+++ b/trove/common/rpc/service.py
@@ -29,6 +29,7 @@ from osprofiler import profiler
 from trove.common import cfg
 from trove.common.i18n import _
 from trove.common import profile
+from trove.common.rpc import secure_serializer as ssz
 from trove import rpc
 
 
@@ -38,9 +39,10 @@ LOG = logging.getLogger(__name__)
 
 class RpcService(service.Service):
 
-    def __init__(self, host=None, binary=None, topic=None, manager=None,
-                 rpc_api_version=None):
+    def __init__(self, key, host=None, binary=None, topic=None, manager=None,
+                 rpc_api_version=None, secure_serializer=ssz.SecureSerializer):
         super(RpcService, self).__init__()
+        self.key = key
         self.host = host or CONF.host
         self.binary = binary or os.path.basename(inspect.stack()[-1][1])
         self.topic = topic or self.binary.rpartition('trove-')[2]
@@ -48,6 +50,7 @@ class RpcService(service.Service):
         self.manager_impl = profiler.trace_cls("rpc")(_manager)
         self.rpc_api_version = rpc_api_version or \
             self.manager_impl.RPC_API_VERSION
+        self.secure_serializer = secure_serializer
         profile.setup_profiler(self.binary, self.host)
 
     def start(self):
@@ -60,7 +63,9 @@ class RpcService(service.Service):
             self.manager_impl.target = target
 
         endpoints = [self.manager_impl]
-        self.rpcserver = rpc.get_server(target, endpoints)
+        self.rpcserver = rpc.get_server(
+            target, endpoints, key=self.key,
+            secure_serializer=self.secure_serializer)
         self.rpcserver.start()
 
         # TODO(hub-cap): Currently the context is none... do we _need_ it here?
diff --git a/trove/conductor/api.py b/trove/conductor/api.py
index 757416b2..be73b2b7 100644
--- a/trove/conductor/api.py
+++ b/trove/conductor/api.py
@@ -16,6 +16,7 @@ from oslo_log import log as logging
 import oslo_messaging as messaging
 
 from trove.common import cfg
+from trove.common.rpc import conductor_guest_serializer as sz
 from trove.common.serializable_notification import SerializableNotification
 from trove import rpc
 
@@ -62,9 +63,10 @@ class API(object):
         self.client = self.get_client(target, version_cap)
 
     def get_client(self, target, version_cap, serializer=None):
-        return rpc.get_client(target,
+        return rpc.get_client(target, key=CONF.instance_rpc_encr_key,
                               version_cap=version_cap,
-                              serializer=serializer)
+                              serializer=serializer,
+                              secure_serializer=sz.ConductorGuestSerializer)
 
     def heartbeat(self, instance_id, payload, sent=None):
         LOG.debug("Making async call to cast heartbeat for instance: %s"
diff --git a/trove/db/models.py b/trove/db/models.py
index 6b8e0475..90dc4800 100644
--- a/trove/db/models.py
+++ b/trove/db/models.py
@@ -13,6 +13,7 @@
 #    under the License.
 
 from oslo_log import log as logging
+from oslo_utils import strutils
 
 from trove.common import exception
 from trove.common.i18n import _
@@ -59,13 +60,15 @@ class DatabaseModelBase(models.ModelBase):
             raise exception.InvalidModelError(errors=self.errors)
         self['updated'] = utils.utcnow()
         LOG.debug("Saving %(name)s: %(dict)s" %
-                  {'name': self.__class__.__name__, 'dict': self.__dict__})
+                  {'name': self.__class__.__name__,
+                   'dict': strutils.mask_dict_password(self.__dict__)})
         return self.db_api.save(self)
 
     def delete(self):
         self['updated'] = utils.utcnow()
         LOG.debug("Deleting %(name)s: %(dict)s" %
-                  {'name': self.__class__.__name__, 'dict': self.__dict__})
+                  {'name': self.__class__.__name__,
+                   'dict': strutils.mask_dict_password(self.__dict__)})
 
         if self.preserve_on_delete:
             self['deleted_at'] = utils.utcnow()
diff --git a/trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py b/trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py
new file mode 100644
index 00000000..7477cfaf
--- /dev/null
+++ b/trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py
@@ -0,0 +1,30 @@
+# Copyright 2016 Tesora, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+from sqlalchemy.schema import Column
+from sqlalchemy.schema import MetaData
+
+from trove.db.sqlalchemy.migrate_repo.schema import String
+from trove.db.sqlalchemy.migrate_repo.schema import Table
+
+
+meta = MetaData()
+
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+    instances = Table('instances', meta, autoload=True)
+    instances.create_column(Column('encrypted_key', String(255)))
diff --git a/trove/guestagent/api.py b/trove/guestagent/api.py
index 180388a0..85be70fa 100644
--- a/trove/guestagent/api.py
+++ b/trove/guestagent/api.py
@@ -69,13 +69,16 @@ class API(object):
 
         version_cap = self.VERSION_ALIASES.get(
             CONF.upgrade_levels.guestagent, CONF.upgrade_levels.guestagent)
-        target = messaging.Target(topic=self._get_routing_key(),
-                                  version=version_cap)
+        self.target = messaging.Target(topic=self._get_routing_key(),
+                                       version=version_cap)
 
-        self.client = self.get_client(target, version_cap)
+        self.client = self.get_client(self.target, version_cap)
 
     def get_client(self, target, version_cap, serializer=None):
-        return rpc.get_client(target,
+        from trove.instance.models import get_instance_encryption_key
+
+        instance_key = get_instance_encryption_key(self.id)
+        return rpc.get_client(target, key=instance_key,
                               version_cap=version_cap,
                               serializer=serializer)
 
@@ -328,12 +331,15 @@ class API(object):
            method do nothing in case a queue is already created by
            the guest
         """
+        from trove.instance.models import DBInstance
         server = None
         target = messaging.Target(topic=self._get_routing_key(),
                                   server=self.id,
                                   version=self.API_BASE_VERSION)
         try:
-            server = rpc.get_server(target, [])
+            instance = DBInstance.get_by(id=self.id)
+            instance_key = instance.key if instance else None
+            server = rpc.get_server(target, [], key=instance_key)
             server.start()
         finally:
             if server is not None:
@@ -352,6 +358,10 @@ class API(object):
         """Recover the guest after upgrading the guest's image."""
         LOG.debug("Recover the guest after upgrading the guest's image.")
         version = self.API_BASE_VERSION
+        LOG.debug("Recycling the client ...")
+        version_cap = self.VERSION_ALIASES.get(
+            CONF.upgrade_levels.guestagent, CONF.upgrade_levels.guestagent)
+        self.client = self.get_client(self.target, version_cap)
 
         self._call("post_upgrade", AGENT_HIGH_TIMEOUT, version=version,
                    upgrade_info=upgrade_info)
diff --git a/trove/instance/models.py b/trove/instance/models.py
index d5cc1519..11656712 100644
--- a/trove/instance/models.py
+++ b/trove/instance/models.py
@@ -26,6 +26,7 @@ from oslo_log import log as logging
 
 from trove.backup.models import Backup
 from trove.common import cfg
+from trove.common import crypto_utils as cu
 from trove.common import exception
 from trove.common.glance_remote import create_glance_client
 from trove.common.i18n import _, _LE, _LI, _LW
@@ -433,6 +434,10 @@ class SimpleInstance(object):
     def region_name(self):
         return self.db_info.region_id
 
+    @property
+    def encrypted_rpc_messaging(self):
+        return True if self.db_info.encrypted_key is not None else False
+
 
 class DetailInstance(SimpleInstance):
     """A detailed view of an Instance.
@@ -749,6 +754,14 @@ class BaseInstance(SimpleInstance):
             "tenant_id=%s\n"
             % (self.id, datastore_manager, self.tenant_id))}
 
+        instance_key = get_instance_encryption_key(self.id)
+        if instance_key:
+            files = {guest_info_file: (
+                "%s"
+                "instance_rpc_encr_key=%s\n" % (
+                    files.get(guest_info_file),
+                    instance_key))}
+
         if os.path.isfile(CONF.get('guest_config')):
             with open(CONF.get('guest_config'), "r") as f:
                 files[os.path.join(injected_config_location,
@@ -1502,7 +1515,8 @@ class DBInstance(dbmodels.DatabaseModelBase):
                     'task_id', 'task_description', 'task_start_time',
                     'volume_id', 'deleted', 'tenant_id',
                     'datastore_version_id', 'configuration_id', 'slave_of_id',
-                    'cluster_id', 'shard_id', 'type', 'region_id']
+                    'cluster_id', 'shard_id', 'type', 'region_id',
+                    'encrypted_key']
 
     def __init__(self, task_status, **kwargs):
         """
@@ -1515,9 +1529,27 @@ class DBInstance(dbmodels.DatabaseModelBase):
         kwargs["task_id"] = task_status.code
         kwargs["task_description"] = task_status.db_text
         kwargs["deleted"] = False
+
+        if CONF.enable_secure_rpc_messaging:
+            key = cu.generate_random_key()
+            kwargs["encrypted_key"] = cu.encode_data(cu.encrypt_data(
+                key, CONF.inst_rpc_key_encr_key))
+            LOG.debug("Generated unique RPC encryption key for "
+                      "instance. key = %s" % key)
+        else:
+            kwargs["encrypted_key"] = None
+
         super(DBInstance, self).__init__(**kwargs)
         self.set_task_status(task_status)
 
+    @property
+    def key(self):
+        if self.encrypted_key is None:
+            return None
+
+        return cu.decrypt_data(cu.decode_data(self.encrypted_key),
+                               CONF.inst_rpc_key_encr_key)
+
     def _validate(self, errors):
         if InstanceTask.from_code(self.task_id) is None:
             errors['task_id'] = "Not valid."
@@ -1534,6 +1566,56 @@ class DBInstance(dbmodels.DatabaseModelBase):
     task_status = property(get_task_status, set_task_status)
 
 
+class instance_encryption_key_cache(object):
+    def __init__(self, func, lru_cache_size=10):
+        self._table = {}
+        self._lru = []
+        self._lru_cache_size = lru_cache_size
+        self._func = func
+
+    def get(self, instance_id):
+        if instance_id in self._table:
+            if self._lru.index(instance_id) > 0:
+                self._lru.remove(instance_id)
+                self._lru.insert(0, instance_id)
+
+            return self._table[instance_id]
+        else:
+            val = self._func(instance_id)
+
+            # BUG(1650518): Cleanup in the Pike release
+            if val is None:
+                return val
+
+            if len(self._lru) == self._lru_cache_size:
+                tail = self._lru.pop()
+                del self._table[tail]
+
+            self._lru.insert(0, instance_id)
+            self._table[instance_id] = val
+            return self._table[instance_id]
+
+    def __getitem__(self, instance_id):
+        return self.get(instance_id)
+
+
+def _get_instance_encryption_key(instance_id):
+    instance = DBInstance.find_by(id=instance_id)
+
+    if instance is not None:
+        return instance.key
+    else:
+        raise exception.NotFound(uuid=id)
+
+
+_instance_encryption_key = instance_encryption_key_cache(
+    func=_get_instance_encryption_key)
+
+
+def get_instance_encryption_key(instance_id):
+    return _instance_encryption_key[instance_id]
+
+
 def persist_instance_fault(notification, event_qualifier):
     """This callback is registered to be fired whenever a
     notification is sent out.
diff --git a/trove/instance/views.py b/trove/instance/views.py
index 6721ec10..30c045c7 100644
--- a/trove/instance/views.py
+++ b/trove/instance/views.py
@@ -127,6 +127,8 @@ class InstanceDetailView(InstanceView):
         if self.context.is_admin:
             result['instance']['server_id'] = self.instance.server_id
             result['instance']['volume_id'] = self.instance.volume_id
+            result['instance']['encrypted_rpc_messaging'] = (
+                self.instance.encrypted_rpc_messaging)
 
         return result
 
diff --git a/trove/rpc.py b/trove/rpc.py
index 03a63d21..dff472ee 100644
--- a/trove/rpc.py
+++ b/trove/rpc.py
@@ -23,7 +23,6 @@ __all__ = [
     'add_extra_exmods',
     'clear_extra_exmods',
     'get_allowed_exmods',
-    'RequestContextSerializer',
     'get_client',
     'get_server',
     'get_notifier',
@@ -32,12 +31,10 @@ __all__ = [
 
 from oslo_config import cfg
 import oslo_messaging as messaging
-from oslo_serialization import jsonutils
-from osprofiler import profiler
 
-from trove.common.context import TroveContext
 import trove.common.exception
-
+from trove.common.rpc import secure_serializer as ssz
+from trove.common.rpc import serializer as sz
 
 CONF = cfg.CONF
 TRANSPORT = None
@@ -56,7 +53,8 @@ def init(conf):
     TRANSPORT = messaging.get_transport(conf,
                                         allowed_remote_exmods=exmods)
 
-    serializer = RequestContextSerializer(JsonPayloadSerializer())
+    serializer = sz.TroveRequestContextSerializer(
+        messaging.JsonPayloadSerializer())
     NOTIFIER = messaging.Notifier(TRANSPORT, serializer=serializer)
 
 
@@ -84,60 +82,26 @@ def get_allowed_exmods():
     return ALLOWED_EXMODS + EXTRA_EXMODS
 
 
-class JsonPayloadSerializer(messaging.NoOpSerializer):
-    @staticmethod
-    def serialize_entity(context, entity):
-        return jsonutils.to_primitive(entity, convert_instances=True)
-
-
-class RequestContextSerializer(messaging.Serializer):
-
-    def __init__(self, base):
-        self._base = base
-
-    def serialize_entity(self, context, entity):
-        if not self._base:
-            return entity
-        return self._base.serialize_entity(context, entity)
-
-    def deserialize_entity(self, context, entity):
-        if not self._base:
-            return entity
-        return self._base.deserialize_entity(context, entity)
-
-    def serialize_context(self, context):
-        _context = context.to_dict()
-        prof = profiler.get()
-        if prof:
-            trace_info = {
-                "hmac_key": prof.hmac_key,
-                "base_id": prof.get_base_id(),
-                "parent_id": prof.get_id()
-            }
-            _context.update({"trace_info": trace_info})
-        return _context
-
-    def deserialize_context(self, context):
-        trace_info = context.pop("trace_info", None)
-        if trace_info:
-            profiler.init(**trace_info)
-        return TroveContext.from_dict(context)
-
-
 def get_transport_url(url_str=None):
     return messaging.TransportURL.parse(CONF, url_str)
 
 
-def get_client(target, version_cap=None, serializer=None):
+def get_client(target, key, version_cap=None, serializer=None,
+               secure_serializer=ssz.SecureSerializer):
     assert TRANSPORT is not None
-    serializer = RequestContextSerializer(serializer)
+    # BUG(1650518): Cleanup in the Pike release
+    # uncomment this (following) line in the pike release
+    # assert key is not None
+    serializer = secure_serializer(
+        sz.TroveRequestContextSerializer(serializer), key)
     return messaging.RPCClient(TRANSPORT,
                                target,
                                version_cap=version_cap,
                                serializer=serializer)
 
 
-def get_server(target, endpoints, serializer=None):
+def get_server(target, endpoints, key, serializer=None,
+               secure_serializer=ssz.SecureSerializer):
     assert TRANSPORT is not None
 
     # Thread module is not monkeypatched if remote debugging is enabled.
@@ -148,7 +112,12 @@ def get_server(target, endpoints, serializer=None):
 
     executor = "blocking" if debug_utils.enabled() else "eventlet"
 
-    serializer = RequestContextSerializer(serializer)
+    # BUG(1650518): Cleanup in the Pike release
+    # uncomment this (following) line in the pike release
+    # assert key is not None
+    serializer = secure_serializer(
+        sz.TroveRequestContextSerializer(serializer), key)
+
     return messaging.get_rpc_server(TRANSPORT,
                                     target,
                                     endpoints,
diff --git a/trove/taskmanager/api.py b/trove/taskmanager/api.py
index 1c1b01aa..437e720b 100644
--- a/trove/taskmanager/api.py
+++ b/trove/taskmanager/api.py
@@ -77,7 +77,12 @@ class API(object):
             cctxt.cast(self.context, method_name, **kwargs)
 
     def get_client(self, target, version_cap, serializer=None):
-        return rpc.get_client(target,
+        if CONF.enable_secure_rpc_messaging:
+            key = CONF.taskmanager_rpc_encr_key
+        else:
+            key = None
+
+        return rpc.get_client(target, key=key,
                               version_cap=version_cap,
                               serializer=serializer)
 
diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py
index 48199618..32deb57f 100755
--- a/trove/taskmanager/models.py
+++ b/trove/taskmanager/models.py
@@ -31,6 +31,7 @@ from trove.cluster.models import Cluster
 from trove.cluster.models import DBCluster
 from trove.cluster import tasks
 from trove.common import cfg
+from trove.common import crypto_utils as cu
 from trove.common import exception
 from trove.common.exception import BackupCreationError
 from trove.common.exception import GuestError
@@ -1420,6 +1421,24 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin):
                 volume_device = self._fix_device_path(
                     volume.attachments[0]['device'])
 
+            # BUG(1650518): Cleanup in the Pike release some instances
+            # that we will be upgrading will be pre secureserialier
+            # and will have no instance_key entries. If this is one of
+            # those instances, make a key. That will make it appear in
+            # the injected files that are generated next. From this
+            # point, and until the guest comes up, attempting to send
+            # messages to it will fail because the RPC framework will
+            # encrypt messages to a guest which potentially doesn't
+            # have the code to handle it.
+            if CONF.enable_secure_rpc_messaging and (
+                    self.db_info.encrypted_key is None):
+                encrypted_key = cu.encode_data(cu.encrypt_data(
+                    cu.generate_random_key(),
+                    CONF.inst_rpc_key_encr_key))
+                self.update_db(encrypted_key=encrypted_key)
+                LOG.debug("Generated unique RPC encryption key for "
+                          "instance = %s, key = %s" % (self.id, encrypted_key))
+
             injected_files = self.get_injected_files(
                 datastore_version.manager)
             LOG.debug("Rebuilding instance %(instance)s with image %(image)s.",
diff --git a/trove/tests/unittests/common/test_conductor_serializer.py b/trove/tests/unittests/common/test_conductor_serializer.py
new file mode 100644
index 00000000..ae5e5ca0
--- /dev/null
+++ b/trove/tests/unittests/common/test_conductor_serializer.py
@@ -0,0 +1,110 @@
+# Copyright 2016 Tesora, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+import mock
+
+from trove.common import cfg
+from trove.common.rpc import conductor_guest_serializer as gsz
+from trove.common.rpc import conductor_host_serializer as hsz
+
+from trove.tests.unittests import trove_testtools
+
+
+CONF = cfg.CONF
+
+
+class FakeInstance(object):
+    def __init__(self):
+        self.uuid = 'a3af1652-686a-4574-a916-2ef7e85136e5'
+
+    @property
+    def key(self):
+        return 'mo79Y86Bp3bzQDWR31ihhVGfLBmeac'
+
+
+class FakeContext(object):
+    def __init__(self, instance_id=None, fields=None):
+        self.instance_id = instance_id
+        self.fields = fields
+
+
+class TestConductorSerializer(trove_testtools.TestCase):
+
+    def setUp(self):
+        self.uuid = 'a3af1652-686a-4574-a916-2ef7e85136e5'
+        self.key = 'mo79Y86Bp3bzQDWR31ihhVGfLBmeac'
+        self.data = 'ELzWd81qtgcj2Gxc1ipbh0HgbvHGrgptDj3n4GNMBN0F2WtNdr'
+        self.context = {'a': 'ij2J8AJLyz0rDqbjxy4jPVINhnK2jsBGpWRKIe3tUnUD',
+                        'b': 32,
+                        'c': {'a': 21, 'b': 22}}
+        self.old_guest_id = gsz.CONF.guest_id
+        gsz.CONF.guest_id = self.uuid
+        super(TestConductorSerializer, self).setUp()
+
+    def tearDown(self):
+        gsz.CONF.guest_id = self.old_guest_id
+        super(TestConductorSerializer, self).tearDown()
+
+    def test_gsz_serialize_entity_nokey(self):
+        sz = gsz.ConductorGuestSerializer(None, None)
+        self.assertEqual(sz.serialize_entity(self.context, self.data),
+                         self.data)
+
+    def test_gsz_serialize_context_nokey(self):
+        sz = gsz.ConductorGuestSerializer(None, None)
+        self.assertEqual(sz.serialize_context(self.context),
+                         self.context)
+
+    @mock.patch('trove.common.rpc.conductor_host_serializer.'
+                'get_instance_encryption_key',
+                return_value='mo79Y86Bp3bzQDWR31ihhVGfLBmeac')
+    def test_hsz_serialize_entity_nokey_noinstance(self, _):
+        sz = hsz.ConductorHostSerializer(None, None)
+        ctxt = FakeContext(instance_id=None)
+        self.assertEqual(sz.serialize_entity(ctxt, self.data),
+                         self.data)
+
+    @mock.patch('trove.common.rpc.conductor_host_serializer.'
+                'get_instance_encryption_key',
+                return_value='mo79Y86Bp3bzQDWR31ihhVGfLBmeac')
+    def test_hsz_serialize_context_nokey_noinstance(self, _):
+        sz = hsz.ConductorHostSerializer(None, None)
+        ctxt = FakeContext(instance_id=None)
+        self.assertEqual(sz.serialize_context(ctxt), ctxt)
+
+    @mock.patch('trove.common.rpc.conductor_host_serializer.'
+                'get_instance_encryption_key',
+                return_value='mo79Y86Bp3bzQDWR31ihhVGfLBmeac')
+    def test_conductor_entity(self, _):
+        guestsz = gsz.ConductorGuestSerializer(None, self.key)
+        hostsz = hsz.ConductorHostSerializer(None, None)
+        encrypted_entity = guestsz.serialize_entity(self.context, self.data)
+        self.assertNotEqual(encrypted_entity, self.data)
+        entity = hostsz.deserialize_entity(self.context, encrypted_entity)
+        self.assertEqual(entity, self.data)
+
+    @mock.patch('trove.common.rpc.conductor_host_serializer.'
+                'get_instance_encryption_key',
+                return_value='mo79Y86Bp3bzQDWR31ihhVGfLBmeac')
+    def test_conductor_context(self, _):
+        guestsz = gsz.ConductorGuestSerializer(None, self.key)
+        hostsz = hsz.ConductorHostSerializer(None, None)
+        encrypted_context = guestsz.serialize_context(self.context)
+        self.assertNotEqual(encrypted_context, self.context)
+        context = hostsz.deserialize_context(encrypted_context)
+        self.assertEqual(context.get('instance_id'), self.uuid)
+        context.pop('instance_id')
+        self.assertDictEqual(context, self.context)
diff --git a/trove/tests/unittests/common/test_secure_serializer.py b/trove/tests/unittests/common/test_secure_serializer.py
new file mode 100644
index 00000000..2eafe96c
--- /dev/null
+++ b/trove/tests/unittests/common/test_secure_serializer.py
@@ -0,0 +1,64 @@
+# Copyright 2016 Tesora, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+from trove.common.rpc import secure_serializer as ssz
+from trove.tests.unittests import trove_testtools
+
+
+class TestSecureSerializer(trove_testtools.TestCase):
+
+    def setUp(self):
+        self.key = 'xuUyAKn5mDANoM5sRxQsb6HGiugWVD'
+        self.data = '5rzFfaKU630rRxL1g3c80EHnHDf534'
+        self.context = {'fld1': 3, 'fld2': 'abc'}
+        super(TestSecureSerializer, self).setUp()
+
+    def tearDown(self):
+        super(TestSecureSerializer, self).tearDown()
+
+    def test_sz_nokey_serialize_entity(self):
+        sz = ssz.SecureSerializer(base=None, key=None)
+        en = sz.serialize_entity(self.context, self.data)
+        self.assertEqual(en, self.data)
+
+    def test_sz_nokey_deserialize_entity(self):
+        sz = ssz.SecureSerializer(base=None, key=None)
+        en = sz.deserialize_entity(self.context, self.data)
+        self.assertEqual(en, self.data)
+
+    def test_sz_nokey_serialize_context(self):
+        sz = ssz.SecureSerializer(base=None, key=None)
+        en = sz.serialize_context(self.context)
+        self.assertEqual(en, self.context)
+
+    def test_sz_nokey_deserialize_context(self):
+        sz = ssz.SecureSerializer(base=None, key=None)
+        en = sz.deserialize_context(self.context)
+        self.assertEqual(en, self.context)
+
+    def test_sz_entity(self):
+        sz = ssz.SecureSerializer(base=None, key=self.key)
+        en = sz.serialize_entity(self.context, self.data)
+        self.assertNotEqual(en, self.data)
+        self.assertEqual(sz.deserialize_entity(self.context, en),
+                         self.data)
+
+    def test_sz_context(self):
+        sz = ssz.SecureSerializer(base=None, key=self.key)
+        sctxt = sz.serialize_context(self.context)
+        self.assertNotEqual(sctxt, self.context)
+        self.assertEqual(sz.deserialize_context(sctxt),
+                         self.context)
diff --git a/trove/tests/unittests/common/test_serializer.py b/trove/tests/unittests/common/test_serializer.py
new file mode 100644
index 00000000..ab4696b3
--- /dev/null
+++ b/trove/tests/unittests/common/test_serializer.py
@@ -0,0 +1,127 @@
+# Copyright 2016 Tesora, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+import mock
+
+from trove.common.rpc import serializer
+from trove.tests.unittests import trove_testtools
+
+
+class TestSerializer(trove_testtools.TestCase):
+
+    def setUp(self):
+        self.data = 'abcdefghijklmnopqrstuvwxyz'
+        self.context = {}
+        super(TestSerializer, self).setUp()
+
+    def tearDown(self):
+        super(TestSerializer, self).tearDown()
+
+    def test_serialize_1(self):
+        base = mock.Mock()
+        sz = serializer.TroveSerializer(base=base)
+        sz.serialize_entity(self.context, self.data)
+        base.serialize_entity.assert_called_with(self.context, self.data)
+
+    def test_serialize_2(self):
+        base = mock.Mock()
+        sz1 = serializer.TroveSerializer(base=base)
+        sz = serializer.TroveSerializer(base=sz1)
+        sz.serialize_entity(self.context, self.data)
+        base.serialize_entity.assert_called_with(self.context, self.data)
+
+    def test_serialize_3(self):
+        base = mock.Mock()
+        sz = serializer.TroveSerializer(base=base)
+        sz.deserialize_entity(self.context, self.data)
+        base.deserialize_entity.assert_called_with(self.context, self.data)
+
+    def test_serialize_4(self):
+        base = mock.Mock()
+        sz1 = serializer.TroveSerializer(base=base)
+        sz = serializer.TroveSerializer(base=sz1)
+        sz.deserialize_entity(self.context, self.data)
+        base.deserialize_entity.assert_called_with(self.context, self.data)
+
+    def test_serialize_5(self):
+        base = mock.Mock()
+        sz = serializer.TroveSerializer(base=base)
+        sz.serialize_context(self.context)
+        base.serialize_context.assert_called_with(self.context)
+
+    def test_serialize_6(self):
+        base = mock.Mock()
+        sz1 = serializer.TroveSerializer(base=base)
+        sz = serializer.TroveSerializer(base=sz1)
+        sz.serialize_context(self.context)
+        base.serialize_context.assert_called_with(self.context)
+
+    def test_serialize_7(self):
+        base = mock.Mock()
+        sz = serializer.TroveSerializer(base=base)
+        sz.deserialize_context(self.context)
+        base.deserialize_context.assert_called_with(self.context)
+
+    def test_serialize_8(self):
+        base = mock.Mock()
+        sz1 = serializer.TroveSerializer(base=base)
+        sz = serializer.TroveSerializer(base=sz1)
+        sz.deserialize_context(self.context)
+        base.deserialize_context.assert_called_with(self.context)
+
+    def test_serialize_9(self):
+        sz = serializer.TroveSerializer(base=None)
+        self.assertEqual(sz.serialize_entity(self.context, self.data),
+                         self.data)
+
+    def test_serialize_10(self):
+        sz = serializer.TroveSerializer(base=None)
+        self.assertEqual(sz.deserialize_entity(self.context, self.data),
+                         self.data)
+
+    def test_serialize_11(self):
+        sz = serializer.TroveSerializer(base=None)
+        self.assertEqual(sz.serialize_context(self.context),
+                         self.context)
+
+    def test_serialize_12(self):
+        sz = serializer.TroveSerializer(base=None)
+        self.assertEqual(sz.deserialize_context(self.context),
+                         self.context)
+
+    def test_serialize_13(self):
+        bz = serializer.TroveSerializer(base=None)
+        sz = serializer.TroveSerializer(base=bz)
+        self.assertEqual(sz.serialize_entity(self.context, self.data),
+                         self.data)
+
+    def test_serialize_14(self):
+        bz = serializer.TroveSerializer(base=None)
+        sz = serializer.TroveSerializer(base=bz)
+        self.assertEqual(sz.deserialize_entity(self.context, self.data),
+                         self.data)
+
+    def test_serialize_15(self):
+        bz = serializer.TroveSerializer(base=None)
+        sz = serializer.TroveSerializer(base=bz)
+        self.assertEqual(sz.serialize_context(self.context),
+                         self.context)
+
+    def test_serialize_16(self):
+        bz = serializer.TroveSerializer(base=None)
+        sz = serializer.TroveSerializer(base=bz)
+        self.assertEqual(sz.deserialize_context(self.context),
+                         self.context)
diff --git a/trove/tests/unittests/conductor/test_conf.py b/trove/tests/unittests/conductor/test_conf.py
index c4305bbe..924dc693 100644
--- a/trove/tests/unittests/conductor/test_conf.py
+++ b/trove/tests/unittests/conductor/test_conf.py
@@ -32,7 +32,8 @@ def mocked_conf(manager):
         'conductor_manager': manager,
         'trove_conductor_workers': 1,
         'host': 'mockhost',
-        'report_interval': 1})
+        'report_interval': 1,
+        'instance_rpc_encr_key': ''})
 
 
 class NoopManager(object):
diff --git a/trove/tests/unittests/guestagent/test_api.py b/trove/tests/unittests/guestagent/test_api.py
index 5390fbea..71efb963 100644
--- a/trove/tests/unittests/guestagent/test_api.py
+++ b/trove/tests/unittests/guestagent/test_api.py
@@ -50,7 +50,9 @@ def _mock_call(cmd, timeout, version=None, username=None, hostname=None,
 
 class ApiTest(trove_testtools.TestCase):
     @mock.patch.object(rpc, 'get_client')
-    def setUp(self, *args):
+    @mock.patch('trove.instance.models.get_instance_encryption_key',
+                return_value='2LMDgren5citVxmSYNiRFCyFfVDjJtDaQT9LYV08')
+    def setUp(self, mock_get_encryption_key, *args):
         super(ApiTest, self).setUp()
         self.context = context.TroveContext()
         self.guest = api.API(self.context, 0)
@@ -58,6 +60,7 @@ class ApiTest(trove_testtools.TestCase):
         self.guest._call = _mock_call
         self.api = api.API(self.context, "instance-id-x23d2d")
         self._mock_rpc_client()
+        mock_get_encryption_key.assert_called()
 
     def test_change_passwords(self):
         self.assertIsNone(self.guest.change_passwords("dummy"))
diff --git a/trove/tests/unittests/guestagent/test_galera_cluster_api.py b/trove/tests/unittests/guestagent/test_galera_cluster_api.py
index 9f79eb56..809d9e1a 100644
--- a/trove/tests/unittests/guestagent/test_galera_cluster_api.py
+++ b/trove/tests/unittests/guestagent/test_galera_cluster_api.py
@@ -37,7 +37,9 @@ def _mock_call(cmd, timeout, version=None, user=None,
 
 class ApiTest(trove_testtools.TestCase):
     @mock.patch.object(rpc, 'get_client')
-    def setUp(self, *args):
+    @mock.patch('trove.instance.models.get_instance_encryption_key',
+                return_value='2LMDgren5citVxmSYNiRFCyFfVDjJtDaQT9LYV08')
+    def setUp(self, mock_get_encryption_key, *args):
         super(ApiTest, self).setUp()
         cluster_guest_api = (GaleraCommonGuestAgentStrategy()
                              .guest_client_class)
@@ -46,6 +48,7 @@ class ApiTest(trove_testtools.TestCase):
         self.guest._call = _mock_call
         self.api = cluster_guest_api(self.context, "instance-id-x23d2d")
         self._mock_rpc_client()
+        mock_get_encryption_key.assert_called()
 
     def test_get_routing_key(self):
         self.assertEqual('guestagent.instance-id-x23d2d',
diff --git a/trove/tests/unittests/guestagent/test_vertica_api.py b/trove/tests/unittests/guestagent/test_vertica_api.py
index b67b9e5f..7c47cc3d 100644
--- a/trove/tests/unittests/guestagent/test_vertica_api.py
+++ b/trove/tests/unittests/guestagent/test_vertica_api.py
@@ -37,13 +37,17 @@ def _mock_call(cmd, timeout, version=None, user=None,
 
 class ApiTest(trove_testtools.TestCase):
     @mock.patch.object(rpc, 'get_client')
-    def setUp(self, *args):
+    @mock.patch('trove.instance.models.get_instance_encryption_key',
+                return_value='2LMDgren5citVxmSYNiRFCyFfVDjJtDaQT9LYV08')
+    def setUp(self, mock_get_encryption_key, *args):
         super(ApiTest, self).setUp()
         self.context = context.TroveContext()
         self.guest = VerticaGuestAgentAPI(self.context, 0)
+
         self.guest._call = _mock_call
         self.api = VerticaGuestAgentAPI(self.context, "instance-id-x23d2d")
         self._mock_rpc_client()
+        mock_get_encryption_key.assert_called()
 
     def test_get_routing_key(self):
         self.assertEqual('guestagent.instance-id-x23d2d',
diff --git a/trove/tests/unittests/instance/test_instance_models.py b/trove/tests/unittests/instance/test_instance_models.py
index c60120be..c089daaa 100644
--- a/trove/tests/unittests/instance/test_instance_models.py
+++ b/trove/tests/unittests/instance/test_instance_models.py
@@ -26,6 +26,7 @@ from trove.instance.models import DBInstance
 from trove.instance.models import DBInstanceFault
 from trove.instance.models import filter_ips
 from trove.instance.models import Instance
+from trove.instance.models import instance_encryption_key_cache
 from trove.instance.models import InstanceServiceStatus
 from trove.instance.models import SimpleInstance
 from trove.instance.tasks import InstanceTasks
@@ -469,3 +470,53 @@ class TestModules(trove_testtools.TestCase):
                             expected_exception,
                             models.validate_modules_for_apply,
                             modules, ds_id, ds_ver_id)
+
+
+def trivial_key_function(id):
+    return id * id
+
+
+class TestInstanceKeyCaching(trove_testtools.TestCase):
+
+    def setUp(self):
+        super(TestInstanceKeyCaching, self).setUp()
+
+    def tearDown(self):
+        super(TestInstanceKeyCaching, self).tearDown()
+
+    def test_basic_caching(self):
+        keycache = instance_encryption_key_cache(trivial_key_function, 5)
+        self.assertEqual(keycache[5], 25)
+        self.assertEqual(keycache[5], 25)
+        self.assertEqual(keycache[25], 625)
+
+    def test_caching(self):
+        keyfn = Mock(return_value=123)
+        keycache = instance_encryption_key_cache(keyfn, 5)
+        self.assertEqual(keycache[5], 123)
+        self.assertEqual(keyfn.call_count, 1)
+        self.assertEqual(keycache[5], 123)
+        self.assertEqual(keyfn.call_count, 1)
+        self.assertEqual(keycache[6], 123)
+        self.assertEqual(keyfn.call_count, 2)
+        self.assertEqual(keycache[7], 123)
+        self.assertEqual(keyfn.call_count, 3)
+        self.assertEqual(keycache[8], 123)
+        self.assertEqual(keyfn.call_count, 4)
+        self.assertEqual(keycache[9], 123)
+        self.assertEqual(keyfn.call_count, 5)
+        self.assertEqual(keycache[10], 123)
+        self.assertEqual(keyfn.call_count, 6)
+        self.assertEqual(keycache[10], 123)
+        self.assertEqual(keyfn.call_count, 6)
+        self.assertEqual(keycache[5], 123)
+        self.assertEqual(keyfn.call_count, 7)
+
+    # BUG(1650518): Cleanup in the Pike release
+    def test_not_caching_none(self):
+        keyfn = Mock(return_value=None)
+        keycache = instance_encryption_key_cache(keyfn, 5)
+        self.assertIsNone(keycache[30])
+        self.assertEqual(keyfn.call_count, 1)
+        self.assertIsNone(keycache[30])
+        self.assertEqual(keyfn.call_count, 2)
diff --git a/trove/tests/unittests/taskmanager/test_models.py b/trove/tests/unittests/taskmanager/test_models.py
index 47297199..1140eaf3 100644
--- a/trove/tests/unittests/taskmanager/test_models.py
+++ b/trove/tests/unittests/taskmanager/test_models.py
@@ -245,7 +245,8 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
             None, None, None, datastore_manager, None, None, None)
         self.assertEqual(server.userdata, self.userdata)
 
-    def test_create_instance_guestconfig(self):
+    @patch.object(DBInstance, 'get_by')
+    def test_create_instance_guestconfig(self, patch_get_by):
         def fake_conf_getter(*args, **kwargs):
             if args[0] == 'guest_config':
                 return self.guestconfig
@@ -268,7 +269,8 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
             self.guestconfig_content,
             files['/etc/trove/conf.d/trove-guestagent.conf'])
 
-    def test_create_instance_guestconfig_compat(self):
+    @patch.object(DBInstance, 'get_by')
+    def test_create_instance_guestconfig_compat(self, patch_get_by):
         def fake_conf_getter(*args, **kwargs):
             if args[0] == 'guest_config':
                 return self.guestconfig
@@ -460,7 +462,8 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
 
     @patch.object(trove.guestagent.api.API, 'attach_replication_slave')
     @patch.object(rpc, 'get_client')
-    def test_attach_replication_slave(self, mock_get_client,
+    @patch.object(DBInstance, 'get_by')
+    def test_attach_replication_slave(self, mock_get_by, mock_get_client,
                                       mock_attach_replication_slave):
         mock_flavor = {'id': 8, 'ram': 768, 'name': 'bigger_flavor'}
         snapshot = {'replication_strategy': 'MysqlGTIDReplication',
@@ -483,6 +486,7 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
     @patch.object(trove.guestagent.api.API, 'attach_replication_slave',
                   side_effect=GuestError)
     @patch('trove.taskmanager.models.LOG')
+    @patch.object(DBInstance, 'get_by')
     def test_error_attach_replication_slave(self, *args):
         mock_flavor = {'id': 8, 'ram': 768, 'name': 'bigger_flavor'}
         snapshot = {'replication_strategy': 'MysqlGTIDReplication',
diff --git a/trove/tests/unittests/upgrade/test_models.py b/trove/tests/unittests/upgrade/test_models.py
index f0459462..9a859de6 100644
--- a/trove/tests/unittests/upgrade/test_models.py
+++ b/trove/tests/unittests/upgrade/test_models.py
@@ -66,7 +66,11 @@ class TestUpgradeModel(trove_testtools.TestCase):
 
     @patch('trove.guestagent.api.API.upgrade')
     @patch.object(rpc, 'get_client')
-    def _assert_create_with_metadata(self, mock_client, api_upgrade_mock,
+    @patch('trove.instance.models.get_instance_encryption_key',
+           return_value='2LMDgren5citVxmSYNiRFCyFfVDjJtDaQT9LYV08')
+    def _assert_create_with_metadata(self, mock_get_encryption_key,
+                                     mock_client,
+                                     api_upgrade_mock,
                                      metadata=None):
         """Exercise UpgradeMessageSender.create() call.
         """
@@ -85,3 +89,4 @@ class TestUpgradeModel(trove_testtools.TestCase):
         func()  # This call should translate to the API call asserted below.
         api_upgrade_mock.assert_called_once_with(instance_version, location,
                                                  metadata)
+        mock_get_encryption_key.assert_called()