diff options
| -rw-r--r-- | doc/source/misc.rst | 8 | ||||
| -rw-r--r-- | etc/proxy-server.conf-sample | 4 | ||||
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | swift/common/middleware/list_endpoints.py | 153 | ||||
| -rw-r--r-- | test/unit/common/middleware/test_list_endpoints.py | 200 |
5 files changed, 367 insertions, 0 deletions
diff --git a/doc/source/misc.rst b/doc/source/misc.rst index 64549570b..9640c3675 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -202,3 +202,11 @@ Static Large Objects .. automodule:: swift.common.middleware.slo :members: :show-inheritance: + + +List Endpoints +============== + +.. automodule:: swift.common.middleware.list_endpoints + :members: + :show-inheritance: diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 86d58c48a..7deddad0b 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -316,6 +316,10 @@ use = egg:swift#name_check # maximum_length = 255 # forbidden_regexp = /\./|/\.\./|/\.$|/\.\.$ +[filter:list-endpoints] +use = egg:swift#list_endpoints +# list_endpoints_path = /endpoints/ + [filter:proxy-logging] use = egg:swift#proxy_logging # If not set, logging directives from [DEFAULT] without "access_" will be used @@ -107,6 +107,8 @@ setup( 'proxy_logging=swift.common.middleware.proxy_logging:' 'filter_factory', 'slo=swift.common.middleware.slo:filter_factory', + 'list_endpoints=swift.common.middleware.list_endpoints:' + 'filter_factory', ], }, ) diff --git a/swift/common/middleware/list_endpoints.py b/swift/common/middleware/list_endpoints.py new file mode 100644 index 000000000..3fa24dcdf --- /dev/null +++ b/swift/common/middleware/list_endpoints.py @@ -0,0 +1,153 @@ +# Copyright (c) 2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +List endpoints for an object, account or container. + +This middleware makes it possible to integrate swift with software +that relies on data locality information to avoid network overhead, +such as Hadoop. + +Answers requests of the form:: + + /endpoints/{account}/{container}/{object} + /endpoints/{account}/{container} + /endpoints/{account} + +with a JSON-encoded list of endpoints of the form:: + + http://{server}:{port}/{dev}/{part}/{acc}/{cont}/{obj} + http://{server}:{port}/{dev}/{part}/{acc}/{cont} + http://{server}:{port}/{dev}/{part}/{acc} + +correspondingly, e.g.:: + + http://10.1.1.1:6000/sda1/2/a/c2/o1 + http://10.1.1.1:6000/sda1/2/a/c2 + http://10.1.1.1:6000/sda1/2/a + +The '/endpoints/' path is customizable ('list_endpoints_path' +configuration parameter). + +Intended for consumption by third-party services living inside the +cluster (as the endpoints make sense only inside the cluster behind +the firewall); potentially written in a different language. + +This is why it's provided as a REST API and not just a Python API: +to avoid requiring clients to write their own ring parsers in their +languages, and to avoid the necessity to distribute the ring file +to clients and keep it up-to-date. + +Note that the call is not authenticated, which means that a proxy +with this middleware enabled should not be open to an untrusted +environment (everyone can query the locality data using this middleware). +""" + +from urllib import quote, unquote + +from swift.common.ring import Ring +from swift.common.utils import json, get_logger, split_path +from swift.common.swob import Request, Response +from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed + + +class ListEndpointsMiddleware(object): + """ + List endpoints for an object, account or container. + + See above for a full description. + + Uses configuration parameter `swift_dir` (default `/etc/swift`). + + :param app: The next WSGI filter or app in the paste.deploy + chain. + :param conf: The configuration dict for the middleware. + """ + + def __init__(self, app, conf): + self.app = app + self.logger = get_logger(conf, log_route='endpoints') + swift_dir = conf.get('swift_dir', '/etc/swift') + self.account_ring = Ring(swift_dir, ring_name='account') + self.container_ring = Ring(swift_dir, ring_name='container') + self.object_ring = Ring(swift_dir, ring_name='object') + self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/') + if not self.endpoints_path.endswith('/'): + self.endpoints_path += '/' + + def __call__(self, env, start_response): + request = Request(env) + + if not request.path.startswith(self.endpoints_path): + return self.app(env, start_response) + + if request.method != 'GET': + return HTTPMethodNotAllowed( + req=request, headers={"Allow": "GET"})(env, start_response) + + try: + clean_path = request.path[len(self.endpoints_path) - 1:] + account, container, obj = \ + split_path(clean_path, 1, 3, True) + except ValueError: + return HTTPBadRequest('No account specified')(env, start_response) + + if account is not None: + account = unquote(account) + if container is not None: + container = unquote(container) + if obj is not None: + obj = unquote(obj) + + if obj is not None: + partition, nodes = self.object_ring.get_nodes( + account, container, obj) + endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ + '{account}/{container}/{obj}' + elif container is not None: + partition, nodes = self.container_ring.get_nodes( + account, container) + endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ + '{account}/{container}' + else: + partition, nodes = self.account_ring.get_nodes( + account) + endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ + '{account}' + + endpoints = [] + for node in nodes: + endpoint = endpoint_template.format( + ip=node['ip'], + port=node['port'], + device=node['device'], + partition=partition, + account=quote(account), + container=quote(container or ''), + obj=quote(obj or '')) + endpoints.append(endpoint) + + return Response(json.dumps(endpoints), + content_type='application/json')(env, start_response) + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def list_endpoints_filter(app): + return ListEndpointsMiddleware(app, conf) + + return list_endpoints_filter diff --git a/test/unit/common/middleware/test_list_endpoints.py b/test/unit/common/middleware/test_list_endpoints.py new file mode 100644 index 000000000..642c01f9d --- /dev/null +++ b/test/unit/common/middleware/test_list_endpoints.py @@ -0,0 +1,200 @@ +# Copyright (c) 2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import unittest +from shutil import rmtree + +import os +from swift.common import ring, utils +from swift.common.utils import json +from swift.common.swob import Request, Response +from swift.common.middleware import list_endpoints + + +class FakeApp(object): + def __call__(self, env, start_response): + return Response(body="FakeApp")(env, start_response) + + +def start_response(*args): + pass + + +class TestListEndpoints(unittest.TestCase): + def setUp(self): + utils.HASH_PATH_SUFFIX = 'endcap' + self.testdir = os.path.join(os.path.dirname(__file__), 'ring') + rmtree(self.testdir, ignore_errors=1) + os.mkdir(self.testdir) + + accountgz = os.path.join(self.testdir, 'account.ring.gz') + containergz = os.path.join(self.testdir, 'container.ring.gz') + objectgz = os.path.join(self.testdir, 'object.ring.gz') + + # Let's make the rings slightly different so we can test + # that the correct ring is consulted (e.g. we don't consult + # the object ring to get nodes for a container) + intended_replica2part2dev_id_a = [ + array.array('H', [3, 1, 3, 1]), + array.array('H', [0, 3, 1, 4]), + array.array('H', [1, 4, 0, 3])] + intended_replica2part2dev_id_c = [ + array.array('H', [4, 3, 0, 1]), + array.array('H', [0, 1, 3, 4]), + array.array('H', [3, 4, 0, 1])] + intended_replica2part2dev_id_o = [ + array.array('H', [0, 1, 0, 1]), + array.array('H', [0, 1, 0, 1]), + array.array('H', [3, 4, 3, 4])] + intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0, + 'ip': '10.1.1.1', 'port': 6000, + 'device': 'sda1'}, + {'id': 1, 'zone': 0, 'weight': 1.0, + 'ip': '10.1.1.1', 'port': 6000, + 'device': 'sdb1'}, + None, + {'id': 3, 'zone': 2, 'weight': 1.0, + 'ip': '10.1.2.1', 'port': 6000, + 'device': 'sdc1'}, + {'id': 4, 'zone': 2, 'weight': 1.0, + 'ip': '10.1.2.2', 'port': 6000, + 'device': 'sdd1'}] + intended_part_shift = 30 + intended_reload_time = 15 + ring.RingData(intended_replica2part2dev_id_a, + intended_devs, intended_part_shift).save(accountgz) + ring.RingData(intended_replica2part2dev_id_c, + intended_devs, intended_part_shift).save(containergz) + ring.RingData(intended_replica2part2dev_id_o, + intended_devs, intended_part_shift).save(objectgz) + + self.app = FakeApp() + self.list_endpoints = list_endpoints.filter_factory( + {'swift_dir': self.testdir})(self.app) + + def tearDown(self): + rmtree(self.testdir, ignore_errors=1) + + def test_get_endpoint(self): + # Expected results for objects taken from test_ring + # Expected results for others computed by manually invoking + # ring.get_nodes(). + resp = Request.blank('/endpoints/a/c/o1').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(json.loads(resp.body), [ + "http://10.1.1.1:6000/sdb1/1/a/c/o1", + "http://10.1.2.2:6000/sdd1/1/a/c/o1" + ]) + + # Here, 'o1/' is the object name. + resp = Request.blank('/endpoints/a/c/o1/').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(json.loads(resp.body), [ + "http://10.1.1.1:6000/sdb1/3/a/c/o1/", + "http://10.1.2.2:6000/sdd1/3/a/c/o1/" + ]) + + resp = Request.blank('/endpoints/a/c2').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(json.loads(resp.body), [ + "http://10.1.1.1:6000/sda1/2/a/c2", + "http://10.1.2.1:6000/sdc1/2/a/c2" + ]) + + resp = Request.blank('/endpoints/a1').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(json.loads(resp.body), [ + "http://10.1.2.1:6000/sdc1/0/a1", + "http://10.1.1.1:6000/sda1/0/a1", + "http://10.1.1.1:6000/sdb1/0/a1" + ]) + + resp = Request.blank('/endpoints/').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 400) + + resp = Request.blank('/endpoints/a/c 2').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(json.loads(resp.body), [ + "http://10.1.1.1:6000/sdb1/3/a/c%202", + "http://10.1.2.2:6000/sdd1/3/a/c%202" + ]) + + resp = Request.blank('/endpoints/a/c%202').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(json.loads(resp.body), [ + "http://10.1.1.1:6000/sdb1/3/a/c%202", + "http://10.1.2.2:6000/sdd1/3/a/c%202" + ]) + + resp = Request.blank('/endpoints/ac%20count/con%20tainer/ob%20ject') \ + .get_response(self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(json.loads(resp.body), [ + "http://10.1.1.1:6000/sdb1/3/ac%20count/con%20tainer/ob%20ject", + "http://10.1.2.2:6000/sdd1/3/ac%20count/con%20tainer/ob%20ject" + ]) + + resp = Request.blank('/endpoints/a/c/o1', {'REQUEST_METHOD': 'POST'}) \ + .get_response(self.list_endpoints) + self.assertEquals(resp.status_int, 405) + self.assertEquals(resp.status, '405 Method Not Allowed') + self.assertEquals(resp.headers['allow'], 'GET') + + resp = Request.blank('/not-endpoints').get_response( + self.list_endpoints) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.status, '200 OK') + self.assertEquals(resp.body, 'FakeApp') + + # test custom path with trailing slash + custom_path_le = list_endpoints.filter_factory({ + 'swift_dir': self.testdir, + 'list_endpoints_path': '/some/another/path/' + })(self.app) + resp = Request.blank('/some/another/path/a/c/o1') \ + .get_response(custom_path_le) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(json.loads(resp.body), [ + "http://10.1.1.1:6000/sdb1/1/a/c/o1", + "http://10.1.2.2:6000/sdd1/1/a/c/o1" + ]) + + # test ustom path without trailing slash + custom_path_le = list_endpoints.filter_factory({ + 'swift_dir': self.testdir, + 'list_endpoints_path': '/some/another/path' + })(self.app) + resp = Request.blank('/some/another/path/a/c/o1') \ + .get_response(custom_path_le) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(json.loads(resp.body), [ + "http://10.1.1.1:6000/sdb1/1/a/c/o1", + "http://10.1.2.2:6000/sdd1/1/a/c/o1" + ]) + + +if __name__ == '__main__': + unittest.main() |