Merged in ctheune/pypi-fix-mirroring (pull request #1)

Fix packages downloading to include LAST SERIAL header

4 files changed, 52 insertions, 6 deletions

diff --git a/.hgignore b/.hgignore
index 733cedf..def1648 100644
--- a/.hgignore
+++ b/.hgignore
@@ -1,7 +1,13 @@
 syntax: glob
+*.swp
+lib
+bin
+local
+include
 **.pyc
 config.ini
 uwsgi-logd.conf
 sshkeys_update
 privkey*
 pubkey*
+pypi.egg*
diff --git a/pypi.wsgi b/pypi.wsgi
index c144b5e..7ed66d6 100644
--- a/pypi.wsgi
+++ b/pypi.wsgi
@@ -73,7 +73,7 @@ def application(environ, start_response):
 def site_fake(app, environ, start_response):
     PATH_INFO = environ['PATH_INFO']
     m = re.match('^/(pypi|simple|daytime|serversig|mirrors|id|oauth|'
-        'security)(.*)', PATH_INFO)
+        'security|packages)(.*)', PATH_INFO)
     if not m:
         start_response("404 not found", [('Content-type', 'text/plain')])
         return ['Not Found: %s' % PATH_INFO]
diff --git a/tools/demodata.py b/tools/demodata.py
index e360be8..178b2e2 100755
--- a/tools/demodata.py
+++ b/tools/demodata.py
@@ -3,6 +3,8 @@ import sys, os, urllib
 
 root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(root)
+# Work around http://sourceforge.net/p/docutils/bugs/214/
+import docutils.utils
 import admin, store, config
 
 cfg = config.Config(root+'/config.ini')
@@ -52,7 +54,7 @@ for version in ('0.1', '0.2', '0.3', '0.4'):
             '_pypi_hidden':version!='0.4'
             })
 
-st.add_file('spam', '1.0', 'THIS IS SOME CONTENT', '1234', 'sdist',
+st.add_file('spam', '1.0', 'THIS IS SOME CONTENT', '12e6ed27f5a127cab06e449171b35b6d', 'sdist',
             'any', '', 'demo.txt', None)
 
 st.commit()
diff --git a/webui.py b/webui.py
index c4262dc..bef4dff 100644
--- a/webui.py
+++ b/webui.py
@@ -65,6 +65,8 @@ safe_username = re.compile(r'^[A-Za-z0-9._]+$')
 safe_email = re.compile(r'^[a-zA-Z0-9._+@-]+$')
 botre = re.compile(r'^$|brains|yeti|myie2|findlinks|ia_archiver|psycheclone|badass|crawler|slurp|spider|bot|scooter|infoseek|looksmart|jeeves', re.I)
 
+packages_path_to_package_name = re.compile(
+    '^/([0-9\.]+|any|source)/./([a-zA-Z0-9][a-zA-Z0-9_\-\.]*)')
 
 class NotFound(Exception):
     pass
@@ -500,7 +502,6 @@ class WebUI:
                          (cssclass, cssclass, self.link_action(action_name), desc))
         return links
 
-
     def inner_run(self):
         ''' Figure out what the request is, and farm off to the appropriate
             handler.
@@ -511,6 +512,8 @@ class WebUI:
             return self.run_simple()
         if script_name and script_name == self.config.simple_sign_script:
             return self.run_simple_sign()
+        if script_name == '/packages':
+            return self.packages()
         if script_name == '/mirrors':
             return self.mirrors()
         if script_name == '/security':
@@ -752,6 +755,10 @@ class WebUI:
             self.handler.set_content_type('text/html; charset=utf-8')
             self.handler.send_header('Content-Length', str(len(html)))
             self.handler.send_header("Surrogate-Key", "simple")
+            # XXX not quite sure whether this is the right thing for empty
+            # mirrors, but anyway.
+            serial = self.store.changelog_last_serial() or 0
+            self.handler.send_header("X-PYPI-LAST-SERIAL", str(serial))
             self.handler.end_headers()
             self.wfile.write(html)
             return
@@ -776,9 +783,8 @@ class WebUI:
 
     def run_simple_sign(self):
         path = self.env.get('PATH_INFO')
-        if not path.endswith('/'):
-            raise Redirect, self.config.simple_sign_script+path+'/'
-        path = path[1:-1]
+        # Helper to support this working with simple WSGI main script.
+        path = path.strip('/')
         if '/' in path:
             raise NotFound, path
         html = self.simple_body(path)
@@ -796,6 +802,38 @@ class WebUI:
         self.handler.end_headers()
         self.wfile.write(sig)
 
+    def packages(self):
+        path = self.env.get('PATH_INFO')
+
+        # I expect that nginx will do the right thing if it doesn't find the 
+        # actual file when resolving the X-accel headers.
+        self.handler.send_response(200, 'OK')
+
+        package = packages_path_to_package_name.match(path)
+        if package:
+            # Make sure that we associate the delivered file with the serial this
+            # is valid for. Intended to support mirrors to more easily achieve
+            # consistency with files that are newer than they may expect.
+            package = package.group(2)
+            serial = self.store.last_serial_for_package(package)
+            self.handler.send_header("X-PYPI-LAST-SERIAL", str(serial))
+
+        # we expect nginx to have configured a location named
+        # '/packages_raw/...' that aliases the original path correctly, see
+        # http://wiki.nginx.org/X-accel and http://wiki.nginx.org/XSendfile for
+        # details.
+        # Sample: (note the missing slash on the alias!)
+        # location /packages_raw {
+        #    alias /path/to/packages/dir;
+        #    add_header X-PYPI-LAST-SERIAL $upstream_http_x_pypi_last_serial;
+        #    internal;
+        #    autoindex on;
+        # }
+	# I tested this using regular http upstreams, so no guarantee this works with uwsgi.
+        self.handler.send_header("X-Accel-Redirect", "/packages_raw" + path)
+
+        self.handler.end_headers()
+
     def run_id(self):
         path = self.env.get('PATH_INFO')
         if not path: