From 561735bc8be151dd1f04c459d5c6ebfa451bc951 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Wed, 20 May 2020 15:59:18 -0500 Subject: stabilize buffer behavior with multiple requests on the same connection previous to this change, a buffer may be reused across requests, and would cause data to be written to disk even if each individual request was not returning much data fixes #265 --- src/waitress/channel.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/waitress/channel.py b/src/waitress/channel.py index a8bc76f..bc9a2bb 100644 --- a/src/waitress/channel.py +++ b/src/waitress/channel.py @@ -174,9 +174,10 @@ class HTTPChannel(wasyncore.dispatcher, object): # there's no current task, so we don't need to try to # lock the outbuf to append to it. outbuf_payload = b"HTTP/1.1 100 Continue\r\n\r\n" + num_bytes = len(outbuf_payload) self.outbufs[-1].append(outbuf_payload) - self.current_outbuf_count += len(outbuf_payload) - self.total_outbufs_len += len(outbuf_payload) + self.current_outbuf_count += num_bytes + self.total_outbufs_len += num_bytes self.sent_continue = True self._flush_some() request.completed = False @@ -311,7 +312,7 @@ class HTTPChannel(wasyncore.dispatcher, object): self.outbufs.append(nextbuf) self.current_outbuf_count = 0 else: - if self.current_outbuf_count > self.adj.outbuf_high_watermark: + if self.current_outbuf_count >= self.adj.outbuf_high_watermark: # rotate to a new buffer if the current buffer has hit # the watermark to avoid it growing unbounded nextbuf = OverflowableBuffer(self.adj.outbuf_overflow) @@ -399,6 +400,15 @@ class HTTPChannel(wasyncore.dispatcher, object): # at the end to account for consecutive service() calls if len(self.requests) > 1: self._flush_outbufs_below_high_watermark() + + # this is a little hacky but basically it's forcing the + # next request to create a new outbuf to avoid sharing + # outbufs across requests which can cause outbufs to + # not be deallocated regularly when a connection is open + # for a long time + if self.current_outbuf_count > 0: + self.current_outbuf_count = self.adj.outbuf_high_watermark + request = self.requests.pop(0) request.close() -- cgit v1.2.1 From a15c7f0eb2b917664bd211d30291ed98cfd35294 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Wed, 20 May 2020 16:05:56 -0500 Subject: add changelog for #300 --- CHANGES.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 701c2b0..c40246b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,14 @@ +Unreleased +---------- + +- Fix an issue with keep-alive connections in which memory usage was higher + than expected because output buffers were being reused across requests on + a long-lived connection and each buffer would not be freed until it was full + or the connection was closed. Buffers are now rotated per-request to + stabilize their behavior. + + See https://github.com/Pylons/waitress/pull/300 + 1.4.3 (2020-02-02) ------------------ -- cgit v1.2.1