diff options
author | Felix Geisendörfer <felix@debuggable.com> | 2014-05-13 17:42:48 +0200 |
---|---|---|
committer | Timothy J Fontaine <tjfontaine@gmail.com> | 2014-06-06 15:04:39 -0700 |
commit | 0dc2f4f82d0f3eae2ad5d36447c9d51a46bfc486 (patch) | |
tree | 4db7ca91f504e95665a0ac5c3bcd3e4d2735a27f | |
parent | f59ec645cb84070a78c96c7144b0fac055d4d1c6 (diff) | |
download | node-0dc2f4f82d0f3eae2ad5d36447c9d51a46bfc486.tar.gz |
string_decoder: Add more comments
-rw-r--r-- | lib/string_decoder.js | 25 |
1 files changed, 25 insertions, 0 deletions
diff --git a/lib/string_decoder.js b/lib/string_decoder.js index 56ae79d42..3fa6a6e59 100644 --- a/lib/string_decoder.js +++ b/lib/string_decoder.js @@ -19,6 +19,14 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. +// StringDecoder provides an interface for efficiently splitting a series of +// buffers into a series of JS strings without breaking apart multi-byte +// characters. CESU-8 is handled as part of the UTF-8 encoding. +// +// @TODO Handling all encodings inside a single object makes it very difficult +// to reason about this code, so it should be split up in the future. +// @TODO There should be a utf8-strict encoding that rejects invalid UTF-8 code +// points as used by CESU-8. var StringDecoder = exports.StringDecoder = function(encoding) { this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); switch (this.encoding) { @@ -37,12 +45,25 @@ var StringDecoder = exports.StringDecoder = function(encoding) { return; } + // Enough space to store all bytes of a single character. UTF-8 needs 4 + // bytes, but CESU-8 may require up to 6 (3 bytes per surrogate). this.charBuffer = new Buffer(6); + // Number of bytes received for the current incomplete multi-byte character. this.charReceived = 0; + // Number of bytes expected for the current incomplete multi-byte character. this.charLength = 0; }; +// write decodes the given buffer and returns it as JS string that is +// guaranteed to not contain any partial multi-byte characters. Any partial +// character found at the end of the buffer is buffered up, and will be +// returned when calling write again with the remaining bytes. +// +// Note: Converting a Buffer containing an orphan surrogate to a String +// currently works, but converting a String to a Buffer (via `new Buffer`, or +// Buffer#write) will replace incomplete surrogates with the unicode +// replacement character. See https://codereview.chromium.org/121173009/ . StringDecoder.prototype.write = function(buffer) { var charStr = ''; // if our last write ended with an incomplete multibyte character @@ -111,6 +132,10 @@ StringDecoder.prototype.write = function(buffer) { return charStr; }; +// detectIncompleteChar determines if there is an incomplete UTF-8 character at +// the end of the given buffer. If so, it sets this.charLength to the byte +// length that character, and sets this.charReceived to the number of bytes +// that are available for this character. StringDecoder.prototype.detectIncompleteChar = function(buffer) { // determine how many bytes we have to check at the end of this buffer var i = (buffer.length >= 3) ? 3 : buffer.length; |