support Unicode line boundaries

jahewson · jahewson · commit 75639ed55bfd · 2016-07-20T23:50:40.000-07:00
diff --git a/README.md b/README.md
@@ -7,6 +7,7 @@
 - node v0.10 `streams2` (transform stream)
 - supports `pipe`
 - supports both UNIX and Windows line endings
+- supports [Unicode UTS #18 line boundaries](http://www.unicode.org/reports/tr18/#Line_Boundaries)
 - can wrap any readable stream
 - can be used as a readable-writable "through-stream" (transform stream)
 - super-simple: `stream = byline(stream);`
diff --git a/lib/byline.js b/lib/byline.js
@@ -99,7 +99,8 @@ LineStream.prototype._transform = function(chunk, encoding, done) {
   }
   this._chunkEncoding = encoding;
   
-  var lines = chunk.split(/\r\n|\r|\n/g);
+  // see: http://www.unicode.org/reports/tr18/#Line_Boundaries
+  var lines = chunk.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
   
   // don't split CRLF which spans chunks
   if (this._lastChunkEndedWithCR && chunk[0] == '\n') {

Original file line number	Diff line number	Diff line change
`@@ -99,7 +99,8 @@ LineStream.prototype._transform = function(chunk, encoding, done) {`
`99`	`99`	`}`
`100`	`100`	`this._chunkEncoding = encoding;`
`101`	`101`
`102`		`- var lines = chunk.split(/\r\n\|\r\|\n/g);`
	`102`	`+ // see: http://www.unicode.org/reports/tr18/#Line_Boundaries`
	`103`	`+ var lines = chunk.split(/\r\n\|[\n\v\f\r\x85\u2028\u2029]/g);`
`103`	`104`
`104`	`105`	`// don't split CRLF which spans chunks`
`105`	`106`	`if (this._lastChunkEndedWithCR && chunk[0] == '\n') {`