Index: src/scanner.h |
diff --git a/src/scanner.h b/src/scanner.h |
index 6e668fd4921ca23e7dbbcd1755404c8269c20bd7..8537c5308c9f73e8dbf2f5f97e8188f195d531cc 100644 |
--- a/src/scanner.h |
+++ b/src/scanner.h |
@@ -121,6 +121,12 @@ class UnicodeCache { |
bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); } |
bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); } |
bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); } |
+ bool IsLineTerminatorSequence(unibrow::uchar c, unibrow::uchar next) { |
+ if (!IsLineTerminator(c)) return false; |
+ if (c == 0x000d && next == 0x000a) return false; // CR with following LF. |
+ return true; |
+ } |
+ |
bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); } |
bool IsWhiteSpaceOrLineTerminator(unibrow::uchar c) { |
return kIsWhiteSpaceOrLineTerminator.get(c); |