Experimental parser: fix two byte to one byte allocation

src/lexer/lexer.h

Index: src/lexer/lexer.h
diff --git a/src/lexer/lexer.h b/src/lexer/lexer.h
index ebc2b77355d8d15f72b38b89384f9113faeee8cd..6b6e99a86d9a5a96a93d3ca8804149b1ee323b86 100644
--- a/src/lexer/lexer.h
+++ b/src/lexer/lexer.h
@@ -351,6 +351,11 @@ class Lexer : public LexerBase {
         (sizeof(Char) == 1 && !token.is_in_primary_range);
   }
 
+  static inline bool MustBeInBufferForAllocation(const TokenDesc& token) {
+    return MustBeInBuffer(token) ||
+        (sizeof(Char) == 2 && token.is_in_primary_range);
+  }
+
  protected:
   virtual void Scan();