ES6 unicode escapes, part 2: Regexps.

src/parser.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/api.h"
 #include "src/ast.h"
 #include "src/bailout-reason.h"
 #include "src/base/platform/platform.h"
@@ skipping 4260 matching lines @@
   }
   isolate()->counters()->total_preparse_skipped()->Increment(
       total_preparse_skipped_);
 }
 
 
 // ----------------------------------------------------------------------------
 // Regular expressions
 
 
-RegExpParser::RegExpParser(FlatStringReader* in,
-                           Handle<String>* error,
-                           bool multiline,
-                           Zone* zone)
+RegExpParser::RegExpParser(FlatStringReader* in, Handle<String>* error,
+                           bool multiline, bool unicode, Zone* zone)
     : isolate_(zone->isolate()),
       zone_(zone),
       error_(error),
       captures_(NULL),
       in_(in),
       current_(kEndMarker),
       next_pos_(0),
       capture_count_(0),
       has_more_(true),
       multiline_(multiline),
+      unicode_(unicode),
       simple_(false),
       contains_anchor_(false),
       is_scanned_for_captures_(false),
       failed_(false) {
   Advance();
 }
 
 
 uc32 RegExpParser::Next() {
   if (has_next()) {
@@ skipping 36 matching lines @@
   next_pos_ += dist - 1;
   Advance();
 }
 
 
 bool RegExpParser::simple() {
   return simple_;
 }
 
 
+bool RegExpParser::IsSyntaxCharacter(uc32 c) {
+  return c == '^' || c == '$' || c == '\\' || c == '.' || c == '*' ||
+         c == '+' || c == '?' || c == '(' || c == ')' || c == '[' || c == ']' ||
+         c == '{' || c == '}' || c == '|';
+}

[mathias, 2015/01/08 12:29:07]

Should `-` be a “syntax character” as well because of its special meaning within
character classes (e.g. `/[a-b]/`)?

[marja, 2015/01/08 13:42:18]

The spec (draft rev 30) says:

SyntaxCharacter :: one of
^ $ \ . * + ? ( ) [ ] { } |

- loses its special meaning if it's the first or the last character in the
character class, so [a-b-] matches a, b, and -.

But idk, hard to say if it's an omission in the spec or intentional. I filed a
bug to ask that: https://bugs.ecmascript.org/show_bug.cgi?id=3519

If they update the spec, I'll update the implementation.

[rossberg, 2015/01/08 14:11:45]

It's not a SyntaxCharacter and that's likely intentional: it has no special
status outside a character class, and inside you can put it first or last, so
you don't need to escape it in either case.

[mathias, 2015/01/21 07:16:54]

/[\-]/u is now allowed: https://bugs.ecmascript.org/show_bug.cgi?id=3519#c1

+
+
 RegExpTree* RegExpParser::ReportError(Vector<const char> message) {
   failed_ = true;
   *error_ = isolate()->factory()->NewStringFromAscii(message).ToHandleChecked();
   // Zip to the end to make sure the no more input is read.
   current_ = kEndMarker;
   next_pos_ = in()->length();
   return NULL;
 }
 
 
@@ skipping 196 matching lines @@
           if (capture == NULL) {
             builder->AddEmpty();
             break;
           }
           RegExpTree* atom = new(zone()) RegExpBackReference(capture);
           builder->AddAtom(atom);
           break;
         }
         uc32 first_digit = Next();
         if (first_digit == '8' || first_digit == '9') {
-          // Treat as identity escape
-          builder->AddCharacter(first_digit);
-          Advance(2);
+          // If the 'u' flag is present, only syntax characters can be escaped,
+          // no other identity escapes are allowed. If the 'u' flag is not
+          // present, all identity escapes are allowed.
+          if (!FLAG_harmony_unicode || !unicode_) {
+            builder->AddCharacter(first_digit);
+            Advance(2);
+          } else {
+            return ReportError(CStrVector("Invalid escape"));
+          }
           break;
         }
       }
       // FALLTHROUGH
       case '0': {
         Advance();
         uc32 octal = ParseOctalLiteral();
         builder->AddCharacter(octal);
         break;
       }
@@ skipping 35 matching lines @@
           Advance(2);
           builder->AddCharacter(controlLetter & 0x1f);
         }
         break;
       }
       case 'x': {
         Advance(2);
         uc32 value;
         if (ParseHexEscape(2, &value)) {
           builder->AddCharacter(value);
+        } else if (!FLAG_harmony_unicode || !unicode_) {
+          builder->AddCharacter('x');
         } else {
-          builder->AddCharacter('x');
+          // If the 'u' flag is present, invalid escapes are not treated as
+          // identity escapes.
+          return ReportError(CStrVector("Invalid escape"));
         }
         break;
       }
       case 'u': {
         Advance(2);
         uc32 value;
-        if (ParseHexEscape(4, &value)) {
+        if (ParseUnicodeEscape(&value)) {
           builder->AddCharacter(value);
+        } else if (!FLAG_harmony_unicode || !unicode_) {
+          builder->AddCharacter('u');
         } else {
-          builder->AddCharacter('u');
+          // If the 'u' flag is present, invalid escapes are not treated as
+          // identity escapes.
+          return ReportError(CStrVector("Invalid unicode escape"));
         }
         break;
       }
       default:
-        // Identity escape.
-        builder->AddCharacter(Next());
-        Advance(2);
+        Advance();
+        // If the 'u' flag is present, only syntax characters can be escaped, no
+        // other identity escapes are allowed. If the 'u' flag is not present,
+        // all identity escapes are allowed.
+        if (!FLAG_harmony_unicode || !unicode_ ||
+            IsSyntaxCharacter(current())) {
+          builder->AddCharacter(current());
+          Advance();
+        } else {
+          return ReportError(CStrVector("Invalid escape"));
+        }
         break;
       }
       break;
     case '{': {
       int dummy;
       if (ParseIntervalQuantifier(&dummy, &dummy)) {
         ReportError(CStrVector("Nothing to repeat") CHECK_FAILED);
       }
       // fallthrough
     }
@@ skipping 222 matching lines @@
     Advance();
     if (value < 32 && '0' <= current() && current() <= '7') {
       value = value * 8 + current() - '0';
       Advance();
     }
   }
   return value;
 }
 
 
-bool RegExpParser::ParseHexEscape(int length, uc32 *value) {
+bool RegExpParser::ParseHexEscape(int length, uc32* value) {
   int start = position();
   uc32 val = 0;
-  bool done = false;
-  for (int i = 0; !done; i++) {
+  for (int i = 0; i < length; ++i) {
     uc32 c = current();
     int d = HexValue(c);
     if (d < 0) {
       Reset(start);
       return false;
     }
     val = val * 16 + d;
     Advance();
-    if (i == length - 1) {
-      done = true;
-    }
   }
   *value = val;
   return true;
 }
 
 
+bool RegExpParser::ParseUnicodeEscape(uc32* value) {
+  // Accept both \uxxxx and \u{xxxxxx} (if harmony unicode escapes are
+  // allowed). In the latter case, the number of hex digits between { } is
+  // arbitrary. \ and u have already been read.
+  if (current() == '{' && FLAG_harmony_unicode && unicode_) {
+    int start = position();
+    Advance();
+    if (ParseUnlimitedLengthHexNumber(0x10ffff, value)) {
+      if (current() == '}') {
+        Advance();
+        return true;
+      }
+    }
+    Reset(start);
+    return false;
+  }
+  // \u but no {, or \u{...} escapes not allowed.
+  return ParseHexEscape(4, value);
+}
+
+
+bool RegExpParser::ParseUnlimitedLengthHexNumber(int max_value, uc32* value) {
+  uc32 x = 0;
+  int d = HexValue(current());
+  if (d < 0) {
+    return false;
+  }
+  while (d >= 0) {
+    x = x * 16 + d;
+    if (x > max_value) {
+      return false;
+    }
+    Advance();
+    d = HexValue(current());
+  }
+  *value = x;
+  return true;
+}
+
+
 uc32 RegExpParser::ParseClassCharacterEscape() {
   DCHECK(current() == '\\');
   DCHECK(has_next() && !IsSpecialClassEscape(Next()));
   Advance();
   switch (current()) {
     case 'b':
       Advance();
       return '\b';
     // ControlEscape :: one of
     //   f n r t v
@@ skipping 34 matching lines @@
       // For compatibility, we interpret a decimal escape that isn't
       // a back reference (and therefore either \0 or not valid according
       // to the specification) as a 1..3 digit octal character code.
       return ParseOctalLiteral();
     case 'x': {
       Advance();
       uc32 value;
       if (ParseHexEscape(2, &value)) {
         return value;
       }
-      // If \x is not followed by a two-digit hexadecimal, treat it
-      // as an identity escape.
-      return 'x';
+      if (!FLAG_harmony_unicode || !unicode_) {
+        // If \x is not followed by a two-digit hexadecimal, treat it
+        // as an identity escape.
+        return 'x';
+      }
+      // If the 'u' flag is present, invalid escapes are not treated as
+      // identity escapes.
+      ReportError(CStrVector("Invalid escape"));
+      return 0;
     }
     case 'u': {
       Advance();
       uc32 value;
-      if (ParseHexEscape(4, &value)) {
+      if (ParseUnicodeEscape(&value)) {
         return value;
       }
-      // If \u is not followed by a four-digit hexadecimal, treat it
-      // as an identity escape.
-      return 'u';
+      if (!FLAG_harmony_unicode || !unicode_) {
+        return 'u';
+      }
+      // If the 'u' flag is present, invalid escapes are not treated as
+      // identity escapes.
+      ReportError(CStrVector("Invalid unicode escape"));
+      return 0;
     }
     default: {
-      // Extended identity escape. We accept any character that hasn't
-      // been matched by a more specific case, not just the subset required
-      // by the ECMAScript specification.
       uc32 result = current();
-      Advance();
-      return result;
+      // If the 'u' flag is present, only syntax characters can be escaped, no
+      // other identity escapes are allowed. If the 'u' flag is not present, all
+      // identity escapes are allowed.
+      if (!FLAG_harmony_unicode || !unicode_ || IsSyntaxCharacter(result)) {
+        Advance();
+        return result;
+      }
+      ReportError(CStrVector("Invalid escape"));
+      return 0;
     }
   }
   return 0;
 }
 
 
 CharacterRange RegExpParser::ParseClassAtom(uc16* char_class) {
   DCHECK_EQ(0, *char_class);
   uc32 first = current();
   if (first == '\\') {
@@ skipping 85 matching lines @@
     ranges->Add(CharacterRange::Everything(), zone());
     is_negated = !is_negated;
   }
   return new(zone()) RegExpCharacterClass(ranges, is_negated);
 }
 
 
 // ----------------------------------------------------------------------------
 // The Parser interface.
 
-bool RegExpParser::ParseRegExp(FlatStringReader* input,
-                               bool multiline,
-                               RegExpCompileData* result,
+bool RegExpParser::ParseRegExp(FlatStringReader* input, bool multiline,
+                               bool unicode, RegExpCompileData* result,
                                Zone* zone) {
   DCHECK(result != NULL);
-  RegExpParser parser(input, &result->error, multiline, zone);
+  RegExpParser parser(input, &result->error, multiline, unicode, zone);
   RegExpTree* tree = parser.ParsePattern();
   if (parser.failed()) {
     DCHECK(tree == NULL);
     DCHECK(!result->error.is_null());
   } else {
     DCHECK(tree != NULL);
     DCHECK(result->error.is_null());
     result->tree = tree;
     int capture_count = parser.captures_started();
     result->simple = tree->IsAtom() && parser.simple() && capture_count == 0;
@@ skipping 189 matching lines @@
     } else {
       const uc16* data = reinterpret_cast<const uc16*>(raw_string->raw_data());
       running_hash = StringHasher::ComputeRunningHash(running_hash, data,
                                                       raw_string->length());
     }
   }
 
   return running_hash;
 }
 } }  // namespace v8::internal