DevTools: Use CodeMirror modes instead of highlight tokenizers

Source/devtools/front_end/SourceJavaScriptTokenizer.re2js

Index: Source/devtools/front_end/SourceJavaScriptTokenizer.re2js
diff --git a/Source/devtools/front_end/SourceJavaScriptTokenizer.re2js b/Source/devtools/front_end/SourceJavaScriptTokenizer.re2js
deleted file mode 100644
index 6b48c05063655613c67312644196b90e1f270925..0000000000000000000000000000000000000000
--- a/Source/devtools/front_end/SourceJavaScriptTokenizer.re2js
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Copyright (C) 2009 Google Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * Generate js file as follows:
-re2c -isc devtools/front_end/SourceJavaScriptTokenizer.re2js \
-  | sed 's|^yy\([^:]*\)*\:|case \1:|' \
-  | sed 's|[*]cursor[+][+]|this._charAt(cursor++)|' \
-  | sed 's|[[*][+][+]cursor|this._charAt(++cursor)|' \
-  | sed 's|[*]cursor|this._charAt(cursor)|' \
-  | sed 's|yych = \*\([^;]*\)|yych = this._charAt\1|' \
-  | sed 's|goto case \([^;]*\)|{ gotoCase = \1; continue; }|' \
-  | sed 's|yych <= \(0x[0-9a-fA-F]*\)|yych \<\= String.fromCharCode(\1)|' \
-  | sed 's|unsigned\ int|var|' \
-  | sed 's|var\ yych|case 1: var yych|' > devtools/front_end/SourceJavaScriptTokenizer.js
- */
-
-/**
- * @constructor
- * @extends {WebInspector.SourceTokenizer}
- */
-WebInspector.SourceJavaScriptTokenizer = function()
-{
-    WebInspector.SourceTokenizer.call(this);
-
-    this._lexConditions = {
-        DIV: 0,
-        NODIV: 1,
-        COMMENT: 2,
-        DSTRING: 3,
-        SSTRING: 4,
-        REGEX: 5
-    };
-
-    this.case_DIV = 1000;
-    this.case_NODIV = 1001;
-    this.case_COMMENT = 1002;
-    this.case_DSTRING = 1003;
-    this.case_SSTRING = 1004;
-    this.case_REGEX = 1005;
-
-    this.condition = this.createInitialCondition();
-}
-
-WebInspector.SourceJavaScriptTokenizer.Keywords = [
-        "null", "true", "false", "break", "case", "catch", "const", "default", "finally", "for",
-        "instanceof", "new", "var", "continue", "function", "return", "void", "delete", "if",
-        "this", "do", "while", "else", "in", "switch", "throw", "try", "typeof", "debugger",
-        "class", "enum", "export", "extends", "import", "super", "get", "set", "with"
-    ].keySet();
-
-WebInspector.SourceJavaScriptTokenizer.GlobalObjectValueProperties = {
-    "NaN": "javascript-nan",
-    "undefined": "javascript-undef",
-    "Infinity": "javascript-inf"
-};
-
-WebInspector.SourceJavaScriptTokenizer.prototype = {
-    createInitialCondition: function()
-    {
-        return { lexCondition: this._lexConditions.NODIV };
-    },
-
-    nextToken: function(cursor)
-    {
-        var cursorOnEnter = cursor;
-        var gotoCase = 1;
-        var YYMARKER;
-        while (1) {
-            switch (gotoCase)
-            // Following comment is replaced with generated state machine.
-            /*!re2c
-                re2c:define:YYCTYPE  = "var";
-                re2c:define:YYCURSOR = cursor;
-                re2c:define:YYGETCONDITION = "this.getLexCondition";
-                re2c:define:YYSETCONDITION = "this.setLexCondition";
-                re2c:condprefix = "case this.case_";
-                re2c:condenumprefix = "this._lexConditions.";
-                re2c:yyfill:enable = 0;
-                re2c:labelprefix = "case ";
-                re2c:indent:top = 2;
-                re2c:indent:string = "    ";
-
-                LineComment = "//" [^\r\n]*;
-                CommentContent = ([^*\r\n] | ("*"+[^/*]))*;
-                Comment = "/*" CommentContent "*"+ "/";
-                CommentStart = "/*" CommentContent [\r\n];
-                CommentEnd = CommentContent "*"+ "/";
-
-                DecimalDigit = [0-9];
-                NonZeroDigit = [1-9];
-                OctalDigit = [0-7];
-                HexDigit = [0-9a-fA-F];
-                SignedInteger = ("+"|"-")? DecimalDigit+;
-                ExponentPart = ("e" | "E") SignedInteger;
-                DecimalIntegerLiteral = "0" | NonZeroDigit DecimalDigit*;
-                DecimalLiteral = DecimalIntegerLiteral "." DecimalDigit* ExponentPart? | "." DecimalDigit+ ExponentPart? | DecimalIntegerLiteral ExponentPart?;
-                HexIntegerLiteral = "0" ("x"|"X") HexDigit+;
-                OctalIntegerLiteral = "0" OctalDigit+;
-                NumericLiteral = DecimalLiteral | HexIntegerLiteral | OctalIntegerLiteral;
-
-                Punctuation = [\!\%\&\(\*\+\,\-\.\:\;\<\=\>\?\[\]\^\{\|\}\~] | "!=" | "!==" | "%=" | "&&" | "&=" | "*=" | "++" | "+=" | "--" | "-=" | "<<" | "<<="  | "<=" | "==" | "===" | ">=" | ">>" | ">>=" | ">>>" | ">>>=" | "^=" | "|=" | "||";
-                Division = "/" | "/=";
-                RightParen = ")";
-
-                Letter = [a-zA-Z\x80-\xFF];
-                UnicodeEscapeSequence = "\\u" HexDigit HexDigit HexDigit HexDigit;
-
-                IdentifierStart = Letter | "_" | "$" | UnicodeEscapeSequence;
-                IdentifierPart = IdentifierStart | DecimalDigit;
-                Identifier = IdentifierStart IdentifierPart *;
-                Spaces = " "+;
-
-                DoubleStringContent = ([^\r\n\"\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*;
-                SingleStringContent = ([^\r\n\'\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*;
-                StringLiteral = "\"" DoubleStringContent "\"" | "'" SingleStringContent "'";
-                DoubleStringStart = "\"" DoubleStringContent "\\" [\r\n];
-                DoubleStringEnd = DoubleStringContent "\"";
-                SingleStringStart = "'" SingleStringContent "\\" [\r\n];
-                SingleStringEnd = SingleStringContent "'";
-
-                BackslashSequence = "\\" [^\r\n];
-                RegexSet = "[" ([^\r\n*\\/] | BackslashSequence)* "]";
-                RegexFirstChar = [^\r\n*\\/\[\]] | BackslashSequence | RegexSet;
-                RegexChar = [^\r\n\\/\[\]] | BackslashSequence | RegexSet;
-                RegexContent = RegexChar*;
-                Regex = "/" RegexFirstChar RegexContent "/" [igm]*;
-                RegexStart = "/" RegexFirstChar RegexContent "\\";
-                RegexEnd = RegexContent "/" [igm]*;
-
-                <DIV,NODIV> LineComment { this.tokenType = "javascript-comment"; return cursor; }
-                <DIV,NODIV> Comment { this.tokenType = "javascript-comment"; return cursor; }
-                <DIV,NODIV> CommentStart => COMMENT { this.tokenType = "javascript-comment"; return cursor; }
-                <COMMENT> CommentContent => COMMENT { this.tokenType = "javascript-comment"; return cursor; }
-                <COMMENT> CommentEnd => NODIV { this.tokenType = "javascript-comment"; return cursor; }
-
-                <DIV,NODIV> Spaces {this.tokenType = "whitespace"; return cursor; }
-                <DIV,NODIV> StringLiteral { this.tokenType = "javascript-string"; return cursor; }
-                <DIV,NODIV> DoubleStringStart => DSTRING { this.tokenType = "javascript-string"; return cursor; }
-                <DSTRING> DoubleStringContent => DSTRING { this.tokenType = "javascript-string"; return cursor; }
-                <DSTRING> DoubleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; }
-                <DIV,NODIV> SingleStringStart => SSTRING { this.tokenType = "javascript-string"; return cursor; }
-                <SSTRING> SingleStringContent => SSTRING { this.tokenType = "javascript-string"; return cursor; }
-                <SSTRING> SingleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; }
-
-                <NODIV> Regex { this.tokenType = "javascript-regexp"; return cursor; }
-                <NODIV> RegexStart => REGEX { this.tokenType = "javascript-regexp"; return cursor; }
-                <REGEX> RegexContent => REGEX { this.tokenType = "javascript-regexp"; return cursor; }
-                <REGEX> RegexEnd => NODIV { this.tokenType = "javascript-regexp"; return cursor; }
-
-                <DIV,NODIV> NumericLiteral => DIV { this.tokenType = "javascript-number"; return cursor; }
-                <DIV,NODIV> Identifier => DIV
-                {
-                    var token = this._line.substring(cursorOnEnter, cursor);
-                    if (WebInspector.SourceJavaScriptTokenizer.GlobalObjectValueProperties.hasOwnProperty(token))
-                        this.tokenType = WebInspector.SourceJavaScriptTokenizer.GlobalObjectValueProperties[token];
-                    else if (WebInspector.SourceJavaScriptTokenizer.Keywords[token] === true && token !== "__proto__")
-                        this.tokenType = "javascript-keyword";
-                    else
-                        this.tokenType = "javascript-ident";
-                    return cursor;
-                }
-                <DIV,NODIV> RightParen => DIV { this.tokenType = "brace-end"; return cursor; }
-                <DIV,NODIV> Punctuation => NODIV
-                {
-                    var token = this._line.charAt(cursorOnEnter);
-                    if (token === "{")
-                        this.tokenType = "block-start";
-                    else if (token === "}")
-                        this.tokenType = "block-end";
-                    else if (token === "(")
-                        this.tokenType = "brace-start";
-                    else this.tokenType = null;
-                    return cursor;
-                }
-                <DIV> Division => NODIV { this.tokenType = null; return cursor; }
-                <*> [^] { this.tokenType = null; return cursor; }
-            */
-        }
-    },
-
-    __proto__: WebInspector.SourceTokenizer.prototype
-}