| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (C) 2009 Google Inc. All rights reserved. | |
| 3 * | |
| 4 * Redistribution and use in source and binary forms, with or without | |
| 5 * modification, are permitted provided that the following conditions are | |
| 6 * met: | |
| 7 * | |
| 8 * * Redistributions of source code must retain the above copyright | |
| 9 * notice, this list of conditions and the following disclaimer. | |
| 10 * * Redistributions in binary form must reproduce the above | |
| 11 * copyright notice, this list of conditions and the following disclaimer | |
| 12 * in the documentation and/or other materials provided with the | |
| 13 * distribution. | |
| 14 * * Neither the name of Google Inc. nor the names of its | |
| 15 * contributors may be used to endorse or promote products derived from | |
| 16 * this software without specific prior written permission. | |
| 17 * | |
| 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 29 */ | |
| 30 | |
| 31 // Generate js file as follows: | |
| 32 // | |
| 33 // re2c -isc devtools/front_end/SourceHTMLTokenizer.re2js \ | |
| 34 // | sed 's|^yy\([^:]*\)*\:|case \1:|' \ | |
| 35 // | sed 's|[*]cursor[+][+]|this._charAt(cursor++)|' \ | |
| 36 // | sed 's|[[*][+][+]cursor|this._charAt(++cursor)|' \ | |
| 37 // | sed 's|[*]cursor|this._charAt(cursor)|' \ | |
| 38 // | sed 's|yych = \*\([^;]*\)|yych = this._charAt\1|' \ | |
| 39 // | sed 's|goto case \([^;]*\)|{ gotoCase = \1; continue; }|' \ | |
| 40 // | sed 's|unsigned\ int|var|' \ | |
| 41 // | sed 's|var\ yych|case 1: var yych|' > devtools/front_end/SourceHTMLTokenize
r.js | |
| 42 | |
| 43 /** | |
| 44 * @constructor | |
| 45 * @extends {WebInspector.SourceTokenizer} | |
| 46 */ | |
| 47 WebInspector.SourceHTMLTokenizer = function() | |
| 48 { | |
| 49 WebInspector.SourceTokenizer.call(this); | |
| 50 | |
| 51 // The order is determined by the generated code. | |
| 52 this._lexConditions = { | |
| 53 INITIAL: 0, | |
| 54 COMMENT: 1, | |
| 55 DOCTYPE: 2, | |
| 56 TAG: 3, | |
| 57 DSTRING: 4, | |
| 58 SSTRING: 5 | |
| 59 }; | |
| 60 this.case_INITIAL = 1000; | |
| 61 this.case_COMMENT = 1001; | |
| 62 this.case_DOCTYPE = 1002; | |
| 63 this.case_TAG = 1003; | |
| 64 this.case_DSTRING = 1004; | |
| 65 this.case_SSTRING = 1005; | |
| 66 | |
| 67 this._parseConditions = { | |
| 68 INITIAL: 0, | |
| 69 ATTRIBUTE: 1, | |
| 70 ATTRIBUTE_VALUE: 2, | |
| 71 LINKIFY: 4, | |
| 72 A_NODE: 8, | |
| 73 SCRIPT: 16, | |
| 74 STYLE: 32 | |
| 75 }; | |
| 76 | |
| 77 this.condition = this.createInitialCondition(); | |
| 78 } | |
| 79 | |
| 80 WebInspector.SourceHTMLTokenizer.prototype = { | |
| 81 createInitialCondition: function() | |
| 82 { | |
| 83 return { lexCondition: this._lexConditions.INITIAL, parseCondition: this
._parseConditions.INITIAL }; | |
| 84 }, | |
| 85 | |
| 86 set line(line) { | |
| 87 if (this._condition.internalJavaScriptTokenizerCondition) { | |
| 88 var match = /<\/script/i.exec(line); | |
| 89 if (match) { | |
| 90 this._internalJavaScriptTokenizer.line = line.substring(0, match
.index); | |
| 91 } else | |
| 92 this._internalJavaScriptTokenizer.line = line; | |
| 93 } else if (this._condition.internalCSSTokenizerCondition) { | |
| 94 var match = /<\/style/i.exec(line); | |
| 95 if (match) { | |
| 96 this._internalCSSTokenizer.line = line.substring(0, match.index)
; | |
| 97 } else | |
| 98 this._internalCSSTokenizer.line = line; | |
| 99 } | |
| 100 this._line = line; | |
| 101 }, | |
| 102 | |
| 103 _isExpectingAttribute: function() | |
| 104 { | |
| 105 return this._condition.parseCondition & this._parseConditions.ATTRIBUTE; | |
| 106 }, | |
| 107 | |
| 108 _isExpectingAttributeValue: function() | |
| 109 { | |
| 110 return this._condition.parseCondition & this._parseConditions.ATTRIBUTE_
VALUE; | |
| 111 }, | |
| 112 | |
| 113 _setExpectingAttribute: function() | |
| 114 { | |
| 115 if (this._isExpectingAttributeValue()) | |
| 116 this._condition.parseCondition ^= this._parseConditions.ATTRIBUTE_VA
LUE; | |
| 117 this._condition.parseCondition |= this._parseConditions.ATTRIBUTE; | |
| 118 }, | |
| 119 | |
| 120 _setExpectingAttributeValue: function() | |
| 121 { | |
| 122 if (this._isExpectingAttribute()) | |
| 123 this._condition.parseCondition ^= this._parseConditions.ATTRIBUTE; | |
| 124 this._condition.parseCondition |= this._parseConditions.ATTRIBUTE_VALUE; | |
| 125 }, | |
| 126 | |
| 127 /** | |
| 128 * @param {boolean=} stringEnds | |
| 129 */ | |
| 130 _stringToken: function(cursor, stringEnds) | |
| 131 { | |
| 132 if (!this._isExpectingAttributeValue()) { | |
| 133 this.tokenType = null; | |
| 134 return cursor; | |
| 135 } | |
| 136 this.tokenType = this._attrValueTokenType(); | |
| 137 if (stringEnds) | |
| 138 this._setExpectingAttribute(); | |
| 139 return cursor; | |
| 140 }, | |
| 141 | |
| 142 _attrValueTokenType: function() | |
| 143 { | |
| 144 if (this._condition.parseCondition & this._parseConditions.LINKIFY) { | |
| 145 if (this._condition.parseCondition & this._parseConditions.A_NODE) | |
| 146 return "html-external-link"; | |
| 147 return "html-resource-link"; | |
| 148 } | |
| 149 return "html-attribute-value"; | |
| 150 }, | |
| 151 | |
| 152 get _internalJavaScriptTokenizer() | |
| 153 { | |
| 154 return WebInspector.SourceTokenizer.Registry.getInstance().getTokenizer(
"text/javascript"); | |
| 155 }, | |
| 156 | |
| 157 get _internalCSSTokenizer() | |
| 158 { | |
| 159 return WebInspector.SourceTokenizer.Registry.getInstance().getTokenizer(
"text/css"); | |
| 160 }, | |
| 161 | |
| 162 scriptStarted: function(cursor) | |
| 163 { | |
| 164 this._condition.internalJavaScriptTokenizerCondition = this._internalJav
aScriptTokenizer.createInitialCondition(); | |
| 165 }, | |
| 166 | |
| 167 scriptEnded: function(cursor) | |
| 168 { | |
| 169 }, | |
| 170 | |
| 171 styleSheetStarted: function(cursor) | |
| 172 { | |
| 173 this._condition.internalCSSTokenizerCondition = this._internalCSSTokeniz
er.createInitialCondition(); | |
| 174 }, | |
| 175 | |
| 176 styleSheetEnded: function(cursor) | |
| 177 { | |
| 178 }, | |
| 179 | |
| 180 nextToken: function(cursor) | |
| 181 { | |
| 182 if (this._condition.internalJavaScriptTokenizerCondition) { | |
| 183 // Re-set line to force </script> detection first. | |
| 184 this.line = this._line; | |
| 185 if (cursor !== this._internalJavaScriptTokenizer._line.length) { | |
| 186 // Tokenizer is stateless, so restore its condition before token
izing and save it after. | |
| 187 this._internalJavaScriptTokenizer.condition = this._condition.in
ternalJavaScriptTokenizerCondition; | |
| 188 var result = this._internalJavaScriptTokenizer.nextToken(cursor)
; | |
| 189 this.tokenType = this._internalJavaScriptTokenizer.tokenType; | |
| 190 this._condition.internalJavaScriptTokenizerCondition = this._int
ernalJavaScriptTokenizer.condition; | |
| 191 return result; | |
| 192 } else if (cursor !== this._line.length) | |
| 193 delete this._condition.internalJavaScriptTokenizerCondition; | |
| 194 } else if (this._condition.internalCSSTokenizerCondition) { | |
| 195 // Re-set line to force </style> detection first. | |
| 196 this.line = this._line; | |
| 197 if (cursor !== this._internalCSSTokenizer._line.length) { | |
| 198 // Tokenizer is stateless, so restore its condition before token
izing and save it after. | |
| 199 this._internalCSSTokenizer.condition = this._condition.internalC
SSTokenizerCondition; | |
| 200 var result = this._internalCSSTokenizer.nextToken(cursor); | |
| 201 this.tokenType = this._internalCSSTokenizer.tokenType; | |
| 202 this._condition.internalCSSTokenizerCondition = this._internalCS
STokenizer.condition; | |
| 203 return result; | |
| 204 } else if (cursor !== this._line.length) | |
| 205 delete this._condition.internalCSSTokenizerCondition; | |
| 206 } | |
| 207 | |
| 208 var cursorOnEnter = cursor; | |
| 209 var gotoCase = 1; | |
| 210 var YYMARKER; | |
| 211 while (1) { | |
| 212 switch (gotoCase) | |
| 213 // Following comment is replaced with generated state machine. | |
| 214 /*!re2c | |
| 215 re2c:define:YYCTYPE = "var"; | |
| 216 re2c:define:YYCURSOR = cursor; | |
| 217 re2c:define:YYGETCONDITION = "this.getLexCondition"; | |
| 218 re2c:define:YYSETCONDITION = "this.setLexCondition"; | |
| 219 re2c:condprefix = "case this.case_"; | |
| 220 re2c:condenumprefix = "this._lexConditions."; | |
| 221 re2c:yyfill:enable = 0; | |
| 222 re2c:labelprefix = "case "; | |
| 223 re2c:indent:top = 2; | |
| 224 re2c:indent:string = " "; | |
| 225 | |
| 226 CommentContent = ([^-\r\n] | ("--" [^>]))*; | |
| 227 Comment = "<!--" CommentContent "-->"; | |
| 228 CommentStart = "<!--" CommentContent [\r\n]; | |
| 229 CommentEnd = CommentContent "-->"; | |
| 230 | |
| 231 DocTypeStart = "<!" [Dd] [Oo] [Cc] [Tt] [Yy] [Pp] [Ee]; | |
| 232 DocTypeContent = [^\r\n>]*; | |
| 233 | |
| 234 ScriptStart = "<" [Ss] [Cc] [Rr] [Ii] [Pp] [Tt]; | |
| 235 ScriptEnd = "</" [Ss] [Cc] [Rr] [Ii] [Pp] [Tt]; | |
| 236 | |
| 237 StyleStart = "<" [Ss] [Tt] [Yy] [Ll] [Ee]; | |
| 238 StyleEnd = "</" [Ss] [Tt] [Yy] [Ll] [Ee]; | |
| 239 | |
| 240 LT = "<" | "</"; | |
| 241 GT = ">"; | |
| 242 EqualSign = "="; | |
| 243 | |
| 244 DoubleStringContent = [^\r\n\"]*; | |
| 245 SingleStringContent = [^\r\n\']*; | |
| 246 StringLiteral = "\"" DoubleStringContent "\"" | "'" SingleString
Content "'"; | |
| 247 DoubleStringStart = "\"" DoubleStringContent [\r\n]; | |
| 248 DoubleStringEnd = DoubleStringContent "\""; | |
| 249 SingleStringStart = "'" SingleStringContent [\r\n]; | |
| 250 SingleStringEnd = SingleStringContent "'"; | |
| 251 | |
| 252 Identifier = [^ \r\n"'<>\[\]=]+; | |
| 253 | |
| 254 <INITIAL> Comment { this.tokenType = "html-comment"; return curs
or; } | |
| 255 <INITIAL> CommentStart => COMMENT { this.tokenType = "html-comme
nt"; return cursor; } | |
| 256 <COMMENT> CommentContent => COMMENT { this.tokenType = "html-com
ment"; return cursor; } | |
| 257 <COMMENT> CommentEnd => INITIAL { this.tokenType = "html-comment
"; return cursor; } | |
| 258 | |
| 259 <INITIAL> DocTypeStart => DOCTYPE { this.tokenType = "html-docty
pe"; return cursor; } | |
| 260 <DOCTYPE> DocTypeContent => DOCTYPE { this.tokenType = "html-doc
type"; return cursor; } | |
| 261 <DOCTYPE> GT => INITIAL { this.tokenType = "html-doctype"; retur
n cursor; } | |
| 262 | |
| 263 <INITIAL> ScriptStart => TAG | |
| 264 { | |
| 265 if (this._condition.parseCondition & this._parseConditions.S
CRIPT) { | |
| 266 // Do not tokenize script tag contents, keep lexer state
, even though processing "<". | |
| 267 this.setLexCondition(this._lexConditions.INITIAL); | |
| 268 this.tokenType = null; | |
| 269 return cursor; | |
| 270 } | |
| 271 this.tokenType = "html-tag"; | |
| 272 this._condition.parseCondition = this._parseConditions.SCRIP
T; | |
| 273 this._setExpectingAttribute(); | |
| 274 return cursor; | |
| 275 } | |
| 276 | |
| 277 <INITIAL> ScriptEnd => TAG | |
| 278 { | |
| 279 this.tokenType = "html-tag"; | |
| 280 this._condition.parseCondition = this._parseConditions.INITI
AL; | |
| 281 this.scriptEnded(cursor - 8); | |
| 282 return cursor; | |
| 283 } | |
| 284 | |
| 285 <INITIAL> StyleStart => TAG | |
| 286 { | |
| 287 if (this._condition.parseCondition & this._parseConditions.S
TYLE) { | |
| 288 // Do not tokenize style tag contents, keep lexer state,
even though processing "<". | |
| 289 this.setLexCondition(this._lexConditions.INITIAL); | |
| 290 this.tokenType = null; | |
| 291 return cursor; | |
| 292 } | |
| 293 this.tokenType = "html-tag"; | |
| 294 this._condition.parseCondition = this._parseConditions.STYLE
; | |
| 295 this._setExpectingAttribute(); | |
| 296 return cursor; | |
| 297 } | |
| 298 | |
| 299 <INITIAL> StyleEnd => TAG | |
| 300 { | |
| 301 this.tokenType = "html-tag"; | |
| 302 this._condition.parseCondition = this._parseConditions.INITI
AL; | |
| 303 this.styleSheetEnded(cursor - 7); | |
| 304 return cursor; | |
| 305 } | |
| 306 | |
| 307 <INITIAL> LT => TAG | |
| 308 { | |
| 309 if (this._condition.parseCondition & (this._parseConditions.
SCRIPT | this._parseConditions.STYLE)) { | |
| 310 // Do not tokenize script and style tag contents, keep l
exer state, even though processing "<". | |
| 311 this.setLexCondition(this._lexConditions.INITIAL); | |
| 312 this.tokenType = null; | |
| 313 return cursor; | |
| 314 } | |
| 315 | |
| 316 this._condition.parseCondition = this._parseConditions.INITI
AL; | |
| 317 this.tokenType = "html-tag"; | |
| 318 return cursor; | |
| 319 } | |
| 320 | |
| 321 <TAG> GT => INITIAL | |
| 322 { | |
| 323 this.tokenType = "html-tag"; | |
| 324 if (this._condition.parseCondition & this._parseConditions.S
CRIPT) { | |
| 325 this.scriptStarted(cursor); | |
| 326 // Do not tokenize script tag contents. | |
| 327 return cursor; | |
| 328 } | |
| 329 | |
| 330 if (this._condition.parseCondition & this._parseConditions.S
TYLE) { | |
| 331 this.styleSheetStarted(cursor); | |
| 332 // Do not tokenize style tag contents. | |
| 333 return cursor; | |
| 334 } | |
| 335 | |
| 336 this._condition.parseCondition = this._parseConditions.INITI
AL; | |
| 337 return cursor; | |
| 338 } | |
| 339 | |
| 340 <TAG> StringLiteral { return this._stringToken(cursor, true); } | |
| 341 <TAG> DoubleStringStart => DSTRING { return this._stringToken(cu
rsor); } | |
| 342 <DSTRING> DoubleStringContent => DSTRING { return this._stringTo
ken(cursor); } | |
| 343 <DSTRING> DoubleStringEnd => TAG { return this._stringToken(curs
or, true); } | |
| 344 <TAG> SingleStringStart => SSTRING { return this._stringToken(cu
rsor); } | |
| 345 <SSTRING> SingleStringContent => SSTRING { return this._stringTo
ken(cursor); } | |
| 346 <SSTRING> SingleStringEnd => TAG { return this._stringToken(curs
or, true); } | |
| 347 | |
| 348 <TAG> EqualSign => TAG | |
| 349 { | |
| 350 if (this._isExpectingAttribute()) | |
| 351 this._setExpectingAttributeValue(); | |
| 352 this.tokenType = null; | |
| 353 return cursor; | |
| 354 } | |
| 355 | |
| 356 <TAG> Identifier | |
| 357 { | |
| 358 if (this._condition.parseCondition === this._parseConditions
.SCRIPT || this._condition.parseCondition === this._parseConditions.STYLE) { | |
| 359 // Fall through if expecting attributes. | |
| 360 this.tokenType = null; | |
| 361 return cursor; | |
| 362 } | |
| 363 | |
| 364 if (this._condition.parseCondition === this._parseConditions
.INITIAL) { | |
| 365 this.tokenType = "html-tag"; | |
| 366 this._setExpectingAttribute(); | |
| 367 var token = this._line.substring(cursorOnEnter, cursor); | |
| 368 if (token === "a") | |
| 369 this._condition.parseCondition |= this._parseConditi
ons.A_NODE; | |
| 370 else if (this._condition.parseCondition & this._parseCon
ditions.A_NODE) | |
| 371 this._condition.parseCondition ^= this._parseConditi
ons.A_NODE; | |
| 372 } else if (this._isExpectingAttribute()) { | |
| 373 var token = this._line.substring(cursorOnEnter, cursor); | |
| 374 if (token === "href" || token === "src") | |
| 375 this._condition.parseCondition |= this._parseConditi
ons.LINKIFY; | |
| 376 else if (this._condition.parseCondition |= this._parseCo
nditions.LINKIFY) | |
| 377 this._condition.parseCondition ^= this._parseConditi
ons.LINKIFY; | |
| 378 this.tokenType = "html-attribute-name"; | |
| 379 } else if (this._isExpectingAttributeValue()) | |
| 380 this.tokenType = this._attrValueTokenType(); | |
| 381 else | |
| 382 this.tokenType = null; | |
| 383 return cursor; | |
| 384 } | |
| 385 <*> [^] { this.tokenType = null; return cursor; } | |
| 386 */ | |
| 387 } | |
| 388 }, | |
| 389 | |
| 390 __proto__: WebInspector.SourceTokenizer.prototype | |
| 391 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 22638008:
DevTools: Use CodeMirror modes instead of highlight tokenizers (Closed)
Base URL: svn://svn.chromium.org/blink/trunk