DevTools: [Documentation] Update parser for WikiText

Source/devtools/front_end/documentation/WikiParser.js

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /**
  * @constructor
  * @param {string} wikiMarkupText
  */
 WebInspector.WikiParser = function(wikiMarkupText)
 {
-    this._position = 0;
-    this._wikiMarkupText = wikiMarkupText;
+    var text = wikiMarkupText.replace(/&lt;/g, "<")

[lushnikov, 2014/09/08 14:08:30]

lets extact this into utilities.js and call it unescapeHTML ( see method
escapeHTML@utilities.js)

+                      .replace(/>/g, ">")
+                      .replace(/:/g, ":")
+                      .replace(/"/g, "\"")
+                      .replace(/&#60;/g, "<")
+                      .replace(/&#62;/g, ">")
+                      .replace(/&amp;/g, "&");
+    this._tokenizer = new WebInspector.WikiParser.Tokenizer(text);
     this._document = this._parse();
-    /** @type {?WebInspector.WikiParser.Tokenizer} */
-    this._tokenizer;
 }
 
 /**
- * @package
- * @enum {string}
- */
-WebInspector.WikiParser.State = {
-    Error: "Error",
-    FirstOpen: "FirstOpen",
-    SecondOpen: "SecondOpen",
-    Title: "Title",
-    PropertyName: "PropertyName",
-    PropertyValue: "PropertyValue",
-    FirstClose: "FirstClose",
-    SecondClose: "SecondClose"
-}
-
-/**
- * @package
- * @enum {string}
- */
-WebInspector.WikiParser.LinkStates = {
-    Error: "Error",
-    LinkUrl: "LinkUrl",
-    LinkName: "LinkName"
-}
-
-/**
- * @package
- * @enum {string}
- */
-WebInspector.WikiParser.HtmlStates = {
-    Error: "Error",
-    Entry: "Entry",
-    InsideTag: "InsideTag",
-    Exit: "Exit"
-}
-
-/**
- * @package
- * @enum {string}
- */
-WebInspector.WikiParser.ValueState = {
-    Error: "Error",
-    Outside: "Outside",
-    InsideSquare: "InsideSquare"
-}
-
-/**
  * @package
  * @enum {string}
  */
 WebInspector.WikiParser.TokenType = {
+    Text: "Text",
+    Table: "Table",

[lushnikov, 2014/09/08 14:08:31]

do we have any tests for table?

[iliia, 2014/09/08 15:20:50]

we don't support tables at the moment.
On 2014/09/08 14:08:31, lushnikov wrote:

+    OpeningBraces: "OpeningBraces",
+    ClosingBraces: "ClosingBraces",
+    Exclamation: "Exclamation",
+    OpeningBrackets: "OpeningBrackets",
+    ClosingBrackets: "ClosingBrackets",
+    EqualSign: "EqualSign",
+    EqualSignInBraces: "EqualSignInBraces",
+    VerticalLine: "VerticalLine",
     TripleQuotes: "TripleQuotes",
-    OpeningBrackets: "OpeningBrackets",
     OpeningCodeTag: "OpeningCodeTag",
-    ClosingBrackets: "ClosingBrackets",
     ClosingCodeTag: "ClosingCodeTag",
     Bullet: "Bullet",
-    Text: "Text",
-    VerticalLine: "VerticalLine",
     LineEnd: "LineEnd",
     CodeBlock: "CodeBlock"
 }
 
 /**
  * @constructor
  * @param {string} result
  * @param {!WebInspector.WikiParser.TokenType} type
  */
 WebInspector.WikiParser.Token = function(result, type)
@@ skipping 12 matching lines @@
     },
 
     /**
      * @return {!WebInspector.WikiParser.TokenType}
      */
     type: function()
     {
         return this._type;
     }
 }
-
 /**
  * @constructor
  * @param {string} str
  */
 WebInspector.WikiParser.Tokenizer = function(str)
 {
     this._text = str;
+    this._token = this._internalNextToken();
 }
 
 WebInspector.WikiParser.Tokenizer.prototype = {
     /**
      * @return {!WebInspector.WikiParser.Token}
      */
-    _nextToken: function()
+    peekToken: function()
+    {
+        return this._token;
+    },
+
+    /**
+     * @return {!WebInspector.WikiParser.Token}
+     */
+    nextToken: function()
+    {
+        var token = this._token;
+        this._token = this._internalNextToken();
+        return token;
+    },
+
+    /**
+     * @return {!WebInspector.WikiParser.Token}
+     */
+    _internalNextToken: function()
     {
         if (WebInspector.WikiParser.newLineWithSpace.test(this._text)) {
             var result = WebInspector.WikiParser.newLineWithSpace.exec(this._text);
-            var begin = result.index + result[0].length;
+            var begin = result.index + result[0].length - 1;
             var end = this._text.length;
             var lineEnd = WebInspector.WikiParser.newLineWithoutSpace.exec(this._text);
             if (lineEnd)
                 end = lineEnd.index;
-            var token = this._text.substring(begin, end).replace(/\n */g, "\n");
+            var token = this._text.substring(begin, end).replace(/\n /g, "\n").replace(/{{=}}/g, "=");
             this._text = this._text.substring(end + 1);
             return new WebInspector.WikiParser.Token(token, WebInspector.WikiParser.TokenType.CodeBlock);
         }
 
         for (var i = 0; i < WebInspector.WikiParser._tokenDescriptors.length; ++i) {
             var result = WebInspector.WikiParser._tokenDescriptors[i].regex.exec(this._text);
             if (result) {
                 this._text = this._text.substring(result.index + result[0].length);
                 return new WebInspector.WikiParser.Token(result[0], WebInspector.WikiParser._tokenDescriptors[i].type);
             }
         }
 
         for (var lastIndex = 0; lastIndex < this._text.length; ++lastIndex) {
             var testString = this._text.substring(lastIndex);
             for (var i = 0; i < WebInspector.WikiParser._tokenDescriptors.length; ++i) {
                 if (WebInspector.WikiParser._tokenDescriptors[i].regex.test(testString)) {
                     var token = this._text.substring(0, lastIndex);
                     this._text = this._text.substring(lastIndex);
                     return new WebInspector.WikiParser.Token(token, WebInspector.WikiParser.TokenType.Text);
                 }
             }
         }
 
         var token = this._text;
         this._text = "";
         return new WebInspector.WikiParser.Token(token, WebInspector.WikiParser.TokenType.Text);
     },
 
     /**
+     * @return {?WebInspector.WikiParser.TokenType}
+     */
+    secondToken: function()
+    {
+        var tokenizer = new WebInspector.WikiParser.Tokenizer(this._text);
+        if (tokenizer.hasMoreTokens())
+            return tokenizer.nextToken().type();
+        return null;
+    },
+
+    /**
      * @return {boolean}
      */
-    _hasMoreTokens: function()
+    hasMoreTokens: function()
     {
         return !!this._text.length;
     }
 }
 
+WebInspector.WikiParser.table = /^{{{!}}/;
+WebInspector.WikiParser.exclamation = /^{{!}}/;
+WebInspector.WikiParser.openingBraces = /^{{/;
+WebInspector.WikiParser.equalSign = /^=/;
+WebInspector.WikiParser.equalSignInBraces = /^{{=}}/;
+WebInspector.WikiParser.closingBraces = /^\s*}}/;
+WebInspector.WikiParser.oneOpeningBracket = /^\n* \[[^\[]/;
+WebInspector.WikiParser.twoOpeningBrackets = /^\n* \[\[/;
+WebInspector.WikiParser.oneClosingBracket = /^\n*\][^\]]/;
+WebInspector.WikiParser.twoClosingBrackets = /^\n*\]\]/;
+WebInspector.WikiParser.tripleQuotes = /^\n*'''/;
+WebInspector.WikiParser.openingCodeTag = /^<\s*code\s*>/;
+WebInspector.WikiParser.closingCodeTag = /^<\s*\/\s*code\s*>/;
+WebInspector.WikiParser.closingBullet = /^\*/;
+WebInspector.WikiParser.lineEnd = /^\n/;
+WebInspector.WikiParser.verticalLine = /^\n*\|/;
+WebInspector.WikiParser.newLineWithSpace = /^\n [^ ]/;
+WebInspector.WikiParser.newLineWithoutSpace = /\n[^ ]/;
+
+/**
+ * @constructor
+ * @param {!RegExp} regex
+ * @param {!WebInspector.WikiParser.TokenType} type
+ */
+WebInspector.WikiParser.TokenDescriptor = function(regex, type)
+{
+    this.regex = regex;
+    this.type = type;
+}
+
+WebInspector.WikiParser._tokenDescriptors = [
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.exclamation, WebInspector.WikiParser.TokenType.Exclamation),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.equalSignInBraces, WebInspector.WikiParser.TokenType.EqualSignInBraces),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.equalSign, WebInspector.WikiParser.TokenType.EqualSign),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.table, WebInspector.WikiParser.TokenType.Table),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.openingBraces, WebInspector.WikiParser.TokenType.OpeningBraces),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.verticalLine, WebInspector.WikiParser.TokenType.VerticalLine),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingBraces, WebInspector.WikiParser.TokenType.ClosingBraces),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.oneOpeningBracket, WebInspector.WikiParser.TokenType.OpeningBrackets),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.twoOpeningBrackets, WebInspector.WikiParser.TokenType.OpeningBrackets),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.oneClosingBracket, WebInspector.WikiParser.TokenType.ClosingBrackets),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.twoClosingBrackets, WebInspector.WikiParser.TokenType.ClosingBrackets),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.newLineWithSpace, WebInspector.WikiParser.TokenType.CodeBlock),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.tripleQuotes, WebInspector.WikiParser.TokenType.TripleQuotes),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.openingCodeTag, WebInspector.WikiParser.TokenType.OpeningCodeTag),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingCodeTag, WebInspector.WikiParser.TokenType.ClosingCodeTag),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingBullet, WebInspector.WikiParser.TokenType.Bullet),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.lineEnd, WebInspector.WikiParser.TokenType.LineEnd)
+]
+
+/**
+ * @package
+ * @enum {string}
+ */
+WebInspector.WikiParser.State = {

[lushnikov, 2014/09/08 14:08:30]

looks like you don't need this

[iliia, 2014/09/08 15:20:50]

Done.

+    Error: "Error",
+    FirstOpen: "FirstOpen",
+    SecondOpen: "SecondOpen",
+    Title: "Title",
+    PropertyName: "PropertyName",
+    PropertyValue: "PropertyValue",
+    FirstClose: "FirstClose",
+    SecondClose: "SecondClose"
+}
+
+/**
+ * @package
+ * @enum {string}
+ */
+WebInspector.WikiParser.HtmlStates = {

[lushnikov, 2014/09/08 14:08:30]

and this

[iliia, 2014/09/08 15:20:51]

Done.

+    Error: "Error",
+    Entry: "Entry",
+    InsideTag: "InsideTag",
+    Exit: "Exit"
+}
+
+/**
+ * @package
+ * @enum {string}
+ */
+WebInspector.WikiParser.ValueState = {

[lushnikov, 2014/09/08 14:08:31]

and even this

[iliia, 2014/09/08 15:20:50]

Done.

+    Error: "Error",
+    Outside: "Outside",
+    InsideSquare: "InsideSquare"
+}
+
 WebInspector.WikiParser.prototype = {
     /**
      * @return {!Object}
      */
     document: function()
     {
         return this._document;
     },
 
     /**
      * @return {!Object}
      */
     _parse: function()
     {
         var obj = {};
-        this._wikiMarkupText = this._wikiMarkupText.replace(/&lt;/g, "<")
-                      .replace(/&gt;/g, ">")
-                      .replace(/&#58;/g, ":")
-                      .replace(/&quot;/g, "\"")
-                      .replace(/&#60;/g, "<")
-                      .replace(/&#62;/g, ">")
-                      .replace(/{{=}}/g, "=")
-                      .replace(/{{!}}/g, "|")
-                      .replace(/&amp;/g, "&");
-        while (this._position < this._wikiMarkupText.length) {
+        while (this._tokenizer.hasMoreTokens()) {
             var field = this._parseField();
             for (var key in field) {
                 console.assert(typeof obj[key] === "undefined", "Duplicate key: " + key);
                 obj[key] = field[key];
             }
         }
         return obj;
     },
 
     /**
+     * @return {?Object}
+     */
+    _parseField: function()
+    {
+        var obj = {};

[lushnikov, 2014/09/08 14:08:30]

obj is not a good name here as it doesn't give any clue. If we parse field here,
lets call it "field"

[iliia, 2014/09/08 15:20:50]

Done.

+        if (!this._tokenizer.hasMoreTokens() || this._tokenizer.nextToken().type() !== WebInspector.WikiParser.TokenType.OpeningBraces)
+            return obj;
+
+        var title = this._deleteTrailingSpaces(this._parseTitle());
+        if (!title.length)
+            return obj;
+
+        obj[title] = {};
+        if (this._tokenizer.peekToken().type() == WebInspector.WikiParser.TokenType.ClosingBraces) {

[lushnikov, 2014/09/08 14:08:31]

===

[iliia, 2014/09/08 15:20:50]

Done.

+            this._tokenizer.nextToken();
+            return obj;
+        }
+        if (!this._tokenizer.secondToken() || this._tokenizer.secondToken() !== WebInspector.WikiParser.TokenType.EqualSign) {
+            var propertyValue = this._parseString();
+            obj[title] = propertyValue;
+        } else {
+            while (this._tokenizer.hasMoreTokens()) {
+                var propertyName = this._parseName();
+                var token = this._tokenizer.peekToken();
+                switch (token.type()) {
+                case WebInspector.WikiParser.TokenType.OpeningBraces:
+                    while (this._tokenizer.peekToken().type() === WebInspector.WikiParser.TokenType.OpeningBraces) {

[lushnikov, 2014/09/08 14:08:30]

this is while, inside case, inside while, inside if.. this is way too
complicated to be read. You should split the logic here into functions

[iliia, 2014/09/08 15:20:50]

Done.

+                        if (!Array.isArray(obj[title][propertyName]))
+                            obj[title][propertyName] = [];
+                        obj[title][propertyName].push(this._parseField());
+                    }
+                    if (this._tokenizer.peekToken().type() === WebInspector.WikiParser.TokenType.VerticalLine)
+                        this._tokenizer.nextToken();
+                    break;
+                case WebInspector.WikiParser.TokenType.LineEnd:
+                    this._tokenizer.nextToken();
+                    break;
+                case WebInspector.WikiParser.TokenType.ClosingBraces:
+                    this._tokenizer.nextToken();
+                    return obj;
+                    break;
+                default:
+                    if (propertyName === "Code")
+                        obj[title][propertyName] = this._parseExampleCode();
+                    else
+                        obj[title][propertyName] = this._parseString();
+                }
+            }
+        }
+        var token = this._tokenizer.nextToken();
+        if (token.type() != WebInspector.WikiParser.TokenType.ClosingBraces)

[lushnikov, 2014/09/08 14:08:30]

!==

[iliia, 2014/09/08 15:20:50]

Done.

+            throw new Error("Two closing braces expected; found " + token.value());
+
+        return obj;
+    },
+
+    /**
      * @return {string}
      */
-    _parseValue: function() {
-        var states = WebInspector.WikiParser.ValueState;
-        var state = states.Outside;
-        var value = "";
-        while (this._position < this._wikiMarkupText.length) {
-            switch (state) {
-            case states.Outside:
-                if (this._wikiMarkupText[this._position] === "|" || (this._wikiMarkupText[this._position] === "}" && this._wikiMarkupText[this._position + 1] === "}"))
-                    return value;
-                switch (this._wikiMarkupText[this._position]) {
-                case "<":
-                    var indexClose = this._wikiMarkupText.indexOf(">", this._position);
-                    if (indexClose !== -1) {
-                        value += this._wikiMarkupText.substring(this._position, indexClose + 1);
-                        this._position = indexClose;
-                    }
-                    break;
-                case "[":
-                    state = states.InsideSquare;
-                    value += this._wikiMarkupText[this._position];
-                    break;
-                default:
-                    value += this._wikiMarkupText[this._position];
-                }
+    _parseTitle: function()
+    {
+        var title = "";
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.peekToken();
+            switch (token.type()) {
+            case WebInspector.WikiParser.TokenType.ClosingBraces:
+                return title;
+            case WebInspector.WikiParser.TokenType.VerticalLine:
+                this._tokenizer.nextToken();
+                return title;
+            case WebInspector.WikiParser.TokenType.Text:
+                title += this._tokenizer.nextToken().value();
                 break;
-            case states.InsideSquare:
-                if (this._wikiMarkupText[this._position] === "[") {
-                    var indexClose = this._wikiMarkupText.indexOf("]]", this._position);
-                    if (indexClose !== -1) {
-                        value += this._wikiMarkupText.substring(this._position, indexClose + 2);
-                        this._position = indexClose + 1;
-                    }
-                } else {
-                    var indexClose = this._wikiMarkupText.indexOf("]", this._position);
-                    if (indexClose !== -1) {
-                        value += this._wikiMarkupText.substring(this._position, indexClose + 1);
-                        this._position = indexClose;
-                    }
-                }
-                state = states.Outside;
-                break;
+            default:
+                return "Wrong title";

[lushnikov, 2014/09/08 14:08:30]

throw exception

[iliia, 2014/09/08 15:20:50]

Done.

[iliia, 2014/09/08 15:20:50]

Done.

             }
-            this._position++;
         }
-        return value;
+        return title;
     },
 
     /**
-     * @return {!Object}
+     * @return {string}
      */
-    _parseField: function()
+    _parseName: function()
     {
-        var obj = {};
-        var title = "";
-        var propertyName = "";
-        var propertyValue = "";
-        var states = WebInspector.WikiParser.State;
-        var state = states.FirstOpen;
-        while (this._position < this._wikiMarkupText.length) {
-            var skipIncrement = false;
-            switch (state) {
-            case states.FirstOpen:
-                if (this._wikiMarkupText[this._position] === "{")
-                    state = states.SecondOpen;
-                else
-                    state = states.Error;
+        var name = "";
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.peekToken();
+            switch (token.type()) {
+            case WebInspector.WikiParser.TokenType.ClosingBraces:
+                return name;
+            case WebInspector.WikiParser.TokenType.EqualSign:
+                this._tokenizer.nextToken();
+                return name;
+            case WebInspector.WikiParser.TokenType.VerticalLine:
+            case WebInspector.WikiParser.TokenType.Text:
+                name += this._tokenizer.nextToken().value();
                 break;
-            case states.SecondOpen:
-                if (this._wikiMarkupText[this._position] === "{")
-                    state = states.Title;
-                else
-                    state = states.Error;
-                break;
-            case states.Title:
-                if (this._wikiMarkupText[this._position] === "|") {
-                    title = this._deleteTrailingSpaces(title);
-                    if (title !== "")
-                        obj[title] = {};
-                    state = states.PropertyName;
-                } else if (this._wikiMarkupText[this._position] === "}") {
-                    title = this._deleteTrailingSpaces(title);
-                    if (title !== "")
-                        obj[title] = {};
-                    state = states.FirstClose;
-                } else {
-                    title += (this._wikiMarkupText[this._position] === "\n" ? "" : this._wikiMarkupText[this._position]);
-                }
-                break;
-            case states.PropertyName:
-                if (this._wikiMarkupText[this._position] === "=") {
-                    state = states.PropertyValue;
-                    this._deleteTrailingSpaces(propertyName);
-                    if (propertyName !== "")
-                        obj[title][propertyName] = [];
-                } else {
-                    if (this._wikiMarkupText[this._position] === "}") {
-                        propertyName = this._deleteTrailingSpaces(propertyName);
-                        obj[title] = propertyName;
-                        state = states.FirstClose;
-                    } else {
-                        propertyName += this._wikiMarkupText[this._position];
-                    }
-                }
-                break;
-            case states.PropertyValue:
-                if (this._wikiMarkupText[this._position] === "{" && this._wikiMarkupText[this._position + 1] === "{") {
-                    propertyValue = this._parseField();
-                    obj[title][propertyName].push(propertyValue);
-                    propertyValue = "";
-                    skipIncrement = true;
-                } else if (this._wikiMarkupText[this._position] === "|") {
-                    propertyValue = this._deleteTrailingSpaces(propertyValue);
-                    if (propertyValue !== "")
-                      obj[title][propertyName] = propertyValue;
-
-                    state = states.PropertyName;
-                    if (Array.isArray(obj[title][propertyName]) && obj[title][propertyName].length === 1) {
-                        var newObj = obj[title][propertyName][0];
-                        obj[title][propertyName] = newObj;
-                    }
-
-                    propertyName = "";
-                    propertyValue = "";
-                } else if (this._position + 1 < this._wikiMarkupText.length && this._wikiMarkupText[this._position] === "}" && this._wikiMarkupText[this._position + 1] === "}") {
-                    propertyValue = this._deleteTrailingSpaces(propertyValue);
-                    if (propertyValue !== "")
-                        obj[title][propertyName].push(propertyValue);
-                    if (Array.isArray(obj[title][propertyName]) && obj[title][propertyName].length === 1) {
-                        var newObj = obj[title][propertyName][0];
-                        obj[title][propertyName] = newObj;
-                    }
-
-                    propertyValue = "";
-                    state = states.FirstClose;
-                } else {
-                    propertyValue = this._parseValue();
-                    skipIncrement = true;
-                }
-                break;
-            case states.FirstClose:
-                if (this._wikiMarkupText[this._position] === "}")
-                    state = states.SecondClose;
-                else
-                    state = states.Error;
-                break;
-            case states.SecondClose:
-                while (this._position < this._wikiMarkupText.length && this._wikiMarkupText[this._position] === "\n")
-                    this._position++;
-                return obj;
-            case states.Error:
-                this._position = this._wikiMarkupText.length;
-                return {};
+            default:
+                return "Wrong name";

[lushnikov, 2014/09/08 14:08:31]

nope, you want to throw here

[iliia, 2014/09/08 15:20:50]

Done.

             }
-            if (!skipIncrement)
-                this._position++;
         }
-        return obj;
+        return name;
     },
 
     /**
-     * @param {string} str
+     * @return {string}
+     */
+    _parseExampleCode: function()
+    {
+        var code = "";
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.peekToken();
+            switch (token.type()) {
+            case WebInspector.WikiParser.TokenType.ClosingBraces:
+                return code;
+            case WebInspector.WikiParser.TokenType.VerticalLine:
+                this._tokenizer.nextToken();
+                return code;
+            case WebInspector.WikiParser.TokenType.Exclamation:
+                this._tokenizer.nextToken();
+                code += '|';
+                break;
+            case WebInspector.WikiParser.TokenType.EqualSignInBraces:

[lushnikov, 2014/09/08 14:08:30]

looks like this branch might be omitted, as Default will cover this case anyway

[iliia, 2014/09/08 15:20:51]

in this case we should add "=", but not "{{=}}".
On 2014/09/08 14:08:30, lushnikov wrote:

+                this._tokenizer.nextToken();
+                code += "=";
+                break;
+            default:
+                this._tokenizer.nextToken();
+                code += token.value();
+            }
+        }
+        return code;
+    },
+
+    /**
      * @return {?WebInspector.WikiParser.Block}
      */
-    parseString: function(str)
+    _parseString: function()
     {
-        this._tokenizer = new WebInspector.WikiParser.Tokenizer(str);
         var children = [];
         var blockChildren = [];
         var text = "";
         var self = this;
 
         function processSimpleText()
         {
             var currentText = self._deleteTrailingSpaces(text);
             if (!currentText.length)
                 return;
             var simpleText = new WebInspector.WikiParser.PlainText(currentText, false);
             blockChildren.push(simpleText);
             text = "";
         }
 
         function processBlock()
         {
             if (blockChildren.length) {
                 children.push(new WebInspector.WikiParser.Block(blockChildren, false));
                 blockChildren = [];
             }
         }
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken();
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.peekToken();
             switch (token.type()) {
+            case WebInspector.WikiParser.TokenType.VerticalLine:
+                this._tokenizer.nextToken();

[lushnikov, 2014/09/08 14:08:30]

why missing break here?

[iliia, 2014/09/08 15:20:51]

because we must do exactly the same as in case of ClosingBrackets, when we met
VerticalLine, we must stop parsing string.
On 2014/09/08 14:08:30, lushnikov wrote:

+            case WebInspector.WikiParser.TokenType.ClosingBraces:
+                processSimpleText();
+                processBlock();
+                return new WebInspector.WikiParser.Block(children, false);
             case WebInspector.WikiParser.TokenType.TripleQuotes:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 var highlightText = this._parseHighlight();
                 blockChildren.push(highlightText)
                 break;
             case WebInspector.WikiParser.TokenType.OpeningBrackets:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 var link = this._parseLink();
                 blockChildren.push(link);
                 break;
             case WebInspector.WikiParser.TokenType.OpeningCodeTag:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 var code = this._parseCode();
                 blockChildren.push(code);
                 break;
             case WebInspector.WikiParser.TokenType.Bullet:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 processBlock();
                 var bulletText = this._parseBullet();
                 children.push(bulletText);
                 break;
             case WebInspector.WikiParser.TokenType.CodeBlock:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 processBlock();
                 var code = new WebInspector.WikiParser.CodeBlock(token.value());
                 children.push(code);
                 break;
             case WebInspector.WikiParser.TokenType.LineEnd:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 processBlock();
                 break;
+            case WebInspector.WikiParser.TokenType.EqualSignInBraces:
+                this._tokenizer.nextToken();
+                text += "=";
+                break;
+            case WebInspector.WikiParser.TokenType.Exclamation:
+                this._tokenizer.nextToken();
+                text += "|";
+                break;
             case WebInspector.WikiParser.TokenType.VerticalLine:
             case WebInspector.WikiParser.TokenType.Text:
+                this._tokenizer.nextToken();
                 text += token.value();
                 break;
             default:
+                this._tokenizer.nextToken();
                 return null;
             }
         }
 
         processSimpleText();
         processBlock();
 
         return new WebInspector.WikiParser.Block(children, false);
     },
 
     /**
      * @return {!WebInspector.WikiParser.Link}
      */
     _parseLink: function()
     {
         var url = "";
         var children = [];
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken();
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken();
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.ClosingBrackets:
                 return new WebInspector.WikiParser.Link(url, children);
             case WebInspector.WikiParser.TokenType.VerticalLine:
+            case WebInspector.WikiParser.TokenType.Exclamation:
                 children.push(this._parseLinkName());
                 return new WebInspector.WikiParser.Link(url, children);
             default:
                 url += token.value();
             }
         }
 
         return new WebInspector.WikiParser.Link(url, children);
     },
 
     /**
      * @return {!WebInspector.WikiParser.Inline}
      */
     _parseLinkName: function()
     {
         var children = [];
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken();
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken();
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.ClosingBrackets:
                 return new WebInspector.WikiParser.Inline(WebInspector.WikiParser.ArticleElement.Type.Inline, children);
             case WebInspector.WikiParser.TokenType.OpeningCodeTag:
                 children.push(this._parseCode());
                 break;
             default:
                 children.push(new WebInspector.WikiParser.PlainText(token.value(), false));
                 break;
             }
         }
 
         return new WebInspector.WikiParser.Inline(WebInspector.WikiParser.ArticleElement.Type.Inline, children);
     },
 
     /**
      * @return {!WebInspector.WikiParser.Inline}
      */
     _parseCode : function()
     {
         var children = [];
         var text = "";
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken();
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken();
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.ClosingCodeTag:
                 text = this._deleteTrailingSpaces(text);
                 if (text.length) {
                     var simpleText = new WebInspector.WikiParser.PlainText(text, false);
                     children.push(simpleText);
                     text = "";
                 }
                 var code = new WebInspector.WikiParser.Inline(WebInspector.WikiParser.ArticleElement.Type.Code, children);
                 return code;
@@ skipping 12 matching lines @@
 
         return new WebInspector.WikiParser.Inline(WebInspector.WikiParser.ArticleElement.Type.Code, children);
     },
 
     /**
      * @return {!WebInspector.WikiParser.Block}
      */
     _parseBullet: function()
     {
         var children = [];
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken()
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken()
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.OpeningBrackets:
                 children.push(this._parseLink());
                 break;
             case WebInspector.WikiParser.TokenType.OpeningCodeTag:
                 children.push(this._parseCode());
                 break;
             case WebInspector.WikiParser.TokenType.LineEnd:
                 return new WebInspector.WikiParser.Block(children, true);
             default:
                 var text = this._deleteTrailingSpaces(token.value());
                 if (text.length) {
                     var simpleText = new WebInspector.WikiParser.PlainText(text, false);
                     children.push(simpleText);
                     text = "";
                 }
             }
         }
 
         return new WebInspector.WikiParser.Block(children, true);
     },
 
     /**
      * @return {!WebInspector.WikiParser.PlainText}
      */
     _parseHighlight: function()
     {
         var text = "";
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken()
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken()
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.TripleQuotes:
                 text = this._deleteTrailingSpaces(text);
                 return new WebInspector.WikiParser.PlainText(text, true);
             default:
                 text += token.value();
             }
         }
         return new WebInspector.WikiParser.PlainText(text, true);
     },
 
     /**
      * @param {string} str
      * @return {string}
      */
     _deleteTrailingSpaces: function(str)
     {
         return str.replace(/[\n\r]*$/gm, "");
     }
 }
 
-WebInspector.WikiParser.oneOpeningBracket = /^\n* \[[^\[]/;
-WebInspector.WikiParser.twoOpeningBrackets = /^\n* \[\[/;
-WebInspector.WikiParser.oneClosingBracket = /^\n*\][^\]] /;
-WebInspector.WikiParser.twoClosingBrackets = /^\n*\]\]/;
-WebInspector.WikiParser.tripleQuotes = /^\n*'''/;
-WebInspector.WikiParser.openingCodeTag = /^<\s*code\s*>/;
-WebInspector.WikiParser.closingCodeTag = /^<\s*\/\s*code\s*>/;
-WebInspector.WikiParser.closingBullet = /^\*/;
-WebInspector.WikiParser.lineEnd = /^\n/;
-WebInspector.WikiParser.verticalLine = /^\|/;
-WebInspector.WikiParser.newLineWithSpace = /^\n /;
-WebInspector.WikiParser.newLineWithoutSpace = /\n[^ ]/;
-
-/**
- * @constructor
- * @param {!RegExp} regex
- * @param {!WebInspector.WikiParser.TokenType} type
- */
-WebInspector.WikiParser.TokenDescriptor = function(regex, type)
-{
-    this.regex = regex;
-    this.type = type;
-}
-
-WebInspector.WikiParser._tokenDescriptors = [
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.newLineWithSpace, WebInspector.WikiParser.TokenType.CodeBlock),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.tripleQuotes, WebInspector.WikiParser.TokenType.TripleQuotes),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.oneOpeningBracket, WebInspector.WikiParser.TokenType.OpeningBrackets),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.twoOpeningBrackets, WebInspector.WikiParser.TokenType.OpeningBrackets),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.oneClosingBracket, WebInspector.WikiParser.TokenType.ClosingBrackets),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.twoClosingBrackets, WebInspector.WikiParser.TokenType.ClosingBrackets),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.openingCodeTag, WebInspector.WikiParser.TokenType.OpeningCodeTag),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingCodeTag, WebInspector.WikiParser.TokenType.ClosingCodeTag),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingBullet, WebInspector.WikiParser.TokenType.Bullet),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.verticalLine, WebInspector.WikiParser.TokenType.VerticalLine),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.lineEnd, WebInspector.WikiParser.TokenType.LineEnd)
-];
-
 /**
  * @constructor
  * @param {!WebInspector.WikiParser.ArticleElement.Type} type
  */
 WebInspector.WikiParser.ArticleElement = function(type)
 {
     this._type = type;
 }
 
 WebInspector.WikiParser.ArticleElement.prototype = {
@@ skipping 147 matching lines @@
     /**
      * @return {string}
      */
     url : function()
     {
         return this._url;
     },
 
     __proto__: WebInspector.WikiParser.Inline.prototype
 }