DevTools: [Documentation] Update parser for WikiText

Source/devtools/front_end/documentation/WikiParser.js

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /**
  * @constructor
  * @param {string} wikiMarkupText
  */
 WebInspector.WikiParser = function(wikiMarkupText)
 {
-    this._position = 0;
-    this._wikiMarkupText = wikiMarkupText;
+    var text = wikiMarkupText.unescapeHTML();
+    this._tokenizer = new WebInspector.WikiParser.Tokenizer(text);
     this._document = this._parse();
-    /** @type {?WebInspector.WikiParser.Tokenizer} */
-    this._tokenizer;
 }
 
 /**
- * @package
- * @enum {string}
+ * @constructor
  */
-WebInspector.WikiParser.State = {
-    Error: "Error",
-    FirstOpen: "FirstOpen",
-    SecondOpen: "SecondOpen",
-    Title: "Title",
-    PropertyName: "PropertyName",
-    PropertyValue: "PropertyValue",
-    FirstClose: "FirstClose",
-    SecondClose: "SecondClose"
+WebInspector.WikiParser.Section = function()
+{
+    /** @type {string} */
+    this.title;
+    /** @type {?WebInspector.WikiParser.Values} */

[apavlov, 2014/09/10 14:09:24]

blank lines before jsdocs

[iliia, 2014/09/11 09:26:30]

Done.

+    this.values;
+    /** @type {?WebInspector.WikiParser.ArticleElement} */
+    this.singleValue;
 }
 
 /**
- * @package
- * @enum {string}
+ * @constructor
  */
-WebInspector.WikiParser.LinkStates = {
-    Error: "Error",
-    LinkUrl: "LinkUrl",
-    LinkName: "LinkName"
+WebInspector.WikiParser.Field = function()
+{
+    /** @type {string} */
+    this.name;
+    /** @type {?WebInspector.WikiParser.FieldValue} */

[apavlov, 2014/09/10 14:09:23]

ditto

[iliia, 2014/09/11 09:26:30]

Done.

+    this.value;
 }
 
-/**
- * @package
- * @enum {string}
- */
-WebInspector.WikiParser.HtmlStates = {
-    Error: "Error",
-    Entry: "Entry",
-    InsideTag: "InsideTag",
-    Exit: "Exit"
-}
-
-/**
- * @package
- * @enum {string}
- */
-WebInspector.WikiParser.ValueState = {
-    Error: "Error",
-    Outside: "Outside",
-    InsideSquare: "InsideSquare"
-}
+/** @typedef {(?WebInspector.WikiParser.ArticleElement|!Array.<!WebInspector.WikiParser.Section>)} */
+WebInspector.WikiParser.FieldValue;
+/** @typedef {?Object.<string, !WebInspector.WikiParser.FieldValue>} */

[apavlov, 2014/09/10 14:09:24]

ditto

[iliia, 2014/09/11 09:26:30]

Done.

+WebInspector.WikiParser.Values;
+/** @typedef {(?WebInspector.WikiParser.Value|?WebInspector.WikiParser.ArticleElement)} */
+WebInspector.WikiParser.Value;
 
 /**
  * @package
  * @enum {string}
  */
 WebInspector.WikiParser.TokenType = {
+    Text: "Text",
+    Table: "Table",
+    OpeningBraces: "OpeningBraces",
+    ClosingBraces: "ClosingBraces",
+    Exclamation: "Exclamation",
+    OpeningBrackets: "OpeningBrackets",
+    ClosingBrackets: "ClosingBrackets",
+    EqualSign: "EqualSign",
+    EqualSignInBraces: "EqualSignInBraces",
+    VerticalLine: "VerticalLine",
     TripleQuotes: "TripleQuotes",
-    OpeningBrackets: "OpeningBrackets",
     OpeningCodeTag: "OpeningCodeTag",
-    ClosingBrackets: "ClosingBrackets",
     ClosingCodeTag: "ClosingCodeTag",
     Bullet: "Bullet",
-    Text: "Text",
-    VerticalLine: "VerticalLine",
     LineEnd: "LineEnd",
     CodeBlock: "CodeBlock"
 }
 
 /**
  * @constructor
  * @param {string} result
  * @param {!WebInspector.WikiParser.TokenType} type
  */
 WebInspector.WikiParser.Token = function(result, type)
@@ skipping 20 matching lines @@
     }
 }
 
 /**
  * @constructor
  * @param {string} str
  */
 WebInspector.WikiParser.Tokenizer = function(str)
 {
     this._text = str;
+    this._token = this._internalNextToken();
 }
 
 WebInspector.WikiParser.Tokenizer.prototype = {
     /**
      * @return {!WebInspector.WikiParser.Token}
      */
-    _nextToken: function()
+    peekToken: function()
+    {
+        return this._token;
+    },
+
+    /**
+     * @return {!WebInspector.WikiParser.Token}
+     */
+    nextToken: function()
+    {
+        var token = this._token;
+        this._token = this._internalNextToken();
+        return token;
+    },
+
+    /**
+     * @return {!WebInspector.WikiParser.Token}
+     */
+    _internalNextToken: function()
     {
         if (WebInspector.WikiParser.newLineWithSpace.test(this._text)) {
             var result = WebInspector.WikiParser.newLineWithSpace.exec(this._text);
-            var begin = result.index + result[0].length;
+            var begin = result.index + result[0].length - 3;
             var end = this._text.length;
             var lineEnd = WebInspector.WikiParser.newLineWithoutSpace.exec(this._text);
             if (lineEnd)
                 end = lineEnd.index;
-            var token = this._text.substring(begin, end).replace(/\n */g, "\n");
+            var token = this._text.substring(begin, end).replace(/\n /g, "\n").replace(/{{=}}/g, "=");
             this._text = this._text.substring(end + 1);
             return new WebInspector.WikiParser.Token(token, WebInspector.WikiParser.TokenType.CodeBlock);
         }
 
         for (var i = 0; i < WebInspector.WikiParser._tokenDescriptors.length; ++i) {
             var result = WebInspector.WikiParser._tokenDescriptors[i].regex.exec(this._text);
             if (result) {
                 this._text = this._text.substring(result.index + result[0].length);
                 return new WebInspector.WikiParser.Token(result[0], WebInspector.WikiParser._tokenDescriptors[i].type);
             }
         }
 
         for (var lastIndex = 0; lastIndex < this._text.length; ++lastIndex) {
             var testString = this._text.substring(lastIndex);
             for (var i = 0; i < WebInspector.WikiParser._tokenDescriptors.length; ++i) {
                 if (WebInspector.WikiParser._tokenDescriptors[i].regex.test(testString)) {
                     var token = this._text.substring(0, lastIndex);
                     this._text = this._text.substring(lastIndex);
                     return new WebInspector.WikiParser.Token(token, WebInspector.WikiParser.TokenType.Text);
                 }
             }
         }
 
         var token = this._text;
         this._text = "";
         return new WebInspector.WikiParser.Token(token, WebInspector.WikiParser.TokenType.Text);
     },
 
     /**
+     * @return {!WebInspector.WikiParser.Tokenizer}
+     */
+    clone: function()
+    {
+        var tokenizer = new WebInspector.WikiParser.Tokenizer(this._text);
+        tokenizer._token = this._token;
+        tokenizer._text = this._text;
+        return tokenizer;
+    },
+
+    /**
      * @return {boolean}
      */
-    _hasMoreTokens: function()
+    hasMoreTokens: function()
     {
         return !!this._text.length;
     }
 }
 
+WebInspector.WikiParser.table = /^{{{!}}/;
+WebInspector.WikiParser.exclamation = /^{{!}}/;
+WebInspector.WikiParser.openingBraces = /^{{/;
+WebInspector.WikiParser.equalSign = /^=/;
+WebInspector.WikiParser.equalSignInBraces = /^{{=}}/;
+WebInspector.WikiParser.closingBraces = /^\s*}}/;
+WebInspector.WikiParser.oneOpeningBracketWithSpace = /^\n* \[/;
+WebInspector.WikiParser.twoOpeningBracketsWithSpace = /^\n* \[\[/;
+WebInspector.WikiParser.oneClosingBracket = /^\n*\]/;
+WebInspector.WikiParser.twoClosingBrackets = /^\n*\]\]/;
+WebInspector.WikiParser.tripleQuotes = /^\n*'''/;
+WebInspector.WikiParser.openingCodeTag = /^<\s*code\s*>/;
+WebInspector.WikiParser.closingCodeTag = /^<\s*\/\s*code\s*>/;
+WebInspector.WikiParser.closingBullet = /^\*/;
+WebInspector.WikiParser.lineEnd = /^\n/;
+WebInspector.WikiParser.verticalLine = /^\n*\|/;
+WebInspector.WikiParser.newLineWithSpace = /^\n [^ ]/;
+WebInspector.WikiParser.newLineWithoutSpace = /\n[^ ]/;
+
+/**
+ * @constructor
+ * @param {!RegExp} regex
+ * @param {!WebInspector.WikiParser.TokenType} type
+ */
+WebInspector.WikiParser.TokenDescriptor = function(regex, type)
+{
+    this.regex = regex;
+    this.type = type;
+}
+
+WebInspector.WikiParser._tokenDescriptors = [
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.exclamation, WebInspector.WikiParser.TokenType.Exclamation),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.equalSignInBraces, WebInspector.WikiParser.TokenType.EqualSignInBraces),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.equalSign, WebInspector.WikiParser.TokenType.EqualSign),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.table, WebInspector.WikiParser.TokenType.Table),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.openingBraces, WebInspector.WikiParser.TokenType.OpeningBraces),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.verticalLine, WebInspector.WikiParser.TokenType.VerticalLine),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingBraces, WebInspector.WikiParser.TokenType.ClosingBraces),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.twoOpeningBracketsWithSpace, WebInspector.WikiParser.TokenType.OpeningBrackets),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.twoClosingBrackets, WebInspector.WikiParser.TokenType.ClosingBrackets),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.oneOpeningBracketWithSpace, WebInspector.WikiParser.TokenType.OpeningBrackets),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.oneClosingBracket, WebInspector.WikiParser.TokenType.ClosingBrackets),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.newLineWithSpace, WebInspector.WikiParser.TokenType.CodeBlock),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.tripleQuotes, WebInspector.WikiParser.TokenType.TripleQuotes),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.openingCodeTag, WebInspector.WikiParser.TokenType.OpeningCodeTag),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingCodeTag, WebInspector.WikiParser.TokenType.ClosingCodeTag),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingBullet, WebInspector.WikiParser.TokenType.Bullet),
+    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.lineEnd, WebInspector.WikiParser.TokenType.LineEnd)
+]
+
 WebInspector.WikiParser.prototype = {
     /**
      * @return {!Object}
      */
     document: function()
     {
         return this._document;
     },
 
     /**
-     * @return {!Object}
+     * @return {?WebInspector.WikiParser.TokenType}
+     */
+    _secondTokenType: function()

[apavlov, 2014/09/10 14:09:23]

why not just save and restore the state of the current tokenizer instead of
cloning it? It's the easiest way to peek a token.

[iliia, 2014/09/11 09:26:31]

Clone is a saving copy of the state of current tokenizer.
On 2014/09/10 14:09:23, apavlov (OOO until Sept.19) wrote:

[apavlov, 2014/09/11 10:40:26]

I know that cloning implicitly allows to retain the current instance state
intact, but is a lot more heavyweight (a new instance creation + destruction
through GC). pushState/popState or plain saving of the state into local
variables is much faster.

+    {
+        var tokenizer = this._tokenizer.clone();
+        if (!tokenizer.hasMoreTokens())
+            return null;
+        tokenizer.nextToken();
+        if (!tokenizer.hasMoreTokens())
+            return null;
+        return tokenizer.nextToken().type();
+    },
+
+    /**
+     * @return {!Object.<string, ?WebInspector.WikiParser.Value>}
      */
     _parse: function()
     {
         var obj = {};
-        this._wikiMarkupText = this._wikiMarkupText.replace(/&lt;/g, "<")
-                      .replace(/&gt;/g, ">")
-                      .replace(/&#58;/g, ":")
-                      .replace(/&quot;/g, "\"")
-                      .replace(/&#60;/g, "<")
-                      .replace(/&#62;/g, ">")
-                      .replace(/{{=}}/g, "=")
-                      .replace(/{{!}}/g, "|")
-                      .replace(/&amp;/g, "&");
-        while (this._position < this._wikiMarkupText.length) {
-            var field = this._parseField();
-            for (var key in field) {
-                console.assert(typeof obj[key] === "undefined", "Duplicate key: " + key);
-                obj[key] = field[key];
+        while (this._tokenizer.hasMoreTokens()) {
+            var section = this._parseSection();
+            if (section.title) {
+                if (section.singleValue)
+                    obj[section.title] = section.singleValue;
+                else
+                    obj[section.title] = section.values;
             }
         }
         return obj;
     },
 
     /**
+     * @return {!WebInspector.WikiParser.Section}
+     */
+    _parseSection: function()
+    {
+        var section = new WebInspector.WikiParser.Section();
+        if (!this._tokenizer.hasMoreTokens() || this._tokenizer.nextToken().type() !== WebInspector.WikiParser.TokenType.OpeningBraces)
+            return section;
+
+        var title = this._deleteTrailingSpaces(this._parseTitle());
+        if (!title.length)
+            return section;
+        section.title = title;
+        if (this._tokenizer.peekToken().type() === WebInspector.WikiParser.TokenType.ClosingBraces) {
+            this._tokenizer.nextToken();
+            return section;
+        }
+        var secondTokenType = this._secondTokenType();
+        if (!secondTokenType || secondTokenType !== WebInspector.WikiParser.TokenType.EqualSign) {
+            section.singleValue = this._parseString();
+        } else {
+            section.values = {};
+            while (this._tokenizer.hasMoreTokens()) {
+                var field = this._parseField();
+                section.values[field.name] = field.value;
+                if (this._tokenizer.peekToken().type() === WebInspector.WikiParser.TokenType.ClosingBraces) {
+                    this._tokenizer.nextToken();
+                    return section;
+                }
+            }
+        }
+        var token = this._tokenizer.nextToken();
+        if (token.type() !== WebInspector.WikiParser.TokenType.ClosingBraces)
+            throw new Error("Two closing braces expected; found " + token.value());
+
+        return section;
+    },
+
+    /**
+     * @return {!WebInspector.WikiParser.Field}
+     */
+    _parseField: function()
+    {
+        var field = new WebInspector.WikiParser.Field;

[apavlov, 2014/09/10 14:09:23]

new ...Field();

[iliia, 2014/09/11 09:26:30]

Done.

+        field.name = this._parseName();
+        var token = this._tokenizer.peekToken();
+        switch (token.type()) {
+        case WebInspector.WikiParser.TokenType.OpeningBraces:
+            field.value = this._parseArray();
+            break;
+        case WebInspector.WikiParser.TokenType.LineEnd:
+            this._tokenizer.nextToken();
+            break;
+        case WebInspector.WikiParser.TokenType.ClosingBraces:
+            return field;
+            break;
+        default:
+            if (field.name === "Code")
+                field.value = this._parseExampleCode();
+            else
+                field.value = this._parseString();
+        }
+        return field;
+    },
+
+    /**
+     * @return {!Array.<!WebInspector.WikiParser.Section>}
+     */
+    _parseArray: function()
+    {
+        var array = [];
+        while (this._tokenizer.peekToken().type() === WebInspector.WikiParser.TokenType.OpeningBraces)
+            array.push(this._parseSection());
+        if (this._tokenizer.peekToken().type() === WebInspector.WikiParser.TokenType.VerticalLine)
+            this._tokenizer.nextToken();
+        return array;
+    },
+
+    /**
      * @return {string}
      */
-    _parseValue: function() {
-        var states = WebInspector.WikiParser.ValueState;
-        var state = states.Outside;
-        var value = "";
-        while (this._position < this._wikiMarkupText.length) {
-            switch (state) {
-            case states.Outside:
-                if (this._wikiMarkupText[this._position] === "|" || (this._wikiMarkupText[this._position] === "}" && this._wikiMarkupText[this._position + 1] === "}"))
-                    return value;
-                switch (this._wikiMarkupText[this._position]) {
-                case "<":
-                    var indexClose = this._wikiMarkupText.indexOf(">", this._position);
-                    if (indexClose !== -1) {
-                        value += this._wikiMarkupText.substring(this._position, indexClose + 1);
-                        this._position = indexClose;
-                    }
-                    break;
-                case "[":
-                    state = states.InsideSquare;
-                    value += this._wikiMarkupText[this._position];
-                    break;
-                default:
-                    value += this._wikiMarkupText[this._position];
-                }
-                break;
-            case states.InsideSquare:
-                if (this._wikiMarkupText[this._position] === "[") {
-                    var indexClose = this._wikiMarkupText.indexOf("]]", this._position);
-                    if (indexClose !== -1) {
-                        value += this._wikiMarkupText.substring(this._position, indexClose + 2);
-                        this._position = indexClose + 1;
-                    }
-                } else {
-                    var indexClose = this._wikiMarkupText.indexOf("]", this._position);
-                    if (indexClose !== -1) {
-                        value += this._wikiMarkupText.substring(this._position, indexClose + 1);
-                        this._position = indexClose;
-                    }
-                }
-                state = states.Outside;
-                break;
-            }
-            this._position++;
-        }
-        return value;
-    },
-
-    /**
-     * @return {!Object}
-     */
-    _parseField: function()
-    {
-        var obj = {};
+    _parseTitle: function()
+    {
         var title = "";
-        var propertyName = "";
-        var propertyValue = "";
-        var states = WebInspector.WikiParser.State;
-        var state = states.FirstOpen;
-        while (this._position < this._wikiMarkupText.length) {
-            var skipIncrement = false;
-            switch (state) {
-            case states.FirstOpen:
-                if (this._wikiMarkupText[this._position] === "{")
-                    state = states.SecondOpen;
-                else
-                    state = states.Error;
-                break;
-            case states.SecondOpen:
-                if (this._wikiMarkupText[this._position] === "{")
-                    state = states.Title;
-                else
-                    state = states.Error;
-                break;
-            case states.Title:
-                if (this._wikiMarkupText[this._position] === "|") {
-                    title = this._deleteTrailingSpaces(title);
-                    if (title !== "")
-                        obj[title] = {};
-                    state = states.PropertyName;
-                } else if (this._wikiMarkupText[this._position] === "}") {
-                    title = this._deleteTrailingSpaces(title);
-                    if (title !== "")
-                        obj[title] = {};
-                    state = states.FirstClose;
-                } else {
-                    title += (this._wikiMarkupText[this._position] === "\n" ? "" : this._wikiMarkupText[this._position]);
-                }
-                break;
-            case states.PropertyName:
-                if (this._wikiMarkupText[this._position] === "=") {
-                    state = states.PropertyValue;
-                    this._deleteTrailingSpaces(propertyName);
-                    if (propertyName !== "")
-                        obj[title][propertyName] = [];
-                } else {
-                    if (this._wikiMarkupText[this._position] === "}") {
-                        propertyName = this._deleteTrailingSpaces(propertyName);
-                        obj[title] = propertyName;
-                        state = states.FirstClose;
-                    } else {
-                        propertyName += this._wikiMarkupText[this._position];
-                    }
-                }
-                break;
-            case states.PropertyValue:
-                if (this._wikiMarkupText[this._position] === "{" && this._wikiMarkupText[this._position + 1] === "{") {
-                    propertyValue = this._parseField();
-                    obj[title][propertyName].push(propertyValue);
-                    propertyValue = "";
-                    skipIncrement = true;
-                } else if (this._wikiMarkupText[this._position] === "|") {
-                    propertyValue = this._deleteTrailingSpaces(propertyValue);
-                    if (propertyValue !== "")
-                      obj[title][propertyName] = propertyValue;
-
-                    state = states.PropertyName;
-                    if (Array.isArray(obj[title][propertyName]) && obj[title][propertyName].length === 1) {
-                        var newObj = obj[title][propertyName][0];
-                        obj[title][propertyName] = newObj;
-                    }
-
-                    propertyName = "";
-                    propertyValue = "";
-                } else if (this._position + 1 < this._wikiMarkupText.length && this._wikiMarkupText[this._position] === "}" && this._wikiMarkupText[this._position + 1] === "}") {
-                    propertyValue = this._deleteTrailingSpaces(propertyValue);
-                    if (propertyValue !== "")
-                        obj[title][propertyName].push(propertyValue);
-                    if (Array.isArray(obj[title][propertyName]) && obj[title][propertyName].length === 1) {
-                        var newObj = obj[title][propertyName][0];
-                        obj[title][propertyName] = newObj;
-                    }
-
-                    propertyValue = "";
-                    state = states.FirstClose;
-                } else {
-                    propertyValue = this._parseValue();
-                    skipIncrement = true;
-                }
-                break;
-            case states.FirstClose:
-                if (this._wikiMarkupText[this._position] === "}")
-                    state = states.SecondClose;
-                else
-                    state = states.Error;
-                break;
-            case states.SecondClose:
-                while (this._position < this._wikiMarkupText.length && this._wikiMarkupText[this._position] === "\n")
-                    this._position++;
-                return obj;
-            case states.Error:
-                this._position = this._wikiMarkupText.length;
-                return {};
-            }
-            if (!skipIncrement)
-                this._position++;
-        }
-        return obj;
-    },
-
-    /**
-     * @param {string} str
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.peekToken();
+            switch (token.type()) {
+            case WebInspector.WikiParser.TokenType.ClosingBraces:
+                return title;
+            case WebInspector.WikiParser.TokenType.VerticalLine:
+                this._tokenizer.nextToken();
+                return title;
+            case WebInspector.WikiParser.TokenType.Text:
+                title += this._tokenizer.nextToken().value();
+                break;

[apavlov, 2014/09/10 14:09:24]

"return title;" for consistency?

[iliia, 2014/09/11 09:26:31]

We must return title only then we met | or }}, so we need to continue and to
read the next token.
On 2014/09/10 14:09:24, apavlov (OOO until Sept.19) wrote:

[apavlov, 2014/09/11 10:40:26]

Oops, misread the code, thanks.

+            default:
+                throw new Error("Title could not be parsed. Unexpected token " + token.value());
+            }
+        }
+        return title;
+    },
+
+    /**
+     * @return {string}
+     */
+    _parseName: function()
+    {
+        var name = "";
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.peekToken();
+            switch (token.type()) {
+            case WebInspector.WikiParser.TokenType.ClosingBraces:
+                return name;
+            case WebInspector.WikiParser.TokenType.EqualSign:
+                this._tokenizer.nextToken();
+                return name;
+            case WebInspector.WikiParser.TokenType.VerticalLine:
+            case WebInspector.WikiParser.TokenType.Text:
+                name += this._tokenizer.nextToken().value();
+                break;
+            default:
+                throw new Error("Name could not be parsed. Unexpected token " + token.value());
+            }
+        }
+        return name;
+    },
+
+    /**
+     * @return {!WebInspector.WikiParser.Block}
+     */
+    _parseExampleCode: function()
+    {
+        var code = "";
+        /**

[apavlov, 2014/09/10 14:09:24]

blank lines between inner function and surrounding code

[iliia, 2014/09/11 09:26:30]

Done.

+         * @return {!WebInspector.WikiParser.Block}
+         */
+        function wrapToArticleElement()

[apavlov, 2014/09/10 14:09:24]

To -> Into

[iliia, 2014/09/11 09:26:30]

Done.

+        {
+            var plainText = new WebInspector.WikiParser.PlainText(code, false);

[apavlov, 2014/09/10 14:09:24]

Can these "false" arguments be made optional in the constructors?

[iliia, 2014/09/11 09:26:31]

Done.

+            var block = new WebInspector.WikiParser.Block([plainText], false)
+            var articleElement = new WebInspector.WikiParser.Block([block], false);
+            return articleElement;
+        }
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.peekToken();
+            switch (token.type()) {
+            case WebInspector.WikiParser.TokenType.ClosingBraces:
+                return wrapToArticleElement();
+            case WebInspector.WikiParser.TokenType.VerticalLine:
+                this._tokenizer.nextToken();
+                return wrapToArticleElement();
+            case WebInspector.WikiParser.TokenType.Exclamation:
+                this._tokenizer.nextToken();
+                code += '|';

[apavlov, 2014/09/10 14:09:23]

double quotes

[iliia, 2014/09/11 09:26:31]

Done.

+                break;
+            case WebInspector.WikiParser.TokenType.EqualSignInBraces:
+                this._tokenizer.nextToken();
+                code += "=";
+                break;
+            default:
+                this._tokenizer.nextToken();
+                code += token.value();
+            }
+        }
+        return wrapToArticleElement();
+    },
+
+    /**
      * @return {?WebInspector.WikiParser.Block}
      */
-    parseString: function(str)
-    {
-        this._tokenizer = new WebInspector.WikiParser.Tokenizer(str);
+    _parseString: function()
+    {
         var children = [];
         var blockChildren = [];
         var text = "";
         var self = this;
 
         function processSimpleText()
         {
             var currentText = self._deleteTrailingSpaces(text);
             if (!currentText.length)
                 return;
             var simpleText = new WebInspector.WikiParser.PlainText(currentText, false);
             blockChildren.push(simpleText);
             text = "";
         }
 
         function processBlock()
         {
             if (blockChildren.length) {
                 children.push(new WebInspector.WikiParser.Block(blockChildren, false));
                 blockChildren = [];
             }
         }
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken();
+
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.peekToken();
             switch (token.type()) {
+            case WebInspector.WikiParser.TokenType.VerticalLine:
+                this._tokenizer.nextToken();
+            case WebInspector.WikiParser.TokenType.ClosingBraces:
+                processSimpleText();
+                processBlock();
+                return new WebInspector.WikiParser.Block(children, false);
             case WebInspector.WikiParser.TokenType.TripleQuotes:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 var highlightText = this._parseHighlight();
                 blockChildren.push(highlightText)
                 break;
             case WebInspector.WikiParser.TokenType.OpeningBrackets:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 var link = this._parseLink();
                 blockChildren.push(link);
                 break;
             case WebInspector.WikiParser.TokenType.OpeningCodeTag:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 var code = this._parseCode();
                 blockChildren.push(code);
                 break;
             case WebInspector.WikiParser.TokenType.Bullet:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 processBlock();
                 var bulletText = this._parseBullet();
                 children.push(bulletText);
                 break;
             case WebInspector.WikiParser.TokenType.CodeBlock:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 processBlock();
-                var code = new WebInspector.WikiParser.CodeBlock(token.value());
+                var code = new WebInspector.WikiParser.CodeBlock(this._deleteFrontLineEnds(token.value()));
                 children.push(code);
                 break;
             case WebInspector.WikiParser.TokenType.LineEnd:
+                this._tokenizer.nextToken();
                 processSimpleText();
                 processBlock();
                 break;
-            case WebInspector.WikiParser.TokenType.VerticalLine:
+            case WebInspector.WikiParser.TokenType.EqualSignInBraces:
+                this._tokenizer.nextToken();
+                text += "=";
+                break;
+            case WebInspector.WikiParser.TokenType.Exclamation:
+                this._tokenizer.nextToken();
+                text += "|";
+                break;
+            case WebInspector.WikiParser.TokenType.ClosingBrackets:
             case WebInspector.WikiParser.TokenType.Text:
+            case WebInspector.WikiParser.TokenType.EqualSign:
+            case WebInspector.WikiParser.TokenType.Table:
+                this._tokenizer.nextToken();
                 text += token.value();
                 break;
             default:
+                this._tokenizer.nextToken();
                 return null;
             }
         }
 
         processSimpleText();
         processBlock();
 
         return new WebInspector.WikiParser.Block(children, false);
     },
 
     /**
      * @return {!WebInspector.WikiParser.Link}
      */
     _parseLink: function()
     {
         var url = "";
         var children = [];
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken();
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken();
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.ClosingBrackets:
                 return new WebInspector.WikiParser.Link(url, children);
             case WebInspector.WikiParser.TokenType.VerticalLine:
+            case WebInspector.WikiParser.TokenType.Exclamation:
                 children.push(this._parseLinkName());
                 return new WebInspector.WikiParser.Link(url, children);
             default:
                 url += token.value();
             }
         }
 
         return new WebInspector.WikiParser.Link(url, children);
     },
 
     /**
      * @return {!WebInspector.WikiParser.Inline}
      */
     _parseLinkName: function()
     {
         var children = [];
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken();
+        var text = "";
+        var self = this;
+        function processSimpleText()
+        {
+            text = self._deleteTrailingSpaces(text);
+            if (!text.length)
+                return;
+            var simpleText = new WebInspector.WikiParser.PlainText(text, false);
+            children.push(simpleText);
+            text = "";
+        }
+
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken();
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.ClosingBrackets:
+                processSimpleText();
                 return new WebInspector.WikiParser.Inline(WebInspector.WikiParser.ArticleElement.Type.Inline, children);

[apavlov, 2014/09/10 14:09:24]

Article element types should never get passed into the constructor of concrete
elements. Rather, their constructors should pass the type into the
superconstructor.

[iliia, 2014/09/11 09:26:30]

So if some of Inlines haven't any special fields or methods, I think it's not a
good idea to create new classes for them.
On 2014/09/10 14:09:24, apavlov (OOO until Sept.19) wrote:

[apavlov, 2014/09/11 10:40:26]

Can an instance of WI.WikiParser.Inline have a type different than "Inline"?

[iliia, 2014/09/11 13:26:28]

Yes, for example, Link.
On 2014/09/11 10:40:26, apavlov (OOO until Sept.19) wrote:

[apavlov, 2014/09/11 19:55:22]

OK, I have chatted with Andrey about it. Leaving this to his discretion.

             case WebInspector.WikiParser.TokenType.OpeningCodeTag:
+                processSimpleText();
                 children.push(this._parseCode());
                 break;
             default:
-                children.push(new WebInspector.WikiParser.PlainText(token.value(), false));
+                text += token.value();
                 break;
             }
         }
 
         return new WebInspector.WikiParser.Inline(WebInspector.WikiParser.ArticleElement.Type.Inline, children);
     },
 
     /**
      * @return {!WebInspector.WikiParser.Inline}
      */
     _parseCode : function()
     {
         var children = [];
         var text = "";
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken();
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken();
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.ClosingCodeTag:
                 text = this._deleteTrailingSpaces(text);
                 if (text.length) {
                     var simpleText = new WebInspector.WikiParser.PlainText(text, false);
                     children.push(simpleText);
                     text = "";
                 }
                 var code = new WebInspector.WikiParser.Inline(WebInspector.WikiParser.ArticleElement.Type.Code, children);
                 return code;
@@ skipping 12 matching lines @@
 
         return new WebInspector.WikiParser.Inline(WebInspector.WikiParser.ArticleElement.Type.Code, children);
     },
 
     /**
      * @return {!WebInspector.WikiParser.Block}
      */
     _parseBullet: function()
     {
         var children = [];
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken()
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken()
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.OpeningBrackets:
                 children.push(this._parseLink());
                 break;
             case WebInspector.WikiParser.TokenType.OpeningCodeTag:
                 children.push(this._parseCode());
                 break;
             case WebInspector.WikiParser.TokenType.LineEnd:
                 return new WebInspector.WikiParser.Block(children, true);
             default:
                 var text = this._deleteTrailingSpaces(token.value());
                 if (text.length) {
                     var simpleText = new WebInspector.WikiParser.PlainText(text, false);
                     children.push(simpleText);
                     text = "";
                 }
             }
         }
 
         return new WebInspector.WikiParser.Block(children, true);
     },
 
     /**
      * @return {!WebInspector.WikiParser.PlainText}
      */
     _parseHighlight: function()
     {
         var text = "";
-        while (this._tokenizer._hasMoreTokens()) {
-            var token = this._tokenizer._nextToken()
+        while (this._tokenizer.hasMoreTokens()) {
+            var token = this._tokenizer.nextToken()
             switch (token.type()) {
             case WebInspector.WikiParser.TokenType.TripleQuotes:
                 text = this._deleteTrailingSpaces(text);
                 return new WebInspector.WikiParser.PlainText(text, true);
             default:
                 text += token.value();
             }
         }
         return new WebInspector.WikiParser.PlainText(text, true);
     },
 
     /**
      * @param {string} str
      * @return {string}
      */
     _deleteTrailingSpaces: function(str)
     {
         return str.replace(/[\n\r]*$/gm, "");
+    },
+
+    /**
+     * @param {string} str
+     * @return {string} str

[apavlov, 2014/09/11 19:55:22]

remove "str"

+     */
+    _deleteFrontLineEnds: function(str)

[apavlov, 2014/09/11 19:55:22]

_trimLeadingNewLines

+    {
+        return str.replace(/^\n*/, "");
     }
 }
 
-WebInspector.WikiParser.oneOpeningBracket = /^\n* \[[^\[]/;
-WebInspector.WikiParser.twoOpeningBrackets = /^\n* \[\[/;
-WebInspector.WikiParser.oneClosingBracket = /^\n*\][^\]] /;
-WebInspector.WikiParser.twoClosingBrackets = /^\n*\]\]/;
-WebInspector.WikiParser.tripleQuotes = /^\n*'''/;
-WebInspector.WikiParser.openingCodeTag = /^<\s*code\s*>/;
-WebInspector.WikiParser.closingCodeTag = /^<\s*\/\s*code\s*>/;
-WebInspector.WikiParser.closingBullet = /^\*/;
-WebInspector.WikiParser.lineEnd = /^\n/;
-WebInspector.WikiParser.verticalLine = /^\|/;
-WebInspector.WikiParser.newLineWithSpace = /^\n /;
-WebInspector.WikiParser.newLineWithoutSpace = /\n[^ ]/;
-
-/**
- * @constructor
- * @param {!RegExp} regex
- * @param {!WebInspector.WikiParser.TokenType} type
- */
-WebInspector.WikiParser.TokenDescriptor = function(regex, type)
-{
-    this.regex = regex;
-    this.type = type;
-}
-
-WebInspector.WikiParser._tokenDescriptors = [
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.newLineWithSpace, WebInspector.WikiParser.TokenType.CodeBlock),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.tripleQuotes, WebInspector.WikiParser.TokenType.TripleQuotes),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.oneOpeningBracket, WebInspector.WikiParser.TokenType.OpeningBrackets),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.twoOpeningBrackets, WebInspector.WikiParser.TokenType.OpeningBrackets),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.oneClosingBracket, WebInspector.WikiParser.TokenType.ClosingBrackets),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.twoClosingBrackets, WebInspector.WikiParser.TokenType.ClosingBrackets),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.openingCodeTag, WebInspector.WikiParser.TokenType.OpeningCodeTag),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingCodeTag, WebInspector.WikiParser.TokenType.ClosingCodeTag),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.closingBullet, WebInspector.WikiParser.TokenType.Bullet),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.verticalLine, WebInspector.WikiParser.TokenType.VerticalLine),
-    new WebInspector.WikiParser.TokenDescriptor(WebInspector.WikiParser.lineEnd, WebInspector.WikiParser.TokenType.LineEnd)
-];
-
 /**
  * @constructor
  * @param {!WebInspector.WikiParser.ArticleElement.Type} type
  */
 WebInspector.WikiParser.ArticleElement = function(type)
 {
     this._type = type;
 }
 
 WebInspector.WikiParser.ArticleElement.prototype = {
@@ skipping 28 matching lines @@
 {
     WebInspector.WikiParser.ArticleElement.call(this, WebInspector.WikiParser.ArticleElement.Type.PlainText);
     this._text = text;
     this._isHighlighted = highlight;
 }
 
 WebInspector.WikiParser.PlainText.prototype = {
     /**
      * @return {string}
      */
     text : function()

[apavlov, 2014/09/10 14:09:24]

no whitespace before colon

[iliia, 2014/09/11 09:26:30]

Done.

     {
         return this._text;
     },
 
     /**
      * @return {boolean}
      */
     isHighlighted: function()
     {
         return this._isHighlighted;
@@ skipping 98 matching lines @@
     /**
      * @return {string}
      */
     url : function()
     {
         return this._url;
     },
 
     __proto__: WebInspector.WikiParser.Inline.prototype
 }