cleanup to Value - fix for StringEscapesTest

frog/parser.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 // TODO(jimhug): Error recovery needs major work!
 /**
  * A simple recursive descent parser for the dart language.
  *
  * This parser is designed to be more permissive than the official
  * Dart grammar.  It is expected that many grammar errors would be
@@ skipping 50 matching lines @@
     _afterParens = <Token>[];
   }
 
   /** Generate an error if [source] has not been completely consumed. */
   void checkEndOfFile() {
     _eat(TokenKind.END_OF_FILE);
   }
 
   /** Guard to break out of parser when an unexpected end of file is found. */
   bool isPrematureEndOfFile() {
-    if (throwOnIncomplete && _maybeEat(TokenKind.END_OF_FILE) ||
-        _maybeEat(TokenKind.INCOMPLETE_MULTILINE_STRING_DQ) ||

[Jennifer Messerly, 2012/01/09 20:16:26]

I think this was REPL support code that had found its way into
isPrematureEndOfFile.

Not sure which cases it was trying to support, though. I'm fine with removing
for now, as we try to simplify the compiler. (one idea I had: REPL could have an
derived parser class or something like that, so it's clear which code is for
normal Dart parsing and which code is for REPL support.)

[jimhug, 2012/01/09 21:19:05]

This was originally my code intended to handle single-line tokenization by
keeping track of a tokenizer state for multiple lines when needed.  It turned
out to be a bad idea both for the tokenizer (at least based on my tip
experience) and to not really work given the complicated states we potentially
have now with string interpolation and nested comments.


On 2012/01/09 20:16:26, John Messerly wrote:

-        _maybeEat(TokenKind.INCOMPLETE_MULTILINE_STRING_SQ)) {
+    if (throwOnIncomplete && _maybeEat(TokenKind.END_OF_FILE)) {
       throw new IncompleteSourceException(_previousToken);
     } else if (_maybeEat(TokenKind.END_OF_FILE)) {
       _error('unexpected end of file', _peekToken.span);
       return true;
     } else {
       return false;
     }
   }
 
   /**
@@ skipping 957 matching lines @@
       return finishPostfixExpression(
           new CallExpression(expr, args, _makeSpan(expr.span.start)));
     }
   }
 
   /** Checks if the given expression is a binary op of the given kind. */
   _isBin(expr, kind) {
     return expr is BinaryExpression && expr.op.kind == kind;
   }
 
-  _boolTypeRef(SourceSpan span) {
-    return new TypeReference(span, world.nonNullBool);
-  }
-
-  _intTypeRef(SourceSpan span) {
-    return new TypeReference(span, world.intType);
-  }
-
-  _doubleTypeRef(SourceSpan span) {
-    return new TypeReference(span, world.doubleType);
-  }
-
-  _stringTypeRef(SourceSpan span) {
-    return new TypeReference(span, world.stringType);
+  _makeLiteral(Value value) {
+    return new LiteralExpression(value, value.span);
   }
 
   primary() {
     int start = _peekToken.start;
     switch (_peek()) {
       case TokenKind.THIS:
         _eat(TokenKind.THIS);
         return new ThisExpression(_makeSpan(start));
 
       case TokenKind.SUPER:
@@ skipping 21 matching lines @@
 
       case TokenKind.LBRACK:
       case TokenKind.INDEX:
         return finishListLiteral(start, false, null);
       case TokenKind.LBRACE:
         return finishMapLiteral(start, false, null);
 
       // Literals
       case TokenKind.NULL:
         _eat(TokenKind.NULL);
-        return new NullExpression(_makeSpan(start));
+        return _makeLiteral(Value.fromNull(_makeSpan(start)));
 
       // TODO(jimhug): Make Literal creation less wasteful - no dup span/text.
       case TokenKind.TRUE:
         _eat(TokenKind.TRUE);
-        return new LiteralExpression(true, _boolTypeRef(_makeSpan(start)),
-          'true', _makeSpan(start));
+        return _makeLiteral(Value.fromBool(true, _makeSpan(start)));
 
       case TokenKind.FALSE:
         _eat(TokenKind.FALSE);
-        return new LiteralExpression(false,_boolTypeRef(_makeSpan(start)),
-          'false', _makeSpan(start));
+        return _makeLiteral(Value.fromBool(false, _makeSpan(start)));
 
       case TokenKind.HEX_INTEGER:
         var t = _next();
-        // Remove the 0x or 0X before parsing the hex number.
-        return new LiteralExpression(parseHex(t.text.substring(2)),
-          _intTypeRef(_makeSpan(start)), t.text, _makeSpan(start));
+        return _makeLiteral(Value.fromInt(t.value, t.span));
 
       case TokenKind.INTEGER:
         var t = _next();
-        return new LiteralExpression(Math.parseInt(t.text),
-          _intTypeRef(_makeSpan(start)), t.text, _makeSpan(start));
+        return _makeLiteral(Value.fromInt(Math.parseInt(t.text), t.span));
 
       case TokenKind.DOUBLE:
         var t = _next();
-        return new LiteralExpression(Math.parseDouble(t.text),
-          _doubleTypeRef(_makeSpan(start)), t.text, _makeSpan(start));
+        return _makeLiteral(
+          Value.fromDouble(Math.parseDouble(t.text), t.span));
 
       case TokenKind.STRING:
-        return stringLiteralExpr();
+      var t = _next();
+      return _makeLiteral(Value.fromString(t.value, t.span));
 
-      case TokenKind.INCOMPLETE_STRING:
+      case TokenKind.STRING_PART:
         return stringInterpolation();
 
       case TokenKind.LT:
         return finishTypedLiteral(start, false);
 
       case TokenKind.VOID:
       case TokenKind.VAR:
       case TokenKind.FINAL:
         return declaredIdentifier(false);
 
       default:
         if (!_peekIdentifier()) {
           // TODO(jimhug): Better error message.
           _errorExpected('expression');
         }
         return new VarExpression(identifier(), _makeSpan(start));
     }
   }
 
   stringInterpolation() {
     int start = _peekToken.start;
-    var lits = [];
+    var pieces = new List<Expression>();
     var startQuote = null, endQuote = null;
-    while(_peekKind(TokenKind.INCOMPLETE_STRING)) {
+    while(_peekKind(TokenKind.STRING_PART)) {
       var token = _next();
-      var text = token.text;
-      if (startQuote == null) {
-        if (isMultilineString(text)) {
-          endQuote = text.substring(0, 3);
-          // TODO(jmesserly): HACK add a newline to everything that's not

[Jennifer Messerly, 2012/01/09 20:16:26]

--hack;

-          // the first multiline string, so we don't incorrectly strip off any
-          // real newlines later in the interpolated string.
-          startQuote = endQuote + '\n';
-        } else {
-          startQuote = endQuote = text[0];
-        }
-        text = text.substring(0, text.length-1) + endQuote; // fix trailing $
-      } else {
-        text = startQuote + text.substring(0, text.length-1) + endQuote;
-      }
-      lits.add(makeStringLiteral(text, token.span));
+      pieces.add(_makeLiteral(Value.fromString(token.value, token.span)));
       if (_maybeEat(TokenKind.LBRACE)) {
-        lits.add(expression());
+        pieces.add(expression());
         _eat(TokenKind.RBRACE);
       } else if (_maybeEat(TokenKind.THIS)) {
-        lits.add(new ThisExpression(_previousToken.span));
+        pieces.add(new ThisExpression(_previousToken.span));
       } else {
         var id = identifier();
-        lits.add(new VarExpression(id, id.span));
+        pieces.add(new VarExpression(id, id.span));
       }
     }
     var tok = _next();
     if (tok.kind != TokenKind.STRING) {
       _errorExpected('interpolated string');
     }
-    var text = startQuote + tok.text;
-    lits.add(makeStringLiteral(text, tok.span));
+    pieces.add(_makeLiteral(Value.fromString(tok.value, tok.span)));
     var span = _makeSpan(start);
-    return new LiteralExpression(lits, _stringTypeRef(span), '\$\$\$', span);
-  }
-
-  makeStringLiteral(String text, SourceSpan span) {
-    return new LiteralExpression(text, _stringTypeRef(span), text, span);
-  }
-
-  stringLiteralExpr() {
-    var token = _next();
-    return makeStringLiteral(token.text, token.span);
+    return new StringInterpExpression(pieces, span);
   }
 
   String maybeStringLiteral() {
     var kind = _peek();
     if (kind == TokenKind.STRING) {
-      return parseStringLiteral(_next().text);
+      var t = _next();
+      return t.value;
     } else if (kind == TokenKind.STRING_PART) {
       _next();
       _errorExpected('string literal, but found interpolated string start');
-    } else if (kind == TokenKind.INCOMPLETE_STRING) {
-      _next();
-      _errorExpected('string literal, but found incomplete string');
     }
     return null;
   }
 
   _parenOrLambda() {
     int start = _peekToken.start;
     if (_atClosureParameters()) {
       var formals = formalParameterList();
       var body = functionBody(true);
       var func = new FunctionDefinition(null, null, null, formals, null, null,
@@ skipping 143 matching lines @@
           name = _typeAsIdentifier(myType);
           myType = null;
         } else {
           // TODO(jimhug): Where do these errors get handled?
         }
       }
     }
     return new DeclaredIdentifier(myType, name, _makeSpan(start));
   }
 
-  // TODO(jimhug): Move this to base <= 36 and into shared code.
-  static int _hexDigit(int c) {
-    if(c >= 48/*0*/ && c <= 57/*9*/) {
-      return c - 48;
-    } else if (c >= 97/*a*/ && c <= 102/*f*/) {
-      return c - 87;
-    } else if (c >= 65/*A*/ && c <= 70/*F*/) {
-      return c - 55;
-    } else {
-      return -1;
-    }
-  }
-
-  static int parseHex(String hex) {
-    var result = 0;
-
-    for (int i = 0; i < hex.length; i++) {
-      var digit = _hexDigit(hex.charCodeAt(i));
-      assert(digit != -1);
-      // Multiply by 16 rather than shift by 4 since that will result in a
-      // correct value for numbers that exceed the 32 bit precision of JS
-      // 'integers'.
-      // TODO: Figure out a better solution to integer truncation. Issue 638.
-      result = (result * 16) + digit;
-    }
-
-    return result;
-  }
-
   finishNewExpression(int start, bool isConst) {
     var type = type();
     var name = null;
     if (_maybeEat(TokenKind.DOT)) {
       name = identifier();
     }
     var args = arguments();
     return new NewExpression(isConst, type, name, args, _makeSpan(start));
   }
 
@@ skipping 230 matching lines @@
         _error('default values only allowed inside [optional] section');
       }
       value = expression();
     } else if (_peekKind(TokenKind.LPAREN)) {
       var formals = formalParameterList();
       var func = new FunctionDefinition(null, type, name, formals,
           null, null, null, _makeSpan(start));
       type = new FunctionTypeReference(false, func, func.span);
     }
     if (inOptionalBlock && value == null) {
-      value = new NullExpression(_makeSpan(start));
+      value = _makeLiteral(Value.fromNull(_makeSpan(start)));
     }
 
     return new FormalNode(isThis, isRest, type, name, value, _makeSpan(start));
   }
 
   formalParameterList() {
     _eatLeftParen();
     var formals = [];
     var inOptionalBlock = false;
     if (!_maybeEat(TokenKind.RPAREN)) {
@@ skipping 101 matching lines @@
   int _pos = 0;
   next() {
     var token = tokens[_pos];
     ++_pos;
     if (_pos == tokens.length) {
       parser.tokenizer = previousTokenizer;
     }
     return token;
   }
 }