TokenPositions on more nodes when running from Kernel

runtime/vm/kernel_to_il.cc

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include <map>
 #include <set>
 #include <string>
 
 #include "vm/kernel_to_il.h"
 
@@ skipping 40 matching lines @@
     } else if ((*outermost_node)->IsConstructor()) {
       parent = Constructor::Cast(*outermost_node)->parent();
     } else if ((*outermost_node)->IsField()) {
       parent = Field::Cast(*outermost_node)->parent();
     }
     if (parent != NULL && parent->IsClass()) *klass = Class::Cast(parent);
   }
 }
 
 
-void ScopeBuilder::EnterScope(TreeNode* node) {
+void ScopeBuilder::EnterScope(TreeNode* node, TokenPosition start_position) {
   scope_ = new (Z) LocalScope(scope_, depth_.function_, depth_.loop_);
+  scope_->set_begin_token_pos(start_position);
   result_->scopes.Insert(node, scope_);
 }
 
 
-void ScopeBuilder::ExitScope() {
+void ScopeBuilder::ExitScope(TokenPosition end_position) {
+  scope_->set_end_token_pos(end_position);
   scope_ = scope_->parent();
 }
 
 
 LocalVariable* ScopeBuilder::MakeVariable(const dart::String& name,
-                                          const AbstractType& type) {
-  return new (Z) LocalVariable(TokenPosition::kNoSource,
-                               TokenPosition::kNoSource, name, type);
+                                          const AbstractType& type,
+                                          TokenPosition declaration_pos,
+                                          TokenPosition token_pos) {
+  return new (Z) LocalVariable(declaration_pos, token_pos, name, type);
 }
 
 
 void ScopeBuilder::AddParameters(FunctionNode* function, intptr_t pos) {
   List<VariableDeclaration>& positional = function->positional_parameters();
   for (intptr_t i = 0; i < positional.length(); ++i) {
     AddParameter(positional[i], pos++);
   }
   List<VariableDeclaration>& named = function->named_parameters();
   for (intptr_t i = 0; i < named.length(); ++i) {
     AddParameter(named[i], pos++);
   }
 }
 
 
 void ScopeBuilder::AddParameter(VariableDeclaration* declaration,
                                 intptr_t pos) {
   LocalVariable* variable = MakeVariable(H.DartSymbol(declaration->name()),
-                                         T.TranslateVariableType(declaration));
+                                         T.TranslateVariableType(declaration),
+                                         declaration->position());
   if (declaration->IsFinal()) {
     variable->set_is_final();
   }
   scope_->InsertParameterAt(pos, variable);
   result_->locals.Insert(declaration, variable);
 
   // The default value may contain 'let' bindings for which the constant
   // evaluator needs scope bindings.
   Expression* defaultValue = declaration->initializer();
   if (defaultValue != NULL) {
@@ skipping 125 matching lines @@
 
 
 void ScopeBuilder::AddVariable(VariableDeclaration* declaration) {
   // In case `declaration->IsConst()` the flow graph building will take care of
   // evaluating the constant and setting it via
   // `declaration->SetConstantValue()`.
   const dart::String& name = declaration->name()->is_empty()
                                  ? GenerateName(":var", name_index_++)
                                  : H.DartSymbol(declaration->name());
   LocalVariable* variable =
-      MakeVariable(name, T.TranslateVariableType(declaration));
+      MakeVariable(name, T.TranslateVariableType(declaration),
+                   declaration->position(), declaration->end_position());
   if (declaration->IsFinal()) {
     variable->set_is_final();
   }
   scope_->AddVariable(variable);
   result_->locals.Insert(declaration, variable);
 }
 
 
 static bool IsStaticInitializer(const Function& function, Zone* zone) {
   return (function.kind() == RawFunction::kImplicitStaticFinalGetter) &&
@@ skipping 29 matching lines @@
                        : NULL;
   ActiveMemberScope active_member(&active_class_, member);
 
 
   LocalScope* enclosing_scope = NULL;
   if (function.IsLocalFunction()) {
     enclosing_scope = LocalScope::RestoreOuterScope(
         ContextScope::Handle(Z, function.context_scope()));
   }
   current_function_scope_ = scope_ = new (Z) LocalScope(enclosing_scope, 0, 0);
+  scope_->set_begin_token_pos(function.token_pos());
+  scope_->set_end_token_pos(function.end_token_pos());
 
   LocalVariable* context_var = parsed_function->current_context_var();
   context_var->set_is_forced_stack();
   scope_->AddVariable(context_var);
   scope_->AddVariable(parsed_function->EnsureExpressionTemp());
 
   parsed_function->SetNodeSequence(
       new SequenceNode(TokenPosition::kNoSource, scope_));
 
   switch (function.kind()) {
@@ skipping 28 matching lines @@
         result_->this_variable = variable;
 
         // We visit instance field initializers because they might contain
         // [Let] expressions and we need to have a mapping.
         if (node_->IsConstructor()) {
           Class* klass = Class::Cast(Constructor::Cast(node_)->parent());
 
           for (intptr_t i = 0; i < klass->fields().length(); i++) {
             Field* field = klass->fields()[i];
             if (!field->IsStatic() && (field->initializer() != NULL)) {
-              EnterScope(field);
+              EnterScope(field, field->position());
               field->initializer()->AcceptExpressionVisitor(this);
-              ExitScope();
+              ExitScope(field->end_position());
             }
           }
         }
       } else if (function.IsFactory()) {
         LocalVariable* variable = MakeVariable(
             Symbols::TypeArgumentsParameter(), AbstractType::dynamic_type());
         scope_->InsertParameterAt(pos++, variable);
         result_->type_arguments_variable = variable;
       }
       AddParameters(node, pos);
@@ skipping 97 matching lines @@
   node->VisitChildren(this);
 }
 
 
 void ScopeBuilder::HandleLocalFunction(TreeNode* parent,
                                        FunctionNode* function) {
   LocalScope* saved_function_scope = current_function_scope_;
   FunctionNode* saved_function_node = current_function_node_;
   ScopeBuilder::DepthState saved_depth_state = depth_;
   depth_ = DepthState(depth_.function_ + 1);
-  EnterScope(parent);
+  EnterScope(parent, function->position());
   current_function_scope_ = scope_;
   current_function_node_ = function;
   if (depth_.function_ == 1) {
     FunctionScope function_scope = {function, scope_};
     result_->function_scopes.Add(function_scope);
   }
   AddParameters(function);
   VisitFunctionNode(function);
-  ExitScope();
+  ExitScope(function->end_position());
   depth_ = saved_depth_state;
   current_function_scope_ = saved_function_scope;
   current_function_node_ = saved_function_node;
 }
 
 
 void ScopeBuilder::HandleSpecialLoad(LocalVariable** variable,
                                      const dart::String& symbol) {
   if (current_function_scope_->parent() != NULL) {
     // We are building the scope tree of a closure function and saw [node]. We
@@ skipping 14 matching lines @@
   }
 }
 
 
 void ScopeBuilder::VisitFunctionExpression(FunctionExpression* node) {
   HandleLocalFunction(node, node->function());
 }
 
 
 void ScopeBuilder::VisitLet(Let* node) {
-  EnterScope(node);
+  EnterScope(node, node->position());
   node->VisitChildren(this);
-  ExitScope();
+  ExitScope(node->end_position());
 }
 
 
 void ScopeBuilder::VisitBlock(Block* node) {
-  EnterScope(node);
+  EnterScope(node, node->position());
   node->VisitChildren(this);
-  ExitScope();
+  ExitScope(node->end_position());
 }
 
 
 void ScopeBuilder::VisitVariableDeclaration(VariableDeclaration* node) {
   AddVariable(node);
   node->VisitChildren(this);
 }
 
 
 void ScopeBuilder::VisitFunctionDeclaration(FunctionDeclaration* node) {
@@ skipping 10 matching lines @@
 
 
 void ScopeBuilder::VisitDoStatement(DoStatement* node) {
   ++depth_.loop_;
   node->VisitChildren(this);
   --depth_.loop_;
 }
 
 
 void ScopeBuilder::VisitForStatement(ForStatement* node) {
-  EnterScope(node);
+  EnterScope(node, node->position());
   List<VariableDeclaration>& variables = node->variables();
   for (intptr_t i = 0; i < variables.length(); ++i) {
     VisitVariableDeclaration(variables[i]);
   }
   ++depth_.loop_;
   if (node->condition() != NULL) {
     node->condition()->AcceptExpressionVisitor(this);
   }
   node->body()->AcceptStatementVisitor(this);
   List<Expression>& updates = node->updates();
   for (intptr_t i = 0; i < updates.length(); ++i) {
     updates[i]->AcceptExpressionVisitor(this);
   }
   --depth_.loop_;
-  ExitScope();
+  ExitScope(node->end_position());
 }
 
 
 void ScopeBuilder::VisitForInStatement(ForInStatement* node) {
   node->iterable()->AcceptExpressionVisitor(this);
   ++depth_.for_in_;
   AddIteratorVariable();
   ++depth_.loop_;
-  EnterScope(node);
+  EnterScope(node, node->position());
   VisitVariableDeclaration(node->variable());
   node->body()->AcceptStatementVisitor(this);
-  ExitScope();
+  ExitScope(node->end_position());
   --depth_.loop_;
   --depth_.for_in_;
 }
 
 
 void ScopeBuilder::AddSwitchVariable() {
   if ((depth_.function_ == 0) && (result_->switch_variable == NULL)) {
     LocalVariable* variable =
         MakeVariable(Symbols::SwitchExpr(), AbstractType::dynamic_type());
     variable->set_is_forced_stack();
@@ skipping 25 matching lines @@
   ++depth_.try_;
   AddTryVariables();
   node->body()->AcceptStatementVisitor(this);
   --depth_.try_;
 
   ++depth_.catch_;
   AddCatchVariables();
   List<Catch>& catches = node->catches();
   for (intptr_t i = 0; i < catches.length(); ++i) {
     Catch* ketch = catches[i];
-    EnterScope(ketch);
+    EnterScope(ketch, ketch->position());
     if (ketch->exception() != NULL) {
       VisitVariableDeclaration(ketch->exception());
     }
     if (ketch->stack_trace() != NULL) {
       VisitVariableDeclaration(ketch->stack_trace());
     }
     ketch->body()->AcceptStatementVisitor(this);
-    ExitScope();
+    ExitScope(ketch->end_position());
   }
   --depth_.catch_;
 }
 
 
 void ScopeBuilder::VisitTryFinally(TryFinally* node) {
   ++depth_.try_;
   ++depth_.finally_;
   AddTryVariables();
   node->body()->AcceptStatementVisitor(this);
@@ skipping 1826 matching lines @@
   PushArgumentInstr* argument = new (Z) PushArgumentInstr(Pop());
   Push(argument);
 
   argument->set_temp_index(argument->temp_index() - 1);
   ++pending_argument_count_;
 
   return Fragment(argument);
 }
 
 
-Fragment FlowGraphBuilder::Return() {
+Fragment FlowGraphBuilder::Return(TokenPosition position) {
   Value* value = Pop();
   ASSERT(stack_ == NULL);
   ReturnInstr* return_instr =
       new (Z) ReturnInstr(TokenPosition::kNoSource, value);
   if (exit_collector_ != NULL) exit_collector_->AddExit(return_instr);
   return Fragment(return_instr).closed();
 }
 
 
 Fragment FlowGraphBuilder::StaticCall(TokenPosition position,
@@ skipping 94 matching lines @@
   Value* value = Pop();
   if (value->BindsToConstant()) {
     emit_store_barrier = kNoStoreBarrier;
   }
   StoreInstanceFieldInstr* store = new (Z) StoreInstanceFieldInstr(
       offset, Pop(), value, emit_store_barrier, TokenPosition::kNoSource);
   return Fragment(store);
 }
 
 
-Fragment FlowGraphBuilder::StoreLocal(LocalVariable* variable) {
+Fragment FlowGraphBuilder::StoreLocal(LocalVariable* variable,
+                                      TokenPosition position) {
   Fragment instructions;
   if (variable->is_captured()) {
     LocalVariable* value = MakeTemporary();
     instructions += LoadContextAt(variable->owner()->context_level());
     instructions += LoadLocal(value);
     instructions +=
         StoreInstanceField(Context::variable_offset(variable->index()));
   } else {
+    Value* value = Pop();
     StoreLocalInstr* store =
-        new (Z) StoreLocalInstr(*variable, Pop(), TokenPosition::kNoSource);
+        new (Z) StoreLocalInstr(*variable, value, position);
     instructions <<= store;
     Push(store);
   }
   return instructions;
 }
 
 
 Fragment FlowGraphBuilder::StoreStaticField(const dart::Field& field) {
   return Fragment(new (Z) StoreStaticFieldInstr(MayCloneField(Z, field), Pop(),
                                                 TokenPosition::kNoSource));
 }
 
 
-Fragment FlowGraphBuilder::StringInterpolate() {
+Fragment FlowGraphBuilder::StringInterpolate(TokenPosition position) {
   Value* array = Pop();
   StringInterpolateInstr* interpolate =
-      new (Z) StringInterpolateInstr(array, TokenPosition::kNoSource);
+      new (Z) StringInterpolateInstr(array, position);
   Push(interpolate);
   return Fragment(interpolate);
 }
 
 
 Fragment FlowGraphBuilder::ThrowTypeError() {
   const dart::Class& klass = dart::Class::ZoneHandle(
       Z, dart::Library::LookupCoreClass(Symbols::TypeError()));
   ASSERT(!klass.IsNull());
   const dart::Function& constructor = dart::Function::ZoneHandle(
@@ skipping 387 matching lines @@
     TargetEntryInstr* non_null_entry;
 
     body += LoadLocal(parameter);
     body += BranchIfNull(&null_entry, &non_null_entry);
 
     // The argument was `null` and the receiver is not the null class (we only
     // go into this branch for user-defined == operators) so we can return
     // false.
     Fragment null_fragment(null_entry);
     null_fragment += Constant(Bool::False());
-    null_fragment += Return();
+    null_fragment += Return(dart_function.end_token_pos());
 
     body = Fragment(body.entry, non_null_entry);
   }
 
   if (dart_function.is_native()) {
     body += NativeFunctionBody(function, dart_function);
   } else if (function->body() != NULL) {
     body += TranslateStatement(function->body());
   }
   if (body.is_open()) {
     body += NullConstant();
-    body += Return();
+    body += Return(dart_function.end_token_pos());
   }
 
   // If functions body contains any yield points build switch statement that
   // selects a continuation point based on the value of :await_jump_var.
   if (!yield_continuations_.is_empty()) {
     // The code we are building will be executed right after we enter
     // the function and before any nested contexts are allocated.
     // Reset current context_depth_ to match this.
     intptr_t current_context_depth = context_depth_;
     context_depth_ = scopes_->yield_jump_variable->owner()->context_level();
@@ skipping 52 matching lines @@
       then->set_try_index(yield_continuations_[i].try_index);
 
       // False branch will contain the next comparison.
       dispatch = Fragment(dispatch.entry, otherwise);
       block = otherwise;
     }
     body = dispatch;
 
     context_depth_ = current_context_depth;
   }
+
   normal_entry->LinkTo(body.entry);
 
   // When compiling for OSR, use a depth first search to prune instructions
   // unreachable from the OSR entry. Catch entries are always considered
   // reachable, even if they become unreachable after OSR.
   if (osr_id_ != Compiler::kNoOSRDeoptId) {
     BitVector* block_marks = new (Z) BitVector(Z, next_block_id_);
     bool found = graph_entry_->PruneUnreachable(graph_entry_, NULL, osr_id_,
                                                 block_marks);
     ASSERT(found);
@@ skipping 334 matching lines @@
       argument_names.SetAt(i, H.DartSymbol(variable->name()));
     }
   }
   // Forward them to the target.
   intptr_t argument_count = positional_argument_count + named_argument_count;
   if (!target.is_static()) ++argument_count;
   body += StaticCall(TokenPosition::kNoSource, target, argument_count,
                      argument_names);
 
   // Return the result.
-  body += Return();
+  body += Return(kernel_function->end_position());
 
   return new (Z) FlowGraph(*parsed_function_, graph_entry_, next_block_id_ - 1);
 }
 
 
 FlowGraph* FlowGraphBuilder::BuildGraphOfNoSuchMethodDispatcher(
     const Function& function) {
   // This function is specialized for a receiver class, a method name, and
   // the arguments descriptor at a call site.
 
@@ skipping 738 matching lines @@
 }
 
 
 void FlowGraphBuilder::VisitVariableGet(VariableGet* node) {
   fragment_ = LoadLocal(LookupVariable(node->variable()));
 }
 
 
 void FlowGraphBuilder::VisitVariableSet(VariableSet* node) {
   Fragment instructions = TranslateExpression(node->expression());
-  instructions += StoreLocal(LookupVariable(node->variable()));
+  instructions +=
+      StoreLocal(LookupVariable(node->variable()), node->position());
   fragment_ = instructions;
 }
 
 
 void FlowGraphBuilder::VisitStaticGet(StaticGet* node) {
   Member* target = node->target();
   if (target->IsField()) {
     Field* kernel_field = Field::Cast(target);
     const dart::Field& field =
         dart::Field::ZoneHandle(Z, H.LookupFieldByKernelField(kernel_field));
@@ skipping 390 matching lines @@
     }
     instructions += PushArgument();  // Type arguments.
 
     instructions += Constant(type);
     instructions += PushArgument();  // Type.
 
     instructions += Constant(Bool::False());
     instructions += PushArgument();  // Negate?.
 
     instructions +=
-        InstanceCall(TokenPosition::kNoSource,
+        InstanceCall(node->position(),
                      dart::Library::PrivateCoreLibName(Symbols::_instanceOf()),
                      Token::kIS, 4);
   }
 
   fragment_ = instructions;
 }
 
 
 void FlowGraphBuilder::VisitAsExpression(AsExpression* node) {
   Fragment instructions = TranslateExpression(node->operand());
@@ skipping 125 matching lines @@
   LocalVariable* array = MakeTemporary();
 
   for (intptr_t i = 0; i < node->expressions().length(); i++) {
     instructions += LoadLocal(array);
     instructions += IntConstant(i);
     instructions += TranslateExpression(node->expressions()[i]);
     instructions += StoreIndexed(kArrayCid);
     instructions += Drop();
   }
 
-  instructions += StringInterpolate();
+  instructions += StringInterpolate(node->position());
 
   fragment_ = instructions;
 }
 
 
 void FlowGraphBuilder::VisitListLiteral(ListLiteral* node) {
   if (node->is_const()) {
     fragment_ = Constant(constant_evaluator_.EvaluateListLiteral(node));
     return;
   }
@@ skipping 176 matching lines @@
                               ? NullConstant()
                               : TranslateExpression(node->expression());
   if (instructions.is_open()) {
     if (inside_try_finally) {
       ASSERT(scopes_->finally_return_variable != NULL);
       instructions += StoreLocal(scopes_->finally_return_variable);
       instructions += Drop();
       instructions += TranslateFinallyFinalizers(NULL, -1);
       if (instructions.is_open()) {
         instructions += LoadLocal(scopes_->finally_return_variable);
-        instructions += Return();
+        instructions += Return(node->position());
       }
     } else {
-      instructions += Return();
+      instructions += Return(node->position());
     }
   } else {
     Pop();
   }
   fragment_ = instructions;
 }
 
 
 void FlowGraphBuilder::VisitExpressionStatement(ExpressionStatement* node) {
   Fragment instructions = TranslateExpression(node->expression());
@@ skipping 12 matching lines @@
   } else {
     if (node->IsConst()) {
       const Instance& constant_value =
           constant_evaluator_.EvaluateExpression(initializer);
       variable->SetConstValue(constant_value);
       instructions += Constant(constant_value);
     } else {
       instructions += TranslateExpression(initializer);
     }
   }
-  instructions += StoreLocal(variable);
+  instructions += StoreLocal(variable, variable->token_pos());
   instructions += Drop();
   fragment_ = instructions;
 }
 
 
 void FlowGraphBuilder::VisitFunctionDeclaration(FunctionDeclaration* node) {
   Fragment instructions = TranslateFunctionNode(node->function(), node);
   instructions += StoreLocal(LookupVariable(node->variable()));
   instructions += Drop();
   fragment_ = instructions;
@@ skipping 749 matching lines @@
 
     continuation = Fragment(continuation.entry, no_error);
   }
 
   fragment_ = continuation;
 }
 
 
 Fragment FlowGraphBuilder::TranslateFunctionNode(FunctionNode* node,
                                                  TreeNode* parent) {
   // The VM has a per-isolate table of functions indexed by the enclosing

[Kevin Millikin (Google), 2017/01/12 13:43:04]

This comment (the part about "we don't have token positions") is wrong.  Update
it or else it will become confusing.

[jensj, 2017/01/13 10:14:44]

Will do.

   // function and token position.  We don't have token positions, so we've
   // simply numbered the immediately-nested functions with respect to the
   // parent.
   Function& function = Function::ZoneHandle(Z);
   for (intptr_t i = 0; i < scopes_->function_scopes.length(); ++i) {
     if (scopes_->function_scopes[i].function != node) continue;
 
+    TokenPosition position = node->position();

[Kevin Millikin (Google), 2017/01/12 13:43:04]

Is there any reason not to just continue numbering them?  Since it doesn't
matter if they're actually token positions or not, it seems simpler than
sometimes using token positions and sometimes a synthetic position based on
numbering them.

[jensj, 2017/01/13 10:14:44]

Numbering them 0, 1, 2 etc is rather weird at it is actually used as that
functions token position, e.g. if you debug that function in Observatory it will
show you the lines between start token position and end token position. So
starting at 0 will show you from the start of the file until whatever end is
set.

+    if (parent->IsFunctionDeclaration()) {
+      position = FunctionDeclaration::Cast(parent)->position();
+    }
+    if (!position.IsReal()) {
+      // Positions has to be unique in regards to the parent.
+      // A non-real at this point is probably -1, we cannot blindly use that
+      // as others might use it too. Create a new dummy non-real TokenPosition.
+      position = TokenPosition(i).ToSynthetic();
+    }
+
     // NOTE: This is not TokenPosition in the general sense!
-    function = I->LookupClosureFunction(parsed_function_->function(),
-                                        TokenPosition(i));
+    function = I->LookupClosureFunction(parsed_function_->function(), position);
     if (function.IsNull()) {
       const dart::String* name;
       if (parent->IsFunctionExpression()) {
         name = &Symbols::AnonymousClosure();
       } else {
         ASSERT(parent->IsFunctionDeclaration());
         name = &H.DartSymbol(
             FunctionDeclaration::Cast(parent)->variable()->name());
       }
       // NOTE: This is not TokenPosition in the general sense!
       function = Function::NewClosureFunction(
-          *name, parsed_function_->function(), TokenPosition(i));
-      function.set_is_debuggable(false);
+          *name, parsed_function_->function(), position);
+      function.set_is_debuggable(node->debuggable());
+      function.set_end_token_pos(node->end_position());
       LocalScope* scope = scopes_->function_scopes[i].scope;
       const ContextScope& context_scope =
           ContextScope::Handle(Z, scope->PreserveOuterScope(context_depth_));
       function.set_context_scope(context_scope);
       function.set_kernel_function(node);
       KernelReader::SetupFunctionParameters(H, T, dart::Class::Handle(Z),
                                             function, node,
                                             false,  // is_method
                                             true);  // is_closure
       // Finalize function type.
@@ skipping 24 matching lines @@
   instructions += LoadLocal(parsed_function_->current_context_var());
   instructions += StoreInstanceField(Closure::context_offset());
 
   return instructions;
 }
 
 
 }  // namespace kernel
 }  // namespace dart
 #endif  // !defined(DART_PRECOMPILED_RUNTIME)