Include source in kernel.

runtime/vm/object.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/object.h"
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "vm/assembler.h"
 #include "vm/become.h"
@@ skipping 3443 matching lines @@
 
 
 TokenPosition Class::ComputeEndTokenPos() const {
   // Return the begin token for synthetic classes.
   if (is_synthesized_class() || IsMixinApplication() || IsTopLevel()) {
     return token_pos();
   }
   Zone* zone = Thread::Current()->zone();
   const Script& scr = Script::Handle(zone, script());
   ASSERT(!scr.IsNull());
+
+  if (scr.kind() == RawScript::kKernelTag) {
+    return TokenPosition::kMinSource;
+  }
+
   const TokenStream& tkns = TokenStream::Handle(zone, scr.tokens());
   if (tkns.IsNull()) {
     ASSERT(Dart::snapshot_kind() == Snapshot::kAppAOT);
     return TokenPosition::kNoSource;
   }
   TokenStream::Iterator tkit(zone, tkns, token_pos(),
                              TokenStream::Iterator::kNoNewlines);
   intptr_t level = 0;
   while (tkit.CurrentTokenKind() != Token::kEOS) {
     if (tkit.CurrentTokenKind() == Token::kLBRACE) {
@@ skipping 5199 matching lines @@
 RawString* Script::Source() const {
   String& source = String::Handle(raw_ptr()->source_);
   if (source.IsNull()) {
     return GenerateSource();
   }
   return raw_ptr()->source_;
 }
 
 
 RawString* Script::GenerateSource() const {
+  if (kind() == RawScript::kKernelTag) {
+    // In kernel it's embedded.
+    return raw_ptr()->source_;
+  }
+
   const TokenStream& token_stream = TokenStream::Handle(tokens());
   if (token_stream.IsNull()) {
     ASSERT(Dart::snapshot_kind() == Snapshot::kAppAOT);
     return String::null();
   }
   return token_stream.GenerateSource();
 }
 
 
 void Script::set_compile_time_constants(const Array& value) const {
   StorePointer(&raw_ptr()->compile_time_constants_, value.raw());
 }
 
 
 RawGrowableObjectArray* Script::GenerateLineNumberArray() const {
   Zone* zone = Thread::Current()->zone();
   const GrowableObjectArray& info =
       GrowableObjectArray::Handle(zone, GrowableObjectArray::New());
   const String& source = String::Handle(zone, Source());
   const String& key = Symbols::Empty();
   const Object& line_separator = Object::Handle(zone);
+  Smi& value = Smi::Handle(zone);
+
+  if (kind() == RawScript::kKernelTag) {
+    const Array& line_starts_array = Array::Handle(line_starts());
+    if (line_starts_array.IsNull()) {
+      // Scripts in the AOT snapshot do not have a line starts array.
+      // We cannot really answer, but at least respond with null-entry.

[Kevin Millikin (Google), 2016/12/20 12:03:34]

Don't write that we cannot really answer, we do answer.  Write something less
conversational:

// Scripts in the AOT snapshot do not have a line starts array.
// A well-formed line number array has a leading null.

[jensj, 2016/12/20 13:30:23]

Done.

+      info.Add(line_separator);  // new line.
+      return info.raw();
+    }
+    intptr_t line_count = line_starts_array.Length();
+    ASSERT(line_count > 0);
+
+    for (int i = 0; i < line_count; i++) {
+      info.Add(line_separator);  // new line.

[Kevin Millikin (Google), 2016/12/20 12:03:34]

Capitalize even these end-of-line comments.

[jensj, 2016/12/20 13:30:23]

Done.

+      value = Smi::New(i + 1);
+      info.Add(value);  // line number.
+      value ^= line_starts_array.At(i);
+      info.Add(value);  // token position.
+      value = Smi::New(1);
+      info.Add(value);  // column.
+    }
+    return info.raw();
+  }
+
   const TokenStream& tkns = TokenStream::Handle(zone, tokens());
-  Smi& value = Smi::Handle(zone);
   String& tokenValue = String::Handle(zone);
   ASSERT(!tkns.IsNull());
   TokenStream::Iterator tkit(zone, tkns, TokenPosition::kMinSource,
                              TokenStream::Iterator::kAllTokens);
   int current_line = -1;
   Scanner s(source, key);
   s.Scan();
   bool skippedNewline = false;
   while (tkit.CurrentTokenKind() != Token::kEOS) {
     if (tkit.CurrentTokenKind() == Token::kNEWLINE) {
@@ skipping 111 matching lines @@
   StoreNonPointer(&raw_ptr()->load_timestamp_, value);
 }
 
 
 void Script::set_tokens(const TokenStream& value) const {
   StorePointer(&raw_ptr()->tokens_, value.raw());
 }
 
 
 void Script::Tokenize(const String& private_key, bool use_shared_tokens) const {
+  if (kind() == RawScript::kKernelTag) {
+    return;
+  }
+
   Thread* thread = Thread::Current();
   Zone* zone = thread->zone();
   const TokenStream& tkns = TokenStream::Handle(zone, tokens());
   if (!tkns.IsNull()) {
     // Already tokenized.
     return;
   }
   // Get the source, scan and allocate the token stream.
   VMTagScope tagScope(thread, VMTag::kCompileScannerTagId);
   CSTAT_TIMER_SCOPE(thread, scanner_timer);
@@ skipping 46 matching lines @@
 
       smi ^= line_starts_array.At(midpoint);
       if (smi.Value() > offset) {
         max = midpoint - 1;
       } else {
         min = midpoint;
       }
     }
     *line = min + 1;
     smi ^= line_starts_array.At(min);
-    *column = offset - smi.Value() + 1;
+    if (column != NULL) {
+      *column = offset - smi.Value() + 1;
+    }
     if (token_len != NULL) {
       *token_len = 1;
     }
     return;
   }
 
   const TokenStream& tkns = TokenStream::Handle(zone, tokens());
   if (tkns.IsNull()) {
     ASSERT((Dart::snapshot_kind() == Snapshot::kAppAOT));
     *line = -1;
@@ skipping 38 matching lines @@
     }
   }
 }
 
 
 void Script::TokenRangeAtLine(intptr_t line_number,
                               TokenPosition* first_token_index,
                               TokenPosition* last_token_index) const {
   ASSERT(first_token_index != NULL && last_token_index != NULL);
   ASSERT(line_number > 0);
+
+  if (kind() == RawScript::kKernelTag) {
+    const Array& line_starts_array = Array::Handle(line_starts());
+    if (line_starts_array.IsNull()) {
+      // Scripts in the AOT snapshot do not have a line starts array.
+      *first_token_index = TokenPosition::kNoSource;
+      *last_token_index = TokenPosition::kNoSource;
+      return;
+    }
+    ASSERT(line_starts_array.Length() >= line_number);
+    Smi& value = Smi::Handle();
+    value ^= line_starts_array.At(line_number - 1);
+    *first_token_index = TokenPosition(value.Value());
+    if (line_starts_array.Length() > line_number) {
+      value ^= line_starts_array.At(line_number);
+      *last_token_index = TokenPosition(value.Value() - 1);
+    } else {
+      // We don't currently have this number.
+      // TODO(jensj): Either use the biggest token position seen, or include
+      // the end of the file in the line-starts array.
+      *last_token_index = TokenPosition(value.Value() + 42);

[Kevin Millikin (Google), 2016/12/20 12:03:34]

42 is too cute.  You could just use value.Value().  Since we have the source
string, isn't it better to use the source's length (or does the source string
include things like new line and whitespace encodings that are not counted
properly in the offsets)?

[jensj, 2016/12/20 13:30:23]

I'll change that. I initially wrote this before including the source code,
and apparently forgot that I could fill this in correctly now.
I certainly hope not. I must admit though, that I have just assumed that not to
be the case. The documentation in front_end says "Return the offset from the
beginning of the file to the first character in the syntactic entity."

+    }
+    return;
+  }
+
   Zone* zone = Thread::Current()->zone();
   *first_token_index = TokenPosition::kNoSource;
   *last_token_index = TokenPosition::kNoSource;
   const TokenStream& tkns = TokenStream::Handle(zone, tokens());
   line_number -= line_offset();
   if (line_number < 1) line_number = 1;
   TokenStream::Iterator tkit(zone, tkns, TokenPosition::kMinSource,
                              TokenStream::Iterator::kAllTokens);
   // Scan through the token stream to the required line.
   intptr_t cur_line = 1;
@@ skipping 13833 matching lines @@
   return UserTag::null();
 }
 
 
 const char* UserTag::ToCString() const {
   const String& tag_label = String::Handle(label());
   return tag_label.ToCString();
 }
 
 }  // namespace dart