Move tar archive parsing to C++

runtime/lib/vmservice.cc

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/bootstrap_natives.h"
-
 #include "vm/dart_api_impl.h"
 #include "vm/exceptions.h"
+#include "vm/growable_array.h"
 #include "vm/message.h"
 #include "vm/native_entry.h"
 #include "vm/object.h"
 #include "vm/port.h"
 #include "vm/service_isolate.h"
 #include "vm/symbols.h"
 
 namespace dart {
 
 DECLARE_FLAG(bool, trace_service);
@@ skipping 125 matching lines @@
   GET_NON_NULL_NATIVE_ARGUMENT(String, stream_id, arguments->NativeArgAt(0));
   Service::CancelStream(stream_id.ToCString());
   return Object::null();
 }
 
 
 DEFINE_NATIVE_ENTRY(VMService_RequestAssets, 0) {
   return Service::RequestAssets();
 }
 
+
+class ByteStream {

[Ivan Posva, 2015/11/25 17:09:04]

How does this differ from class ReadStream in datastream.h?

[zra, 2015/11/25 19:06:15]

There isn't a big difference, and I changed to use ReadStream.

+ public:
+  ByteStream(uint8_t* bytes, intptr_t bytes_length) :
+      bytes_(bytes), bytes_length_(bytes_length), cursor_(0) {}
+
+  intptr_t length() const { return bytes_length_; }
+
+  intptr_t remaining() const { return length() - cursor_; }
+
+  intptr_t cursor() const { return cursor_; }
+
+  uint8_t* current() const {
+    return bytes_ + cursor_;
+  }
+
+  uint8_t PeekByte(intptr_t index = 0) const {
+    return bytes_[cursor_ + index];
+  }
+
+  uint8_t ReadByte() {
+    uint8_t r = PeekByte();
+    advance(1);
+    return r;
+  }
+
+  intptr_t ReadBytes(uint8_t* bytes, intptr_t num_bytes) {
+    uint8_t* src = bytes_ + cursor_;
+    num_bytes = num_bytes > remaining() ? remaining() : num_bytes;
+    memmove(bytes, src, num_bytes);
+    advance(num_bytes);
+    return num_bytes;
+  }
+
+  void Skip(intptr_t bytes) {
+    advance(bytes);
+  }
+
+  void SeekToNextBlock(intptr_t blockSize) {
+    intptr_t remainder = blockSize - (cursor_ % blockSize);
+    advance(remainder);
+  }
+
+  void SetCursor(intptr_t cursor) {
+    cursor_ = cursor;
+    if (cursor_ > length()) {
+      cursor_ = length();
+    }
+  }
+
+ private:
+  void advance(intptr_t bytes) {
+    cursor_ += bytes;
+    if (cursor_ > length()) {
+      cursor_ = length();
+    }
+  }
+
+  uint8_t* bytes_;
+  intptr_t bytes_length_;
+  intptr_t cursor_;
+
+  DISALLOW_COPY_AND_ASSIGN(ByteStream);
+};
+
+
+class TarArchive {
+ public:
+  static const intptr_t tarHeaderSize = 512;

[Ivan Posva, 2015/11/25 17:09:04]

enum instead of a set of constants? Then you can also type the return value of
the ReadType() method.

Do the constants have to be public?

[zra, 2015/11/25 19:06:15]

Changed to private enums.

+  static const intptr_t tarHeaderFilenameSize = 100;
+  static const intptr_t tarHeaderFilenameOffset = 0;
+  static const intptr_t tarHeaderSizeSize = 12;
+  static const intptr_t tarHeaderSizeOffset = 124;
+  static const intptr_t tarHeaderTypeSize = 1;
+  static const intptr_t tarHeaderTypeOffset = 156;
+  static const intptr_t tarHeaderFileType = 0x30;
+
+  TarArchive(uint8_t* bytes, intptr_t bytes_length)
+      : bs_(bytes, bytes_length) {}
+
+  void Read() {
+    while (HasNext()) {
+      char* filename;
+      uint8_t* data;
+      intptr_t data_length;
+      if (Next(&filename, &data, &data_length)) {
+        filenames_.Add(filename);
+        contents_.Add(data);
+        content_lengths_.Add(data_length);
+      }
+    }
+  }
+
+  char* NextFilename() {
+    return filenames_.RemoveLast();
+  }
+
+  uint8_t* NextContent() {
+    return contents_.RemoveLast();
+  }
+
+  intptr_t NextContentLength() {
+    return content_lengths_.RemoveLast();
+  }
+
+  bool HasMore() const {
+    return filenames_.length() > 0;
+  }
+
+  intptr_t Length() const { return filenames_.length(); }
+
+ private:
+  bool HasNext() const {
+    return !EndOfArchive();
+  }
+
+  bool Next(char** filename, uint8_t** data, intptr_t* data_length) {
+    intptr_t startOfBlock = bs_.cursor();
+    *filename = ReadFilename();
+    bs_.SetCursor(startOfBlock + tarHeaderSizeOffset);
+    intptr_t size = ReadSize();
+    bs_.SetCursor(startOfBlock + tarHeaderTypeOffset);
+    uint8_t type = ReadType();
+    bs_.SeekToNextBlock(tarHeaderSize);
+    if (type != tarHeaderFileType) {
+      SkipContents(size);
+      return false;
+    }
+    ReadContents(data, size);
+    *data_length = size;
+    return true;
+  }
+
+  bool EndOfArchive() const {
+    if (bs_.remaining() < (tarHeaderSize * 2)) {
+      return true;
+    }
+    for (intptr_t i = 0; i < (tarHeaderSize * 2); i++) {
+      if (bs_.PeekByte(i) != 0) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  uint8_t ReadType() {
+    return bs_.ReadByte();
+  }
+
+  void SkipContents(intptr_t size) {
+    bs_.Skip(size);
+    bs_.SeekToNextBlock(tarHeaderSize);
+  }
+
+  intptr_t ReadCString(char** s, intptr_t length) {
+    intptr_t to_read = Utils::Minimum(length, bs_.remaining());
+    char* result = new char[to_read + 1];
+    strncpy(result, reinterpret_cast<char*>(bs_.current()), to_read);
+    result[to_read] = '\0';
+    bs_.SetCursor(bs_.cursor() + to_read);
+    *s = result;
+    return to_read;
+  }
+
+  intptr_t ReadSize() {
+    char* octalSize;
+    unsigned int size;
+
+    ReadCString(&octalSize, tarHeaderSizeSize);
+    int result = sscanf(octalSize, "%o", &size);
+    delete[] octalSize;
+
+    if (result != 1) {
+      return 0;
+    }
+    return size;
+  }
+
+  char* ReadFilename() {
+    char* result;
+    intptr_t result_length = ReadCString(&result, tarHeaderFilenameSize);
+    if (result[0] == '/') {
+      return result;
+    }
+    char* fixed_result = new char[result_length + 2];  // '/' + '\0'.
+    fixed_result[0] = '/';
+    strncpy(&fixed_result[1], result, result_length);
+    fixed_result[result_length + 1] = '\0';
+    delete[] result;
+    return fixed_result;
+  }
+
+  void ReadContents(uint8_t** data, intptr_t size) {
+    uint8_t* result = new uint8_t[size];
+    bs_.ReadBytes(result, size);
+    bs_.SeekToNextBlock(tarHeaderSize);
+    *data = result;
+  }
+
+  ByteStream bs_;
+  GrowableArray<char*> filenames_;
+  GrowableArray<uint8_t*> contents_;
+  GrowableArray<intptr_t> content_lengths_;
+
+  DISALLOW_COPY_AND_ASSIGN(TarArchive);
+};
+
+static void ContentsFinalizer(void* isolate_callback_data,
+                              Dart_WeakPersistentHandle handle,
+                              void* peer) {
+  uint8_t* data = reinterpret_cast<uint8_t*>(peer);

[Ivan Posva, 2015/11/25 17:36:44]

You also should delete the handle here. No?

[zra, 2015/11/25 19:06:15]

Looking at //runtime/bin/io_buffer.h the Finalizer there does not delete the
WeakPersistentHandle.

[zra, 2015/11/25 20:52:58]

We verified offline that WeakPersistentHandles are deleted by the VM after
calling the finalizer.

+  delete[] data;
+}
+
+DEFINE_NATIVE_ENTRY(VMService_DecodeAssets, 1) {
+  GET_NON_NULL_NATIVE_ARGUMENT(TypedData, data, arguments->NativeArgAt(0));
+  Api::Scope scope(thread);
+
+  Dart_Handle data_handle = Api::NewHandle(thread->isolate(), data.raw());
+
+  Dart_TypedData_Type typ;
+  void* bytes;
+  intptr_t length;
+  Dart_Handle err = Dart_TypedDataAcquireData(
+      data_handle, &typ, &bytes, &length);

[Ivan Posva, 2015/11/25 17:36:44]

As discussed we should check if there is a possibility to just keep the raw
pointer that was read from file (or is in the .text section) and make all
external arrays/strings point into that instead reducing the memory overhead
even further. That would mean that we have to guarantee that the assets data is
not removed from memory.

[zra, 2015/11/25 19:06:16]

Filed issue #25041 and added a TODO.

+  ASSERT(!Dart_IsError(err));
+
+  TarArchive archive(reinterpret_cast<uint8_t*>(bytes), length);
+  archive.Read();
+
+  err = Dart_TypedDataReleaseData(data_handle);
+  ASSERT(!Dart_IsError(err));
+
+  intptr_t archive_size = archive.Length();
+
+  const Array& result_list = Array::Handle(thread->zone(),
+    Array::New(archive_size));
+
+  intptr_t idx = 0;
+  while (archive.HasMore()) {
+    char* filename = archive.NextFilename();
+    uint8_t* contents = archive.NextContent();
+    intptr_t contents_length = archive.NextContentLength();
+
+    const Array& pair = Array::Handle(thread->zone(), Array::New(2));
+
+    const String& dart_filename = String::Handle(thread->zone(),
+        String::New(filename));
+    delete[] filename;
+
+    Dart_Handle dart_contents = Dart_NewExternalTypedData(
+        Dart_TypedData_kUint8, contents, contents_length);
+    ASSERT(!Dart_IsError(dart_contents));
+    Dart_NewWeakPersistentHandle(
+        dart_contents, contents, contents_length, ContentsFinalizer);
+
+    pair.SetAt(0, dart_filename);
+    pair.SetAt(1, Api::UnwrapExternalTypedDataHandle(
+      thread->zone(), dart_contents));
+    result_list.SetAt(idx, pair);
+    idx++;
+  }
+
+  return result_list.raw();
+}
+
 }  // namespace dart