Add support for loading scripts from http

runtime/bin/dartutils.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+#include <errno.h>  // NOLINT
+
 #include "bin/dartutils.h"
 
 #include "include/dart_api.h"
 
 #include "platform/assert.h"
 #include "platform/globals.h"
 
 #include "bin/extensions.h"
 #include "bin/directory.h"
 #include "bin/file.h"
 #include "bin/io_buffer.h"
 #include "bin/utils.h"
+#include "bin/socket.h"
 
 namespace dart {
 namespace bin {
 
 const char* DartUtils::original_working_directory = NULL;
 const char* DartUtils::kDartScheme = "dart:";
 const char* DartUtils::kDartExtensionScheme = "dart-ext:";
 const char* DartUtils::kAsyncLibURL = "dart:async";
 const char* DartUtils::kBuiltinLibURL = "dart:builtin";
 const char* DartUtils::kCoreLibURL = "dart:core";
 const char* DartUtils::kIOLibURL = "dart:io";
 const char* DartUtils::kIOLibPatchURL = "dart:io-patch";
 const char* DartUtils::kUriLibURL = "dart:uri";
+const char* DartUtils::kHttpScheme = "http:";
 
 const char* DartUtils::kIdFieldName = "_id";
 
 uint8_t DartUtils::magic_number[] = { 0xf5, 0xf5, 0xdc, 0xdc };
 
 static bool IsWindowsHost() {
 #if defined(TARGET_OS_WINDOWS)
   return true;
 #else  // defined(TARGET_OS_WINDOWS)
   return false;
@@ skipping 95 matching lines @@
 
 
 bool DartUtils::IsDartSchemeURL(const char* url_name) {
   static const intptr_t kDartSchemeLen = strlen(kDartScheme);
   // If the URL starts with "dart:" then it is considered as a special
   // library URL which is handled differently from other URLs.
   return (strncmp(url_name, kDartScheme, kDartSchemeLen) == 0);
 }
 
 
+bool DartUtils::IsHttpSchemeURL(const char* url_name) {
+  static const intptr_t kHttpSchemeLen = strlen(kHttpScheme);
+  return (strncmp(url_name, kHttpScheme, kHttpSchemeLen) == 0);
+}
+
+
 bool DartUtils::IsDartExtensionSchemeURL(const char* url_name) {
   static const intptr_t kDartExtensionSchemeLen = strlen(kDartExtensionScheme);
   // If the URL starts with "dartext:" then it is considered as a special
   // extension library URL which is handled differently from other URLs.
   return
       (strncmp(url_name, kDartExtensionScheme, kDartExtensionSchemeLen) == 0);
 }
 
 
 bool DartUtils::IsDartIOLibURL(const char* url_name) {
@@ skipping 71 matching lines @@
   File* file_stream = reinterpret_cast<File*>(stream);
   bool bytes_written = file_stream->WriteFully(buffer, num_bytes);
   ASSERT(bytes_written);
 }
 
 
 void DartUtils::CloseFile(void* stream) {
   delete reinterpret_cast<File*>(stream);
 }
 
+// Writes string into socket. Spins until socket connects.
+static intptr_t SocketWriteString(intptr_t socket, const char* str,
+                                  intptr_t len) {
+    int r;
+    intptr_t cursor = 0;
+    do {
+      r = Socket::Write(socket, &str[cursor], len);
+      if (r < 0) {
+        if (errno == ENOTCONN) {

[Ivan Posva, 2013/05/24 17:53:43]

Spinning is in general not a good idea. Also there is a Socket::SetBlocking
call, did that not work for you?

Is ENOTCONN supported on Windows?

[Cutch, 2013/05/24 17:59:56]

Socket::CreateConnect sets the socket to be non-blocking before the connect
call. Socket::SetNonBlocking takes effect after the connection. I do call
Socket::SetNonBlocking before the read. Alternatively, I can add a new
Socket::CreateConnectNonBlocking.
There is WSAENOTCONN and my impression from googling for this is that it is
supported. Need to verify that though.

[siva, 2013/05/24 23:23:18]

I agree the notion of spinning seems contrived.

Maybe we should change the socket API to separate Create from
Connect. i.e have a Socket::Create(...) function and a
Socket::Connect(...) function. That way we will have the flexibility of making
the connect a blocking or non blocking operation.

On 2013/05/24 17:59:56, Cutch wrote:

[Cutch, 2013/05/25 01:37:42]

Done.

+          // CreateConnect sets the socket to be non-blocking, we may get here
+          // before the connection has occurred. Spin here.
+          continue;
+        }
+        return cursor;
+      }
+      cursor += r;
+      len -= r;
+    } while (len > 0);
+    return cursor;
+}
+
+static uint8_t* SocketReadUntilEOF(intptr_t socket, intptr_t* response_len) {
+  intptr_t buffer_size = 16 * 1024;

[siva, 2013/05/24 23:23:18]

const intptr_t kbufferSize = 16 * KB;

to be uniform.

[Cutch, 2013/05/25 01:37:42]

Done.

+  uint8_t* buffer = reinterpret_cast<uint8_t*>(malloc(buffer_size));
+  intptr_t buffer_cursor = 0;
+  do {
+    if (buffer_cursor == buffer_size-1) {
+      // Double buffer size.
+      buffer_size *= 2;
+      buffer = reinterpret_cast<uint8_t*>(realloc(buffer, buffer_size));
+    }

[siva, 2013/05/24 23:23:18]

This check for buffer overflow could be at the bottom of the do .. while loop
just after buffer_cursor += bytes_read;

[Cutch, 2013/05/25 01:37:42]

Done.

+    ssize_t bytes_read = read(socket, &buffer[buffer_cursor],
+                              buffer_size-buffer_cursor-1);

[siva, 2013/05/24 23:23:18]

We normally have a space between operators in our sources:
e.g:
buffer_size - buffer_cursor - 1

[Cutch, 2013/05/25 01:37:42]

Done.

+    if (bytes_read < 0) {
+      if (errno == EWOULDBLOCK) {
+        continue;
+      }
+      free(buffer);
+      return NULL;
+    }
+
+    buffer_cursor += bytes_read;

[siva, 2013/05/24 23:23:18]

The increment could be after the if (bytes_read == 0) check.

[Cutch, 2013/05/25 01:37:42]

It can't be because I need to add the zero character to the correct position of
the buffer and set the response_length to the correct value.

+
+    if (bytes_read == 0) {
+      *response_len = buffer_cursor;
+      buffer[buffer_cursor] = '\0';
+      break;
+    }
+  } while (true);
+  return buffer;
+}
+
+static bool HttpGetRequestOkay(const char* response) {
+  static const char* kOkayReply = "HTTP/1.0 200 OK";

[siva, 2013/05/24 23:23:18]

Are we going with HTTP 1.0 and not 1.1

There seems to be some overlap in these messages with the stuff in
vmstats_impl.cc, maybe we could factor some of these out. Not for this CL but in
the future.

[Cutch, 2013/05/25 01:37:42]

Yes, we don't need anything from 1.1.
Agreed that there is overlap with the vmstats code. However, I think the correct
long term strategy is to not implement our own HTTP client and server code and
adapt the Dart implementation to be easily called from C or use a small third
party library blessed by Google.

+  static const intptr_t kOkayReplyLen = strlen(kOkayReply);
+  return (strncmp(response, kOkayReply, kOkayReplyLen) == 0);
+}
+
+static const uint8_t* HttpRequestGetPayload(const char* response) {
+  const char* split = strstr(response, "\r\n\r\n");
+  if (split != NULL) {
+    return reinterpret_cast<const uint8_t*>(split+4);
+  }
+  return NULL;
+}
+
+// TODO(iposva): Allocate from the zone instead of leaking error string
+// here. On the other hand the binary is about the exit anyway.

[siva, 2013/05/24 23:23:18]

Is this comment valid for this use case? The binary doesn't appear to be
exiting.

[Cutch, 2013/05/25 01:37:42]

I believe what Ivan was saying is that the binary will be exiting almost
immediately after this happens (because the initial source or an imported source
file could not be loaded.).

+#define SET_ERROR_MSG(error_msg, format, ...)                                  \
+  intptr_t len = snprintf(NULL, 0, format, __VA_ARGS__);                       \
+  char *msg = reinterpret_cast<char*>(malloc(len + 1));                        \
+  snprintf(msg, len + 1, format, __VA_ARGS__);                                 \
+  *error_msg = msg
+
+static const uint8_t* HttpGetRequest(const char* domain, const char* path,
+                                     int port, intptr_t* response_len,
+                                     const char** error_msg) {
+  OSError* error = NULL;
+  SocketAddresses* addresses = Socket::LookupAddress(domain,
+                                                     -1,
+                                                     &error);
+  if (addresses == NULL || addresses->count() == 0) {
+    SET_ERROR_MSG(error_msg, "Unable to resolve %s", domain);
+    return NULL;
+  }
+  int preferred_address = 0;
+  for (int i = 0; i < addresses->count(); i++) {
+    SocketAddress* address = addresses->GetAt(i);
+    if (address->GetType() == SocketAddress::ADDRESS_LOOPBACK_IP_V4) {
+      preferred_address = i;
+      break;
+    }
+  }
+  intptr_t tcp_client = Socket::CreateConnect(
+    addresses->GetAt(preferred_address)->addr(),
+    port);
+  if (tcp_client < 0) {
+    SET_ERROR_MSG(error_msg, "Unable to connect to %s:%d", domain, port);
+    return NULL;
+  }
+  // Send get request.
+  {
+    const char* format =
+        "GET %s HTTP/1.0\r\nUser-Agent: Dart VM\r\nHost: %s\r\n\r\n";
+    intptr_t len = snprintf(NULL, 0, format, path, domain);
+    char* get_request = reinterpret_cast<char*>(malloc(len + 1));
+    snprintf(get_request, len + 1, format, path, domain);
+    intptr_t r = SocketWriteString(tcp_client, get_request, len);
+    free(get_request);
+    if (r != len) {
+      SET_ERROR_MSG(error_msg, "Unable to write to %s:%d", domain, port);
+      Socket::Close(tcp_client);
+      return NULL;
+    }
+  }
+  // Consume response.
+  Socket::SetNonBlocking(tcp_client);
+  uint8_t* response = SocketReadUntilEOF(tcp_client, response_len);
+  // Close socket.
+  Socket::Close(tcp_client);
+  if (response == NULL) {
+    SET_ERROR_MSG(error_msg, "Unable to read from %s:%d", domain, port);
+    return NULL;
+  }
+  if (HttpGetRequestOkay(reinterpret_cast<const char*>(response)) == false) {
+    SET_ERROR_MSG(error_msg, "Invalid HTTP response from %s:%d", domain, port);
+    free(response);
+    return NULL;
+  }
+  return response;
+}
+
+
+Dart_Handle DartUtils::ReadStringFromHttp(const char* script_uri) {
+  Dart_Handle result;
+  Dart_Handle uri = NewString(script_uri);
+  Dart_Handle builtin_lib =
+      Builtin::LoadAndCheckLibrary(Builtin::kBuiltinLibrary);
+  Dart_Handle path = PathFromUri(uri, builtin_lib);
+  Dart_Handle domain = DomainFromUri(uri, builtin_lib);
+  Dart_Handle port = PortFromUri(uri, builtin_lib);

[siva, 2013/05/24 23:23:18]

path, domain and port could be Error objects because Dart_Invoke(...) could have
returned Error.

[Cutch, 2013/05/25 01:37:42]

Good catch. Done.

+  const char* path_str = NULL;
+  const char* domain_str = NULL;
+  int64_t port_int = 0;
+  result = Dart_StringToCString(path, &path_str);
+  if (Dart_IsError(result)) {
+    return result;
+  }
+  result = Dart_StringToCString(domain, &domain_str);
+  if (Dart_IsError(result)) {
+    return result;
+  }
+  if (GetInt64Value(port, &port_int) == false) {
+    return Dart_Error("Invalid port");
+  }
+  const char* error_msg = NULL;
+  intptr_t len;
+  const uint8_t* text_buffer = HttpGetRequest(domain_str, path_str, port_int,
+                                              &len, &error_msg);
+  if (text_buffer == NULL) {
+    return Dart_Error(error_msg);
+  }
+  const uint8_t* payload = HttpRequestGetPayload(
+      reinterpret_cast<const char*>(text_buffer));
+  if (payload == NULL) {
+    return Dart_Error("Invalid HTTP response.");
+  }
+  // Subtract HTTP response from length.
+  len -= (payload-text_buffer);
+  ASSERT(len >= 0);
+  Dart_Handle str = Dart_NewStringFromUTF8(payload, len);
+  return str;
+}
+
 
 static const uint8_t* ReadFileFully(const char* filename,
                                     intptr_t* file_len,
                                     const char** error_msg) {
   void* stream = DartUtils::OpenFile(filename, false);
   if (stream == NULL) {
-    const char* format = "Unable to open file: %s";
-    intptr_t len = snprintf(NULL, 0, format, filename);
-    // TODO(iposva): Allocate from the zone instead of leaking error string
-    // here. On the other hand the binary is about the exit anyway.
-    char* msg = reinterpret_cast<char*>(malloc(len + 1));
-    snprintf(msg, len + 1, format, filename);
-    *error_msg = msg;
+    SET_ERROR_MSG(error_msg, "Unable to open file: %s", filename);
     return NULL;
   }
   *file_len = -1;
   const uint8_t* text_buffer = NULL;
   DartUtils::ReadFile(&text_buffer, file_len, stream);
   if (text_buffer == NULL || *file_len == -1) {
     *error_msg = "Unable to read file contents";
     text_buffer = NULL;
   }
   DartUtils::CloseFile(stream);
@@ skipping 33 matching lines @@
   Dart_Handle dart_args[kNumArgs];
   dart_args[0] = script_uri;
   dart_args[1] = (IsWindowsHost() ? Dart_True() : Dart_False());
   return Dart_Invoke(builtin_lib,
                      NewString("_filePathFromUri"),
                      kNumArgs,
                      dart_args);
 }
 
 
+Dart_Handle DartUtils::PathFromUri(Dart_Handle script_uri,
+                                   Dart_Handle builtin_lib) {
+  const int kNumArgs = 1;
+  Dart_Handle dart_args[kNumArgs];
+  dart_args[0] = script_uri;
+  return Dart_Invoke(builtin_lib,
+                     NewString("_pathFromHttpUri"),
+                     kNumArgs,
+                     dart_args);
+}
+
+
+Dart_Handle DartUtils::DomainFromUri(Dart_Handle script_uri,
+                                     Dart_Handle builtin_lib) {
+  const int kNumArgs = 1;
+  Dart_Handle dart_args[kNumArgs];
+  dart_args[0] = script_uri;
+  return Dart_Invoke(builtin_lib,
+                     NewString("_domainFromHttpUri"),
+                     kNumArgs,
+                     dart_args);
+}
+
+
+Dart_Handle DartUtils::PortFromUri(Dart_Handle script_uri,
+                                   Dart_Handle builtin_lib) {
+  const int kNumArgs = 1;
+  Dart_Handle dart_args[kNumArgs];
+  dart_args[0] = script_uri;
+  return Dart_Invoke(builtin_lib,
+                     NewString("_portFromHttpUri"),
+                     kNumArgs,
+                     dart_args);
+}

[siva, 2013/05/24 23:23:18]

Seems like these three functions could have been factored into just one with an
additional 'const char*' parameter for the function name to invoke.

[Cutch, 2013/05/25 01:37:42]

Done.

+
+
 Dart_Handle DartUtils::ResolveUri(Dart_Handle library_url,
                                   Dart_Handle url,
                                   Dart_Handle builtin_lib) {
   const int kNumArgs = 2;
   Dart_Handle dart_args[kNumArgs];
   dart_args[0] = library_url;
   dart_args[1] = url;
   return Dart_Invoke(
       builtin_lib, NewString("_resolveUri"), kNumArgs, dart_args);
 }
@@ skipping 107 matching lines @@
 }
 
 
 void DartUtils::WriteMagicNumber(File* file) {
   // Write a magic number and version information into the snapshot file.
   bool bytes_written = file->WriteFully(magic_number, sizeof(magic_number));
   ASSERT(bytes_written);
 }
 
 
+Dart_Handle DartUtils::LoadScriptHttp(const char* script_uri,
+                                      Dart_Handle builtin_lib) {
+  Dart_Handle uri = NewString(script_uri);
+  Dart_Handle path = PathFromUri(uri, builtin_lib);
+  Dart_Handle domain = DomainFromUri(uri, builtin_lib);
+  Dart_Handle port = PortFromUri(uri, builtin_lib);
+  const char* path_str = NULL;
+  const char* domain_str = NULL;
+  int64_t port_int = 0;
+  Dart_Handle result;
+  result = Dart_StringToCString(path, &path_str);
+  if (Dart_IsError(result)) {
+    return result;
+  }
+  result = Dart_StringToCString(domain, &domain_str);
+  if (Dart_IsError(result)) {
+    return result;
+  }
+  if (GetInt64Value(port, &port_int) == false) {
+    return Dart_Error("Invalid port");
+  }
+  const char* error_msg = NULL;
+  intptr_t len;
+  const uint8_t* text_buffer;
+  text_buffer = HttpGetRequest(domain_str, path_str, port_int, &len,
+                               &error_msg);
+  if (text_buffer == NULL) {
+    return Dart_Error(error_msg);
+  }
+  const uint8_t* payload = HttpRequestGetPayload(
+      reinterpret_cast<const char*>(text_buffer));
+  if (payload == NULL) {
+    return Dart_Error("Invalid HTTP response.");
+  }
+  // Subtract HTTP response from length.
+  len -= (payload-text_buffer);
+  ASSERT(len >= 0);
+  // At this point we have received a valid HTTP 200 reply and
+  // payload points at the beginning of the script or snapshot.
+  bool is_snapshot = false;
+  payload = SniffForMagicNumber(payload, &len, &is_snapshot);
+  if (is_snapshot) {
+    return Dart_LoadScriptFromSnapshot(payload, len);
+  } else {
+    Dart_Handle source = Dart_NewStringFromUTF8(payload, len);
+    if (Dart_IsError(source)) {
+      return source;
+    }
+    return Dart_LoadScript(uri, source, 0, 0);
+  }
+}
+
+
 Dart_Handle DartUtils::LoadScript(const char* script_uri,
                                   Dart_Handle builtin_lib) {
+  if (DartUtils::IsHttpSchemeURL(script_uri)) {
+    return LoadScriptHttp(script_uri, builtin_lib);
+  }
   Dart_Handle resolved_script_uri;
   resolved_script_uri = ResolveScriptUri(NewString(script_uri), builtin_lib);
   if (Dart_IsError(resolved_script_uri)) {
     return resolved_script_uri;
   }
   Dart_Handle script_path = DartUtils::FilePathFromUri(resolved_script_uri,
                                                        builtin_lib);
   if (Dart_IsError(script_path)) {
     return script_path;
   }
@@ skipping 19 matching lines @@
     return Dart_LoadScript(resolved_script_uri, source, 0, 0);
   }
 }
 
 
 Dart_Handle DartUtils::LoadSource(CommandLineOptions* url_mapping,
                                   Dart_Handle library,
                                   Dart_Handle url,
                                   Dart_LibraryTag tag,
                                   const char* url_string) {
+  bool is_http_scheme_url = DartUtils::IsHttpSchemeURL(url_string);
   if (url_mapping != NULL && IsDartSchemeURL(url_string)) {
     const char* mapped_url_string = MapLibraryUrl(url_mapping, url_string);
     if (mapped_url_string == NULL) {
       return Dart_Error("Do not know how to load %s", url_string);
     }
     // We have a URL mapping specified, just read the file that the
     // URL mapping specifies and load it.
     url_string = mapped_url_string;
   }
-  // The tag is either an import or a source tag.
-  // Read the file and load it according to the specified tag.
-  Dart_Handle source = DartUtils::ReadStringFromFile(url_string);
+  Dart_Handle source;
+  if (is_http_scheme_url) {
+    // Read the file over http.
+    source = DartUtils::ReadStringFromHttp(url_string);
+  } else {
+    // Read the file.
+    source = DartUtils::ReadStringFromFile(url_string);
+  }
   if (Dart_IsError(source)) {
     return source;  // source contains the error string.
   }
+  // The tag is either an import or a source tag.
+  // Load it according to the specified tag.
   if (tag == kImportTag) {
     // Return library object or an error string.
     return Dart_LoadLibrary(url, source);
   } else if (tag == kSourceTag) {
     return Dart_LoadSource(library, url, source);
   }
   return Dart_Error("wrong tag");
 }
 
 
@@ skipping 312 matching lines @@
       new CObjectString(CObject::NewString(os_error->message()));
   CObjectArray* result = new CObjectArray(CObject::NewArray(3));
   result->SetAt(0, new CObjectInt32(CObject::NewInt32(kOSError)));
   result->SetAt(1, new CObjectInt32(CObject::NewInt32(os_error->code())));
   result->SetAt(2, error_message);
   return result;
 }
 
 }  // namespace bin
 }  // namespace dart