Add support for HTTP proxy in the HTTP client

runtime/bin/http_impl.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 class _HttpHeaders implements HttpHeaders {
   _HttpHeaders() : _headers = new Map<String, List<String>>();
 
   List<String> operator[](String name) {
     name = name.toLowerCase();
     return _headers[name];
@@ skipping 828 matching lines @@
     List<String> headerValues = headers["cookie"];
     if (headerValues != null) {
       headerValues.forEach((headerValue) => _parseCookieString(headerValue));
     }
     return _cookies;
   }
 
   InputStream get inputStream {
     if (_inputStream == null) {
       _inputStream = new _HttpInputStream(this);
+      _inputStream._streamMarkedClosed = _dataEndCalled;
     }
     return _inputStream;
   }
 
   String get protocolVersion => _protocolVersion;
 
   void _onRequestStart(String method, String uri, String version) {
     _method = method;
     _uri = uri;
     _parseRequestUri(uri);
   }
 
   void _onHeaderReceived(String name, String value) {
     _headers.add(name, value);
   }
 
   void _onHeadersComplete() {
     _headers._mutable = false;
     // Prepare for receiving data.
     _buffer = new _BufferList();
   }
 
   void _onDataReceived(List<int> data) {
     _buffer.add(data);
     if (_inputStream != null) _inputStream._dataReceived();
   }
 
   void _onDataEnd() {
     if (_inputStream != null) _inputStream._closeReceived();
+    _dataEndCalled = true;
   }
 
   // Escaped characters in uri are expected to have been parsed.
   void _parseRequestUri(String uri) {
     int position;
     position = uri.indexOf("?", 0);
     if (position == -1) {
       _path = _HttpUtils.decodeUrlEncodedString(_uri);
       _queryString = null;
       _queryParameters = new Map();
@@ skipping 22 matching lines @@
     _streamErrorHandler = callback;
   }
 
   String _method;
   String _uri;
   String _path;
   String _queryString;
   Map<String, String> _queryParameters;
   _HttpInputStream _inputStream;
   _BufferList _buffer;
+  bool _dataEndCalled = false;
   Function _streamErrorHandler;
 }
 
 
 // HTTP response object for sending a HTTP response.
 class _HttpResponse extends _HttpRequestResponseBase implements HttpResponse {
   _HttpResponse(_HttpConnection httpConnection)
       : super(httpConnection),
         _statusCode = HttpStatus.OK;
 
@@ skipping 607 matching lines @@
   Set<_HttpConnection> _connections;  // Set of currently connected clients.
   List<_RequestHandlerRegistration> _handlers;
   Object _defaultHandler;
   Function _onError;
 }
 
 
 class _HttpClientRequest
     extends _HttpRequestResponseBase implements HttpClientRequest {
   _HttpClientRequest(String this._method,
-                     String this._uri,
+                     Uri this._uri,
                      _HttpClientConnection connection)
       : super(connection) {
     _connection = connection;
     // Default GET and HEAD requests to have no content.
     if (_method == "GET" || _method == "HEAD") {
       _contentLength = 0;
     }
   }
 
   void set contentLength(int contentLength) {
@@ skipping 48 matching lines @@
     _streamErrorHandler = callback;
   }
 
   void _writeHeader() {
     List<int> data;
 
     // Write request line.
     data = _method.toString().charCodes();
     _httpConnection._write(data);
     _writeSP();
-    data = _uri.toString().charCodes();
+    // Send the path for direct connections and the whole URL for
+    // proxy connections.
+    if (!_connection._usingProxy) {
+      String path;
+      if (_uri.query != "") {
+        if (_uri.fragment != "") {
+          path = "${_uri.path}?${_uri.query}#${_uri.fragment}";
+        } else {
+          path = "${_uri.path}?${_uri.query}";
+        }
+      } else {
+         path = _uri.path;
+      }
+      data = path.charCodes();
+    } else {
+      data = _uri.toString().charCodes();
+    }
     _httpConnection._write(data);
     _writeSP();
     _httpConnection._write(_Const.HTTP11);
     _writeCRLF();
 
     // Determine the value of the "Transfer-Encoding" header based on
     // whether the content length is known. If there is no content
     // neither "Content-Length" nor "Transfer-Encoding" is set
     if (_contentLength > 0) {
       _headers.set(HttpHeaders.CONTENT_LENGTH, _contentLength.toString());
@@ skipping 12 matching lines @@
       }
       _headers.add("cookie", sb.toString());
     }
 
     // Write headers.
     _writeHeaders();
     _state = HEADER_SENT;
   }
 
   String _method;
-  String _uri;
+  Uri _uri;
   _HttpClientConnection _connection;
   _HttpOutputStream _outputStream;
   Function _streamErrorHandler;
 }
 
 
 class _HttpClientResponse
     extends _HttpRequestResponseBase implements HttpClientResponse {
   _HttpClientResponse(_HttpClientConnection connection)
       : super(connection) {
@@ skipping 18 matching lines @@
       values.forEach((value) {
         _cookies.add(new Cookie.fromSetCookieValue(value));
       });
     }
     return _cookies;
   }
 
   InputStream get inputStream {
     if (_inputStream == null) {
       _inputStream = new _HttpInputStream(this);
+      _inputStream._streamMarkedClosed = _dataEndCalled;
     }
     return _inputStream;
   }
 
   void _onRequestStart(String method, String uri, String version) {
     // TODO(sgjesse): Error handling
   }
 
   void _onResponseStart(int statusCode, String reasonPhrase, String version) {
     _statusCode = statusCode;
@@ skipping 41 matching lines @@
   }
 
   void _onDataReceived(List<int> data) {
     _buffer.add(data);
     if (_inputStream != null) _inputStream._dataReceived();
   }
 
   void _onDataEnd() {
     _connection._responseDone();
     if (_inputStream != null) _inputStream._closeReceived();
+    _dataEndCalled = true;
   }
 
   // Delegate functions for the HttpInputStream implementation.
   int _streamAvailable() {
     return _buffer.length;
   }
 
   List<int> _streamRead(int bytesToRead) {
     return _buffer.readBytes(bytesToRead);
   }
 
   int _streamReadInto(List<int> buffer, int offset, int len) {
     List<int> data = _buffer.readBytes(len);
     buffer.setRange(offset, data.length, data);
     return data.length;
   }
 
   void _streamSetErrorHandler(callback(e)) {
     _streamErrorHandler = callback;
   }
 
   int _statusCode;
   String _reasonPhrase;
 
   _HttpClientConnection _connection;
   _HttpInputStream _inputStream;
   _BufferList _buffer;
+  bool _dataEndCalled = false;
+
   Function _streamErrorHandler;
 }
 
 
 class _HttpClientConnection
     extends _HttpConnectionBase implements HttpClientConnection {
   _HttpClientConnection(_HttpClient this._client);
 
   void _connectionEstablished(_SocketConnection socketConn) {
     super._connectionEstablished(socketConn._socket);
@@ skipping 17 matching lines @@
       if (_socket != null) {
         _socket.close();
       }
     } else {
       _client._returnSocketConnection(_socketConn);
     }
     _socket = null;
     _socketConn = null;
   }
 
-  HttpClientRequest open(String method, String uri) {
+  HttpClientRequest open(String method, Uri uri) {
     _method = method;
     // Tell the HTTP parser the method it is expecting a response to.
     _httpParser.responseToMethod = method;
     _request = new _HttpClientRequest(method, uri, this);
     _response = new _HttpClientResponse(this);
     return _request;
   }
 
   DetachedSocket detachSocket() {
     return _detachSocket();
@@ skipping 82 matching lines @@
 
   Function _onRequest;
   Function _onResponse;
   Function _onErrorCallback;
 
   _HttpClient _client;
   _SocketConnection _socketConn;
   HttpClientRequest _request;
   HttpClientResponse _response;
   String _method;
+  bool _usingProxy;
 
   // Redirect handling
   bool followRedirects = true;
   int maxRedirects = 5;
   List<_RedirectInfo> _redirects;
 
   // Callbacks.
   var requestReceived;
 }
 
@@ skipping 15 matching lines @@
   Duration _idleTime(Date now) => now.difference(_returnTime);
 
   int hashCode() => _socket.hashCode();
 
   String _host;
   int _port;
   Socket _socket;
   Date _returnTime;
 }
 
+class _ProxyConfiguration {
+  static const String PROXY_PREFIX = "PROXY ";
+  static const String DIRECT_PREFIX = "DIRECT";
+
+      _ProxyConfiguration(String configuration) : proxies = new List<_Proxy>() {
+    if (configuration == null) {
+      throw new HttpException("Invalid proxy configuration $configuration");
+    }
+    List<String> list = configuration.split(";");
+    list.forEach((String proxy) {
+      proxy = proxy.trim();
+      if (!proxy.isEmpty()) {
+        if (proxy.startsWith(PROXY_PREFIX)) {
+          int colon = proxy.indexOf(":");
+          if (colon == -1 || colon == 0 || colon == proxy.length - 1) {
+            throw new HttpException(
+                "Invalid proxy configuration $configuration");
+          }
+          // Skip the "PROXY " prefix.
+          String host = proxy.substring(PROXY_PREFIX.length, colon).trim();
+          int port = int.parse(proxy.substring(colon + 1).trim());

[Anders Johnsen, 2012/10/08 11:05:25]

Try around parse (or the hole method)?

[Søren Gjesse, 2012/10/08 12:17:30]

Done.

+          proxies.add(new _Proxy(host, port));
+        } else if (proxy.trim() == DIRECT_PREFIX) {
+          proxies.add(new _Proxy.direct());
+        } else {
+          throw new HttpException("Invalid proxy configuration $configuration");
+        }
+      }
+    });
+  }
+  const _ProxyConfiguration.direct()
+      : proxies = [const _Proxy.direct()];
+
+  final List<_Proxy> proxies;
+}
+
+class _Proxy {
+  const _Proxy(this.host, this.port) : direct = false;
+  const _Proxy.direct() : host = null, port = null, direct = true;
+
+  final String host;
+  final int port;
+  final bool direct;
+}
 
 class _HttpClient implements HttpClient {
   static const int DEFAULT_EVICTION_TIMEOUT = 60000;
 
   _HttpClient() : _openSockets = new Map(),
                   _activeSockets = new Set(),
                   _shutdown = false;
 
   HttpClientConnection open(
       String method, String host, int port, String path) {
     // TODO(sgjesse): The path set here can contain both query and
     // fragment. They should be cracked and set correctly.
     return _open(method, new Uri.fromComponents(
         scheme: "http", domain: host, port: port, path: path));
   }
 
   HttpClientConnection _open(String method,
                              Uri uri,
                              [_HttpClientConnection connection]) {
     if (_shutdown) throw new HttpException("HttpClient shutdown");
     if (method == null || uri.domain.isEmpty() == null) {
       throw new ArgumentError(null);
     }
     return _prepareHttpClientConnection(method, uri, connection);
   }
 
   HttpClientConnection openUrl(String method, Uri url) {
-    _openUrl(method, url);
+    return _openUrl(method, url);

[Anders Johnsen, 2012/10/08 11:05:25]

Nice catch :)

   }
 
   HttpClientConnection _openUrl(String method,
                                 Uri url,
                                 [_HttpClientConnection connection]) {
     if (url.scheme != "http") {
       throw new HttpException("Unsupported URL scheme ${url.scheme}");
     }
     if (url.userInfo != "") {
       throw new HttpException("Unsupported user info ${url.userInfo}");
     }
     return _open(method, url, connection);
   }
 
   HttpClientConnection get(String host, int port, String path) {
     return open("GET", host, port, path);
   }
 
   HttpClientConnection getUrl(Uri url) => _openUrl("GET", url);
 
   HttpClientConnection post(String host, int port, String path) {
     return open("POST", host, port, path);
   }
 
   HttpClientConnection postUrl(Uri url) => _openUrl("POST", url);
 
+  set findProxy(String f(Uri uri)) => _findProxy = f;
+
   void shutdown() {
      _openSockets.forEach((String key, Queue<_SocketConnection> connections) {
        while (!connections.isEmpty()) {
          _SocketConnection socketConn = connections.removeFirst();
          socketConn._socket.close();
        }
      });
      _activeSockets.forEach((_SocketConnection socketConn) {
        socketConn._socket.close();
      });
@@ skipping 12 matching lines @@
 
   HttpClientConnection _prepareHttpClientConnection(
     String method,
     Uri url,
     [_HttpClientConnection connection]) {
 
     String host = url.domain;
     int port = url.port == 0 ? HttpClient.DEFAULT_HTTP_PORT : url.port;
 
     void _connectionOpened(_SocketConnection socketConn,
-                           _HttpClientConnection connection) {
+                           _HttpClientConnection connection,
+                           bool usingProxy) {
+      connection._usingProxy = usingProxy;
       connection._connectionEstablished(socketConn);
-      String path;
-      if (url.query != "") {
-        if (url.fragment != "") {
-          path = "${url.path}?${url.query}#${url.fragment}";
-        } else {
-          path = "${url.path}?${url.query}";
-        }
-      } else {
-        path = url.path;
-      }
-      HttpClientRequest request = connection.open(method, path);
+      HttpClientRequest request = connection.open(method, url);
       request.headers.host = host;
       request.headers.port = port;
       if (connection._onRequest != null) {
         connection._onRequest(request);
       } else {
         request.outputStream.close();
       }
     }
 
     // Create a new connection if we are not re-using an existing one.
     if (connection == null) {
       connection = new _HttpClientConnection(this);
     }
     connection.onDetach = () => _activeSockets.remove(connection._socketConn);
 
+    // Check to see if a proxy server should be used for this connection.
+    _ProxyConfiguration proxyConfiguration = const _ProxyConfiguration.direct();
+    if (_findProxy != null) {
+      // TODO(sgjesse): Keep a map of these as normally only a few

[Anders Johnsen, 2012/10/08 11:05:25]

That's only valid if it won't change over time. Is that a fair assumption?

[Søren Gjesse, 2012/10/08 12:17:30]

In most cases only a few different strings will be returned from findProxy. What
I am suggesting is to have a map from these strings to their _ProxyConfiguration
decoding.

[Anders Johnsen, 2012/10/08 12:27:29]

Ahh okay!

+      // configuration strings will be used.
+      proxyConfiguration = new _ProxyConfiguration(_findProxy(url));
+    }
+
+    // Determine the actual host to connect to.
+    String connectHost;
+    int connectPort;
+    if (proxyConfiguration.proxies[0].direct) {

[Anders Johnsen, 2012/10/08 11:05:25]

proxyConfiguration.proxies[0] is used quite a bit. Can you extract it early and
reuse throughout?

[Søren Gjesse, 2012/10/08 12:17:30]

Done.

+      connectHost = host;
+      connectPort = port;
+    } else {
+      connectHost = proxyConfiguration.proxies[0].host;
+      connectPort = proxyConfiguration.proxies[0].port;
+    }
+
     // If there are active connections for this key get the first one
     // otherwise create a new one.
-    String key = _connectionKey(host, port);
+    String key = _connectionKey(connectHost, connectPort);
     Queue socketConnections = _openSockets[key];
     if (socketConnections == null || socketConnections.isEmpty()) {
-      Socket socket = new Socket(host, port);
+      Socket socket = new Socket(connectHost, connectPort);
       // Until the connection is established handle connection errors
       // here as the HttpClientConnection object is not yet associated
       // with the socket.
       socket.onError = (e) {
         // Report the error through the HttpClientConnection object to
         // the client.
         connection._onError(e);
       };
       socket.onConnect = () {
         // When the connection is established, clear the error
         // callback as it will now be handled by the
         // HttpClientConnection object which will be associated with
         // the connected socket.
         socket.onError = null;
         _SocketConnection socketConn =
-            new _SocketConnection(host, port, socket);
+            new _SocketConnection(connectHost, connectPort, socket);
         _activeSockets.add(socketConn);
-        _connectionOpened(socketConn, connection);
+        _connectionOpened(socketConn,
+                          connection,
+                          !proxyConfiguration.proxies[0].direct);
       };
     } else {
       _SocketConnection socketConn = socketConnections.removeFirst();
       _activeSockets.add(socketConn);
-      new Timer(0, (ignored) => _connectionOpened(socketConn, connection));
+      new Timer(0, (ignored) =>
+                _connectionOpened(socketConn,
+                                  connection,
+                                  !proxyConfiguration.proxies[0].direct));
 
       // Get rid of eviction timer if there are no more active connections.
       if (socketConnections.isEmpty()) _openSockets.remove(key);
       if (_openSockets.isEmpty()) _cancelEvictionTimer();
     }
 
     return connection;
   }
 
   void _returnSocketConnection(_SocketConnection socketConn) {
@@ skipping 77 matching lines @@
 
 
 class _RedirectInfo implements RedirectInfo {
   const _RedirectInfo(int this.statusCode,
                       String this.method,
                       Uri this.location);
   final int statusCode;
   final String method;
   final Uri location;
 }