AU: Manual proxy support

multi_http_fetcher.h

Index: multi_http_fetcher.h
diff --git a/multi_http_fetcher.h b/multi_http_fetcher.h
index 533998b033b74d287943f959d9d30b787f13fb08..8680137c29e4d889139275edd020609cec066c40 100644
--- a/multi_http_fetcher.h
+++ b/multi_http_fetcher.h
@@ -24,9 +24,11 @@ template<typename BaseHttpFetcher>
 class MultiHttpFetcher : public HttpFetcher, public HttpFetcherDelegate {
  public:
   typedef std::vector<std::pair<off_t, off_t> > RangesVect;
+  typedef std::vector<std::tr1::shared_ptr<BaseHttpFetcher> > FetchersVect;
 
-  MultiHttpFetcher()
-      : sent_transfer_complete_(false),
+  MultiHttpFetcher(ProxyResolver* proxy_resolver)
+      : HttpFetcher(proxy_resolver),
+        sent_transfer_complete_(false),
         pending_next_fetcher_(false),
         current_index_(0),
         bytes_received_this_fetcher_(0) {}
@@ -38,9 +40,12 @@ class MultiHttpFetcher : public HttpFetcher, public HttpFetcherDelegate {
     for (typename std::vector<std::tr1::shared_ptr<BaseHttpFetcher>
              >::iterator it = fetchers_.begin(), e = fetchers_.end();
          it != e; ++it) {
-      (*it) = std::tr1::shared_ptr<BaseHttpFetcher>(new BaseHttpFetcher);
+      (*it) = std::tr1::shared_ptr<BaseHttpFetcher>(
+          new BaseHttpFetcher(proxy_resolver_));
       (*it)->set_delegate(this);
+      LOG(INFO) << "list of fetchers: " << (uint64_t)(it->get());

[petkov, 2010/11/19 05:37:15]

do you need this new logging? and below?

[adlr, 2010/11/20 02:52:29]

oops. yeah, when i use c-style casts i'm being quick for debugging... removed

     }
+    LOG(INFO) << "done w/ list";
   }
 
   void SetOffset(off_t offset) {}  // for now, doesn't support this
@@ -134,6 +139,14 @@ class MultiHttpFetcher : public HttpFetcher, public HttpFetcherDelegate {
       (*it)->SetBuildType(is_official);
     }
   }
+  
+  virtual void SetProxies(const std::vector<std::string>& proxies) {

[petkov, 2010/11/19 05:37:15]

Instead of overloading this method, can't you just do SetProxies right before
calling BeginTransfer on the next fetcher? You might want to change SetProxies
to take begin/end iterators or implement set_proxies or switch deque to
vector...

[adlr, 2010/11/20 02:52:29]

We could do that. We would probably also want to do it w/ the other Set*()
methods, tho. Perhaps we should have an HttpFetcher cleanup to revamp all this?
For now I'm inclined to keep this inline w/ the other Set*() methods.
I tried to make SetProxies take iterators, but their type would be a template
type to the function, which makes it not able to be virtual. The nice thing
about having a deque is it handles large proxy lists better (arguably not a
likely case in practice). Just kind feels a little weird to "downgrade" to
vector...

[petkov, 2010/11/22 17:49:00]

Yeah, we should have a cleanup. And we could rename it to HTTPFetcher :-)
That's OK. The difference vs. the other Set* methods is that this one is
implemented in the base class (and also provides a data field in it).
You could also "upgrade" vector to deque :-) Doesn't matter much so feel free to
ignore.

+    for (typename FetchersVect::iterator it = fetchers_.begin(),
+             e = fetchers_.end();
+         it != e; ++it) {

[petkov, 2010/11/19 05:37:15]

move up?

[adlr, 2010/11/20 02:52:29]

Done.

+      (*it)->SetProxies(proxies);
+    }
+  }
 
  private:
   void SendTransferComplete(HttpFetcher* fetcher, bool successful) {
@@ -163,6 +176,12 @@ class MultiHttpFetcher : public HttpFetcher, public HttpFetcherDelegate {
     TEST_AND_RETURN(current_index_ < ranges_.size());
     TEST_AND_RETURN(fetcher == fetchers_[current_index_].get());
     TEST_AND_RETURN(!pending_next_fetcher_);
+    if (fetcher != fetchers_[current_index_].get()) {

[petkov, 2010/11/19 05:37:15]

When is this true given that there's a TEST_AND_RETURN for this above?

[adlr, 2010/11/20 02:52:29]

i think this may have been an issue w/ a merge. removed.

+      LOG(INFO) << "current index: " << current_index_;
+      LOG(WARNING) << "Received bytes from invalid fetcher";
+      LOG(WARNING) << (uint64_t)(fetcher) << " exp " << (uint64_t)(fetchers_[current_index_].get());
+      return;
+    }
     off_t next_size = length;
     if (ranges_[current_index_].second >= 0) {
       next_size = std::min(next_size,
@@ -220,7 +239,7 @@ class MultiHttpFetcher : public HttpFetcher, public HttpFetcherDelegate {
   bool pending_next_fetcher_;
 
   RangesVect ranges_;
-  std::vector<std::tr1::shared_ptr<BaseHttpFetcher> > fetchers_;
+  FetchersVect fetchers_;
 
   RangesVect::size_type current_index_;  // index into ranges_, fetchers_
   off_t bytes_received_this_fetcher_;