Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(260)

Side by Side Diff: content/common/net/url_fetcher.cc

Issue 8403017: Rename URLFetcher to be URLFetcherImpl, now that we have the content::URLFetcher interface. (Closed) Base URL: svn://chrome-svn/chrome/trunk/src/
Patch Set: '' Created 9 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « content/common/net/url_fetcher.h ('k') | content/common/net/url_fetcher_impl.h » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/common/net/url_fetcher.h"
6
7 #include <set>
8
9 #include "base/bind.h"
10 #include "base/compiler_specific.h"
11 #include "base/file_path.h"
12 #include "base/file_util_proxy.h"
13 #include "base/lazy_instance.h"
14 #include "base/memory/scoped_ptr.h"
15 #include "base/memory/weak_ptr.h"
16 #include "base/message_loop_proxy.h"
17 #include "base/platform_file.h"
18 #include "base/stl_util.h"
19 #include "base/string_util.h"
20 #include "base/threading/thread.h"
21 #include "content/public/common/url_fetcher_delegate.h"
22 #include "content/public/common/url_fetcher_factory.h"
23 #include "googleurl/src/gurl.h"
24 #include "net/base/host_port_pair.h"
25 #include "net/base/io_buffer.h"
26 #include "net/base/load_flags.h"
27 #include "net/base/net_errors.h"
28 #include "net/http/http_request_headers.h"
29 #include "net/http/http_response_headers.h"
30 #include "net/url_request/url_request.h"
31 #include "net/url_request/url_request_context.h"
32 #include "net/url_request/url_request_context_getter.h"
33 #include "net/url_request/url_request_throttler_manager.h"
34
35 static const int kBufferSize = 4096;
36
37 class URLFetcher::Core
38 : public base::RefCountedThreadSafe<URLFetcher::Core>,
39 public net::URLRequest::Delegate {
40 public:
41 // For POST requests, set |content_type| to the MIME type of the content
42 // and set |content| to the data to upload. |flags| are flags to apply to
43 // the load operation--these should be one or more of the LOAD_* flags
44 // defined in net/base/load_flags.h.
45 Core(URLFetcher* fetcher,
46 const GURL& original_url,
47 RequestType request_type,
48 content::URLFetcherDelegate* d);
49
50 // Starts the load. It's important that this not happen in the constructor
51 // because it causes the IO thread to begin AddRef()ing and Release()ing
52 // us. If our caller hasn't had time to fully construct us and take a
53 // reference, the IO thread could interrupt things, run a task, Release()
54 // us, and destroy us, leaving the caller with an already-destroyed object
55 // when construction finishes.
56 void Start();
57
58 // Stops any in-progress load and ensures no callback will happen. It is
59 // safe to call this multiple times.
60 void Stop();
61
62 // Reports that the received content was malformed (i.e. failed parsing
63 // or validation). This makes the throttling logic that does exponential
64 // back-off when servers are having problems treat the current request as
65 // a failure. Your call to this method will be ignored if your request is
66 // already considered a failure based on the HTTP response code or response
67 // headers.
68 void ReceivedContentWasMalformed();
69
70 // Overridden from net::URLRequest::Delegate:
71 virtual void OnResponseStarted(net::URLRequest* request);
72 virtual void OnReadCompleted(net::URLRequest* request, int bytes_read);
73
74 content::URLFetcherDelegate* delegate() const { return delegate_; }
75 static void CancelAll();
76
77 private:
78 friend class base::RefCountedThreadSafe<URLFetcher::Core>;
79
80 class Registry {
81 public:
82 Registry();
83 ~Registry();
84
85 void AddURLFetcherCore(Core* core);
86 void RemoveURLFetcherCore(Core* core);
87
88 void CancelAll();
89
90 int size() const {
91 return fetchers_.size();
92 }
93
94 private:
95 std::set<Core*> fetchers_;
96
97 DISALLOW_COPY_AND_ASSIGN(Registry);
98 };
99
100 // Class TempFileWriter encapsulates all state involved in writing
101 // response bytes to a temporary file. It is only used if
102 // |Core::response_destination_| == TEMP_FILE. Each instance of
103 // TempFileWriter is owned by a URLFetcher::Core, which manages
104 // its lifetime and never transfers ownership. While writing to
105 // a file, all function calls happen on the IO thread.
106 class TempFileWriter {
107 public:
108 TempFileWriter(
109 URLFetcher::Core* core,
110 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy);
111
112 ~TempFileWriter();
113 void CreateTempFile();
114 void DidCreateTempFile(base::PlatformFileError error_code,
115 base::PassPlatformFile file_handle,
116 FilePath file_path);
117
118 // Record |num_bytes_| response bytes in |core_->buffer_| to the file.
119 void WriteBuffer(int num_bytes);
120
121 // Called when a write has been done. Continues writing if there are
122 // any more bytes to write. Otherwise, initiates a read in core_.
123 void ContinueWrite(base::PlatformFileError error_code,
124 int bytes_written);
125
126 // Drop ownership of the file at path |temp_file_|. This class
127 // will not delete it or write to it again.
128 void DisownTempFile();
129
130 // Close the temp file if it is open.
131 void CloseTempFileAndCompleteRequest();
132
133 // Remove the temp file if we we created one.
134 void RemoveTempFile();
135
136 const FilePath& temp_file() const { return temp_file_; }
137 int64 total_bytes_written() { return total_bytes_written_; }
138 base::PlatformFileError error_code() const { return error_code_; }
139
140 private:
141 // Callback which gets the result of closing the temp file.
142 void DidCloseTempFile(base::PlatformFileError error);
143
144 // The URLFetcher::Core which instantiated this class.
145 URLFetcher::Core* core_;
146
147 // The last error encountered on a file operation. base::PLATFORM_FILE_OK
148 // if no error occurred.
149 base::PlatformFileError error_code_;
150
151 // Callbacks are created for use with base::FileUtilProxy.
152 base::WeakPtrFactory<URLFetcher::Core::TempFileWriter> weak_factory_;
153
154 // Message loop on which file operations should happen.
155 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy_;
156
157 // Path to the temporary file. This path is empty when there
158 // is no temp file.
159 FilePath temp_file_;
160
161 // Handle to the temp file.
162 base::PlatformFile temp_file_handle_;
163
164 // We always append to the file. Track the total number of bytes
165 // written, so that writes know the offset to give.
166 int64 total_bytes_written_;
167
168 // How many bytes did the last Write() try to write? Needed so
169 // that if not all the bytes get written on a Write(), we can
170 // call Write() again with the rest.
171 int pending_bytes_;
172
173 // When writing, how many bytes from the buffer have been successfully
174 // written so far?
175 int buffer_offset_;
176 };
177
178 virtual ~Core();
179
180 // Wrapper functions that allow us to ensure actions happen on the right
181 // thread.
182 void StartOnIOThread();
183 void StartURLRequest();
184 void StartURLRequestWhenAppropriate();
185 void CancelURLRequest();
186 void OnCompletedURLRequest(base::TimeDelta backoff_delay);
187 void InformDelegateFetchIsComplete();
188 void NotifyMalformedContent();
189 void RetryOrCompleteUrlFetch();
190
191 // Deletes the request, removes it from the registry, and removes the
192 // destruction observer.
193 void ReleaseRequest();
194
195 // Returns the max value of exponential back-off release time for
196 // |original_url_| and |url_|.
197 base::TimeTicks GetBackoffReleaseTime();
198
199 void CompleteAddingUploadDataChunk(const std::string& data,
200 bool is_last_chunk);
201
202 // Adds a block of data to be uploaded in a POST body. This can only be
203 // called after Start().
204 void AppendChunkToUpload(const std::string& data, bool is_last_chunk);
205
206 // Store the response bytes in |buffer_| in the container indicated by
207 // |response_destination_|. Return true if the write has been
208 // done, and another read can overwrite |buffer_|. If this function
209 // returns false, it will post a task that will read more bytes once the
210 // write is complete.
211 bool WriteBuffer(int num_bytes);
212
213 // Read response bytes from the request.
214 void ReadResponse();
215
216 // Drop ownership of any temp file managed by |temp_file_|.
217 void DisownTempFile();
218
219 URLFetcher* fetcher_; // Corresponding fetcher object
220 GURL original_url_; // The URL we were asked to fetch
221 GURL url_; // The URL we eventually wound up at
222 RequestType request_type_; // What type of request is this?
223 net::URLRequestStatus status_; // Status of the request
224 content::URLFetcherDelegate* delegate_; // Object to notify on completion
225 scoped_refptr<base::MessageLoopProxy> delegate_loop_proxy_;
226 // Message loop proxy of the creating
227 // thread.
228 scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
229 // The message loop proxy for the thread
230 // on which the request IO happens.
231 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy_;
232 // The message loop proxy for the thread
233 // on which file access happens.
234 scoped_ptr<net::URLRequest> request_; // The actual request this wraps
235 int load_flags_; // Flags for the load operation
236 int response_code_; // HTTP status code for the request
237 std::string data_; // Results of the request, when we are
238 // storing the response as a string.
239 scoped_refptr<net::IOBuffer> buffer_;
240 // Read buffer
241 scoped_refptr<net::URLRequestContextGetter> request_context_getter_;
242 // Cookie/cache info for the request
243 net::ResponseCookies cookies_; // Response cookies
244 net::HttpRequestHeaders extra_request_headers_;
245 scoped_refptr<net::HttpResponseHeaders> response_headers_;
246 bool was_fetched_via_proxy_;
247 net::HostPortPair socket_address_;
248
249 std::string upload_content_; // HTTP POST payload
250 std::string upload_content_type_; // MIME type of POST payload
251 std::string referrer_; // HTTP Referer header value
252 bool is_chunked_upload_; // True if using chunked transfer encoding
253
254 // Used to determine how long to wait before making a request or doing a
255 // retry.
256 // Both of them can only be accessed on the IO thread.
257 // We need not only the throttler entry for |original_URL|, but also the one
258 // for |url|. For example, consider the case that URL A redirects to URL B,
259 // for which the server returns a 500 response. In this case, the exponential
260 // back-off release time of URL A won't increase. If we retry without
261 // considering the back-off constraint of URL B, we may send out too many
262 // requests for URL A in a short period of time.
263 scoped_refptr<net::URLRequestThrottlerEntryInterface>
264 original_url_throttler_entry_;
265 scoped_refptr<net::URLRequestThrottlerEntryInterface> url_throttler_entry_;
266
267 // |num_retries_| indicates how many times we've failed to successfully
268 // fetch this URL. Once this value exceeds the maximum number of retries
269 // specified by the owner URLFetcher instance, we'll give up.
270 int num_retries_;
271
272 // True if the URLFetcher has been cancelled.
273 bool was_cancelled_;
274
275 // If writing results to a file, |temp_file_writer_| will manage creation,
276 // writing, and destruction of that file.
277 scoped_ptr<TempFileWriter> temp_file_writer_;
278
279 // Where should responses be saved?
280 ResponseDestinationType response_destination_;
281
282 // If |automatically_retry_on_5xx_| is false, 5xx responses will be
283 // propagated to the observer, if it is true URLFetcher will automatically
284 // re-execute the request, after the back-off delay has expired.
285 // true by default.
286 bool automatically_retry_on_5xx_;
287 // Maximum retries allowed.
288 int max_retries_;
289 // Back-off time delay. 0 by default.
290 base::TimeDelta backoff_delay_;
291
292 static base::LazyInstance<Registry> g_registry;
293
294 friend class URLFetcher;
295 DISALLOW_COPY_AND_ASSIGN(Core);
296 };
297
298 URLFetcher::Core::Registry::Registry() {}
299 URLFetcher::Core::Registry::~Registry() {}
300
301 void URLFetcher::Core::Registry::AddURLFetcherCore(Core* core) {
302 DCHECK(!ContainsKey(fetchers_, core));
303 fetchers_.insert(core);
304 }
305
306 void URLFetcher::Core::Registry::RemoveURLFetcherCore(Core* core) {
307 DCHECK(ContainsKey(fetchers_, core));
308 fetchers_.erase(core);
309 }
310
311 void URLFetcher::Core::Registry::CancelAll() {
312 while (!fetchers_.empty())
313 (*fetchers_.begin())->CancelURLRequest();
314 }
315
316 // static
317 base::LazyInstance<URLFetcher::Core::Registry>
318 URLFetcher::Core::g_registry(base::LINKER_INITIALIZED);
319
320 URLFetcher::Core::TempFileWriter::TempFileWriter(
321 URLFetcher::Core* core,
322 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy)
323 : core_(core),
324 error_code_(base::PLATFORM_FILE_OK),
325 ALLOW_THIS_IN_INITIALIZER_LIST(weak_factory_(this)),
326 file_message_loop_proxy_(file_message_loop_proxy),
327 temp_file_handle_(base::kInvalidPlatformFileValue) {
328 }
329
330 URLFetcher::Core::TempFileWriter::~TempFileWriter() {
331 RemoveTempFile();
332 }
333
334 void URLFetcher::Core::TempFileWriter::CreateTempFile() {
335 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread());
336 CHECK(file_message_loop_proxy_.get());
337 base::FileUtilProxy::CreateTemporary(
338 file_message_loop_proxy_,
339 0, // No additional file flags.
340 base::Bind(&URLFetcher::Core::TempFileWriter::DidCreateTempFile,
341 weak_factory_.GetWeakPtr()));
342 }
343
344 void URLFetcher::Core::TempFileWriter::DidCreateTempFile(
345 base::PlatformFileError error_code,
346 base::PassPlatformFile file_handle,
347 FilePath file_path) {
348 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread());
349
350 if (base::PLATFORM_FILE_OK != error_code) {
351 error_code_ = error_code;
352 RemoveTempFile();
353 core_->delegate_loop_proxy_->PostTask(
354 FROM_HERE, base::Bind(&Core::InformDelegateFetchIsComplete, core_));
355 return;
356 }
357
358 temp_file_ = file_path;
359 temp_file_handle_ = file_handle.ReleaseValue();
360 total_bytes_written_ = 0;
361
362 core_->io_message_loop_proxy_->PostTask(
363 FROM_HERE, base::Bind(&Core::StartURLRequestWhenAppropriate, core_));
364 }
365
366 void URLFetcher::Core::TempFileWriter::WriteBuffer(int num_bytes) {
367 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread());
368
369 // Start writing to the temp file by setting the initial state
370 // of |pending_bytes_| and |buffer_offset_| to indicate that the
371 // entire buffer has not yet been written.
372 pending_bytes_ = num_bytes;
373 buffer_offset_ = 0;
374 ContinueWrite(base::PLATFORM_FILE_OK, 0);
375 }
376
377 void URLFetcher::Core::TempFileWriter::ContinueWrite(
378 base::PlatformFileError error_code,
379 int bytes_written) {
380 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread());
381
382 if (base::PLATFORM_FILE_OK != error_code) {
383 error_code_ = error_code;
384 RemoveTempFile();
385 core_->delegate_loop_proxy_->PostTask(
386 FROM_HERE, base::Bind(&Core::InformDelegateFetchIsComplete, core_));
387 return;
388 }
389
390 total_bytes_written_ += bytes_written;
391 buffer_offset_ += bytes_written;
392 pending_bytes_ -= bytes_written;
393
394 if (pending_bytes_ > 0) {
395 base::FileUtilProxy::Write(
396 file_message_loop_proxy_, temp_file_handle_,
397 total_bytes_written_, // Append to the end
398 (core_->buffer_->data() + buffer_offset_), pending_bytes_,
399 base::Bind(&URLFetcher::Core::TempFileWriter::ContinueWrite,
400 weak_factory_.GetWeakPtr()));
401 } else {
402 // Finished writing core_->buffer_ to the file. Read some more.
403 core_->ReadResponse();
404 }
405 }
406
407 void URLFetcher::Core::TempFileWriter::DisownTempFile() {
408 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread());
409
410 // Disowning is done by the delegate's OnURLFetchComplete method.
411 // The temp file should be closed by the time that method is called.
412 DCHECK(temp_file_handle_ == base::kInvalidPlatformFileValue);
413
414 // Forget about any temp file by reseting the path.
415 temp_file_ = FilePath();
416 }
417
418 void URLFetcher::Core::TempFileWriter::CloseTempFileAndCompleteRequest() {
419 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread());
420
421 if (temp_file_handle_ != base::kInvalidPlatformFileValue) {
422 base::FileUtilProxy::Close(
423 file_message_loop_proxy_, temp_file_handle_,
424 base::Bind(&URLFetcher::Core::TempFileWriter::DidCloseTempFile,
425 weak_factory_.GetWeakPtr()));
426 temp_file_handle_ = base::kInvalidPlatformFileValue;
427 }
428 }
429
430 void URLFetcher::Core::TempFileWriter::DidCloseTempFile(
431 base::PlatformFileError error_code) {
432 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread());
433
434 if (base::PLATFORM_FILE_OK != error_code) {
435 error_code_ = error_code;
436 RemoveTempFile();
437 core_->delegate_loop_proxy_->PostTask(
438 FROM_HERE, base::Bind(&Core::InformDelegateFetchIsComplete, core_));
439 return;
440 }
441
442 // If the file was successfully closed, then the URL request is complete.
443 core_->RetryOrCompleteUrlFetch();
444 }
445
446 void URLFetcher::Core::TempFileWriter::RemoveTempFile() {
447 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread());
448
449 // Close the temp file if it is open.
450 if (temp_file_handle_ != base::kInvalidPlatformFileValue) {
451 base::FileUtilProxy::Close(
452 file_message_loop_proxy_, temp_file_handle_,
453 base::FileUtilProxy::StatusCallback()); // No callback: Ignore errors.
454 temp_file_handle_ = base::kInvalidPlatformFileValue;
455 }
456
457 if (!temp_file_.empty()) {
458 base::FileUtilProxy::Delete(
459 file_message_loop_proxy_, temp_file_,
460 false, // No need to recurse, as the path is to a file.
461 base::FileUtilProxy::StatusCallback()); // No callback: Ignore errors.
462 DisownTempFile();
463 }
464 }
465
466 static content::URLFetcherFactory* g_factory = NULL;
467 static bool g_interception_enabled = false;
468
469 // static
470 content::URLFetcher* content::URLFetcher::Create(
471 const GURL& url,
472 RequestType request_type,
473 content::URLFetcherDelegate* d) {
474 return new ::URLFetcher(url, request_type, d);
475 }
476
477 // static
478 content::URLFetcher* content::URLFetcher::Create(
479 int id,
480 const GURL& url,
481 RequestType request_type,
482 content::URLFetcherDelegate* d) {
483 return g_factory ? g_factory->CreateURLFetcher(id, url, request_type, d) :
484 new ::URLFetcher(url, request_type, d);
485 }
486
487 // static
488 void content::URLFetcher::CancelAll() {
489 ::URLFetcher::CancelAll();
490 }
491
492 // static
493 void content::URLFetcher::SetEnableInterceptionForTests(bool enabled) {
494 g_interception_enabled = enabled;
495 }
496
497
498 URLFetcher::URLFetcher(const GURL& url,
499 RequestType request_type,
500 content::URLFetcherDelegate* d)
501 : ALLOW_THIS_IN_INITIALIZER_LIST(
502 core_(new Core(this, url, request_type, d))) {
503 }
504
505 URLFetcher::~URLFetcher() {
506 core_->Stop();
507 }
508
509 URLFetcher::Core::Core(URLFetcher* fetcher,
510 const GURL& original_url,
511 RequestType request_type,
512 content::URLFetcherDelegate* d)
513 : fetcher_(fetcher),
514 original_url_(original_url),
515 request_type_(request_type),
516 delegate_(d),
517 delegate_loop_proxy_(
518 base::MessageLoopProxy::current()),
519 request_(NULL),
520 load_flags_(net::LOAD_NORMAL),
521 response_code_(RESPONSE_CODE_INVALID),
522 buffer_(new net::IOBuffer(kBufferSize)),
523 was_fetched_via_proxy_(false),
524 is_chunked_upload_(false),
525 num_retries_(0),
526 was_cancelled_(false),
527 response_destination_(STRING),
528 automatically_retry_on_5xx_(true),
529 max_retries_(0) {
530 }
531
532 URLFetcher::Core::~Core() {
533 // |request_| should be NULL. If not, it's unsafe to delete it here since we
534 // may not be on the IO thread.
535 DCHECK(!request_.get());
536 }
537
538 void URLFetcher::Core::Start() {
539 DCHECK(delegate_loop_proxy_);
540 CHECK(request_context_getter_) << "We need an URLRequestContext!";
541 if (io_message_loop_proxy_) {
542 DCHECK_EQ(io_message_loop_proxy_,
543 request_context_getter_->GetIOMessageLoopProxy());
544 } else {
545 io_message_loop_proxy_ = request_context_getter_->GetIOMessageLoopProxy();
546 }
547 CHECK(io_message_loop_proxy_.get()) << "We need an IO message loop proxy";
548
549 io_message_loop_proxy_->PostTask(
550 FROM_HERE, base::Bind(&Core::StartOnIOThread, this));
551 }
552
553 void URLFetcher::Core::StartOnIOThread() {
554 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
555
556 switch (response_destination_) {
557 case STRING:
558 StartURLRequestWhenAppropriate();
559 break;
560
561 case TEMP_FILE:
562 DCHECK(file_message_loop_proxy_.get())
563 << "Need to set the file message loop proxy.";
564
565 temp_file_writer_.reset(
566 new TempFileWriter(this, file_message_loop_proxy_));
567
568 // If the temp file is successfully created,
569 // Core::StartURLRequestWhenAppropriate() will be called.
570 temp_file_writer_->CreateTempFile();
571 break;
572
573 default:
574 NOTREACHED();
575 }
576 }
577
578 void URLFetcher::Core::Stop() {
579 if (delegate_loop_proxy_) { // May be NULL in tests.
580 DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
581 }
582 delegate_ = NULL;
583 fetcher_ = NULL;
584 if (io_message_loop_proxy_.get()) {
585 io_message_loop_proxy_->PostTask(
586 FROM_HERE, base::Bind(&Core::CancelURLRequest, this));
587 }
588 }
589
590 void URLFetcher::Core::ReceivedContentWasMalformed() {
591 DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
592 if (io_message_loop_proxy_.get()) {
593 io_message_loop_proxy_->PostTask(
594 FROM_HERE, base::Bind(&Core::NotifyMalformedContent, this));
595 }
596 }
597
598 void URLFetcher::Core::CancelAll() {
599 g_registry.Get().CancelAll();
600 }
601
602 void URLFetcher::Core::OnResponseStarted(net::URLRequest* request) {
603 DCHECK_EQ(request, request_.get());
604 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
605 if (request_->status().is_success()) {
606 response_code_ = request_->GetResponseCode();
607 response_headers_ = request_->response_headers();
608 socket_address_ = request_->GetSocketAddress();
609 was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
610 }
611
612 ReadResponse();
613 }
614
615 void URLFetcher::Core::CompleteAddingUploadDataChunk(
616 const std::string& content, bool is_last_chunk) {
617 DCHECK(is_chunked_upload_);
618 DCHECK(request_.get());
619 DCHECK(!content.empty());
620 request_->AppendChunkToUpload(content.data(),
621 static_cast<int>(content.length()),
622 is_last_chunk);
623 }
624
625 void URLFetcher::Core::AppendChunkToUpload(const std::string& content,
626 bool is_last_chunk) {
627 DCHECK(delegate_loop_proxy_);
628 CHECK(io_message_loop_proxy_.get());
629 io_message_loop_proxy_->PostTask(
630 FROM_HERE,
631 base::Bind(&Core::CompleteAddingUploadDataChunk, this, content,
632 is_last_chunk));
633 }
634
635 // Return true if the write was done and reading may continue.
636 // Return false if the write is pending, and the next read will
637 // be done later.
638 bool URLFetcher::Core::WriteBuffer(int num_bytes) {
639 bool write_complete = false;
640 switch (response_destination_) {
641 case STRING:
642 data_.append(buffer_->data(), num_bytes);
643 write_complete = true;
644 break;
645
646 case TEMP_FILE:
647 temp_file_writer_->WriteBuffer(num_bytes);
648 // WriteBuffer() sends a request the file thread.
649 // The write is not done yet.
650 write_complete = false;
651 break;
652
653 default:
654 NOTREACHED();
655 }
656 return write_complete;
657 }
658
659 void URLFetcher::Core::OnReadCompleted(net::URLRequest* request,
660 int bytes_read) {
661 DCHECK(request == request_);
662 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
663
664 url_ = request->url();
665 url_throttler_entry_ =
666 net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(url_);
667
668 bool waiting_on_write = false;
669 do {
670 if (!request_->status().is_success() || bytes_read <= 0)
671 break;
672
673 if (!WriteBuffer(bytes_read)) {
674 // If WriteBuffer() returns false, we have a pending write to
675 // wait on before reading further.
676 waiting_on_write = true;
677 break;
678 }
679 } while (request_->Read(buffer_, kBufferSize, &bytes_read));
680
681 const net::URLRequestStatus status = request_->status();
682
683 if (status.is_success())
684 request_->GetResponseCookies(&cookies_);
685
686 // See comments re: HEAD requests in ReadResponse().
687 if ((!status.is_io_pending() && !waiting_on_write) ||
688 (request_type_ == HEAD)) {
689 status_ = status;
690 ReleaseRequest();
691
692 // If a temp file is open, close it.
693 if (temp_file_writer_.get()) {
694 // If the file is open, close it. After closing the file,
695 // RetryOrCompleteUrlFetch() will be called.
696 temp_file_writer_->CloseTempFileAndCompleteRequest();
697 } else {
698 // Otherwise, complete or retry the URL request directly.
699 RetryOrCompleteUrlFetch();
700 }
701 }
702 }
703
704 void URLFetcher::Core::RetryOrCompleteUrlFetch() {
705 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
706 base::TimeDelta backoff_delay;
707
708 // Checks the response from server.
709 if (response_code_ >= 500 ||
710 status_.error() == net::ERR_TEMPORARILY_THROTTLED) {
711 // When encountering a server error, we will send the request again
712 // after backoff time.
713 ++num_retries_;
714
715 // Note that backoff_delay may be 0 because (a) the URLRequestThrottler
716 // code does not necessarily back off on the first error, and (b) it
717 // only backs off on some of the 5xx status codes.
718 base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
719 backoff_delay = backoff_release_time - base::TimeTicks::Now();
720 if (backoff_delay < base::TimeDelta())
721 backoff_delay = base::TimeDelta();
722
723 if (automatically_retry_on_5xx_ &&
724 num_retries_ <= max_retries_) {
725 StartOnIOThread();
726 return;
727 }
728 } else {
729 backoff_delay = base::TimeDelta();
730 }
731 request_context_getter_ = NULL;
732 bool posted = delegate_loop_proxy_->PostTask(
733 FROM_HERE, base::Bind(&Core::OnCompletedURLRequest, this, backoff_delay));
734
735 // If the delegate message loop does not exist any more, then the delegate
736 // should be gone too.
737 DCHECK(posted || !delegate_);
738 }
739
740 void URLFetcher::Core::ReadResponse() {
741 // Some servers may treat HEAD requests as GET requests. To free up the
742 // network connection as soon as possible, signal that the request has
743 // completed immediately, without trying to read any data back (all we care
744 // about is the response code and headers, which we already have).
745 int bytes_read = 0;
746 if (request_->status().is_success() && (request_type_ != HEAD))
747 request_->Read(buffer_, kBufferSize, &bytes_read);
748 OnReadCompleted(request_.get(), bytes_read);
749 }
750
751 void URLFetcher::Core::DisownTempFile() {
752 temp_file_writer_->DisownTempFile();
753 }
754
755 void URLFetcher::Core::StartURLRequest() {
756 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
757
758 if (was_cancelled_) {
759 // Since StartURLRequest() is posted as a *delayed* task, it may
760 // run after the URLFetcher was already stopped.
761 return;
762 }
763
764 CHECK(request_context_getter_);
765 DCHECK(!request_.get());
766
767 g_registry.Get().AddURLFetcherCore(this);
768 request_.reset(new net::URLRequest(original_url_, this));
769 int flags = request_->load_flags() | load_flags_;
770 if (!g_interception_enabled) {
771 flags = flags | net::LOAD_DISABLE_INTERCEPT;
772 }
773 if (is_chunked_upload_)
774 request_->EnableChunkedUpload();
775 request_->set_load_flags(flags);
776 request_->set_context(request_context_getter_->GetURLRequestContext());
777 request_->set_referrer(referrer_);
778
779 switch (request_type_) {
780 case GET:
781 break;
782
783 case POST:
784 DCHECK(!upload_content_.empty() || is_chunked_upload_);
785 DCHECK(!upload_content_type_.empty());
786
787 request_->set_method("POST");
788 extra_request_headers_.SetHeader(net::HttpRequestHeaders::kContentType,
789 upload_content_type_);
790 if (!upload_content_.empty()) {
791 request_->AppendBytesToUpload(
792 upload_content_.data(), static_cast<int>(upload_content_.length()));
793 }
794 break;
795
796 case HEAD:
797 request_->set_method("HEAD");
798 break;
799
800 default:
801 NOTREACHED();
802 }
803
804 if (!extra_request_headers_.IsEmpty())
805 request_->SetExtraRequestHeaders(extra_request_headers_);
806
807 // There might be data left over from a previous request attempt.
808 data_.clear();
809
810 // If we are writing the response to a file, the only caller
811 // of this function should have created it and not written yet.
812 CHECK(!temp_file_writer_.get() ||
813 temp_file_writer_->total_bytes_written() == 0);
814
815 request_->Start();
816 }
817
818 void URLFetcher::Core::StartURLRequestWhenAppropriate() {
819 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
820
821 if (was_cancelled_)
822 return;
823
824 if (original_url_throttler_entry_ == NULL) {
825 original_url_throttler_entry_ =
826 net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(
827 original_url_);
828 }
829
830 int64 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
831 GetBackoffReleaseTime());
832 if (delay == 0) {
833 StartURLRequest();
834 } else {
835 MessageLoop::current()->PostDelayedTask(
836 FROM_HERE, base::Bind(&Core::StartURLRequest, this), delay);
837 }
838 }
839
840 void URLFetcher::Core::CancelURLRequest() {
841 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
842
843 if (request_.get()) {
844 request_->Cancel();
845 ReleaseRequest();
846 }
847 // Release the reference to the request context. There could be multiple
848 // references to URLFetcher::Core at this point so it may take a while to
849 // delete the object, but we cannot delay the destruction of the request
850 // context.
851 request_context_getter_ = NULL;
852 was_cancelled_ = true;
853 temp_file_writer_.reset();
854 }
855
856 void URLFetcher::Core::OnCompletedURLRequest(
857 base::TimeDelta backoff_delay) {
858 DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
859
860 // Save the status and backoff_delay so that delegates can read it.
861 if (delegate_) {
862 backoff_delay_ = backoff_delay;
863 InformDelegateFetchIsComplete();
864 }
865 }
866
867 void URLFetcher::Core::InformDelegateFetchIsComplete() {
868 CHECK(delegate_loop_proxy_->BelongsToCurrentThread());
869 if (delegate_) {
870 delegate_->OnURLFetchComplete(fetcher_);
871 }
872 }
873
874 void URLFetcher::Core::NotifyMalformedContent() {
875 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
876 if (url_throttler_entry_ != NULL) {
877 int status_code = response_code_;
878 if (status_code == RESPONSE_CODE_INVALID) {
879 // The status code will generally be known by the time clients
880 // call the |ReceivedContentWasMalformed()| function (which ends up
881 // calling the current function) but if it's not, we need to assume
882 // the response was successful so that the total failure count
883 // used to calculate exponential back-off goes up.
884 status_code = 200;
885 }
886 url_throttler_entry_->ReceivedContentWasMalformed(status_code);
887 }
888 }
889
890 void URLFetcher::Core::ReleaseRequest() {
891 request_.reset();
892 g_registry.Get().RemoveURLFetcherCore(this);
893 }
894
895 base::TimeTicks URLFetcher::Core::GetBackoffReleaseTime() {
896 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
897 DCHECK(original_url_throttler_entry_ != NULL);
898
899 base::TimeTicks original_url_backoff =
900 original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
901 base::TimeTicks destination_url_backoff;
902 if (url_throttler_entry_ != NULL &&
903 original_url_throttler_entry_ != url_throttler_entry_) {
904 destination_url_backoff =
905 url_throttler_entry_->GetExponentialBackoffReleaseTime();
906 }
907
908 return original_url_backoff > destination_url_backoff ?
909 original_url_backoff : destination_url_backoff;
910 }
911
912 void URLFetcher::SetUploadData(const std::string& upload_content_type,
913 const std::string& upload_content) {
914 DCHECK(!core_->is_chunked_upload_);
915 core_->upload_content_type_ = upload_content_type;
916 core_->upload_content_ = upload_content;
917 }
918
919 void URLFetcher::SetChunkedUpload(const std::string& content_type) {
920 DCHECK(core_->is_chunked_upload_ ||
921 (core_->upload_content_type_.empty() &&
922 core_->upload_content_.empty()));
923 core_->upload_content_type_ = content_type;
924 core_->upload_content_.clear();
925 core_->is_chunked_upload_ = true;
926 }
927
928 void URLFetcher::AppendChunkToUpload(const std::string& data,
929 bool is_last_chunk) {
930 DCHECK(data.length());
931 core_->AppendChunkToUpload(data, is_last_chunk);
932 }
933
934 const std::string& URLFetcher::upload_data() const {
935 return core_->upload_content_;
936 }
937
938 void URLFetcher::SetReferrer(const std::string& referrer) {
939 core_->referrer_ = referrer;
940 }
941
942 void URLFetcher::SetLoadFlags(int load_flags) {
943 core_->load_flags_ = load_flags;
944 }
945
946 int URLFetcher::GetLoadFlags() const {
947 return core_->load_flags_;
948 }
949
950 void URLFetcher::SetExtraRequestHeaders(
951 const std::string& extra_request_headers) {
952 core_->extra_request_headers_.Clear();
953 core_->extra_request_headers_.AddHeadersFromString(extra_request_headers);
954 }
955
956 void URLFetcher::GetExtraRequestHeaders(net::HttpRequestHeaders* headers) {
957 headers->CopyFrom(core_->extra_request_headers_);
958 }
959
960 void URLFetcher::SetRequestContext(
961 net::URLRequestContextGetter* request_context_getter) {
962 DCHECK(!core_->request_context_getter_);
963 core_->request_context_getter_ = request_context_getter;
964 }
965
966 void URLFetcher::SetAutomaticallyRetryOn5xx(bool retry) {
967 core_->automatically_retry_on_5xx_ = retry;
968 }
969
970 void URLFetcher::SetMaxRetries(int max_retries) {
971 core_->max_retries_ = max_retries;
972 }
973
974 int URLFetcher::GetMaxRetries() const {
975 return core_->max_retries_;
976 }
977
978
979 base::TimeDelta URLFetcher::GetBackoffDelay() const {
980 return core_->backoff_delay_;
981 }
982
983 void URLFetcher::SaveResponseToTemporaryFile(
984 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy) {
985 core_->file_message_loop_proxy_ = file_message_loop_proxy;
986 core_->response_destination_ = TEMP_FILE;
987 }
988
989 net::HttpResponseHeaders* URLFetcher::GetResponseHeaders() const {
990 return core_->response_headers_;
991 }
992
993 void URLFetcher::set_response_headers(
994 scoped_refptr<net::HttpResponseHeaders> headers) {
995 core_->response_headers_ = headers;
996 }
997
998 // TODO(panayiotis): socket_address_ is written in the IO thread,
999 // if this is accessed in the UI thread, this could result in a race.
1000 // Same for response_headers_ above and was_fetched_via_proxy_ below.
1001 net::HostPortPair URLFetcher::GetSocketAddress() const {
1002 return core_->socket_address_;
1003 }
1004
1005 bool URLFetcher::WasFetchedViaProxy() const {
1006 return core_->was_fetched_via_proxy_;
1007 }
1008
1009 void URLFetcher::set_was_fetched_via_proxy(bool flag) {
1010 core_->was_fetched_via_proxy_ = flag;
1011 }
1012
1013 void URLFetcher::Start() {
1014 core_->Start();
1015 }
1016
1017 void URLFetcher::StartWithRequestContextGetter(
1018 net::URLRequestContextGetter* request_context_getter) {
1019 SetRequestContext(request_context_getter);
1020 core_->Start();
1021 }
1022
1023 const GURL& URLFetcher::GetOriginalUrl() const {
1024 return core_->original_url_;
1025 }
1026
1027 const GURL& URLFetcher::GetUrl() const {
1028 return core_->url_;
1029 }
1030
1031 const net::URLRequestStatus& URLFetcher::GetStatus() const {
1032 return core_->status_;
1033 }
1034
1035 int URLFetcher::GetResponseCode() const {
1036 return core_->response_code_;
1037 }
1038
1039 const net::ResponseCookies& URLFetcher::GetCookies() const {
1040 return core_->cookies_;
1041 }
1042
1043 bool URLFetcher::FileErrorOccurred(
1044 base::PlatformFileError* out_error_code) const {
1045
1046 // Can't have a file error if no file is being created or written to.
1047 if (!core_->temp_file_writer_.get()) {
1048 return false;
1049 }
1050
1051 base::PlatformFileError error_code = core_->temp_file_writer_->error_code();
1052 if (error_code == base::PLATFORM_FILE_OK)
1053 return false;
1054
1055 *out_error_code = error_code;
1056 return true;
1057 }
1058
1059 void URLFetcher::ReceivedContentWasMalformed() {
1060 core_->ReceivedContentWasMalformed();
1061 }
1062
1063 bool URLFetcher::GetResponseAsString(std::string* out_response_string) const {
1064 if (core_->response_destination_ != STRING)
1065 return false;
1066
1067 *out_response_string = core_->data_;
1068 return true;
1069 }
1070
1071 bool URLFetcher::GetResponseAsFilePath(bool take_ownership,
1072 FilePath* out_response_path) const {
1073 DCHECK(core_->delegate_loop_proxy_->BelongsToCurrentThread());
1074 if (core_->response_destination_ != TEMP_FILE ||
1075 !core_->temp_file_writer_.get())
1076 return false;
1077
1078 *out_response_path = core_->temp_file_writer_->temp_file();
1079
1080 if (take_ownership) {
1081 core_->io_message_loop_proxy_->PostTask(
1082 FROM_HERE, base::Bind(&Core::DisownTempFile, core_.get()));
1083 }
1084 return true;
1085 }
1086
1087 // static
1088 void URLFetcher::CancelAll() {
1089 Core::CancelAll();
1090 }
1091
1092 // static
1093 int URLFetcher::GetNumFetcherCores() {
1094 return Core::g_registry.Get().size();
1095 }
1096
1097 content::URLFetcherDelegate* URLFetcher::delegate() const {
1098 return core_->delegate();
1099 }
1100
1101 // static
1102 content::URLFetcherFactory* URLFetcher::factory() {
1103 return g_factory;
1104 }
1105
1106 // static
1107 void URLFetcher::set_factory(content::URLFetcherFactory* factory) {
1108 g_factory = factory;
1109 }
OLDNEW
« no previous file with comments | « content/common/net/url_fetcher.h ('k') | content/common/net/url_fetcher_impl.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698