OLD | NEW |
| (Empty) |
1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "content/common/net/url_fetcher.h" | |
6 | |
7 #include <set> | |
8 | |
9 #include "base/bind.h" | |
10 #include "base/compiler_specific.h" | |
11 #include "base/file_path.h" | |
12 #include "base/file_util_proxy.h" | |
13 #include "base/lazy_instance.h" | |
14 #include "base/memory/scoped_ptr.h" | |
15 #include "base/memory/weak_ptr.h" | |
16 #include "base/message_loop_proxy.h" | |
17 #include "base/platform_file.h" | |
18 #include "base/stl_util.h" | |
19 #include "base/string_util.h" | |
20 #include "base/threading/thread.h" | |
21 #include "content/public/common/url_fetcher_delegate.h" | |
22 #include "content/public/common/url_fetcher_factory.h" | |
23 #include "googleurl/src/gurl.h" | |
24 #include "net/base/host_port_pair.h" | |
25 #include "net/base/io_buffer.h" | |
26 #include "net/base/load_flags.h" | |
27 #include "net/base/net_errors.h" | |
28 #include "net/http/http_request_headers.h" | |
29 #include "net/http/http_response_headers.h" | |
30 #include "net/url_request/url_request.h" | |
31 #include "net/url_request/url_request_context.h" | |
32 #include "net/url_request/url_request_context_getter.h" | |
33 #include "net/url_request/url_request_throttler_manager.h" | |
34 | |
35 static const int kBufferSize = 4096; | |
36 | |
37 class URLFetcher::Core | |
38 : public base::RefCountedThreadSafe<URLFetcher::Core>, | |
39 public net::URLRequest::Delegate { | |
40 public: | |
41 // For POST requests, set |content_type| to the MIME type of the content | |
42 // and set |content| to the data to upload. |flags| are flags to apply to | |
43 // the load operation--these should be one or more of the LOAD_* flags | |
44 // defined in net/base/load_flags.h. | |
45 Core(URLFetcher* fetcher, | |
46 const GURL& original_url, | |
47 RequestType request_type, | |
48 content::URLFetcherDelegate* d); | |
49 | |
50 // Starts the load. It's important that this not happen in the constructor | |
51 // because it causes the IO thread to begin AddRef()ing and Release()ing | |
52 // us. If our caller hasn't had time to fully construct us and take a | |
53 // reference, the IO thread could interrupt things, run a task, Release() | |
54 // us, and destroy us, leaving the caller with an already-destroyed object | |
55 // when construction finishes. | |
56 void Start(); | |
57 | |
58 // Stops any in-progress load and ensures no callback will happen. It is | |
59 // safe to call this multiple times. | |
60 void Stop(); | |
61 | |
62 // Reports that the received content was malformed (i.e. failed parsing | |
63 // or validation). This makes the throttling logic that does exponential | |
64 // back-off when servers are having problems treat the current request as | |
65 // a failure. Your call to this method will be ignored if your request is | |
66 // already considered a failure based on the HTTP response code or response | |
67 // headers. | |
68 void ReceivedContentWasMalformed(); | |
69 | |
70 // Overridden from net::URLRequest::Delegate: | |
71 virtual void OnResponseStarted(net::URLRequest* request); | |
72 virtual void OnReadCompleted(net::URLRequest* request, int bytes_read); | |
73 | |
74 content::URLFetcherDelegate* delegate() const { return delegate_; } | |
75 static void CancelAll(); | |
76 | |
77 private: | |
78 friend class base::RefCountedThreadSafe<URLFetcher::Core>; | |
79 | |
80 class Registry { | |
81 public: | |
82 Registry(); | |
83 ~Registry(); | |
84 | |
85 void AddURLFetcherCore(Core* core); | |
86 void RemoveURLFetcherCore(Core* core); | |
87 | |
88 void CancelAll(); | |
89 | |
90 int size() const { | |
91 return fetchers_.size(); | |
92 } | |
93 | |
94 private: | |
95 std::set<Core*> fetchers_; | |
96 | |
97 DISALLOW_COPY_AND_ASSIGN(Registry); | |
98 }; | |
99 | |
100 // Class TempFileWriter encapsulates all state involved in writing | |
101 // response bytes to a temporary file. It is only used if | |
102 // |Core::response_destination_| == TEMP_FILE. Each instance of | |
103 // TempFileWriter is owned by a URLFetcher::Core, which manages | |
104 // its lifetime and never transfers ownership. While writing to | |
105 // a file, all function calls happen on the IO thread. | |
106 class TempFileWriter { | |
107 public: | |
108 TempFileWriter( | |
109 URLFetcher::Core* core, | |
110 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy); | |
111 | |
112 ~TempFileWriter(); | |
113 void CreateTempFile(); | |
114 void DidCreateTempFile(base::PlatformFileError error_code, | |
115 base::PassPlatformFile file_handle, | |
116 FilePath file_path); | |
117 | |
118 // Record |num_bytes_| response bytes in |core_->buffer_| to the file. | |
119 void WriteBuffer(int num_bytes); | |
120 | |
121 // Called when a write has been done. Continues writing if there are | |
122 // any more bytes to write. Otherwise, initiates a read in core_. | |
123 void ContinueWrite(base::PlatformFileError error_code, | |
124 int bytes_written); | |
125 | |
126 // Drop ownership of the file at path |temp_file_|. This class | |
127 // will not delete it or write to it again. | |
128 void DisownTempFile(); | |
129 | |
130 // Close the temp file if it is open. | |
131 void CloseTempFileAndCompleteRequest(); | |
132 | |
133 // Remove the temp file if we we created one. | |
134 void RemoveTempFile(); | |
135 | |
136 const FilePath& temp_file() const { return temp_file_; } | |
137 int64 total_bytes_written() { return total_bytes_written_; } | |
138 base::PlatformFileError error_code() const { return error_code_; } | |
139 | |
140 private: | |
141 // Callback which gets the result of closing the temp file. | |
142 void DidCloseTempFile(base::PlatformFileError error); | |
143 | |
144 // The URLFetcher::Core which instantiated this class. | |
145 URLFetcher::Core* core_; | |
146 | |
147 // The last error encountered on a file operation. base::PLATFORM_FILE_OK | |
148 // if no error occurred. | |
149 base::PlatformFileError error_code_; | |
150 | |
151 // Callbacks are created for use with base::FileUtilProxy. | |
152 base::WeakPtrFactory<URLFetcher::Core::TempFileWriter> weak_factory_; | |
153 | |
154 // Message loop on which file operations should happen. | |
155 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy_; | |
156 | |
157 // Path to the temporary file. This path is empty when there | |
158 // is no temp file. | |
159 FilePath temp_file_; | |
160 | |
161 // Handle to the temp file. | |
162 base::PlatformFile temp_file_handle_; | |
163 | |
164 // We always append to the file. Track the total number of bytes | |
165 // written, so that writes know the offset to give. | |
166 int64 total_bytes_written_; | |
167 | |
168 // How many bytes did the last Write() try to write? Needed so | |
169 // that if not all the bytes get written on a Write(), we can | |
170 // call Write() again with the rest. | |
171 int pending_bytes_; | |
172 | |
173 // When writing, how many bytes from the buffer have been successfully | |
174 // written so far? | |
175 int buffer_offset_; | |
176 }; | |
177 | |
178 virtual ~Core(); | |
179 | |
180 // Wrapper functions that allow us to ensure actions happen on the right | |
181 // thread. | |
182 void StartOnIOThread(); | |
183 void StartURLRequest(); | |
184 void StartURLRequestWhenAppropriate(); | |
185 void CancelURLRequest(); | |
186 void OnCompletedURLRequest(base::TimeDelta backoff_delay); | |
187 void InformDelegateFetchIsComplete(); | |
188 void NotifyMalformedContent(); | |
189 void RetryOrCompleteUrlFetch(); | |
190 | |
191 // Deletes the request, removes it from the registry, and removes the | |
192 // destruction observer. | |
193 void ReleaseRequest(); | |
194 | |
195 // Returns the max value of exponential back-off release time for | |
196 // |original_url_| and |url_|. | |
197 base::TimeTicks GetBackoffReleaseTime(); | |
198 | |
199 void CompleteAddingUploadDataChunk(const std::string& data, | |
200 bool is_last_chunk); | |
201 | |
202 // Adds a block of data to be uploaded in a POST body. This can only be | |
203 // called after Start(). | |
204 void AppendChunkToUpload(const std::string& data, bool is_last_chunk); | |
205 | |
206 // Store the response bytes in |buffer_| in the container indicated by | |
207 // |response_destination_|. Return true if the write has been | |
208 // done, and another read can overwrite |buffer_|. If this function | |
209 // returns false, it will post a task that will read more bytes once the | |
210 // write is complete. | |
211 bool WriteBuffer(int num_bytes); | |
212 | |
213 // Read response bytes from the request. | |
214 void ReadResponse(); | |
215 | |
216 // Drop ownership of any temp file managed by |temp_file_|. | |
217 void DisownTempFile(); | |
218 | |
219 URLFetcher* fetcher_; // Corresponding fetcher object | |
220 GURL original_url_; // The URL we were asked to fetch | |
221 GURL url_; // The URL we eventually wound up at | |
222 RequestType request_type_; // What type of request is this? | |
223 net::URLRequestStatus status_; // Status of the request | |
224 content::URLFetcherDelegate* delegate_; // Object to notify on completion | |
225 scoped_refptr<base::MessageLoopProxy> delegate_loop_proxy_; | |
226 // Message loop proxy of the creating | |
227 // thread. | |
228 scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_; | |
229 // The message loop proxy for the thread | |
230 // on which the request IO happens. | |
231 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy_; | |
232 // The message loop proxy for the thread | |
233 // on which file access happens. | |
234 scoped_ptr<net::URLRequest> request_; // The actual request this wraps | |
235 int load_flags_; // Flags for the load operation | |
236 int response_code_; // HTTP status code for the request | |
237 std::string data_; // Results of the request, when we are | |
238 // storing the response as a string. | |
239 scoped_refptr<net::IOBuffer> buffer_; | |
240 // Read buffer | |
241 scoped_refptr<net::URLRequestContextGetter> request_context_getter_; | |
242 // Cookie/cache info for the request | |
243 net::ResponseCookies cookies_; // Response cookies | |
244 net::HttpRequestHeaders extra_request_headers_; | |
245 scoped_refptr<net::HttpResponseHeaders> response_headers_; | |
246 bool was_fetched_via_proxy_; | |
247 net::HostPortPair socket_address_; | |
248 | |
249 std::string upload_content_; // HTTP POST payload | |
250 std::string upload_content_type_; // MIME type of POST payload | |
251 std::string referrer_; // HTTP Referer header value | |
252 bool is_chunked_upload_; // True if using chunked transfer encoding | |
253 | |
254 // Used to determine how long to wait before making a request or doing a | |
255 // retry. | |
256 // Both of them can only be accessed on the IO thread. | |
257 // We need not only the throttler entry for |original_URL|, but also the one | |
258 // for |url|. For example, consider the case that URL A redirects to URL B, | |
259 // for which the server returns a 500 response. In this case, the exponential | |
260 // back-off release time of URL A won't increase. If we retry without | |
261 // considering the back-off constraint of URL B, we may send out too many | |
262 // requests for URL A in a short period of time. | |
263 scoped_refptr<net::URLRequestThrottlerEntryInterface> | |
264 original_url_throttler_entry_; | |
265 scoped_refptr<net::URLRequestThrottlerEntryInterface> url_throttler_entry_; | |
266 | |
267 // |num_retries_| indicates how many times we've failed to successfully | |
268 // fetch this URL. Once this value exceeds the maximum number of retries | |
269 // specified by the owner URLFetcher instance, we'll give up. | |
270 int num_retries_; | |
271 | |
272 // True if the URLFetcher has been cancelled. | |
273 bool was_cancelled_; | |
274 | |
275 // If writing results to a file, |temp_file_writer_| will manage creation, | |
276 // writing, and destruction of that file. | |
277 scoped_ptr<TempFileWriter> temp_file_writer_; | |
278 | |
279 // Where should responses be saved? | |
280 ResponseDestinationType response_destination_; | |
281 | |
282 // If |automatically_retry_on_5xx_| is false, 5xx responses will be | |
283 // propagated to the observer, if it is true URLFetcher will automatically | |
284 // re-execute the request, after the back-off delay has expired. | |
285 // true by default. | |
286 bool automatically_retry_on_5xx_; | |
287 // Maximum retries allowed. | |
288 int max_retries_; | |
289 // Back-off time delay. 0 by default. | |
290 base::TimeDelta backoff_delay_; | |
291 | |
292 static base::LazyInstance<Registry> g_registry; | |
293 | |
294 friend class URLFetcher; | |
295 DISALLOW_COPY_AND_ASSIGN(Core); | |
296 }; | |
297 | |
298 URLFetcher::Core::Registry::Registry() {} | |
299 URLFetcher::Core::Registry::~Registry() {} | |
300 | |
301 void URLFetcher::Core::Registry::AddURLFetcherCore(Core* core) { | |
302 DCHECK(!ContainsKey(fetchers_, core)); | |
303 fetchers_.insert(core); | |
304 } | |
305 | |
306 void URLFetcher::Core::Registry::RemoveURLFetcherCore(Core* core) { | |
307 DCHECK(ContainsKey(fetchers_, core)); | |
308 fetchers_.erase(core); | |
309 } | |
310 | |
311 void URLFetcher::Core::Registry::CancelAll() { | |
312 while (!fetchers_.empty()) | |
313 (*fetchers_.begin())->CancelURLRequest(); | |
314 } | |
315 | |
316 // static | |
317 base::LazyInstance<URLFetcher::Core::Registry> | |
318 URLFetcher::Core::g_registry(base::LINKER_INITIALIZED); | |
319 | |
320 URLFetcher::Core::TempFileWriter::TempFileWriter( | |
321 URLFetcher::Core* core, | |
322 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy) | |
323 : core_(core), | |
324 error_code_(base::PLATFORM_FILE_OK), | |
325 ALLOW_THIS_IN_INITIALIZER_LIST(weak_factory_(this)), | |
326 file_message_loop_proxy_(file_message_loop_proxy), | |
327 temp_file_handle_(base::kInvalidPlatformFileValue) { | |
328 } | |
329 | |
330 URLFetcher::Core::TempFileWriter::~TempFileWriter() { | |
331 RemoveTempFile(); | |
332 } | |
333 | |
334 void URLFetcher::Core::TempFileWriter::CreateTempFile() { | |
335 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); | |
336 CHECK(file_message_loop_proxy_.get()); | |
337 base::FileUtilProxy::CreateTemporary( | |
338 file_message_loop_proxy_, | |
339 0, // No additional file flags. | |
340 base::Bind(&URLFetcher::Core::TempFileWriter::DidCreateTempFile, | |
341 weak_factory_.GetWeakPtr())); | |
342 } | |
343 | |
344 void URLFetcher::Core::TempFileWriter::DidCreateTempFile( | |
345 base::PlatformFileError error_code, | |
346 base::PassPlatformFile file_handle, | |
347 FilePath file_path) { | |
348 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); | |
349 | |
350 if (base::PLATFORM_FILE_OK != error_code) { | |
351 error_code_ = error_code; | |
352 RemoveTempFile(); | |
353 core_->delegate_loop_proxy_->PostTask( | |
354 FROM_HERE, base::Bind(&Core::InformDelegateFetchIsComplete, core_)); | |
355 return; | |
356 } | |
357 | |
358 temp_file_ = file_path; | |
359 temp_file_handle_ = file_handle.ReleaseValue(); | |
360 total_bytes_written_ = 0; | |
361 | |
362 core_->io_message_loop_proxy_->PostTask( | |
363 FROM_HERE, base::Bind(&Core::StartURLRequestWhenAppropriate, core_)); | |
364 } | |
365 | |
366 void URLFetcher::Core::TempFileWriter::WriteBuffer(int num_bytes) { | |
367 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); | |
368 | |
369 // Start writing to the temp file by setting the initial state | |
370 // of |pending_bytes_| and |buffer_offset_| to indicate that the | |
371 // entire buffer has not yet been written. | |
372 pending_bytes_ = num_bytes; | |
373 buffer_offset_ = 0; | |
374 ContinueWrite(base::PLATFORM_FILE_OK, 0); | |
375 } | |
376 | |
377 void URLFetcher::Core::TempFileWriter::ContinueWrite( | |
378 base::PlatformFileError error_code, | |
379 int bytes_written) { | |
380 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); | |
381 | |
382 if (base::PLATFORM_FILE_OK != error_code) { | |
383 error_code_ = error_code; | |
384 RemoveTempFile(); | |
385 core_->delegate_loop_proxy_->PostTask( | |
386 FROM_HERE, base::Bind(&Core::InformDelegateFetchIsComplete, core_)); | |
387 return; | |
388 } | |
389 | |
390 total_bytes_written_ += bytes_written; | |
391 buffer_offset_ += bytes_written; | |
392 pending_bytes_ -= bytes_written; | |
393 | |
394 if (pending_bytes_ > 0) { | |
395 base::FileUtilProxy::Write( | |
396 file_message_loop_proxy_, temp_file_handle_, | |
397 total_bytes_written_, // Append to the end | |
398 (core_->buffer_->data() + buffer_offset_), pending_bytes_, | |
399 base::Bind(&URLFetcher::Core::TempFileWriter::ContinueWrite, | |
400 weak_factory_.GetWeakPtr())); | |
401 } else { | |
402 // Finished writing core_->buffer_ to the file. Read some more. | |
403 core_->ReadResponse(); | |
404 } | |
405 } | |
406 | |
407 void URLFetcher::Core::TempFileWriter::DisownTempFile() { | |
408 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); | |
409 | |
410 // Disowning is done by the delegate's OnURLFetchComplete method. | |
411 // The temp file should be closed by the time that method is called. | |
412 DCHECK(temp_file_handle_ == base::kInvalidPlatformFileValue); | |
413 | |
414 // Forget about any temp file by reseting the path. | |
415 temp_file_ = FilePath(); | |
416 } | |
417 | |
418 void URLFetcher::Core::TempFileWriter::CloseTempFileAndCompleteRequest() { | |
419 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); | |
420 | |
421 if (temp_file_handle_ != base::kInvalidPlatformFileValue) { | |
422 base::FileUtilProxy::Close( | |
423 file_message_loop_proxy_, temp_file_handle_, | |
424 base::Bind(&URLFetcher::Core::TempFileWriter::DidCloseTempFile, | |
425 weak_factory_.GetWeakPtr())); | |
426 temp_file_handle_ = base::kInvalidPlatformFileValue; | |
427 } | |
428 } | |
429 | |
430 void URLFetcher::Core::TempFileWriter::DidCloseTempFile( | |
431 base::PlatformFileError error_code) { | |
432 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); | |
433 | |
434 if (base::PLATFORM_FILE_OK != error_code) { | |
435 error_code_ = error_code; | |
436 RemoveTempFile(); | |
437 core_->delegate_loop_proxy_->PostTask( | |
438 FROM_HERE, base::Bind(&Core::InformDelegateFetchIsComplete, core_)); | |
439 return; | |
440 } | |
441 | |
442 // If the file was successfully closed, then the URL request is complete. | |
443 core_->RetryOrCompleteUrlFetch(); | |
444 } | |
445 | |
446 void URLFetcher::Core::TempFileWriter::RemoveTempFile() { | |
447 DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); | |
448 | |
449 // Close the temp file if it is open. | |
450 if (temp_file_handle_ != base::kInvalidPlatformFileValue) { | |
451 base::FileUtilProxy::Close( | |
452 file_message_loop_proxy_, temp_file_handle_, | |
453 base::FileUtilProxy::StatusCallback()); // No callback: Ignore errors. | |
454 temp_file_handle_ = base::kInvalidPlatformFileValue; | |
455 } | |
456 | |
457 if (!temp_file_.empty()) { | |
458 base::FileUtilProxy::Delete( | |
459 file_message_loop_proxy_, temp_file_, | |
460 false, // No need to recurse, as the path is to a file. | |
461 base::FileUtilProxy::StatusCallback()); // No callback: Ignore errors. | |
462 DisownTempFile(); | |
463 } | |
464 } | |
465 | |
466 static content::URLFetcherFactory* g_factory = NULL; | |
467 static bool g_interception_enabled = false; | |
468 | |
469 // static | |
470 content::URLFetcher* content::URLFetcher::Create( | |
471 const GURL& url, | |
472 RequestType request_type, | |
473 content::URLFetcherDelegate* d) { | |
474 return new ::URLFetcher(url, request_type, d); | |
475 } | |
476 | |
477 // static | |
478 content::URLFetcher* content::URLFetcher::Create( | |
479 int id, | |
480 const GURL& url, | |
481 RequestType request_type, | |
482 content::URLFetcherDelegate* d) { | |
483 return g_factory ? g_factory->CreateURLFetcher(id, url, request_type, d) : | |
484 new ::URLFetcher(url, request_type, d); | |
485 } | |
486 | |
487 // static | |
488 void content::URLFetcher::CancelAll() { | |
489 ::URLFetcher::CancelAll(); | |
490 } | |
491 | |
492 // static | |
493 void content::URLFetcher::SetEnableInterceptionForTests(bool enabled) { | |
494 g_interception_enabled = enabled; | |
495 } | |
496 | |
497 | |
498 URLFetcher::URLFetcher(const GURL& url, | |
499 RequestType request_type, | |
500 content::URLFetcherDelegate* d) | |
501 : ALLOW_THIS_IN_INITIALIZER_LIST( | |
502 core_(new Core(this, url, request_type, d))) { | |
503 } | |
504 | |
505 URLFetcher::~URLFetcher() { | |
506 core_->Stop(); | |
507 } | |
508 | |
509 URLFetcher::Core::Core(URLFetcher* fetcher, | |
510 const GURL& original_url, | |
511 RequestType request_type, | |
512 content::URLFetcherDelegate* d) | |
513 : fetcher_(fetcher), | |
514 original_url_(original_url), | |
515 request_type_(request_type), | |
516 delegate_(d), | |
517 delegate_loop_proxy_( | |
518 base::MessageLoopProxy::current()), | |
519 request_(NULL), | |
520 load_flags_(net::LOAD_NORMAL), | |
521 response_code_(RESPONSE_CODE_INVALID), | |
522 buffer_(new net::IOBuffer(kBufferSize)), | |
523 was_fetched_via_proxy_(false), | |
524 is_chunked_upload_(false), | |
525 num_retries_(0), | |
526 was_cancelled_(false), | |
527 response_destination_(STRING), | |
528 automatically_retry_on_5xx_(true), | |
529 max_retries_(0) { | |
530 } | |
531 | |
532 URLFetcher::Core::~Core() { | |
533 // |request_| should be NULL. If not, it's unsafe to delete it here since we | |
534 // may not be on the IO thread. | |
535 DCHECK(!request_.get()); | |
536 } | |
537 | |
538 void URLFetcher::Core::Start() { | |
539 DCHECK(delegate_loop_proxy_); | |
540 CHECK(request_context_getter_) << "We need an URLRequestContext!"; | |
541 if (io_message_loop_proxy_) { | |
542 DCHECK_EQ(io_message_loop_proxy_, | |
543 request_context_getter_->GetIOMessageLoopProxy()); | |
544 } else { | |
545 io_message_loop_proxy_ = request_context_getter_->GetIOMessageLoopProxy(); | |
546 } | |
547 CHECK(io_message_loop_proxy_.get()) << "We need an IO message loop proxy"; | |
548 | |
549 io_message_loop_proxy_->PostTask( | |
550 FROM_HERE, base::Bind(&Core::StartOnIOThread, this)); | |
551 } | |
552 | |
553 void URLFetcher::Core::StartOnIOThread() { | |
554 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
555 | |
556 switch (response_destination_) { | |
557 case STRING: | |
558 StartURLRequestWhenAppropriate(); | |
559 break; | |
560 | |
561 case TEMP_FILE: | |
562 DCHECK(file_message_loop_proxy_.get()) | |
563 << "Need to set the file message loop proxy."; | |
564 | |
565 temp_file_writer_.reset( | |
566 new TempFileWriter(this, file_message_loop_proxy_)); | |
567 | |
568 // If the temp file is successfully created, | |
569 // Core::StartURLRequestWhenAppropriate() will be called. | |
570 temp_file_writer_->CreateTempFile(); | |
571 break; | |
572 | |
573 default: | |
574 NOTREACHED(); | |
575 } | |
576 } | |
577 | |
578 void URLFetcher::Core::Stop() { | |
579 if (delegate_loop_proxy_) { // May be NULL in tests. | |
580 DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); | |
581 } | |
582 delegate_ = NULL; | |
583 fetcher_ = NULL; | |
584 if (io_message_loop_proxy_.get()) { | |
585 io_message_loop_proxy_->PostTask( | |
586 FROM_HERE, base::Bind(&Core::CancelURLRequest, this)); | |
587 } | |
588 } | |
589 | |
590 void URLFetcher::Core::ReceivedContentWasMalformed() { | |
591 DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); | |
592 if (io_message_loop_proxy_.get()) { | |
593 io_message_loop_proxy_->PostTask( | |
594 FROM_HERE, base::Bind(&Core::NotifyMalformedContent, this)); | |
595 } | |
596 } | |
597 | |
598 void URLFetcher::Core::CancelAll() { | |
599 g_registry.Get().CancelAll(); | |
600 } | |
601 | |
602 void URLFetcher::Core::OnResponseStarted(net::URLRequest* request) { | |
603 DCHECK_EQ(request, request_.get()); | |
604 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
605 if (request_->status().is_success()) { | |
606 response_code_ = request_->GetResponseCode(); | |
607 response_headers_ = request_->response_headers(); | |
608 socket_address_ = request_->GetSocketAddress(); | |
609 was_fetched_via_proxy_ = request_->was_fetched_via_proxy(); | |
610 } | |
611 | |
612 ReadResponse(); | |
613 } | |
614 | |
615 void URLFetcher::Core::CompleteAddingUploadDataChunk( | |
616 const std::string& content, bool is_last_chunk) { | |
617 DCHECK(is_chunked_upload_); | |
618 DCHECK(request_.get()); | |
619 DCHECK(!content.empty()); | |
620 request_->AppendChunkToUpload(content.data(), | |
621 static_cast<int>(content.length()), | |
622 is_last_chunk); | |
623 } | |
624 | |
625 void URLFetcher::Core::AppendChunkToUpload(const std::string& content, | |
626 bool is_last_chunk) { | |
627 DCHECK(delegate_loop_proxy_); | |
628 CHECK(io_message_loop_proxy_.get()); | |
629 io_message_loop_proxy_->PostTask( | |
630 FROM_HERE, | |
631 base::Bind(&Core::CompleteAddingUploadDataChunk, this, content, | |
632 is_last_chunk)); | |
633 } | |
634 | |
635 // Return true if the write was done and reading may continue. | |
636 // Return false if the write is pending, and the next read will | |
637 // be done later. | |
638 bool URLFetcher::Core::WriteBuffer(int num_bytes) { | |
639 bool write_complete = false; | |
640 switch (response_destination_) { | |
641 case STRING: | |
642 data_.append(buffer_->data(), num_bytes); | |
643 write_complete = true; | |
644 break; | |
645 | |
646 case TEMP_FILE: | |
647 temp_file_writer_->WriteBuffer(num_bytes); | |
648 // WriteBuffer() sends a request the file thread. | |
649 // The write is not done yet. | |
650 write_complete = false; | |
651 break; | |
652 | |
653 default: | |
654 NOTREACHED(); | |
655 } | |
656 return write_complete; | |
657 } | |
658 | |
659 void URLFetcher::Core::OnReadCompleted(net::URLRequest* request, | |
660 int bytes_read) { | |
661 DCHECK(request == request_); | |
662 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
663 | |
664 url_ = request->url(); | |
665 url_throttler_entry_ = | |
666 net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(url_); | |
667 | |
668 bool waiting_on_write = false; | |
669 do { | |
670 if (!request_->status().is_success() || bytes_read <= 0) | |
671 break; | |
672 | |
673 if (!WriteBuffer(bytes_read)) { | |
674 // If WriteBuffer() returns false, we have a pending write to | |
675 // wait on before reading further. | |
676 waiting_on_write = true; | |
677 break; | |
678 } | |
679 } while (request_->Read(buffer_, kBufferSize, &bytes_read)); | |
680 | |
681 const net::URLRequestStatus status = request_->status(); | |
682 | |
683 if (status.is_success()) | |
684 request_->GetResponseCookies(&cookies_); | |
685 | |
686 // See comments re: HEAD requests in ReadResponse(). | |
687 if ((!status.is_io_pending() && !waiting_on_write) || | |
688 (request_type_ == HEAD)) { | |
689 status_ = status; | |
690 ReleaseRequest(); | |
691 | |
692 // If a temp file is open, close it. | |
693 if (temp_file_writer_.get()) { | |
694 // If the file is open, close it. After closing the file, | |
695 // RetryOrCompleteUrlFetch() will be called. | |
696 temp_file_writer_->CloseTempFileAndCompleteRequest(); | |
697 } else { | |
698 // Otherwise, complete or retry the URL request directly. | |
699 RetryOrCompleteUrlFetch(); | |
700 } | |
701 } | |
702 } | |
703 | |
704 void URLFetcher::Core::RetryOrCompleteUrlFetch() { | |
705 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
706 base::TimeDelta backoff_delay; | |
707 | |
708 // Checks the response from server. | |
709 if (response_code_ >= 500 || | |
710 status_.error() == net::ERR_TEMPORARILY_THROTTLED) { | |
711 // When encountering a server error, we will send the request again | |
712 // after backoff time. | |
713 ++num_retries_; | |
714 | |
715 // Note that backoff_delay may be 0 because (a) the URLRequestThrottler | |
716 // code does not necessarily back off on the first error, and (b) it | |
717 // only backs off on some of the 5xx status codes. | |
718 base::TimeTicks backoff_release_time = GetBackoffReleaseTime(); | |
719 backoff_delay = backoff_release_time - base::TimeTicks::Now(); | |
720 if (backoff_delay < base::TimeDelta()) | |
721 backoff_delay = base::TimeDelta(); | |
722 | |
723 if (automatically_retry_on_5xx_ && | |
724 num_retries_ <= max_retries_) { | |
725 StartOnIOThread(); | |
726 return; | |
727 } | |
728 } else { | |
729 backoff_delay = base::TimeDelta(); | |
730 } | |
731 request_context_getter_ = NULL; | |
732 bool posted = delegate_loop_proxy_->PostTask( | |
733 FROM_HERE, base::Bind(&Core::OnCompletedURLRequest, this, backoff_delay)); | |
734 | |
735 // If the delegate message loop does not exist any more, then the delegate | |
736 // should be gone too. | |
737 DCHECK(posted || !delegate_); | |
738 } | |
739 | |
740 void URLFetcher::Core::ReadResponse() { | |
741 // Some servers may treat HEAD requests as GET requests. To free up the | |
742 // network connection as soon as possible, signal that the request has | |
743 // completed immediately, without trying to read any data back (all we care | |
744 // about is the response code and headers, which we already have). | |
745 int bytes_read = 0; | |
746 if (request_->status().is_success() && (request_type_ != HEAD)) | |
747 request_->Read(buffer_, kBufferSize, &bytes_read); | |
748 OnReadCompleted(request_.get(), bytes_read); | |
749 } | |
750 | |
751 void URLFetcher::Core::DisownTempFile() { | |
752 temp_file_writer_->DisownTempFile(); | |
753 } | |
754 | |
755 void URLFetcher::Core::StartURLRequest() { | |
756 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
757 | |
758 if (was_cancelled_) { | |
759 // Since StartURLRequest() is posted as a *delayed* task, it may | |
760 // run after the URLFetcher was already stopped. | |
761 return; | |
762 } | |
763 | |
764 CHECK(request_context_getter_); | |
765 DCHECK(!request_.get()); | |
766 | |
767 g_registry.Get().AddURLFetcherCore(this); | |
768 request_.reset(new net::URLRequest(original_url_, this)); | |
769 int flags = request_->load_flags() | load_flags_; | |
770 if (!g_interception_enabled) { | |
771 flags = flags | net::LOAD_DISABLE_INTERCEPT; | |
772 } | |
773 if (is_chunked_upload_) | |
774 request_->EnableChunkedUpload(); | |
775 request_->set_load_flags(flags); | |
776 request_->set_context(request_context_getter_->GetURLRequestContext()); | |
777 request_->set_referrer(referrer_); | |
778 | |
779 switch (request_type_) { | |
780 case GET: | |
781 break; | |
782 | |
783 case POST: | |
784 DCHECK(!upload_content_.empty() || is_chunked_upload_); | |
785 DCHECK(!upload_content_type_.empty()); | |
786 | |
787 request_->set_method("POST"); | |
788 extra_request_headers_.SetHeader(net::HttpRequestHeaders::kContentType, | |
789 upload_content_type_); | |
790 if (!upload_content_.empty()) { | |
791 request_->AppendBytesToUpload( | |
792 upload_content_.data(), static_cast<int>(upload_content_.length())); | |
793 } | |
794 break; | |
795 | |
796 case HEAD: | |
797 request_->set_method("HEAD"); | |
798 break; | |
799 | |
800 default: | |
801 NOTREACHED(); | |
802 } | |
803 | |
804 if (!extra_request_headers_.IsEmpty()) | |
805 request_->SetExtraRequestHeaders(extra_request_headers_); | |
806 | |
807 // There might be data left over from a previous request attempt. | |
808 data_.clear(); | |
809 | |
810 // If we are writing the response to a file, the only caller | |
811 // of this function should have created it and not written yet. | |
812 CHECK(!temp_file_writer_.get() || | |
813 temp_file_writer_->total_bytes_written() == 0); | |
814 | |
815 request_->Start(); | |
816 } | |
817 | |
818 void URLFetcher::Core::StartURLRequestWhenAppropriate() { | |
819 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
820 | |
821 if (was_cancelled_) | |
822 return; | |
823 | |
824 if (original_url_throttler_entry_ == NULL) { | |
825 original_url_throttler_entry_ = | |
826 net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl( | |
827 original_url_); | |
828 } | |
829 | |
830 int64 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest( | |
831 GetBackoffReleaseTime()); | |
832 if (delay == 0) { | |
833 StartURLRequest(); | |
834 } else { | |
835 MessageLoop::current()->PostDelayedTask( | |
836 FROM_HERE, base::Bind(&Core::StartURLRequest, this), delay); | |
837 } | |
838 } | |
839 | |
840 void URLFetcher::Core::CancelURLRequest() { | |
841 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
842 | |
843 if (request_.get()) { | |
844 request_->Cancel(); | |
845 ReleaseRequest(); | |
846 } | |
847 // Release the reference to the request context. There could be multiple | |
848 // references to URLFetcher::Core at this point so it may take a while to | |
849 // delete the object, but we cannot delay the destruction of the request | |
850 // context. | |
851 request_context_getter_ = NULL; | |
852 was_cancelled_ = true; | |
853 temp_file_writer_.reset(); | |
854 } | |
855 | |
856 void URLFetcher::Core::OnCompletedURLRequest( | |
857 base::TimeDelta backoff_delay) { | |
858 DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); | |
859 | |
860 // Save the status and backoff_delay so that delegates can read it. | |
861 if (delegate_) { | |
862 backoff_delay_ = backoff_delay; | |
863 InformDelegateFetchIsComplete(); | |
864 } | |
865 } | |
866 | |
867 void URLFetcher::Core::InformDelegateFetchIsComplete() { | |
868 CHECK(delegate_loop_proxy_->BelongsToCurrentThread()); | |
869 if (delegate_) { | |
870 delegate_->OnURLFetchComplete(fetcher_); | |
871 } | |
872 } | |
873 | |
874 void URLFetcher::Core::NotifyMalformedContent() { | |
875 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
876 if (url_throttler_entry_ != NULL) { | |
877 int status_code = response_code_; | |
878 if (status_code == RESPONSE_CODE_INVALID) { | |
879 // The status code will generally be known by the time clients | |
880 // call the |ReceivedContentWasMalformed()| function (which ends up | |
881 // calling the current function) but if it's not, we need to assume | |
882 // the response was successful so that the total failure count | |
883 // used to calculate exponential back-off goes up. | |
884 status_code = 200; | |
885 } | |
886 url_throttler_entry_->ReceivedContentWasMalformed(status_code); | |
887 } | |
888 } | |
889 | |
890 void URLFetcher::Core::ReleaseRequest() { | |
891 request_.reset(); | |
892 g_registry.Get().RemoveURLFetcherCore(this); | |
893 } | |
894 | |
895 base::TimeTicks URLFetcher::Core::GetBackoffReleaseTime() { | |
896 DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); | |
897 DCHECK(original_url_throttler_entry_ != NULL); | |
898 | |
899 base::TimeTicks original_url_backoff = | |
900 original_url_throttler_entry_->GetExponentialBackoffReleaseTime(); | |
901 base::TimeTicks destination_url_backoff; | |
902 if (url_throttler_entry_ != NULL && | |
903 original_url_throttler_entry_ != url_throttler_entry_) { | |
904 destination_url_backoff = | |
905 url_throttler_entry_->GetExponentialBackoffReleaseTime(); | |
906 } | |
907 | |
908 return original_url_backoff > destination_url_backoff ? | |
909 original_url_backoff : destination_url_backoff; | |
910 } | |
911 | |
912 void URLFetcher::SetUploadData(const std::string& upload_content_type, | |
913 const std::string& upload_content) { | |
914 DCHECK(!core_->is_chunked_upload_); | |
915 core_->upload_content_type_ = upload_content_type; | |
916 core_->upload_content_ = upload_content; | |
917 } | |
918 | |
919 void URLFetcher::SetChunkedUpload(const std::string& content_type) { | |
920 DCHECK(core_->is_chunked_upload_ || | |
921 (core_->upload_content_type_.empty() && | |
922 core_->upload_content_.empty())); | |
923 core_->upload_content_type_ = content_type; | |
924 core_->upload_content_.clear(); | |
925 core_->is_chunked_upload_ = true; | |
926 } | |
927 | |
928 void URLFetcher::AppendChunkToUpload(const std::string& data, | |
929 bool is_last_chunk) { | |
930 DCHECK(data.length()); | |
931 core_->AppendChunkToUpload(data, is_last_chunk); | |
932 } | |
933 | |
934 const std::string& URLFetcher::upload_data() const { | |
935 return core_->upload_content_; | |
936 } | |
937 | |
938 void URLFetcher::SetReferrer(const std::string& referrer) { | |
939 core_->referrer_ = referrer; | |
940 } | |
941 | |
942 void URLFetcher::SetLoadFlags(int load_flags) { | |
943 core_->load_flags_ = load_flags; | |
944 } | |
945 | |
946 int URLFetcher::GetLoadFlags() const { | |
947 return core_->load_flags_; | |
948 } | |
949 | |
950 void URLFetcher::SetExtraRequestHeaders( | |
951 const std::string& extra_request_headers) { | |
952 core_->extra_request_headers_.Clear(); | |
953 core_->extra_request_headers_.AddHeadersFromString(extra_request_headers); | |
954 } | |
955 | |
956 void URLFetcher::GetExtraRequestHeaders(net::HttpRequestHeaders* headers) { | |
957 headers->CopyFrom(core_->extra_request_headers_); | |
958 } | |
959 | |
960 void URLFetcher::SetRequestContext( | |
961 net::URLRequestContextGetter* request_context_getter) { | |
962 DCHECK(!core_->request_context_getter_); | |
963 core_->request_context_getter_ = request_context_getter; | |
964 } | |
965 | |
966 void URLFetcher::SetAutomaticallyRetryOn5xx(bool retry) { | |
967 core_->automatically_retry_on_5xx_ = retry; | |
968 } | |
969 | |
970 void URLFetcher::SetMaxRetries(int max_retries) { | |
971 core_->max_retries_ = max_retries; | |
972 } | |
973 | |
974 int URLFetcher::GetMaxRetries() const { | |
975 return core_->max_retries_; | |
976 } | |
977 | |
978 | |
979 base::TimeDelta URLFetcher::GetBackoffDelay() const { | |
980 return core_->backoff_delay_; | |
981 } | |
982 | |
983 void URLFetcher::SaveResponseToTemporaryFile( | |
984 scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy) { | |
985 core_->file_message_loop_proxy_ = file_message_loop_proxy; | |
986 core_->response_destination_ = TEMP_FILE; | |
987 } | |
988 | |
989 net::HttpResponseHeaders* URLFetcher::GetResponseHeaders() const { | |
990 return core_->response_headers_; | |
991 } | |
992 | |
993 void URLFetcher::set_response_headers( | |
994 scoped_refptr<net::HttpResponseHeaders> headers) { | |
995 core_->response_headers_ = headers; | |
996 } | |
997 | |
998 // TODO(panayiotis): socket_address_ is written in the IO thread, | |
999 // if this is accessed in the UI thread, this could result in a race. | |
1000 // Same for response_headers_ above and was_fetched_via_proxy_ below. | |
1001 net::HostPortPair URLFetcher::GetSocketAddress() const { | |
1002 return core_->socket_address_; | |
1003 } | |
1004 | |
1005 bool URLFetcher::WasFetchedViaProxy() const { | |
1006 return core_->was_fetched_via_proxy_; | |
1007 } | |
1008 | |
1009 void URLFetcher::set_was_fetched_via_proxy(bool flag) { | |
1010 core_->was_fetched_via_proxy_ = flag; | |
1011 } | |
1012 | |
1013 void URLFetcher::Start() { | |
1014 core_->Start(); | |
1015 } | |
1016 | |
1017 void URLFetcher::StartWithRequestContextGetter( | |
1018 net::URLRequestContextGetter* request_context_getter) { | |
1019 SetRequestContext(request_context_getter); | |
1020 core_->Start(); | |
1021 } | |
1022 | |
1023 const GURL& URLFetcher::GetOriginalUrl() const { | |
1024 return core_->original_url_; | |
1025 } | |
1026 | |
1027 const GURL& URLFetcher::GetUrl() const { | |
1028 return core_->url_; | |
1029 } | |
1030 | |
1031 const net::URLRequestStatus& URLFetcher::GetStatus() const { | |
1032 return core_->status_; | |
1033 } | |
1034 | |
1035 int URLFetcher::GetResponseCode() const { | |
1036 return core_->response_code_; | |
1037 } | |
1038 | |
1039 const net::ResponseCookies& URLFetcher::GetCookies() const { | |
1040 return core_->cookies_; | |
1041 } | |
1042 | |
1043 bool URLFetcher::FileErrorOccurred( | |
1044 base::PlatformFileError* out_error_code) const { | |
1045 | |
1046 // Can't have a file error if no file is being created or written to. | |
1047 if (!core_->temp_file_writer_.get()) { | |
1048 return false; | |
1049 } | |
1050 | |
1051 base::PlatformFileError error_code = core_->temp_file_writer_->error_code(); | |
1052 if (error_code == base::PLATFORM_FILE_OK) | |
1053 return false; | |
1054 | |
1055 *out_error_code = error_code; | |
1056 return true; | |
1057 } | |
1058 | |
1059 void URLFetcher::ReceivedContentWasMalformed() { | |
1060 core_->ReceivedContentWasMalformed(); | |
1061 } | |
1062 | |
1063 bool URLFetcher::GetResponseAsString(std::string* out_response_string) const { | |
1064 if (core_->response_destination_ != STRING) | |
1065 return false; | |
1066 | |
1067 *out_response_string = core_->data_; | |
1068 return true; | |
1069 } | |
1070 | |
1071 bool URLFetcher::GetResponseAsFilePath(bool take_ownership, | |
1072 FilePath* out_response_path) const { | |
1073 DCHECK(core_->delegate_loop_proxy_->BelongsToCurrentThread()); | |
1074 if (core_->response_destination_ != TEMP_FILE || | |
1075 !core_->temp_file_writer_.get()) | |
1076 return false; | |
1077 | |
1078 *out_response_path = core_->temp_file_writer_->temp_file(); | |
1079 | |
1080 if (take_ownership) { | |
1081 core_->io_message_loop_proxy_->PostTask( | |
1082 FROM_HERE, base::Bind(&Core::DisownTempFile, core_.get())); | |
1083 } | |
1084 return true; | |
1085 } | |
1086 | |
1087 // static | |
1088 void URLFetcher::CancelAll() { | |
1089 Core::CancelAll(); | |
1090 } | |
1091 | |
1092 // static | |
1093 int URLFetcher::GetNumFetcherCores() { | |
1094 return Core::g_registry.Get().size(); | |
1095 } | |
1096 | |
1097 content::URLFetcherDelegate* URLFetcher::delegate() const { | |
1098 return core_->delegate(); | |
1099 } | |
1100 | |
1101 // static | |
1102 content::URLFetcherFactory* URLFetcher::factory() { | |
1103 return g_factory; | |
1104 } | |
1105 | |
1106 // static | |
1107 void URLFetcher::set_factory(content::URLFetcherFactory* factory) { | |
1108 g_factory = factory; | |
1109 } | |
OLD | NEW |