OLD | NEW |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "headless/public/util/generic_url_request_job.h" | 5 #include "headless/public/util/generic_url_request_job.h" |
6 | 6 |
7 #include <string.h> | 7 #include <string.h> |
8 #include <algorithm> | 8 #include <algorithm> |
9 | 9 |
10 #include "base/logging.h" | 10 #include "base/logging.h" |
| 11 #include "content/public/browser/devtools_agent_host.h" |
| 12 #include "content/public/browser/resource_request_info.h" |
| 13 #include "content/public/browser/web_contents.h" |
11 #include "headless/public/util/url_request_dispatcher.h" | 14 #include "headless/public/util/url_request_dispatcher.h" |
12 #include "net/base/io_buffer.h" | 15 #include "net/base/io_buffer.h" |
13 #include "net/base/net_errors.h" | 16 #include "net/base/net_errors.h" |
14 #include "net/base/registry_controlled_domains/registry_controlled_domain.h" | 17 #include "net/base/registry_controlled_domains/registry_controlled_domain.h" |
| 18 #include "net/base/upload_bytes_element_reader.h" |
15 #include "net/cookies/cookie_store.h" | 19 #include "net/cookies/cookie_store.h" |
16 #include "net/http/http_response_headers.h" | 20 #include "net/http/http_response_headers.h" |
17 #include "net/url_request/url_request_context.h" | 21 #include "net/url_request/url_request_context.h" |
18 | 22 |
19 namespace headless { | 23 namespace headless { |
20 namespace { | 24 namespace { |
21 | 25 |
22 // True if the request method is "safe" (per section 4.2.1 of RFC 7231). | 26 // True if the request method is "safe" (per section 4.2.1 of RFC 7231). |
23 bool IsMethodSafe(const std::string& method) { | 27 bool IsMethodSafe(const std::string& method) { |
24 return method == "GET" || method == "HEAD" || method == "OPTIONS" || | 28 return method == "GET" || method == "HEAD" || method == "OPTIONS" || |
(...skipping 10 matching lines...) Expand all Loading... |
35 net::NetworkDelegate* network_delegate, | 39 net::NetworkDelegate* network_delegate, |
36 URLRequestDispatcher* url_request_dispatcher, | 40 URLRequestDispatcher* url_request_dispatcher, |
37 std::unique_ptr<URLFetcher> url_fetcher, | 41 std::unique_ptr<URLFetcher> url_fetcher, |
38 Delegate* delegate) | 42 Delegate* delegate) |
39 : ManagedDispatchURLRequestJob(request, | 43 : ManagedDispatchURLRequestJob(request, |
40 network_delegate, | 44 network_delegate, |
41 url_request_dispatcher), | 45 url_request_dispatcher), |
42 url_fetcher_(std::move(url_fetcher)), | 46 url_fetcher_(std::move(url_fetcher)), |
43 origin_task_runner_(base::ThreadTaskRunnerHandle::Get()), | 47 origin_task_runner_(base::ThreadTaskRunnerHandle::Get()), |
44 delegate_(delegate), | 48 delegate_(delegate), |
| 49 request_resource_info_( |
| 50 content::ResourceRequestInfo::ForRequest(request_)), |
45 weak_factory_(this) {} | 51 weak_factory_(this) {} |
46 | 52 |
47 GenericURLRequestJob::~GenericURLRequestJob() { | 53 GenericURLRequestJob::~GenericURLRequestJob() { |
48 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | 54 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); |
49 } | 55 } |
50 | 56 |
51 void GenericURLRequestJob::SetExtraRequestHeaders( | 57 void GenericURLRequestJob::SetExtraRequestHeaders( |
52 const net::HttpRequestHeaders& headers) { | 58 const net::HttpRequestHeaders& headers) { |
53 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | 59 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); |
54 extra_request_headers_ = headers; | 60 extra_request_headers_ = headers; |
55 | 61 |
56 if (extra_request_headers_.GetHeader(kDevtoolsRequestId, | 62 // TODO(alexclarke): Remove kDevtoolsRequestId |
57 &devtools_request_id_)) { | 63 extra_request_headers_.RemoveHeader(kDevtoolsRequestId); |
58 extra_request_headers_.RemoveHeader(kDevtoolsRequestId); | |
59 } | |
60 } | 64 } |
61 | 65 |
62 void GenericURLRequestJob::Start() { | 66 void GenericURLRequestJob::Start() { |
63 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | 67 PrepareCookies(request_->url(), request_->method(), |
64 if (!delegate_->BlockOrRewriteRequest( | 68 url::Origin(request_->first_party_for_cookies()), |
65 request_->url(), devtools_request_id_, request_->method(), | 69 base::Bind(&Delegate::OnPendingRequest, |
66 request_->referrer(), | 70 base::Unretained(delegate_), this)); |
67 base::Bind(&GenericURLRequestJob::OnRewriteResult, | |
68 weak_factory_.GetWeakPtr(), origin_task_runner_))) { | |
69 PrepareCookies(request_->url(), request_->method(), | |
70 url::Origin(request_->first_party_for_cookies())); | |
71 } | |
72 } | 71 } |
73 | 72 |
74 // static | |
75 void GenericURLRequestJob::OnRewriteResult( | |
76 base::WeakPtr<GenericURLRequestJob> weak_this, | |
77 const scoped_refptr<base::SingleThreadTaskRunner>& origin_task_runner, | |
78 RewriteResult result, | |
79 const GURL& url, | |
80 const std::string& method) { | |
81 if (!origin_task_runner->RunsTasksOnCurrentThread()) { | |
82 origin_task_runner->PostTask( | |
83 FROM_HERE, | |
84 base::Bind(&GenericURLRequestJob::OnRewriteResultOnOriginThread, | |
85 weak_this, result, url, method)); | |
86 return; | |
87 } | |
88 if (weak_this) | |
89 weak_this->OnRewriteResultOnOriginThread(result, url, method); | |
90 } | |
91 | |
92 void GenericURLRequestJob::OnRewriteResultOnOriginThread( | |
93 RewriteResult result, | |
94 const GURL& url, | |
95 const std::string& method) { | |
96 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | |
97 switch (result) { | |
98 case RewriteResult::kAllow: | |
99 // Note that we use the rewritten url for selecting cookies. | |
100 // Also, rewriting does not affect the request initiator. | |
101 PrepareCookies(url, method, url::Origin(url)); | |
102 break; | |
103 case RewriteResult::kDeny: | |
104 DispatchStartError(net::ERR_FILE_NOT_FOUND); | |
105 break; | |
106 case RewriteResult::kFailure: | |
107 DispatchStartError(net::ERR_UNEXPECTED); | |
108 break; | |
109 default: | |
110 DCHECK(false); | |
111 } | |
112 }; | |
113 | |
114 void GenericURLRequestJob::PrepareCookies(const GURL& rewritten_url, | 73 void GenericURLRequestJob::PrepareCookies(const GURL& rewritten_url, |
115 const std::string& method, | 74 const std::string& method, |
116 const url::Origin& site_for_cookies) { | 75 const url::Origin& site_for_cookies, |
| 76 const base::Closure& done_callback) { |
117 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | 77 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); |
118 net::CookieStore* cookie_store = request_->context()->cookie_store(); | 78 net::CookieStore* cookie_store = request_->context()->cookie_store(); |
119 net::CookieOptions options; | 79 net::CookieOptions options; |
120 options.set_include_httponly(); | 80 options.set_include_httponly(); |
121 | 81 |
122 // See net::URLRequestHttpJob::AddCookieHeaderAndStart(). | 82 // See net::URLRequestHttpJob::AddCookieHeaderAndStart(). |
123 url::Origin requested_origin(rewritten_url); | 83 url::Origin requested_origin(rewritten_url); |
124 if (net::registry_controlled_domains::SameDomainOrHost( | 84 if (net::registry_controlled_domains::SameDomainOrHost( |
125 requested_origin, site_for_cookies, | 85 requested_origin, site_for_cookies, |
126 net::registry_controlled_domains::INCLUDE_PRIVATE_REGISTRIES)) { | 86 net::registry_controlled_domains::INCLUDE_PRIVATE_REGISTRIES)) { |
127 if (net::registry_controlled_domains::SameDomainOrHost( | 87 if (net::registry_controlled_domains::SameDomainOrHost( |
128 requested_origin, request_->initiator(), | 88 requested_origin, request_->initiator(), |
129 net::registry_controlled_domains::INCLUDE_PRIVATE_REGISTRIES)) { | 89 net::registry_controlled_domains::INCLUDE_PRIVATE_REGISTRIES)) { |
130 options.set_same_site_cookie_mode( | 90 options.set_same_site_cookie_mode( |
131 net::CookieOptions::SameSiteCookieMode::INCLUDE_STRICT_AND_LAX); | 91 net::CookieOptions::SameSiteCookieMode::INCLUDE_STRICT_AND_LAX); |
132 } else if (IsMethodSafe(request_->method())) { | 92 } else if (IsMethodSafe(request_->method())) { |
133 options.set_same_site_cookie_mode( | 93 options.set_same_site_cookie_mode( |
134 net::CookieOptions::SameSiteCookieMode::INCLUDE_LAX); | 94 net::CookieOptions::SameSiteCookieMode::INCLUDE_LAX); |
135 } | 95 } |
136 } | 96 } |
137 | 97 |
138 cookie_store->GetCookieListWithOptionsAsync( | 98 cookie_store->GetCookieListWithOptionsAsync( |
139 rewritten_url, options, | 99 rewritten_url, options, |
140 base::Bind(&GenericURLRequestJob::OnCookiesAvailable, | 100 base::Bind(&GenericURLRequestJob::OnCookiesAvailable, |
141 weak_factory_.GetWeakPtr(), rewritten_url, method)); | 101 weak_factory_.GetWeakPtr(), rewritten_url, method, |
| 102 done_callback)); |
142 } | 103 } |
143 | 104 |
144 void GenericURLRequestJob::OnCookiesAvailable( | 105 void GenericURLRequestJob::OnCookiesAvailable( |
145 const GURL& rewritten_url, | 106 const GURL& rewritten_url, |
146 const std::string& method, | 107 const std::string& method, |
| 108 const base::Closure& done_callback, |
147 const net::CookieList& cookie_list) { | 109 const net::CookieList& cookie_list) { |
148 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | 110 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); |
149 // TODO(alexclarke): Set user agent. | 111 // TODO(alexclarke): Set user agent. |
150 // Pass cookies, the referrer and any extra headers into the fetch request. | 112 // Pass cookies, the referrer and any extra headers into the fetch request. |
151 extra_request_headers_.SetHeader( | 113 extra_request_headers_.SetHeader( |
152 net::HttpRequestHeaders::kCookie, | 114 net::HttpRequestHeaders::kCookie, |
153 net::CookieStore::BuildCookieLine(cookie_list)); | 115 net::CookieStore::BuildCookieLine(cookie_list)); |
154 | 116 |
155 extra_request_headers_.SetHeader(net::HttpRequestHeaders::kReferer, | 117 extra_request_headers_.SetHeader(net::HttpRequestHeaders::kReferer, |
156 request_->referrer()); | 118 request_->referrer()); |
157 | 119 |
158 // The resource may have been supplied in the request. | 120 done_callback.Run(); |
159 const HttpResponse* matched_resource = delegate_->MaybeMatchResource( | |
160 rewritten_url, devtools_request_id_, method, extra_request_headers_); | |
161 | |
162 if (matched_resource) { | |
163 OnFetchCompleteExtractHeaders( | |
164 matched_resource->final_url, matched_resource->http_response_code, | |
165 matched_resource->response_data, matched_resource->response_data_size); | |
166 } else { | |
167 url_fetcher_->StartFetch(rewritten_url, method, extra_request_headers_, | |
168 devtools_request_id_, this); | |
169 } | |
170 } | 121 } |
171 | 122 |
172 void GenericURLRequestJob::OnFetchStartError(net::Error error) { | 123 void GenericURLRequestJob::OnFetchStartError(net::Error error) { |
173 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | 124 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); |
174 DispatchStartError(error); | 125 DispatchStartError(error); |
| 126 delegate_->OnResourceLoadFailed(this, error); |
175 } | 127 } |
176 | 128 |
177 void GenericURLRequestJob::OnFetchComplete( | 129 void GenericURLRequestJob::OnFetchComplete( |
178 const GURL& final_url, | 130 const GURL& final_url, |
179 int http_response_code, | 131 int http_response_code, |
180 scoped_refptr<net::HttpResponseHeaders> response_headers, | 132 scoped_refptr<net::HttpResponseHeaders> response_headers, |
181 const char* body, | 133 const char* body, |
182 size_t body_size) { | 134 size_t body_size) { |
183 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | 135 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); |
184 response_time_ = base::TimeTicks::Now(); | 136 response_time_ = base::TimeTicks::Now(); |
185 http_response_code_ = http_response_code; | 137 http_response_code_ = http_response_code; |
186 response_headers_ = response_headers; | 138 response_headers_ = response_headers; |
187 body_ = body; | 139 body_ = body; |
188 body_size_ = body_size; | 140 body_size_ = body_size; |
189 | 141 |
190 DispatchHeadersComplete(); | 142 DispatchHeadersComplete(); |
191 | 143 |
192 std::string mime_type; | 144 delegate_->OnResourceLoadComplete(this, final_url, http_response_code, |
193 GetMimeType(&mime_type); | 145 response_headers_, body_, body_size_); |
194 | |
195 delegate_->OnResourceLoadComplete(final_url, devtools_request_id_, mime_type, | |
196 http_response_code); | |
197 } | 146 } |
198 | 147 |
199 int GenericURLRequestJob::ReadRawData(net::IOBuffer* buf, int buf_size) { | 148 int GenericURLRequestJob::ReadRawData(net::IOBuffer* buf, int buf_size) { |
200 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); | 149 DCHECK(origin_task_runner_->RunsTasksOnCurrentThread()); |
201 // TODO(skyostil): Implement ranged fetches. | 150 // TODO(skyostil): Implement ranged fetches. |
202 // TODO(alexclarke): Add coverage for all the cases below. | 151 // TODO(alexclarke): Add coverage for all the cases below. |
203 size_t bytes_available = body_size_ - read_offset_; | 152 size_t bytes_available = body_size_ - read_offset_; |
204 size_t bytes_to_copy = | 153 size_t bytes_to_copy = |
205 std::min(static_cast<size_t>(buf_size), bytes_available); | 154 std::min(static_cast<size_t>(buf_size), bytes_available); |
206 if (bytes_to_copy) { | 155 if (bytes_to_copy) { |
(...skipping 22 matching lines...) Expand all Loading... |
229 return false; | 178 return false; |
230 return response_headers_->GetCharset(charset); | 179 return response_headers_->GetCharset(charset); |
231 } | 180 } |
232 | 181 |
233 void GenericURLRequestJob::GetLoadTimingInfo( | 182 void GenericURLRequestJob::GetLoadTimingInfo( |
234 net::LoadTimingInfo* load_timing_info) const { | 183 net::LoadTimingInfo* load_timing_info) const { |
235 // TODO(alexclarke): Investigate setting the other members too where possible. | 184 // TODO(alexclarke): Investigate setting the other members too where possible. |
236 load_timing_info->receive_headers_end = response_time_; | 185 load_timing_info->receive_headers_end = response_time_; |
237 } | 186 } |
238 | 187 |
| 188 const net::URLRequest* GenericURLRequestJob::GetURLRequest() const { |
| 189 return request_; |
| 190 } |
| 191 |
| 192 int GenericURLRequestJob::GetFrameTreeNodeId() const { |
| 193 return request_resource_info_->GetFrameTreeNodeId(); |
| 194 } |
| 195 |
| 196 std::string GenericURLRequestJob::GetDevToolsAgentHostId() const { |
| 197 return content::DevToolsAgentHost::GetOrCreateFor( |
| 198 request_resource_info_->GetWebContentsGetterForRequest().Run()) |
| 199 ->GetId(); |
| 200 } |
| 201 |
| 202 Request::ResourceType GenericURLRequestJob::GetResourceType() const { |
| 203 switch (request_resource_info_->GetResourceType()) { |
| 204 case content::RESOURCE_TYPE_MAIN_FRAME: |
| 205 return Request::ResourceType::MAIN_FRAME; |
| 206 case content::RESOURCE_TYPE_SUB_FRAME: |
| 207 return Request::ResourceType::SUB_FRAME; |
| 208 case content::RESOURCE_TYPE_STYLESHEET: |
| 209 return Request::ResourceType::STYLESHEET; |
| 210 case content::RESOURCE_TYPE_SCRIPT: |
| 211 return Request::ResourceType::SCRIPT; |
| 212 case content::RESOURCE_TYPE_IMAGE: |
| 213 return Request::ResourceType::IMAGE; |
| 214 case content::RESOURCE_TYPE_FONT_RESOURCE: |
| 215 return Request::ResourceType::FONT_RESOURCE; |
| 216 case content::RESOURCE_TYPE_SUB_RESOURCE: |
| 217 return Request::ResourceType::SUB_RESOURCE; |
| 218 case content::RESOURCE_TYPE_OBJECT: |
| 219 return Request::ResourceType::OBJECT; |
| 220 case content::RESOURCE_TYPE_MEDIA: |
| 221 return Request::ResourceType::MEDIA; |
| 222 case content::RESOURCE_TYPE_WORKER: |
| 223 return Request::ResourceType::WORKER; |
| 224 case content::RESOURCE_TYPE_SHARED_WORKER: |
| 225 return Request::ResourceType::SHARED_WORKER; |
| 226 case content::RESOURCE_TYPE_PREFETCH: |
| 227 return Request::ResourceType::PREFETCH; |
| 228 case content::RESOURCE_TYPE_FAVICON: |
| 229 return Request::ResourceType::FAVICON; |
| 230 case content::RESOURCE_TYPE_XHR: |
| 231 return Request::ResourceType::XHR; |
| 232 case content::RESOURCE_TYPE_PING: |
| 233 return Request::ResourceType::PING; |
| 234 case content::RESOURCE_TYPE_SERVICE_WORKER: |
| 235 return Request::ResourceType::SERVICE_WORKER; |
| 236 case content::RESOURCE_TYPE_CSP_REPORT: |
| 237 return Request::ResourceType::CSP_REPORT; |
| 238 case content::RESOURCE_TYPE_PLUGIN_RESOURCE: |
| 239 return Request::ResourceType::PLUGIN_RESOURCE; |
| 240 default: |
| 241 NOTREACHED() << "Unrecognized resource type"; |
| 242 return Request::ResourceType::MAIN_FRAME; |
| 243 } |
| 244 } |
| 245 |
| 246 namespace { |
| 247 std::string GetUploadData(net::URLRequest* request) { |
| 248 if (!request->has_upload()) |
| 249 return ""; |
| 250 |
| 251 const net::UploadDataStream* stream = request->get_upload(); |
| 252 if (!stream->GetElementReaders()) |
| 253 return ""; |
| 254 |
| 255 DCHECK_EQ(1u, stream->GetElementReaders()->size()); |
| 256 const net::UploadBytesElementReader* reader = |
| 257 (*stream->GetElementReaders())[0]->AsBytesReader(); |
| 258 return std::string(reader->bytes(), reader->length()); |
| 259 } |
| 260 } // namespace |
| 261 |
| 262 const Request* GenericURLRequestJob::GetRequest() const { |
| 263 return this; |
| 264 } |
| 265 |
| 266 void GenericURLRequestJob::AllowRequest() { |
| 267 if (!origin_task_runner_->RunsTasksOnCurrentThread()) { |
| 268 origin_task_runner_->PostTask( |
| 269 FROM_HERE, base::Bind(&GenericURLRequestJob::AllowRequest, |
| 270 weak_factory_.GetWeakPtr())); |
| 271 return; |
| 272 } |
| 273 |
| 274 url_fetcher_->StartFetch(request_->url(), request_->method(), |
| 275 GetUploadData(request_), extra_request_headers_, |
| 276 this); |
| 277 } |
| 278 |
| 279 void GenericURLRequestJob::BlockRequest(net::Error error) { |
| 280 if (!origin_task_runner_->RunsTasksOnCurrentThread()) { |
| 281 origin_task_runner_->PostTask( |
| 282 FROM_HERE, base::Bind(&GenericURLRequestJob::BlockRequest, |
| 283 weak_factory_.GetWeakPtr(), error)); |
| 284 return; |
| 285 } |
| 286 |
| 287 DispatchStartError(error); |
| 288 } |
| 289 |
| 290 void GenericURLRequestJob::ModifyRequest( |
| 291 const GURL& url, |
| 292 const std::string& method, |
| 293 const std::string& post_data, |
| 294 const net::HttpRequestHeaders& request_headers) { |
| 295 if (!origin_task_runner_->RunsTasksOnCurrentThread()) { |
| 296 origin_task_runner_->PostTask( |
| 297 FROM_HERE, base::Bind(&GenericURLRequestJob::ModifyRequest, |
| 298 weak_factory_.GetWeakPtr(), url, method, |
| 299 post_data, request_headers)); |
| 300 return; |
| 301 } |
| 302 |
| 303 extra_request_headers_ = request_headers; |
| 304 PrepareCookies( |
| 305 request_->url(), request_->method(), |
| 306 url::Origin(request_->first_party_for_cookies()), |
| 307 base::Bind(&URLFetcher::StartFetch, base::Unretained(url_fetcher_.get()), |
| 308 url, method, post_data, request_headers, this)); |
| 309 } |
| 310 |
| 311 void GenericURLRequestJob::MockResponse( |
| 312 std::unique_ptr<MockResponseData> mock_response) { |
| 313 if (!origin_task_runner_->RunsTasksOnCurrentThread()) { |
| 314 origin_task_runner_->PostTask( |
| 315 FROM_HERE, base::Bind(&GenericURLRequestJob::MockResponse, |
| 316 weak_factory_.GetWeakPtr(), |
| 317 base::Passed(std::move(mock_response)))); |
| 318 return; |
| 319 } |
| 320 |
| 321 mock_response_ = std::move(mock_response); |
| 322 |
| 323 OnFetchCompleteExtractHeaders(request_->url(), |
| 324 mock_response_->http_response_code, |
| 325 mock_response_->response_data.data(), |
| 326 mock_response_->response_data.size()); |
| 327 } |
| 328 |
239 } // namespace headless | 329 } // namespace headless |
OLD | NEW |