Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(322)

Side by Side Diff: net/filter/sdch_policy_delegate.cc

Issue 2368433002: Add net::SdchSourceStream and net::SdchPolicyDelegate (Closed)
Patch Set: Fix histograms Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « net/filter/sdch_policy_delegate.h ('k') | net/filter/sdch_policy_delegate_unittest.cc » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "net/filter/sdch_policy_delegate.h"
6
7 #include "base/metrics/histogram_macros.h"
8 #include "base/strings/string_util.h"
9 #include "base/values.h"
10 #include "net/base/sdch_problem_codes.h"
11 #include "net/log/net_log.h"
12 #include "net/log/net_log_event_type.h"
13 #include "net/url_request/url_request_http_job.h"
14
15 namespace net {
16
17 namespace {
18
19 const char kRefreshHtml[] =
20 "<head><META HTTP-EQUIV=\"Refresh\" CONTENT=\"0\"></head>";
21 // Mime types:
22 const char kTextHtml[] = "text/html";
23
24 // Disambiguate various types of responses that trigger a meta-refresh,
25 // failure, or fallback to pass-through.
26 enum ResponseCorruptionDetectionCause {
27 RESPONSE_NONE,
28
29 // 404 Http Response Code
30 RESPONSE_404 = 1,
31
32 // Not a 200 Http Response Code
33 RESPONSE_NOT_200 = 2,
34
35 // Cached before dictionary retrieved.
36 RESPONSE_OLD_UNENCODED = 3,
37
38 // Speculative but incorrect SDCH filtering was added added.
39 RESPONSE_TENTATIVE_SDCH = 4,
40
41 // Missing correct dict for decoding.
42 RESPONSE_NO_DICTIONARY = 5,
43
44 // Not an SDCH response but should be.
45 RESPONSE_CORRUPT_SDCH = 6,
46
47 // No dictionary was advertised with the request, the server claims
48 // to have encoded with SDCH anyway, but it isn't an SDCH response.
49 RESPONSE_ENCODING_LIE = 7,
50
51 RESPONSE_MAX,
52 };
53
54 const char* ResponseCorruptionDetectionCauseToString(
55 ResponseCorruptionDetectionCause cause) {
56 const char* cause_string = "<unknown>";
57 switch (cause) {
58 case RESPONSE_NONE:
59 cause_string = "NONE";
60 break;
61 case RESPONSE_404:
62 cause_string = "404";
63 break;
64 case RESPONSE_NOT_200:
65 cause_string = "NOT_200";
66 break;
67 case RESPONSE_OLD_UNENCODED:
68 cause_string = "OLD_UNENCODED";
69 break;
70 case RESPONSE_TENTATIVE_SDCH:
71 cause_string = "TENTATIVE_SDCH";
72 break;
73 case RESPONSE_NO_DICTIONARY:
74 cause_string = "NO_DICTIONARY";
75 break;
76 case RESPONSE_CORRUPT_SDCH:
77 cause_string = "CORRUPT_SDCH";
78 break;
79 case RESPONSE_ENCODING_LIE:
80 cause_string = "ENCODING_LIE";
81 break;
82 case RESPONSE_MAX:
83 cause_string = "<Error: max enum value>";
84 break;
85 }
86 return cause_string;
87 }
88
89 std::unique_ptr<base::Value> NetLogResponseCorruptionDetectionCallback(
90 ResponseCorruptionDetectionCause cause,
91 bool cached,
92 NetLogCaptureMode capture_mode) {
93 std::unique_ptr<base::DictionaryValue> dict(new base::DictionaryValue());
94 dict->SetString("cause", ResponseCorruptionDetectionCauseToString(cause));
95 dict->SetBoolean("cached", cached);
96 return std::move(dict);
97 }
98
99 void LogCorruptionDetection(const NetLogWithSource& net_log,
100 bool is_cached_content,
101 ResponseCorruptionDetectionCause cause) {
102 // Use if statement rather than ?: because UMA_HISTOGRAM_ENUMERATION
103 // caches the histogram name based on the call site.
104 if (is_cached_content) {
105 UMA_HISTOGRAM_ENUMERATION("Sdch3.ResponseCorruptionDetection.Cached", cause,
106 RESPONSE_MAX);
107 } else {
108 UMA_HISTOGRAM_ENUMERATION("Sdch3.ResponseCorruptionDetection.Uncached",
109 cause, RESPONSE_MAX);
110 }
111 net_log.AddEvent(NetLogEventType::SDCH_RESPONSE_CORRUPTION_DETECTION,
112 base::Bind(&NetLogResponseCorruptionDetectionCallback, cause,
113 is_cached_content));
114 }
115
116 } // namespace
117
118 SdchPolicyDelegate::SdchPolicyDelegate(
119 bool possible_pass_through,
120 URLRequestHttpJob* job,
121 std::string mime_type,
122 const GURL& url,
123 bool is_cached_content,
124 SdchManager* sdch_manager,
125 std::unique_ptr<SdchManager::DictionarySet> dictionary_set,
126 int response_code,
127 const NetLogWithSource& net_log)
128 : possible_pass_through_(possible_pass_through),
129 job_(job),
130 mime_type_(mime_type),
131 url_(url),
132 is_cached_content_(is_cached_content),
133 sdch_manager_(sdch_manager),
134 dictionary_set_(std::move(dictionary_set)),
135 response_code_(response_code),
136 net_log_(net_log) {}
137
138 SdchPolicyDelegate::~SdchPolicyDelegate() {}
139
140 // static
141 void SdchPolicyDelegate::FixUpSdchContentEncodings(
142 const NetLogWithSource& net_log,
143 const std::string& mime_type,
144 SdchManager::DictionarySet* dictionary_set,
145 std::vector<SourceStream::SourceType>* types) {
146 if (!dictionary_set) {
147 // If sdch dictionary is advertised, we might need to add some decoding, as
148 // some proxies strip encoding completely.
149 // It was not an SDCH request, so we'll just record stats.
150 if (1 < types->size()) {
151 // Multiple filters were intended to only be used for SDCH (thus far!)
152 SdchManager::LogSdchProblem(net_log,
153 SDCH_MULTIENCODING_FOR_NON_SDCH_REQUEST);
154 }
155 if ((1 == types->size()) && (SourceStream::TYPE_SDCH == types->front())) {
156 SdchManager::LogSdchProblem(
157 net_log, SDCH_SDCH_CONTENT_ENCODE_FOR_NON_SDCH_REQUEST);
158 }
159 return;
160 }
161 // If content encoding included SDCH, then everything is "relatively" fine.
162 if (!types->empty() && types->front() == SourceStream::TYPE_SDCH) {
163 // Some proxies (found currently in Argentina) strip the Content-Encoding
164 // text from "sdch,gzip" to a mere "sdch" without modifying the compressed
165 // payload. To handle this gracefully, we simulate the "probably" deleted
166 // ",gzip" by appending a tentative gzip decode, which will default to a
167 // no-op pass through filter if it doesn't get gzip headers where
168 // expected.
169 if (1 == types->size()) {
170 types->push_back(SourceStream::TYPE_GZIP_FALLBACK);
171 SdchManager::LogSdchProblem(net_log, SDCH_OPTIONAL_GUNZIP_ENCODING_ADDED);
172 }
173 return;
174 }
175
176 // There are now several cases to handle for an SDCH request. Foremost, if
177 // the outbound request was stripped so as not to advertise support for
178 // encodings, we might get back content with no encoding, or (for example)
179 // just gzip. We have to be sure that any changes we make allow for such
180 // minimal coding to work. That issue is why we use POSSIBLE_SDCH filters,
181 // as those filters sniff the content, and act as pass-through filters if
182 // headers are not found. TODO(xunjieli): Currently POSSIBLE_SDCH filters
183 // issue meta-refresh instead of pass-throughs to match old code. Change it
184 // to do pass-throughs.
185
186 // If the outbound GET is not modified, then the server will generally try to
187 // send us SDCH encoded content. As that content returns, there are several
188 // corruptions of the header "content-encoding" that proxies may perform (and
189 // have been detected in the wild). We already dealt with the a honest
190 // content encoding of "sdch,gzip" being corrupted into "sdch" with no change
191 // of the actual content. Another common corruption is to either discard
192 // the accurate content encoding, or to replace it with gzip only (again, with
193 // no change in actual content). The last observed corruption it to actually
194 // change the content, such as by re-gzipping it, and that may happen along
195 // with corruption of the stated content encoding (wow!).
196
197 // The one unresolved failure mode comes when we advertise a dictionary, and
198 // the server tries to *send* a gzipped file (not gzip encode content), and
199 // then we could do a gzip decode :-(.
200 // We will gather a lot of stats as we perform the fixups.
201 if (base::StartsWith(mime_type, kTextHtml,
202 base::CompareCase::INSENSITIVE_ASCII)) {
203 // Suspicious case: Advertised dictionary, but server didn't use sdch, and
204 // we're HTML tagged.
205 if (types->empty()) {
206 SdchManager::LogSdchProblem(net_log, SDCH_ADDED_CONTENT_ENCODING);
207 } else if (1 == types->size()) {
208 SdchManager::LogSdchProblem(net_log, SDCH_FIXED_CONTENT_ENCODING);
209 } else {
210 SdchManager::LogSdchProblem(net_log, SDCH_FIXED_CONTENT_ENCODINGS);
211 }
212 } else {
213 // Remarkable case!?! We advertised an SDCH dictionary, content-encoding
214 // was not marked for SDCH processing: Why did the server suggest an SDCH
215 // dictionary in the first place??. Also, the content isn't
216 // tagged as HTML, despite the fact that SDCH encoding is mostly likely
217 // for HTML: Did some anti-virus system strip this tag (sometimes they
218 // strip accept-encoding headers on the request)?? Does the content
219 // encoding not start with "text/html" for some other reason?? We'll
220 // report this as a fixup to a binary file, but it probably really is
221 // text/html (some how).
222 if (types->empty()) {
223 SdchManager::LogSdchProblem(net_log, SDCH_BINARY_ADDED_CONTENT_ENCODING);
224 } else if (1 == types->size()) {
225 SdchManager::LogSdchProblem(net_log, SDCH_BINARY_FIXED_CONTENT_ENCODING);
226 } else {
227 SdchManager::LogSdchProblem(net_log, SDCH_BINARY_FIXED_CONTENT_ENCODINGS);
228 }
229 }
230
231 // Leave the existing encoding type to be processed first, and add our
232 // tentative decodings to be done afterwards. Vodaphone UK reportedly will
233 // perform a second layer of gzip encoding atop the server's sdch,gzip
234 // encoding, and then claim that the content encoding is a mere gzip. As a
235 // result we'll need (in that case) to do the gunzip, plus our tentative
236 // gunzip and tentative SDCH decoding. This approach nicely handles the
237 // empty() list as well, and should work with other (as yet undiscovered)
238 // proxies the choose to re-compressed with some other encoding (such as
239 // bzip2, etc.).
240 types->insert(types->begin(), SourceStream::TYPE_GZIP_FALLBACK);
241 types->insert(types->begin(), SourceStream::TYPE_SDCH_POSSIBLE);
242 }
243
244 // Dictionary id errors are often the first indication that the SDCH stream has
245 // become corrupt. There are many possible causes: non-200 response codes, a
246 // cached non-SDCH-ified reply, or a response that claims to be SDCH but isn't
247 // actually. These are handled here by issuing a meta-refresh or swapping to the
248 // "passthrough" mode if appropriate, or failing the request if the error is
249 // unrecoverable.
250 SdchPolicyDelegate::ErrorRecovery SdchPolicyDelegate::OnDictionaryIdError(
251 std::string* replace_output) {
252 if (possible_pass_through_) {
253 LogCorruptionDetection(net_log_, is_cached_content_,
254 RESPONSE_TENTATIVE_SDCH);
255 // Ideally we should return PASS_THROUGH here, but this is done to match
256 // the old behavior in sdch_filter.cc.
257 }
258 // HTTP 404 might be an unencoded error page, so if decoding failed, pass it
259 // through. TODO(xunjieli): Remove this. crbug.com/516773.
260 if (response_code_ == 404) {
261 SdchManager::LogSdchProblem(net_log_, SDCH_PASS_THROUGH_404_CODE);
262 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_404);
263 return PASS_THROUGH;
264 }
265
266 // HTTP !200 gets a meta-refresh for HTML.
267 // TODO(xunjieli): remove this. crbug.com/654393.
268 if (response_code_ != 200) {
269 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_NOT_200);
270 return IssueMetaRefreshIfPossible(replace_output);
271 }
272
273 // If this is a cached result and the source hasn't requested a dictionary, it
274 // probably never had a dictionary to begin and is an unencoded response from
275 // earlier.
276 if (is_cached_content_) {
277 SdchManager::LogSdchProblem(net_log_, SDCH_PASS_THROUGH_OLD_CACHED);
278 LogCorruptionDetection(net_log_, is_cached_content_,
279 RESPONSE_OLD_UNENCODED);
280 return PASS_THROUGH;
281 }
282
283 // The original request didn't advertise any dictionaries, but the
284 // response claimed to be SDCH. There is no way to repair this situation: the
285 // original request already didn't advertise any dictionaries, and retrying it
286 // would likely have the/ same result. Blacklist the domain and try passing
287 // through.
288 if (!dictionary_set_) {
289 sdch_manager_->BlacklistDomain(url_, SDCH_PASSING_THROUGH_NON_SDCH);
290 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_ENCODING_LIE);
291 return PASS_THROUGH;
292 }
293 // Since SDCH dictionaries are advertised, this is a corrupt SDCH response.
294 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_CORRUPT_SDCH);
295 return IssueMetaRefreshIfPossible(replace_output);
296 }
297
298 // Dictionary fails to load when we have a plausible dictionay id. There are
299 // many possible causes: a cached SDCH-ified reply for which the SdchManager did
300 // not have the dictionary or a corrupted response. These are handled here by
301 // issuing a meta-refresh except the case where response code is 404.
302 SdchPolicyDelegate::ErrorRecovery SdchPolicyDelegate::OnGetDictionaryError(
303 std::string* replace_output) {
304 if (possible_pass_through_) {
305 LogCorruptionDetection(net_log_, is_cached_content_,
306 RESPONSE_TENTATIVE_SDCH);
307 // Ideally we should return PASS_THROUGH here, but this is done to match
308 // the old behavior in sdch_filter.cc.
309 }
310 // HTTP 404 might be an unencoded error page, so if decoding failed, pass it
311 // through. TODO(xunjieli): Remove this case crbug.com/516773.
312 if (response_code_ == 404) {
313 SdchManager::LogSdchProblem(net_log_, SDCH_PASS_THROUGH_404_CODE);
314 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_404);
315 return PASS_THROUGH;
316 }
317 SdchManager::LogSdchProblem(net_log_, SDCH_DICTIONARY_HASH_NOT_FOUND);
318 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_NO_DICTIONARY);
319 return IssueMetaRefreshIfPossible(replace_output);
320 }
321
322 SdchPolicyDelegate::ErrorRecovery SdchPolicyDelegate::OnDecodingError(
323 std::string* replace_output) {
324 // A decoding error, as opposed to a dictionary error, indicates a
325 // decompression failure partway through the payload of the SDCH stream,
326 // which means that the filter already witnessed a valid dictionary ID and
327 // successfully retrieved a dictionary for it. Decoding errors are not
328 // recoverable and it is not appropriate to stop decoding, so there are
329 // relatively few error cases here.
330 //
331 // In particular, a decoding error for an HTML payload is recoverable by
332 // issuing a meta-refresh, but to avoid having that happen too often, this
333 // class also temporarily blacklists the domain. A decoding error for a
334 // non-HTML payload is unrecoverable, so such an error gets a permanent
335 // blacklist entry. If the content was cached, no blacklisting is needed.
336 // TODO(xunjieli): This case should be removed. crbug.com/651821.
337 return IssueMetaRefreshIfPossible(replace_output);
338 }
339
340 bool SdchPolicyDelegate::OnGetDictionary(const std::string& server_id,
341 const std::string** text) {
342 if (dictionary_set_) {
343 *text = dictionary_set_->GetDictionaryText(server_id);
344 if (*text) {
345 server_id_ = server_id;
346 return true;
347 }
348 }
349 // This is a hack. Naively, the dictionaries available for
350 // decoding should be only the ones advertised. However, there are
351 // cases, specifically resources encoded with old dictionaries living
352 // in the cache, that mean the full set of dictionaries should be made
353 // available for decoding. It's not known how often this happens;
354 // if it happens rarely enough, this code can be removed.
355 //
356 // TODO(rdsmith): Long-term, a better solution is necessary, since
357 // an entry in the cache being encoded with the dictionary doesn't
358 // guarantee that the dictionary is present. That solution probably
359 // involves storing unencoded resources in the cache, but might
360 // involve evicting encoded resources on dictionary removal.
361 // See http://crbug.com/383405.
362 SdchProblemCode rv = SDCH_OK;
363 unexpected_dictionary_set_ =
364 sdch_manager_->GetDictionarySetByHash(url_, server_id, &rv);
365 if (unexpected_dictionary_set_) {
366 *text = unexpected_dictionary_set_->GetDictionaryText(server_id);
367 SdchManager::LogSdchProblem(
368 net_log_, is_cached_content_ ? SDCH_UNADVERTISED_DICTIONARY_USED_CACHED
369 : SDCH_UNADVERTISED_DICTIONARY_USED);
370 if (*text) {
371 server_id_ = server_id;
372 return true;
373 }
374 }
375 return false;
376 }
377
378 void SdchPolicyDelegate::OnStreamDestroyed(
379 SdchSourceStream::InputState input_state,
380 bool buffered_output_present,
381 bool decoding_not_finished) {
382 if (decoding_not_finished) {
383 SdchManager::LogSdchProblem(net_log_, SDCH_INCOMPLETE_SDCH_CONTENT);
384 // Make it possible for the user to hit reload, and get non-sdch content.
385 // Note this will "wear off" quickly enough, and is just meant to assure
386 // in some rare case that the user is not stuck.
387 sdch_manager_->BlacklistDomain(url_, SDCH_INCOMPLETE_SDCH_CONTENT);
388 }
389 // Filter chaining error, or premature teardown.
390 if (buffered_output_present)
391 SdchManager::LogSdchProblem(net_log_, SDCH_UNFLUSHED_CONTENT);
392
393 // Only record packet stats for non-cached content.
394 if (is_cached_content_) {
395 // Not a real error, but it is useful to have this tally.
396 // TODO(jar): Remove this stat after SDCH stability is validated.
397 SdchManager::LogSdchProblem(net_log_, SDCH_CACHE_DECODED);
398 return; // We don't need timing stats, and we aready got ratios.
399 }
400 switch (input_state) {
401 case SdchSourceStream::STATE_DECODE: {
402 job_->RecordPacketStats(FilterContext::StatisticSelector::SDCH_DECODE);
403 // Allow latency experiments to proceed.
404 sdch_manager_->SetAllowLatencyExperiment(url_, true);
405
406 // Notify successful dictionary usage.
407 DCHECK(!server_id_.empty());
408 sdch_manager_->OnDictionaryUsed(server_id_);
409 return;
410 }
411 case SdchSourceStream::STATE_LOAD_DICTIONARY:
412 SdchManager::LogSdchProblem(net_log_, SDCH_PRIOR_TO_DICTIONARY);
413 return;
414 case SdchSourceStream::STATE_PASS_THROUGH:
415 job_->RecordPacketStats(
416 FilterContext::StatisticSelector::SDCH_PASSTHROUGH);
417 return;
418 case SdchSourceStream::STATE_OUTPUT_REPLACE:
419 // This is meta refresh case. Already accounted for when set.
420 return;
421 } // end of switch.
422 }
423
424 // TODO(xunjieli): Remove meta refresh. crbug.com/651821.
425 SdchPolicyDelegate::ErrorRecovery
426 SdchPolicyDelegate::IssueMetaRefreshIfPossible(std::string* replace_output) {
427 // Errors for non-HTML payloads are unrecoverable and get the domain
428 // blacklisted indefinitely.
429 if (mime_type_.npos == mime_type_.find("text/html")) {
430 SdchProblemCode problem =
431 (is_cached_content_ ? SDCH_CACHED_META_REFRESH_UNSUPPORTED
432 : SDCH_META_REFRESH_UNSUPPORTED);
433 sdch_manager_->BlacklistDomainForever(url_, problem);
434 SdchManager::LogSdchProblem(net_log_, problem);
435 return NONE;
436 }
437
438 if (is_cached_content_) {
439 // Cached content is a probably startup tab, so just get the fresh content
440 // and try again, without disabling SDCH.
441 SdchManager::LogSdchProblem(net_log_, SDCH_META_REFRESH_CACHED_RECOVERY);
442 } else {
443 // Since it wasn't in the cache, blacklist for some period to get the
444 // correct content.
445 sdch_manager_->BlacklistDomain(url_, SDCH_META_REFRESH_RECOVERY);
446 SdchManager::LogSdchProblem(net_log_, SDCH_META_REFRESH_RECOVERY);
447 }
448
449 *replace_output = std::string(kRefreshHtml, strlen(kRefreshHtml));
450 return REPLACE_OUTPUT;
451 }
452
453 } // namespace net
OLDNEW
« no previous file with comments | « net/filter/sdch_policy_delegate.h ('k') | net/filter/sdch_policy_delegate_unittest.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698