Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(232)

Side by Side Diff: net/filter/sdch_policy_delegate.cc

Issue 2368433002: Add net::SdchSourceStream and net::SdchPolicyDelegate (Closed)
Patch Set: Address Randy's comments Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "net/filter/sdch_policy_delegate.h"
6
7 #include "base/metrics/histogram_macros.h"
8 #include "base/strings/string_util.h"
9 #include "base/values.h"
10 #include "net/base/sdch_problem_codes.h"
11 #include "net/log/net_log.h"
12 #include "net/log/net_log_event_type.h"
13 #include "net/url_request/url_request_http_job.h"
14 #include "sdch/open-vcdiff/src/google/vcdecoder.h"
15
16 namespace net {
17
18 namespace {
19
20 const char kRefreshHtml[] =
21 "<head><META HTTP-EQUIV=\"Refresh\" CONTENT=\"0\"></head>";
22 // Mime types:
23 const char kTextHtml[] = "text/html";
24
25 // Disambiguate various types of responses that trigger a meta-refresh,
26 // failure, or fallback to pass-through.
27 enum ResponseCorruptionDetectionCause {
28 RESPONSE_NONE,
29
30 // 404 Http Response Code
31 RESPONSE_404 = 1,
32
33 // Not a 200 Http Response Code
34 RESPONSE_NOT_200 = 2,
35
36 // Cached before dictionary retrieved.
37 RESPONSE_OLD_UNENCODED = 3,
38
39 // Speculative but incorrect SDCH filtering was added added.
40 RESPONSE_TENTATIVE_SDCH = 4,
41
42 // Missing correct dict for decoding.
43 RESPONSE_NO_DICTIONARY = 5,
44
45 // Not an SDCH response but should be.
46 RESPONSE_CORRUPT_SDCH = 6,
47
48 // No dictionary was advertised with the request, the server claims
49 // to have encoded with SDCH anyway, but it isn't an SDCH response.
50 RESPONSE_ENCODING_LIE = 7,
51
52 RESPONSE_MAX,
53 };
54
55 const char* ResponseCorruptionDetectionCauseToString(
56 ResponseCorruptionDetectionCause cause) {
57 const char* cause_string = "<unknown>";
58 switch (cause) {
59 case RESPONSE_NONE:
60 cause_string = "NONE";
61 break;
62 case RESPONSE_404:
63 cause_string = "404";
64 break;
65 case RESPONSE_NOT_200:
66 cause_string = "NOT_200";
67 break;
68 case RESPONSE_OLD_UNENCODED:
69 cause_string = "OLD_UNENCODED";
70 break;
71 case RESPONSE_TENTATIVE_SDCH:
72 cause_string = "TENTATIVE_SDCH";
73 break;
74 case RESPONSE_NO_DICTIONARY:
75 cause_string = "NO_DICTIONARY";
76 break;
77 case RESPONSE_CORRUPT_SDCH:
78 cause_string = "CORRUPT_SDCH";
79 break;
80 case RESPONSE_ENCODING_LIE:
81 cause_string = "ENCODING_LIE";
82 break;
83 case RESPONSE_MAX:
84 cause_string = "<Error: max enum value>";
85 break;
86 }
87 return cause_string;
88 }
89
90 std::unique_ptr<base::Value> NetLogResponseCorruptionDetectionCallback(
91 ResponseCorruptionDetectionCause cause,
92 bool cached,
93 NetLogCaptureMode capture_mode) {
94 std::unique_ptr<base::DictionaryValue> dict(new base::DictionaryValue());
95 dict->SetString("cause", ResponseCorruptionDetectionCauseToString(cause));
96 dict->SetBoolean("cached", cached);
97 return std::move(dict);
98 }
99
100 void LogCorruptionDetection(const NetLogWithSource& net_log,
101 bool is_cached_content,
102 ResponseCorruptionDetectionCause cause) {
103 // Use if statement rather than ?: because UMA_HISTOGRAM_ENUMERATION
104 // caches the histogram name based on the call site.
105 if (is_cached_content) {
106 UMA_HISTOGRAM_ENUMERATION("Sdch3.ResponseCorruptionDetection.Cached", cause,
107 RESPONSE_MAX);
108 } else {
109 UMA_HISTOGRAM_ENUMERATION("Sdch3.ResponseCorruptionDetection.Uncached",
110 cause, RESPONSE_MAX);
111 }
112 net_log.AddEvent(NetLogEventType::SDCH_RESPONSE_CORRUPTION_DETECTION,
113 base::Bind(&NetLogResponseCorruptionDetectionCallback, cause,
114 is_cached_content));
115 }
116
117 } // namespace
118
119 SdchPolicyDelegate::SdchPolicyDelegate(
120 bool possible_pass_through,
121 URLRequestHttpJob* job,
122 std::string mime_type,
123 const GURL& url,
124 bool is_cached_content,
125 SdchManager* sdch_manager,
126 std::unique_ptr<SdchManager::DictionarySet> dictionary_set,
127 int response_code,
128 const NetLogWithSource& net_log)
129 : possible_pass_through_(possible_pass_through),
130 job_(job),
131 mime_type_(mime_type),
132 url_(url),
133 is_cached_content_(is_cached_content),
134 sdch_manager_(sdch_manager),
135 dictionary_set_(std::move(dictionary_set)),
136 response_code_(response_code),
137 net_log_(net_log) {}
138
139 SdchPolicyDelegate::~SdchPolicyDelegate() {}
140
141 // static
142 void SdchPolicyDelegate::FixUpSdchContentEncodings(
143 const NetLogWithSource& net_log,
144 const std::string& mime_type,
145 SdchManager::DictionarySet* dictionary_set,
146 std::vector<SourceStream::SourceType>* types) {
147 // If sdch dictionary is advertised, we might need to add some decoding, as
148 // some proxies strip encoding completely.
Randy Smith (Not in Mondays) 2016/10/19 19:49:41 nit, suggestion: I think this comment would be cle
xunjieli 2016/10/20 00:09:14 Done.
149 if (!dictionary_set) {
150 // It was not an SDCH request, so we'll just record stats.
151 if (1 < types->size()) {
152 // Multiple filters were intended to only be used for SDCH (thus far!)
153 SdchManager::LogSdchProblem(net_log,
154 SDCH_MULTIENCODING_FOR_NON_SDCH_REQUEST);
155 }
156 if ((1 == types->size()) && (SourceStream::TYPE_SDCH == types->front())) {
157 SdchManager::LogSdchProblem(
158 net_log, SDCH_SDCH_CONTENT_ENCODE_FOR_NON_SDCH_REQUEST);
159 }
160 return;
161 }
162 // If content encoding included SDCH, then everything is "relatively" fine.
163 if (!types->empty() && types->front() == SourceStream::TYPE_SDCH) {
164 // Some proxies (found currently in Argentina) strip the Content-Encoding
165 // text from "sdch,gzip" to a mere "sdch" without modifying the compressed
166 // payload. To handle this gracefully, we simulate the "probably" deleted
167 // ",gzip" by appending a tentative gzip decode, which will default to a
168 // no-op pass through filter if it doesn't get gzip headers where
169 // expected.
170 if (1 == types->size()) {
171 types->push_back(SourceStream::TYPE_GZIP_FALLBACK);
172 SdchManager::LogSdchProblem(net_log, SDCH_OPTIONAL_GUNZIP_ENCODING_ADDED);
173 }
174 return;
175 }
176
177 // There are now several cases to handle for an SDCH request. Foremost, if
178 // the outbound request was stripped so as not to advertise support for
179 // encodings, we might get back content with no encoding, or (for example)
180 // just gzip. We have to be sure that any changes we make allow for such
181 // minimal coding to work. That issue is why we use POSSIBLE_SDCH filters,
182 // as those filters sniff the content, and act as pass-through filters if
183 // headers are not found. TODO(xunjieli): Currently POSSIBLE_SDCH filters
184 // issue meta-refresh instead of pass-throughs to match old code. Change it
185 // to do pass-throughs.
186
187 // If the outbound GET is not modified, then the server will generally try to
188 // send us SDCH encoded content. As that content returns, there are several
189 // corruptions of the header "content-encoding" that proxies may perform (and
190 // have been detected in the wild). We already dealt with the a honest
191 // content encoding of "sdch,gzip" being corrupted into "sdch" with no change
192 // of the actual content. Another common corruption is to either discard
193 // the accurate content encoding, or to replace it with gzip only (again, with
194 // no change in actual content). The last observed corruption it to actually
195 // change the content, such as by re-gzipping it, and that may happen along
196 // with corruption of the stated content encoding (wow!).
197
198 // The one unresolved failure mode comes when we advertise a dictionary, and
199 // the server tries to *send* a gzipped file (not gzip encode content), and
200 // then we could do a gzip decode :-(.
201 // We will gather a lot of stats as we perform the fixups.
202 if (base::StartsWith(mime_type, kTextHtml,
203 base::CompareCase::INSENSITIVE_ASCII)) {
204 // Suspicious case: Advertised dictionary, but server didn't use sdch, and
205 // we're HTML tagged.
206 if (types->empty()) {
207 SdchManager::LogSdchProblem(net_log, SDCH_ADDED_CONTENT_ENCODING);
208 } else if (1 == types->size()) {
209 SdchManager::LogSdchProblem(net_log, SDCH_FIXED_CONTENT_ENCODING);
210 } else {
211 SdchManager::LogSdchProblem(net_log, SDCH_FIXED_CONTENT_ENCODINGS);
212 }
213 } else {
214 // Remarkable case!?! We advertised an SDCH dictionary, content-encoding
215 // was not marked for SDCH processing: Why did the server suggest an SDCH
216 // dictionary in the first place??. Also, the content isn't
217 // tagged as HTML, despite the fact that SDCH encoding is mostly likely
218 // for HTML: Did some anti-virus system strip this tag (sometimes they
219 // strip accept-encoding headers on the request)?? Does the content
220 // encoding not start with "text/html" for some other reason?? We'll
221 // report this as a fixup to a binary file, but it probably really is
222 // text/html (some how).
223 if (types->empty()) {
224 SdchManager::LogSdchProblem(net_log, SDCH_BINARY_ADDED_CONTENT_ENCODING);
225 } else if (1 == types->size()) {
226 SdchManager::LogSdchProblem(net_log, SDCH_BINARY_FIXED_CONTENT_ENCODING);
227 } else {
228 SdchManager::LogSdchProblem(net_log, SDCH_BINARY_FIXED_CONTENT_ENCODINGS);
229 }
230 }
231
232 // Leave the existing encoding type to be processed first, and add our
233 // tentative decodings to be done afterwards. Vodaphone UK reportedly will
234 // perform a second layer of gzip encoding atop the server's sdch,gzip
235 // encoding, and then claim that the content encoding is a mere gzip. As a
236 // result we'll need (in that case) to do the gunzip, plus our tentative
237 // gunzip and tentative SDCH decoding. This approach nicely handles the
238 // empty() list as well, and should work with other (as yet undiscovered)
239 // proxies the choose to re-compressed with some other encoding (such as
240 // bzip2, etc.).
241 types->insert(types->begin(), SourceStream::TYPE_GZIP_FALLBACK);
242 types->insert(types->begin(), SourceStream::TYPE_SDCH_POSSIBLE);
243 }
244
245 // Dictionary id errors are often the first indication that the SDCH stream has
246 // become corrupt. There are many possible causes: non-200 response codes, a
247 // cached non-SDCH-ified reply, or a response that claims to be SDCH but isn't
248 // actually. These are handled here by issuing a meta-refresh or swapping to the
249 // "passthrough" mode if appropriate, or failing the request if the error is
250 // unrecoverable.
251 SdchPolicyDelegate::ErrorRecovery SdchPolicyDelegate::OnDictionaryIdError(
252 std::string* replace_output) {
253 if (possible_pass_through_) {
254 LogCorruptionDetection(net_log_, is_cached_content_,
255 RESPONSE_TENTATIVE_SDCH);
256 // Ideally we should return PASS_THROUGH here, but this is done to match
257 // the old behavior in sdch_filter.cc.
258 }
259 // HTTP 404 might be an unencoded error page, so if decoding failed, pass it
260 // through. TODO(xunjieli): Remove this. crbug.com/516773.
261 if (response_code_ == 404) {
262 SdchManager::LogSdchProblem(net_log_, SDCH_PASS_THROUGH_404_CODE);
263 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_404);
264 return PASS_THROUGH;
265 }
266
267 // HTTP !200 gets a meta-refresh for HTML.
268 // TODO(xunjieli): remove this. crbug.com/654393.
269 if (response_code_ != 200) {
270 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_NOT_200);
271 return IssueMetaRefreshIfPossible(replace_output);
272 }
273
274 // If this is a cached result and the source hasn't requested a dictionary, it
275 // probably never had a dictionary to begin and is an unencoded response from
276 // earlier.
277 if (is_cached_content_) {
278 SdchManager::LogSdchProblem(net_log_, SDCH_PASS_THROUGH_OLD_CACHED);
279 LogCorruptionDetection(net_log_, is_cached_content_,
280 RESPONSE_OLD_UNENCODED);
281 return PASS_THROUGH;
282 }
283
284 // The original request didn't advertise any dictionaries, but the
285 // response claimed to be SDCH. There is no way to repair this situation: the
286 // original request already didn't advertise any dictionaries, and retrying it
287 // would likely have the/ same result. Blacklist the domain and try passing
288 // through.
289 if (!dictionary_set_) {
290 sdch_manager_->BlacklistDomain(url_, SDCH_PASSING_THROUGH_NON_SDCH);
291 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_ENCODING_LIE);
292 return PASS_THROUGH;
293 }
294 // Since SDCH dictionaries are advertised, this is a corrupt SDCH response.
295 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_CORRUPT_SDCH);
296 return IssueMetaRefreshIfPossible(replace_output);
297 }
298
299 // Dictionary fails to load when we have a plausible dictionay id. There are
300 // many possible causes: a cached SDCH-ified reply for which the SdchManager did
301 // not have the dictionary or a corrupted response. These are handled here by
302 // issuing a meta-refresh except the case where response code is 404.
303 SdchPolicyDelegate::ErrorRecovery SdchPolicyDelegate::OnGetDictionaryError(
304 std::string* replace_output) {
305 if (possible_pass_through_) {
306 LogCorruptionDetection(net_log_, is_cached_content_,
307 RESPONSE_TENTATIVE_SDCH);
308 // Ideally we should return PASS_THROUGH here, but this is done to match
309 // the old behavior in sdch_filter.cc.
310 }
311 // HTTP 404 might be an unencoded error page, so if decoding failed, pass it
312 // through. TODO(xunjieli): Remove this case crbug.com/516773.
313 if (response_code_ == 404) {
314 SdchManager::LogSdchProblem(net_log_, SDCH_PASS_THROUGH_404_CODE);
315 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_404);
316 return PASS_THROUGH;
317 }
318 SdchManager::LogSdchProblem(net_log_, SDCH_DICTIONARY_HASH_NOT_FOUND);
319 LogCorruptionDetection(net_log_, is_cached_content_, RESPONSE_NO_DICTIONARY);
320 return IssueMetaRefreshIfPossible(replace_output);
321 }
322
323 SdchPolicyDelegate::ErrorRecovery SdchPolicyDelegate::OnDecodingError(
324 std::string* replace_output) {
325 // A decoding error, as opposed to a dictionary error, indicates a
326 // decompression failure partway through the payload of the SDCH stream,
327 // which means that the filter already witnessed a valid dictionary ID and
328 // successfully retrieved a dictionary for it. Decoding errors are not
329 // recoverable and it is not appropriate to stop decoding, so there are
330 // relatively few error cases here.
331 //
332 // In particular, a decoding error for an HTML payload is recoverable by
333 // issuing a meta-refresh, but to avoid having that happen too often, this
334 // class also temporarily blacklists the domain. A decoding error for a
335 // non-HTML payload is unrecoverable, so such an error gets a permanent
336 // blacklist entry. If the content was cached, no blacklisting is needed.
337 // TODO(xunjieli): This case should be removed. crbug.com/651821.
338 return IssueMetaRefreshIfPossible(replace_output);
339 }
340
341 bool SdchPolicyDelegate::OnGetDictionary(const std::string& server_id,
342 const std::string** text) {
343 if (dictionary_set_) {
344 *text = dictionary_set_->GetDictionaryText(server_id);
345 if (*text) {
346 server_id_ = server_id;
347 return true;
348 }
349 }
350 // This is a hack. Naively, the dictionaries available for
351 // decoding should be only the ones advertised. However, there are
352 // cases, specifically resources encoded with old dictionaries living
353 // in the cache, that mean the full set of dictionaries should be made
354 // available for decoding. It's not known how often this happens;
355 // if it happens rarely enough, this code can be removed.
356 //
357 // TODO(rdsmith): Long-term, a better solution is necessary, since
358 // an entry in the cache being encoded with the dictionary doesn't
359 // guarantee that the dictionary is present. That solution probably
360 // involves storing unencoded resources in the cache, but might
361 // involve evicting encoded resources on dictionary removal.
362 // See http://crbug.com/383405.
363 SdchProblemCode rv = SDCH_OK;
364 unexpected_dictionary_set_ =
365 sdch_manager_->GetDictionarySetByHash(url_, server_id, &rv);
366 if (unexpected_dictionary_set_) {
367 *text = unexpected_dictionary_set_->GetDictionaryText(server_id);
368 SdchManager::LogSdchProblem(
369 net_log_, is_cached_content_ ? SDCH_UNADVERTISED_DICTIONARY_USED_CACHED
370 : SDCH_UNADVERTISED_DICTIONARY_USED);
371 if (*text) {
372 server_id_ = server_id;
373 return true;
374 }
375 }
376 return false;
377 }
378
379 void SdchPolicyDelegate::OnStreamDestroyed(
380 SdchSourceStream::InputState input_state,
381 const std::string& buffered_output,
382 open_vcdiff::VCDiffStreamingDecoder* decoder) {
383 if (decoder) {
384 if (!decoder->FinishDecoding()) {
385 SdchManager::LogSdchProblem(net_log_, SDCH_INCOMPLETE_SDCH_CONTENT);
386 // Make it possible for the user to hit reload, and get non-sdch content.
387 // Note this will "wear off" quickly enough, and is just meant to assure
388 // in some rare case that the user is not stuck.
389 sdch_manager_->BlacklistDomain(url_, SDCH_INCOMPLETE_SDCH_CONTENT);
390 }
391 }
392 // Filter chaining error, or premature teardown.
393 if (!buffered_output.empty())
394 SdchManager::LogSdchProblem(net_log_, SDCH_UNFLUSHED_CONTENT);
395
396 // Only record packet stats for non-cached content.
397 if (is_cached_content_) {
398 // Not a real error, but it is useful to have this tally.
399 // TODO(jar): Remove this stat after SDCH stability is validated.
400 SdchManager::LogSdchProblem(net_log_, SDCH_CACHE_DECODED);
401 return; // We don't need timing stats, and we aready got ratios.
402 }
403 switch (input_state) {
404 case SdchSourceStream::STATE_DECODE: {
405 job_->RecordPacketStats(FilterContext::StatisticSelector::SDCH_DECODE);
406 // Allow latency experiments to proceed.
407 sdch_manager_->SetAllowLatencyExperiment(url_, true);
408
409 // Notify successful dictionary usage.
410 DCHECK(!server_id_.empty());
411 sdch_manager_->OnDictionaryUsed(server_id_);
412 return;
413 }
414 case SdchSourceStream::STATE_LOAD_DICTIONARY:
415 SdchManager::LogSdchProblem(net_log_, SDCH_PRIOR_TO_DICTIONARY);
416 return;
417 case SdchSourceStream::STATE_PASS_THROUGH:
418 job_->RecordPacketStats(
419 FilterContext::StatisticSelector::SDCH_PASSTHROUGH);
420 return;
421 case SdchSourceStream::STATE_OUTPUT_REPLACE:
422 // This is meta refresh case. Already accounted for when set.
423 return;
424 } // end of switch.
425 }
426
427 // TODO(xunjieli): Remove meta refresh. crbug.com/651821.
428 SdchPolicyDelegate::ErrorRecovery
429 SdchPolicyDelegate::IssueMetaRefreshIfPossible(std::string* replace_output) {
430 // Errors for non-HTML payloads are unrecoverable and get the domain
431 // blacklisted indefinitely.
432 if (mime_type_.npos == mime_type_.find("text/html")) {
433 SdchProblemCode problem =
434 (is_cached_content_ ? SDCH_CACHED_META_REFRESH_UNSUPPORTED
435 : SDCH_META_REFRESH_UNSUPPORTED);
436 sdch_manager_->BlacklistDomainForever(url_, problem);
437 SdchManager::LogSdchProblem(net_log_, problem);
438 return NONE;
439 }
440
441 if (is_cached_content_) {
442 // Cached content is a probably startup tab, so just get the fresh content
443 // and try again, without disabling SDCH.
444 SdchManager::LogSdchProblem(net_log_, SDCH_META_REFRESH_CACHED_RECOVERY);
445 } else {
446 // Since it wasn't in the cache, blacklist for some period to get the
447 // correct content.
448 sdch_manager_->BlacklistDomain(url_, SDCH_META_REFRESH_RECOVERY);
449 SdchManager::LogSdchProblem(net_log_, SDCH_META_REFRESH_RECOVERY);
450 }
451
452 *replace_output = std::string(kRefreshHtml, strlen(kRefreshHtml));
453 return REPLACE_OUTPUT;
454 }
455
456 } // namespace net
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698