| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include "net/url_request/url_request_throttler_entry.h" | |
| 6 | |
| 7 #include <cmath> | |
| 8 | |
| 9 #include "base/logging.h" | |
| 10 #include "base/metrics/field_trial.h" | |
| 11 #include "base/metrics/histogram.h" | |
| 12 #include "base/rand_util.h" | |
| 13 #include "base/strings/string_number_conversions.h" | |
| 14 #include "base/values.h" | |
| 15 #include "net/base/load_flags.h" | |
| 16 #include "net/base/net_log.h" | |
| 17 #include "net/url_request/url_request.h" | |
| 18 #include "net/url_request/url_request_context.h" | |
| 19 #include "net/url_request/url_request_throttler_header_interface.h" | |
| 20 #include "net/url_request/url_request_throttler_manager.h" | |
| 21 | |
| 22 namespace net { | |
| 23 | |
| 24 const int URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs = 2000; | |
| 25 const int URLRequestThrottlerEntry::kDefaultMaxSendThreshold = 20; | |
| 26 | |
| 27 // This set of back-off parameters will (at maximum values, i.e. without | |
| 28 // the reduction caused by jitter) add 0-41% (distributed uniformly | |
| 29 // in that range) to the "perceived downtime" of the remote server, once | |
| 30 // exponential back-off kicks in and is throttling requests for more than | |
| 31 // about a second at a time. Once the maximum back-off is reached, the added | |
| 32 // perceived downtime decreases rapidly, percentage-wise. | |
| 33 // | |
| 34 // Another way to put it is that the maximum additional perceived downtime | |
| 35 // with these numbers is a couple of seconds shy of 15 minutes, and such | |
| 36 // a delay would not occur until the remote server has been actually | |
| 37 // unavailable at the end of each back-off period for a total of about | |
| 38 // 48 minutes. | |
| 39 // | |
| 40 // Ignoring the first couple of errors is just a conservative measure to | |
| 41 // avoid false positives. It should help avoid back-off from kicking in e.g. | |
| 42 // on flaky connections. | |
| 43 const int URLRequestThrottlerEntry::kDefaultNumErrorsToIgnore = 2; | |
| 44 const int URLRequestThrottlerEntry::kDefaultInitialDelayMs = 700; | |
| 45 const double URLRequestThrottlerEntry::kDefaultMultiplyFactor = 1.4; | |
| 46 const double URLRequestThrottlerEntry::kDefaultJitterFactor = 0.4; | |
| 47 const int URLRequestThrottlerEntry::kDefaultMaximumBackoffMs = 15 * 60 * 1000; | |
| 48 const int URLRequestThrottlerEntry::kDefaultEntryLifetimeMs = 2 * 60 * 1000; | |
| 49 const char URLRequestThrottlerEntry::kExponentialThrottlingHeader[] = | |
| 50 "X-Chrome-Exponential-Throttling"; | |
| 51 const char URLRequestThrottlerEntry::kExponentialThrottlingDisableValue[] = | |
| 52 "disable"; | |
| 53 | |
| 54 // Returns NetLog parameters when a request is rejected by throttling. | |
| 55 base::Value* NetLogRejectedRequestCallback(const std::string* url_id, | |
| 56 int num_failures, | |
| 57 const base::TimeDelta& release_after, | |
| 58 NetLog::LogLevel /* log_level */) { | |
| 59 base::DictionaryValue* dict = new base::DictionaryValue(); | |
| 60 dict->SetString("url", *url_id); | |
| 61 dict->SetInteger("num_failures", num_failures); | |
| 62 dict->SetInteger("release_after_ms", | |
| 63 static_cast<int>(release_after.InMilliseconds())); | |
| 64 return dict; | |
| 65 } | |
| 66 | |
| 67 URLRequestThrottlerEntry::URLRequestThrottlerEntry( | |
| 68 URLRequestThrottlerManager* manager, | |
| 69 const std::string& url_id) | |
| 70 : sliding_window_period_( | |
| 71 base::TimeDelta::FromMilliseconds(kDefaultSlidingWindowPeriodMs)), | |
| 72 max_send_threshold_(kDefaultMaxSendThreshold), | |
| 73 is_backoff_disabled_(false), | |
| 74 backoff_entry_(&backoff_policy_), | |
| 75 manager_(manager), | |
| 76 url_id_(url_id), | |
| 77 net_log_(BoundNetLog::Make( | |
| 78 manager->net_log(), NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING)) { | |
| 79 DCHECK(manager_); | |
| 80 Initialize(); | |
| 81 } | |
| 82 | |
| 83 URLRequestThrottlerEntry::URLRequestThrottlerEntry( | |
| 84 URLRequestThrottlerManager* manager, | |
| 85 const std::string& url_id, | |
| 86 int sliding_window_period_ms, | |
| 87 int max_send_threshold, | |
| 88 int initial_backoff_ms, | |
| 89 double multiply_factor, | |
| 90 double jitter_factor, | |
| 91 int maximum_backoff_ms) | |
| 92 : sliding_window_period_( | |
| 93 base::TimeDelta::FromMilliseconds(sliding_window_period_ms)), | |
| 94 max_send_threshold_(max_send_threshold), | |
| 95 is_backoff_disabled_(false), | |
| 96 backoff_entry_(&backoff_policy_), | |
| 97 manager_(manager), | |
| 98 url_id_(url_id) { | |
| 99 DCHECK_GT(sliding_window_period_ms, 0); | |
| 100 DCHECK_GT(max_send_threshold_, 0); | |
| 101 DCHECK_GE(initial_backoff_ms, 0); | |
| 102 DCHECK_GT(multiply_factor, 0); | |
| 103 DCHECK_GE(jitter_factor, 0.0); | |
| 104 DCHECK_LT(jitter_factor, 1.0); | |
| 105 DCHECK_GE(maximum_backoff_ms, 0); | |
| 106 DCHECK(manager_); | |
| 107 | |
| 108 Initialize(); | |
| 109 backoff_policy_.initial_delay_ms = initial_backoff_ms; | |
| 110 backoff_policy_.multiply_factor = multiply_factor; | |
| 111 backoff_policy_.jitter_factor = jitter_factor; | |
| 112 backoff_policy_.maximum_backoff_ms = maximum_backoff_ms; | |
| 113 backoff_policy_.entry_lifetime_ms = -1; | |
| 114 backoff_policy_.num_errors_to_ignore = 0; | |
| 115 backoff_policy_.always_use_initial_delay = false; | |
| 116 } | |
| 117 | |
| 118 bool URLRequestThrottlerEntry::IsEntryOutdated() const { | |
| 119 // This function is called by the URLRequestThrottlerManager to determine | |
| 120 // whether entries should be discarded from its url_entries_ map. We | |
| 121 // want to ensure that it does not remove entries from the map while there | |
| 122 // are clients (objects other than the manager) holding references to | |
| 123 // the entry, otherwise separate clients could end up holding separate | |
| 124 // entries for a request to the same URL, which is undesirable. Therefore, | |
| 125 // if an entry has more than one reference (the map will always hold one), | |
| 126 // it should not be considered outdated. | |
| 127 // | |
| 128 // We considered whether to make URLRequestThrottlerEntry objects | |
| 129 // non-refcounted, but since any means of knowing whether they are | |
| 130 // currently in use by others than the manager would be more or less | |
| 131 // equivalent to a refcount, we kept them refcounted. | |
| 132 if (!HasOneRef()) | |
| 133 return false; | |
| 134 | |
| 135 // If there are send events in the sliding window period, we still need this | |
| 136 // entry. | |
| 137 if (!send_log_.empty() && | |
| 138 send_log_.back() + sliding_window_period_ > ImplGetTimeNow()) { | |
| 139 return false; | |
| 140 } | |
| 141 | |
| 142 return GetBackoffEntry()->CanDiscard(); | |
| 143 } | |
| 144 | |
| 145 void URLRequestThrottlerEntry::DisableBackoffThrottling() { | |
| 146 is_backoff_disabled_ = true; | |
| 147 } | |
| 148 | |
| 149 void URLRequestThrottlerEntry::DetachManager() { | |
| 150 manager_ = NULL; | |
| 151 } | |
| 152 | |
| 153 bool URLRequestThrottlerEntry::ShouldRejectRequest( | |
| 154 const URLRequest& request, | |
| 155 NetworkDelegate* network_delegate) const { | |
| 156 bool reject_request = false; | |
| 157 if (!is_backoff_disabled_ && !ExplicitUserRequest(request.load_flags()) && | |
| 158 (!network_delegate || network_delegate->CanThrottleRequest(request)) && | |
| 159 GetBackoffEntry()->ShouldRejectRequest()) { | |
| 160 net_log_.AddEvent( | |
| 161 NetLog::TYPE_THROTTLING_REJECTED_REQUEST, | |
| 162 base::Bind(&NetLogRejectedRequestCallback, | |
| 163 &url_id_, | |
| 164 GetBackoffEntry()->failure_count(), | |
| 165 GetBackoffEntry()->GetTimeUntilRelease())); | |
| 166 reject_request = true; | |
| 167 } | |
| 168 | |
| 169 int reject_count = reject_request ? 1 : 0; | |
| 170 UMA_HISTOGRAM_ENUMERATION( | |
| 171 "Throttling.RequestThrottled", reject_count, 2); | |
| 172 | |
| 173 return reject_request; | |
| 174 } | |
| 175 | |
| 176 int64 URLRequestThrottlerEntry::ReserveSendingTimeForNextRequest( | |
| 177 const base::TimeTicks& earliest_time) { | |
| 178 base::TimeTicks now = ImplGetTimeNow(); | |
| 179 | |
| 180 // If a lot of requests were successfully made recently, | |
| 181 // sliding_window_release_time_ may be greater than | |
| 182 // exponential_backoff_release_time_. | |
| 183 base::TimeTicks recommended_sending_time = | |
| 184 std::max(std::max(now, earliest_time), | |
| 185 std::max(GetBackoffEntry()->GetReleaseTime(), | |
| 186 sliding_window_release_time_)); | |
| 187 | |
| 188 DCHECK(send_log_.empty() || | |
| 189 recommended_sending_time >= send_log_.back()); | |
| 190 // Log the new send event. | |
| 191 send_log_.push(recommended_sending_time); | |
| 192 | |
| 193 sliding_window_release_time_ = recommended_sending_time; | |
| 194 | |
| 195 // Drop the out-of-date events in the event list. | |
| 196 // We don't need to worry that the queue may become empty during this | |
| 197 // operation, since the last element is sliding_window_release_time_. | |
| 198 while ((send_log_.front() + sliding_window_period_ <= | |
| 199 sliding_window_release_time_) || | |
| 200 send_log_.size() > static_cast<unsigned>(max_send_threshold_)) { | |
| 201 send_log_.pop(); | |
| 202 } | |
| 203 | |
| 204 // Check if there are too many send events in recent time. | |
| 205 if (send_log_.size() == static_cast<unsigned>(max_send_threshold_)) | |
| 206 sliding_window_release_time_ = send_log_.front() + sliding_window_period_; | |
| 207 | |
| 208 return (recommended_sending_time - now).InMillisecondsRoundedUp(); | |
| 209 } | |
| 210 | |
| 211 base::TimeTicks | |
| 212 URLRequestThrottlerEntry::GetExponentialBackoffReleaseTime() const { | |
| 213 // If a site opts out, it's likely because they have problems that trigger | |
| 214 // the back-off mechanism when it shouldn't be triggered, in which case | |
| 215 // returning the calculated back-off release time would probably be the | |
| 216 // wrong thing to do (i.e. it would likely be too long). Therefore, we | |
| 217 // return "now" so that retries are not delayed. | |
| 218 if (is_backoff_disabled_) | |
| 219 return ImplGetTimeNow(); | |
| 220 | |
| 221 return GetBackoffEntry()->GetReleaseTime(); | |
| 222 } | |
| 223 | |
| 224 void URLRequestThrottlerEntry::UpdateWithResponse( | |
| 225 const std::string& host, | |
| 226 const URLRequestThrottlerHeaderInterface* response) { | |
| 227 if (IsConsideredError(response->GetResponseCode())) { | |
| 228 GetBackoffEntry()->InformOfRequest(false); | |
| 229 } else { | |
| 230 GetBackoffEntry()->InformOfRequest(true); | |
| 231 | |
| 232 std::string throttling_header = response->GetNormalizedValue( | |
| 233 kExponentialThrottlingHeader); | |
| 234 if (!throttling_header.empty()) | |
| 235 HandleThrottlingHeader(throttling_header, host); | |
| 236 } | |
| 237 } | |
| 238 | |
| 239 void URLRequestThrottlerEntry::ReceivedContentWasMalformed(int response_code) { | |
| 240 // A malformed body can only occur when the request to fetch a resource | |
| 241 // was successful. Therefore, in such a situation, we will receive one | |
| 242 // call to ReceivedContentWasMalformed() and one call to | |
| 243 // UpdateWithResponse() with a response categorized as "good". To end | |
| 244 // up counting one failure, we need to count two failures here against | |
| 245 // the one success in UpdateWithResponse(). | |
| 246 // | |
| 247 // We do nothing for a response that is already being considered an error | |
| 248 // based on its status code (otherwise we would count 3 errors instead of 1). | |
| 249 if (!IsConsideredError(response_code)) { | |
| 250 GetBackoffEntry()->InformOfRequest(false); | |
| 251 GetBackoffEntry()->InformOfRequest(false); | |
| 252 } | |
| 253 } | |
| 254 | |
| 255 URLRequestThrottlerEntry::~URLRequestThrottlerEntry() { | |
| 256 } | |
| 257 | |
| 258 void URLRequestThrottlerEntry::Initialize() { | |
| 259 sliding_window_release_time_ = base::TimeTicks::Now(); | |
| 260 backoff_policy_.num_errors_to_ignore = kDefaultNumErrorsToIgnore; | |
| 261 backoff_policy_.initial_delay_ms = kDefaultInitialDelayMs; | |
| 262 backoff_policy_.multiply_factor = kDefaultMultiplyFactor; | |
| 263 backoff_policy_.jitter_factor = kDefaultJitterFactor; | |
| 264 backoff_policy_.maximum_backoff_ms = kDefaultMaximumBackoffMs; | |
| 265 backoff_policy_.entry_lifetime_ms = kDefaultEntryLifetimeMs; | |
| 266 backoff_policy_.always_use_initial_delay = false; | |
| 267 } | |
| 268 | |
| 269 bool URLRequestThrottlerEntry::IsConsideredError(int response_code) { | |
| 270 // We throttle only for the status codes most likely to indicate the server | |
| 271 // is failing because it is too busy or otherwise are likely to be | |
| 272 // because of DDoS. | |
| 273 // | |
| 274 // 500 is the generic error when no better message is suitable, and | |
| 275 // as such does not necessarily indicate a temporary state, but | |
| 276 // other status codes cover most of the permanent error states. | |
| 277 // 503 is explicitly documented as a temporary state where the server | |
| 278 // is either overloaded or down for maintenance. | |
| 279 // 509 is the (non-standard but widely implemented) Bandwidth Limit Exceeded | |
| 280 // status code, which might indicate DDoS. | |
| 281 // | |
| 282 // We do not back off on 502 or 504, which are reported by gateways | |
| 283 // (proxies) on timeouts or failures, because in many cases these requests | |
| 284 // have not made it to the destination server and so we do not actually | |
| 285 // know that it is down or busy. One degenerate case could be a proxy on | |
| 286 // localhost, where you are not actually connected to the network. | |
| 287 return (response_code == 500 || | |
| 288 response_code == 503 || | |
| 289 response_code == 509); | |
| 290 } | |
| 291 | |
| 292 base::TimeTicks URLRequestThrottlerEntry::ImplGetTimeNow() const { | |
| 293 return base::TimeTicks::Now(); | |
| 294 } | |
| 295 | |
| 296 void URLRequestThrottlerEntry::HandleThrottlingHeader( | |
| 297 const std::string& header_value, | |
| 298 const std::string& host) { | |
| 299 if (header_value == kExponentialThrottlingDisableValue) { | |
| 300 DisableBackoffThrottling(); | |
| 301 if (manager_) | |
| 302 manager_->AddToOptOutList(host); | |
| 303 } | |
| 304 } | |
| 305 | |
| 306 const BackoffEntry* URLRequestThrottlerEntry::GetBackoffEntry() const { | |
| 307 return &backoff_entry_; | |
| 308 } | |
| 309 | |
| 310 BackoffEntry* URLRequestThrottlerEntry::GetBackoffEntry() { | |
| 311 return &backoff_entry_; | |
| 312 } | |
| 313 | |
| 314 // static | |
| 315 bool URLRequestThrottlerEntry::ExplicitUserRequest(const int load_flags) { | |
| 316 return (load_flags & LOAD_MAYBE_USER_GESTURE) != 0; | |
| 317 } | |
| 318 | |
| 319 } // namespace net | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 992733002:
Remove //net (except for Android test stuff) and sdch (Closed)
Base URL: git@github.com:domokit/mojo.git@master