Add management related code to SafeBrowsingNavigationObserverManager

chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager.h

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef CHROME_BROWSER_SAFE_BROWSING_SAFE_BROWSING_NAVIGATION_OBSERVER_MANAGER_H_
 #define CHROME_BROWSER_SAFE_BROWSING_SAFE_BROWSING_NAVIGATION_OBSERVER_MANAGER_H_
 
+#include "chrome/common/safe_browsing/csd.pb.h"
 #include "content/public/browser/notification_observer.h"
 #include "content/public/browser/notification_registrar.h"
 #include "content/public/browser/web_contents_observer.h"
 #include "url/gurl.h"
 
 namespace safe_browsing {
 
 class SafeBrowsingNavigationObserver;
 struct NavigationEvent;
 struct ResolvedIPAddress;
 
 // Manager class for SafeBrowsingNavigationObserver, which is in charge of
-// cleaning up stale navigation events, and identifing landing page/landing
+// cleaning up stale navigation events, and identifying landing page/landing
 // referrer for a specific download.
 // TODO(jialiul): For now, SafeBrowsingNavigationObserverManager also listens to
 // NOTIFICATION_RETARGETING as a way to detect cross frame/tab navigation.
 // Remove base class content::NotificationObserver when
 // WebContentsObserver::DidOpenRequestedURL() covers all retargeting cases.
 class SafeBrowsingNavigationObserverManager
     : public content::NotificationObserver,
       public base::RefCountedThreadSafe<SafeBrowsingNavigationObserverManager> {
  public:
+  // For UMA histogram counting. Do NOT change order.
+  enum AttributionResult {
+    SUCCESS = 1,                   // Identified referrer chain is not empty.
+    SUCCESS_LANDING_PAGE = 2,      // Successfully identified landing page.
+    SUCCESS_LANDING_REFERRER = 3,  // Successfully identified landing referrer.
+    INVALID_URL = 4,
+    NAVIGATION_EVENT_NOT_FOUND = 5,
+
+    // Always at the end.
+    ATTRIBUTION_FAILURE_TYPE_MAX
+  };
+
   // Helper function to check if user gesture is older than
   // kUserGestureTTLInSecond.
   static bool IsUserGestureExpired(const base::Time& timestamp);
   // Helper function to strip empty ref fragment from a URL. Many pages
   // end up with a "#" at the end of their URLs due to navigation triggered by
   // href="#" and javascript onclick function. We don't want to have separate
   // entries for these cases in the maps.
   static GURL ClearEmptyRef(const GURL& url);
 
   SafeBrowsingNavigationObserverManager();
 
   // Add |nav_event| to |navigation_map_| based on |nav_event_key|. Object
   // pointed to by |nav_event| will be no longer accessible after this function.
   void RecordNavigationEvent(const GURL& nav_event_key,
                              NavigationEvent* nav_event);
   void RecordUserGestureForWebContents(content::WebContents* web_contents,
                                        const base::Time& timestamp);
   void OnUserGestureConsumed(content::WebContents* web_contents,
                              const base::Time& timestamp);
   void RecordHostToIpMapping(const std::string& host, const std::string& ip);
+
   // Clean-ups need to be done when a WebContents gets destroyed.
   void OnWebContentDestroyed(content::WebContents* web_contents);
 
-  // TODO(jialiul): more functions are coming for managing navigation_map_.
+  // Remove all the observed NavigationEvents, user gestures, and resolved IP
+  // addresses that are older than kNavigationFootprintTTLInSecond.
+  void CleanUpStaleNavigationFootprints();
+
+  // Based on the |target_url| and |target_tab_id|, backtrace observed

[Charlie Reis, 2016/12/14 07:43:58]

nit: s/backtrace/trace back the/

[Jialiu Lin, 2016/12/14 19:06:29]

Done.

+  // NavigationEvents in navigation_map_ to identify the sequence of navigations

[Charlie Reis, 2016/12/14 07:43:58]

user-initiated navigations?

[Jialiu Lin, 2016/12/14 19:06:29]

Not only user-initiated navigations but also redirects in between. 

+  // lead to the target with the coverage limited to |user_gesture_count_limit|

[Charlie Reis, 2016/12/14 07:43:58]

nit: s/lead/leading/
nit: comma after target

[Jialiu Lin, 2016/12/14 19:06:29]

Done.

+  // number of user gestures. Then convert these identified NavigationEvents
+  // into ReferrerChainEntrys and append them to |out_referrer_chain|.
+  AttributionResult IdentifyReferrerChain(
+      const GURL& target_url,
+      int target_tab_id,  // -1 if tab id is not valid
+      int user_gesture_count_limit,
+      std::vector<ReferrerChainEntry>* out_referrer_chain);
+
+  // Identify and add referrer chain info of a download to ClientDownloadRequest
+  // proto. This function also record UMA stats of download attribution result.
+  // TODO(jialiul): This function will be moved to DownloadProtectionService
+  // class shortly.
+  void AddReferrerChainToClientDownloadRequest(
+      const GURL& download_url,
+      content::WebContents* source_contents,
+      ClientDownloadRequest* out_request);
 
  private:
   friend class base::RefCountedThreadSafe<
       SafeBrowsingNavigationObserverManager>;
   friend class TestNavigationObserverManager;
   friend class SBNavigationObserverBrowserTest;
   friend class SBNavigationObserverTest;
 
   struct GurlHash {
     std::size_t operator()(const GURL& url) const {
@@ skipping 13 matching lines @@
   void Observe(int type,
                const content::NotificationSource& source,
                const content::NotificationDetails& details) override;
 
   void RecordRetargeting(const content::NotificationDetails& details);
 
   NavigationMap* navigation_map() { return &navigation_map_; }
 
   HostToIpMap* host_to_ip_map() { return &host_to_ip_map_; }
 
+  // Remove stale entries from navigation_map_ if they are older than
+  // kNavigationFootprintTTLInSecond (2 minutes).
+  void CleanUpNavigationEvents();
+
+  // Remove stale entries from user_gesture_map_ if they are older than
+  // kUserGestureTTLInSecond (1 sec).
+  void CleanUpUserGestures();
+
+  // Remove stale entries from host_to_ip_map_ if they are older than
+  // kNavigationFootprintTTLInSecond (2 minutes).
+  void CleanUpIpAddresses();
+
+  bool IsCleanUpScheduled() const;
+
+  void ScheduleNextCleanUpAfterInterval(base::TimeDelta interval);
+
+  // Find the most recent navigation event that navigated to |target_url| in the
+  // tab with ID |target_tab_id|. If |target_tab_id| is not available (-1), we
+  // look for all tabs for the most recent navigation to |target_url|.
+  // For some cases, the most recent navigation to |target_url| may not be
+  // relevant.
+  // For example, url1 in window A opens url2 in window B, url1 then opens an
+  // about::blank page window C and injects script code in it to trigger a

[Charlie Reis, 2016/12/14 07:43:58]

nit: about:blank

[Jialiu Lin, 2016/12/14 19:06:29]

Oops. Fixed.

+  // delayed download in Window D. Before the download occurs, url2 in window B
+  // opens a different about:blank page in window C.
+  // A ---- C - D
+  //   \   /
+  //     B
+  // In this case, FindNavigationEvent() will think url2 in Window B is the
+  // referrer of about::blank in Window C since this navigation is more recent.
+  // However, it does not prevent us to attribute url1 in Window A as the cause
+  // of all these navigations.
+  NavigationEvent* FindNavigationEvent(const GURL& target_url,
+                                       int target_tab_id);
+
+  void AddToReferrerChain(std::vector<ReferrerChainEntry>* referrer_chain,
+                          NavigationEvent* nav_event,
+                          ReferrerChainEntry::URLType type);
+
   // navigation_map_ keeps track of all the observed navigations. This map is
   // keyed on the resolved request url. In other words, in case of server
   // redirects, its key is the last server redirect url, otherwise, it is the
   // original target url. Since the same url can be requested multiple times
   // across different tabs and frames, the value of this map is a vector of
   // NavigationEvent ordered by navigation finish time.
   // TODO(jialiul): Entries in navigation_map_ will be removed if they are older
   // than 2 minutes since their corresponding navigations finish.
   NavigationMap navigation_map_;
 
   // user_gesture_map_ keeps track of the timestamp of last user gesture in
   // in each WebContents. We assume for majority of cases, a navigation
   // shortly after a user gesture indicate this navigation is user initiated.
   UserGestureMap user_gesture_map_;
 
   // Host to timestamped IP addresses map that covers all the main frame and
   // subframe URLs' hosts. Since it is possible for a host to resolve to more
   // than one IP in even a short period of time, we map a single host to a
   // vector of ResolvedIPAddresss. This map is used to fill in ip_address field
   // in URLChainEntry in ClientDownloadRequest.
   HostToIpMap host_to_ip_map_;
 
   content::NotificationRegistrar registrar_;
 
+  base::OneShotTimer cleanup_timer_;
+
   DISALLOW_COPY_AND_ASSIGN(SafeBrowsingNavigationObserverManager);
 };
 }  // namespace safe_browsing
 
 #endif  // CHROME_BROWSER_SAFE_BROWSING_SAFE_BROWSING_NAVIGATION_OBSERVER_MANAGER_H_