predictors: Add origin learning.

chrome/browser/predictors/resource_prefetch_predictor.h

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_
 #define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_
 
 #include <stddef.h>
 
 #include <map>
@@ skipping 86 matching lines @@
 //
 // TODO(zhenw): Currently only main frame requests/redirects/responses are
 // recorded. Consider recording sub-frame responses independently or together
 // with main frame.
 class ResourcePrefetchPredictor
     : public KeyedService,
       public history::HistoryServiceObserver,
       public base::SupportsWeakPtr<ResourcePrefetchPredictor>,
       public precache::PrecacheManager::Delegate {
  public:
+  struct OriginRequestSummary {

[alexilin, 2017/04/10 14:58:28]

Do we really need this struct?
Can't we derive the summary from a vector of URLRequestSummary?

[Benoit L, 2017/04/11 09:04:05]

You're right, this is not strictly necessary.
This is mostly here for readability, and also because we do not store all the
matching URLRequestSummary. For instance, redirects and no-store resources.

I would prefer to keep it, if you don't mind.
wdyt?

[alexilin, 2017/04/11 09:42:19]

Sgtm. Anyway, we would have to reconstruct this map in LearnOrigins().

+    OriginRequestSummary();
+    OriginRequestSummary(const OriginRequestSummary& other);
+    ~OriginRequestSummary();
+
+    GURL origin;
+    bool always_access_network;
+    bool accessed_network;
+    int first_occurrence;
+  };
+
   // Stores the data that we need to get from the URLRequest.
   struct URLRequestSummary {
     URLRequestSummary();
     URLRequestSummary(const URLRequestSummary& other);
     ~URLRequestSummary();
 
     NavigationID navigation_id;
     GURL resource_url;
     content::ResourceType resource_type;
     net::RequestPriority priority;
 
     // Only for responses.
     std::string mime_type;
     bool was_cached;
     GURL redirect_url;  // Empty unless request was redirected to a valid url.
 
     bool has_validators;
     bool always_revalidate;
+    bool is_no_store;
+    bool network_accessed;
 
     // Initializes a |URLRequestSummary| from a |URLRequest| response.
     // Returns true for success. Note: NavigationID is NOT initialized
     // by this function.
     static bool SummarizeResponse(const net::URLRequest& request,
                                   URLRequestSummary* summary);
   };
 
   // Stores the data learned from a single navigation.
   struct PageRequestSummary {
     explicit PageRequestSummary(const GURL& main_frame_url);
     PageRequestSummary(const PageRequestSummary& other);
     ~PageRequestSummary();
 
     GURL main_frame_url;
     GURL initial_url;
 
     // Stores all subresource requests within a single navigation, from initial
     // main frame request to navigation completion.
     std::vector<URLRequestSummary> subresource_requests;
+    // Map of origin -> OriginRequestSummary. Only one instance of each origin
+    // is kept per navigation, but the summary is updated several times.
+    std::map<GURL, OriginRequestSummary> origins;
   };
 
   // Stores a result of prediction. Essentially, |subresource_urls| is main
   // result and other fields are used for diagnosis and histograms reporting.
   struct Prediction {
     Prediction();
     Prediction(const Prediction& other);
     ~Prediction();
 
     bool is_host;
@@ skipping 118 matching lines @@
                            TestPrefetchingDurationHistogram);
 
   enum InitializationState {
     NOT_INITIALIZED = 0,
     INITIALIZING = 1,
     INITIALIZED = 2
   };
   typedef ResourcePrefetchPredictorTables::PrefetchDataMap PrefetchDataMap;
   typedef ResourcePrefetchPredictorTables::RedirectDataMap RedirectDataMap;
   typedef ResourcePrefetchPredictorTables::ManifestDataMap ManifestDataMap;
+  typedef ResourcePrefetchPredictorTables::OriginDataMap OriginDataMap;
 
   typedef std::map<NavigationID, std::unique_ptr<PageRequestSummary>>
       NavigationMap;
 
   // Returns true if the main page request is supported for prediction.
   static bool IsHandledMainPage(net::URLRequest* request);
 
   // Returns true if the subresource request is supported for prediction.
   static bool IsHandledSubresource(net::URLRequest* request,
                                    content::ResourceType resource_type);
 
   // Returns true if the subresource has a supported type.
   static bool IsHandledResourceType(content::ResourceType resource_type,
                                     const std::string& mime_type);
 
   // Returns true if the request (should have a response in it) is "no-store".
-  static bool IsNoStore(const net::URLRequest* request);
+  static bool IsNoStore(const net::URLRequest& request);
 
   // Returns true iff |redirect_data_map| contains confident redirect endpoint
   // for |entry_point| and assigns it to the |redirect_endpoint|.
   static bool GetRedirectEndpoint(const std::string& entry_point,
                                   const RedirectDataMap& redirect_data_map,
                                   std::string* redirect_endpoint);
 
   static void SetAllowPortInUrlsForTesting(bool state);
 
   // KeyedService methods override.
   void Shutdown() override;
 
   // Functions called on different network events pertaining to the loading of
   // main frame resource or sub resources.
   void OnMainFrameRequest(const URLRequestSummary& request);
   void OnMainFrameResponse(const URLRequestSummary& response);
   void OnMainFrameRedirect(const URLRequestSummary& response);
   void OnSubresourceResponse(const URLRequestSummary& response);
+  void OnSubresourceRedirect(const URLRequestSummary& response);
 
   // Called when onload completes for a navigation. We treat this point as the
   // "completion" of the navigation. The resources requested by the page up to
   // this point are the only ones considered for prefetching.
   void OnNavigationComplete(const NavigationID& nav_id_without_timing_info);
 
   // Returns true iff there is PrefetchData that can be used for a
   // |main_frame_url| and fills |prediction| with resources that need to be
   // prefetched. |prediction| pointer may be equal nullptr to get return value
   // only.
   bool GetPrefetchData(const GURL& main_frame_url,
                        Prediction* prediction) const;
 
   // Returns true iff the |data_map| contains PrefetchData that can be used
   // for a |main_frame_key| and fills |urls| with resources that need to be
   // prefetched. |urls| pointer may be equal nullptr to get return value only.
   bool PopulatePrefetcherRequest(const std::string& main_frame_key,
                                  const PrefetchDataMap& data_map,
                                  std::vector<GURL>* urls) const;
 
   // Callback for task to read predictor database. Takes ownership of
   // all arguments.
   void CreateCaches(std::unique_ptr<PrefetchDataMap> url_data_map,
                     std::unique_ptr<PrefetchDataMap> host_data_map,
                     std::unique_ptr<RedirectDataMap> url_redirect_data_map,
                     std::unique_ptr<RedirectDataMap> host_redirect_data_map,
-                    std::unique_ptr<ManifestDataMap> manifest_data_map);
+                    std::unique_ptr<ManifestDataMap> manifest_data_map,
+                    std::unique_ptr<OriginDataMap> origin_data_map);
 
   // Called during initialization when history is read and the predictor
   // database has been read.
   void OnHistoryAndCacheLoaded();
 
   // Cleanup inflight_navigations_, inflight_prefetches_, and prefetcher_stats_.
   void CleanupAbandonedNavigations(const NavigationID& navigation_id);
 
   // Deletes all URLs from the predictor database, the caches and removes all
   // inflight navigations.
   void DeleteAllUrls();
 
   // Deletes data for the input |urls| and their corresponding hosts from the
   // predictor database and caches.
   void DeleteUrls(const history::URLRows& urls);
 
   // Callback for GetUrlVisitCountTask.
   void OnVisitCountLookup(size_t url_visit_count,
                           const PageRequestSummary& summary);
 
   // Removes the oldest entry in the input |data_map|, also deleting it from the
   // predictor database.
   void RemoveOldestEntryInPrefetchDataMap(PrefetchKeyType key_type,
                                           PrefetchDataMap* data_map);
 
   void RemoveOldestEntryInRedirectDataMap(PrefetchKeyType key_type,
                                           RedirectDataMap* data_map);
 
   void RemoveOldestEntryInManifestDataMap(ManifestDataMap* data_map);
+  void RemoveOldestEntryInOriginDataMap(OriginDataMap* data_map);
 
   // Merges resources in |new_resources| into the |data_map| and correspondingly
   // updates the predictor database. Also calls LearnRedirect if relevant.
   void LearnNavigation(const std::string& key,
                        PrefetchKeyType key_type,
                        const std::vector<URLRequestSummary>& new_resources,
                        size_t max_data_map_size,
                        PrefetchDataMap* data_map,
                        const std::string& key_before_redirects,
                        RedirectDataMap* redirect_map);
 
   // Updates information about final redirect destination for |key| in
   // |redirect_map| and correspondingly updates the predictor database.
   void LearnRedirect(const std::string& key,
                      PrefetchKeyType key_type,
                      const std::string& final_redirect,
                      size_t max_redirect_map_size,
                      RedirectDataMap* redirect_map);
 
+  void LearnOrigins(const std::string& host,
+                    const std::map<GURL, OriginRequestSummary>& summaries,
+                    size_t max_data_map_size,
+                    OriginDataMap* data_map);
+
   // Reports database readiness metric defined as percentage of navigated hosts
   // found in DB for last X entries in history.
   void ReportDatabaseReadiness(const history::TopHostsList& top_hosts) const;
 
   // history::HistoryServiceObserver:
   void OnURLsDeleted(history::HistoryService* history_service,
                      bool all_history,
                      bool expired,
                      const history::URLRows& deleted_rows,
                      const std::set<GURL>& favicon_urls) override;
@@ skipping 16 matching lines @@
   scoped_refptr<ResourcePrefetchPredictorTables> tables_;
   scoped_refptr<ResourcePrefetcherManager> prefetch_manager_;
   base::CancelableTaskTracker history_lookup_consumer_;
 
   // Copy of the data in the predictor tables.
   std::unique_ptr<PrefetchDataMap> url_table_cache_;
   std::unique_ptr<PrefetchDataMap> host_table_cache_;
   std::unique_ptr<RedirectDataMap> url_redirect_table_cache_;
   std::unique_ptr<RedirectDataMap> host_redirect_table_cache_;
   std::unique_ptr<ManifestDataMap> manifest_table_cache_;
+  std::unique_ptr<OriginDataMap> origin_table_cache_;
 
   std::map<GURL, base::TimeTicks> inflight_prefetches_;
   NavigationMap inflight_navigations_;
 
   std::map<GURL, std::unique_ptr<ResourcePrefetcher::PrefetcherStats>>
       prefetcher_stats_;
 
   ScopedObserver<history::HistoryService, history::HistoryServiceObserver>
       history_service_observer_;
 
@@ skipping 26 matching lines @@
 
  private:
   ResourcePrefetchPredictor* predictor_;
 
   DISALLOW_COPY_AND_ASSIGN(TestObserver);
 };
 
 }  // namespace predictors
 
 #endif  // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_