predictors: Store protobufs in the database.

tools/resource_prefetch_predictor/prefetch_predictor_tool.py

Index: tools/resource_prefetch_predictor/prefetch_predictor_tool.py
diff --git a/tools/resource_prefetch_predictor/prefetch_predictor_tool.py b/tools/resource_prefetch_predictor/prefetch_predictor_tool.py
index fad71305bbbe46a8bccdd3ffa286023bb0eb9ebb..fe7976414bc3f9e3e5b89e809ae59fcaae5b63ff 100755
--- a/tools/resource_prefetch_predictor/prefetch_predictor_tool.py
+++ b/tools/resource_prefetch_predictor/prefetch_predictor_tool.py
@@ -15,46 +15,29 @@ import argparse
 import operator
 import sqlite3
 
-
-class ResourceType(object):
-  """Partially mirrors content::ResourceType."""
-  STYLESHEET = 2
-  SCRIPT = 3
-  FONT_RESOURCE = 5
+from resource_prefetch_predictor_pb2 import ResourceData
 
 
 class Entry(object):
   """Represents an entry in the predictor database."""
-  HEADER = (
-      'score,main_page_url,resource_type,number_of_hits,number_of_misses,'
-      'consecutive_misses,average_position,confidence,has_validators,'
-      'always_revalidate,resource_url')
-
   def __init__(
-      self, main_page_url, resource_url, resource_type, number_of_hits,
-      number_of_misses, consecutive_misses, average_position, priority,
-      has_validators, always_revalidate):
+      self, main_page_url, resource_url, proto_buffer):
     self.main_page_url = main_page_url
     self.resource_url = resource_url
-    self.resource_type = resource_type
-    self.number_of_hits = int(number_of_hits)
-    self.number_of_misses = int(number_of_misses)
-    self.consecutive_misses = int(consecutive_misses)
-    self.average_position = int(average_position)
-    self.priority = int(priority)
-    self.has_validators = bool(int(has_validators))
-    self.always_revalidate = bool(int(always_revalidate))
-    self.confidence = float(number_of_hits) / (
-        number_of_hits + number_of_misses)
+    self.proto = ResourceData()
+    self.proto.ParseFromString(proto_buffer)
+    self.confidence = float(self.proto.number_of_hits / (
+        self.proto.number_of_hits + self.proto.number_of_misses))
     self.score = self._Score()
 
   def _Score(self):
     """Mirrors ResourcePrefetchPredictorTables::ResourceRow::UpdateScore."""
     multiplier = 1
-    if self.resource_type in (ResourceType.STYLESHEET, ResourceType.SCRIPT,
-                              ResourceType.FONT_RESOURCE):
+    if self.proto.resource_type in (ResourceData.RESOURCE_TYPE_STYLESHEET,
+                                    ResourceData.RESOURCE_TYPE_SCRIPT,
+                                    ResourceData.RESOURCE_TYPE_FONT_RESOURCE):
       multiplier = 2
-    return multiplier * 100 - self.average_position
+    return multiplier * 100 - self.proto.average_position
 
   @classmethod
   def FromRow(cls, row):
@@ -62,11 +45,8 @@ class Entry(object):
     return Entry(*row)
 
   def __str__(self):
-    return '%f,%s,%d,%d,%d,%d,%d,%f,%d,%d,%d\t%s' % (
-        self.score, self.main_page_url, self.resource_type,
-        self.number_of_hits, self.number_of_misses, self.consecutive_misses,
-        self.average_position, self.confidence, self.priority,
-        self.has_validators, self.always_revalidate, self.resource_url)
+    return 'score: %s\nmain_page_url: %s\nconfidence: %f"\n%s' % (
+        self.score, self.main_page_url, self.confidence, self.proto)
 
 
 def FilterAndSort(entries, domain):
@@ -77,20 +57,17 @@ def FilterAndSort(entries, domain):
   result = filter(
       lambda x: ((domain is None or x.main_page_url == domain)
                  and x.confidence > .7
-                 and x.number_of_hits >= 2), entries)
+                 and x.proto.number_of_hits >= 2), entries)
   return sorted(result, key=operator.attrgetter('score'), reverse=True)
 
 
 def DatabaseStats(filename, domain):
   connection = sqlite3.connect(filename)
   c = connection.cursor()
-  query = ('SELECT main_page_url, resource_url, resource_type, number_of_hits, '
-           'number_of_misses, consecutive_misses, average_position, priority, '
-           'has_validators, always_revalidate '
+  query = ('SELECT main_page_url, resource_url, proto '
            'FROM resource_prefetch_predictor_host')
   entries = [Entry.FromRow(row) for row in c.execute(query)]
   prefetched = FilterAndSort(entries, domain)
-  print Entry.HEADER

[pasko, 2016/08/22 15:51:28]

it was in CSV format, so I guess there was value of importing it as a CSV, but
now it is going to be something that:
a) only lists a part of data
b) contains newlines and does not contain a CSV header

I am probably OK with (b), even though I do not see reasoning behind it, but it
seems like we still want to be able to see all the data in the database, so (a)
is a problem.

Am I missing something?

[Benoit L, 2016/08/23 09:46:05]

The previous format was just there because it was simple. The new format is
actually a lot better, as the enum values are clearly shown.

Example:

score: 122.0
main_page_url: www.hardware.fr
confidence: 1.000000"
primary_key: ""
resource_url:
"http://www.hardware.fr/js/SocialSharePrivacy/stylesheets/modules/twitter.css"
resource_type: RESOURCE_TYPE_STYLESHEET
number_of_hits: 2
number_of_misses: 0
consecutive_misses: 0
average_position: 78.0
priority: REQUEST_PRIORITY_IDLE
has_validators: true
always_revalidate: false
score: 122.0
main_page_url: www.hardware.fr
confidence: 1.000000"
primary_key: ""
resource_url:
"http://www.hardware.fr/js/SocialSharePrivacy/stylesheets/modules/twitter.css"
resource_type: RESOURCE_TYPE_STYLESHEET
number_of_hits: 2
number_of_misses: 0
consecutive_misses: 0
average_position: 78.0
priority: REQUEST_PRIORITY_IDLE
has_validators: true
always_revalidate: false

   for x in prefetched:
     print x