Add scripts for distillability modelling

heuristics/distillable/server.py

Index: heuristics/distillable/server.py
diff --git a/foo/server.py b/heuristics/distillable/server.py
old mode 100644
new mode 100755
similarity index 80%
rename from foo/server.py
rename to heuristics/distillable/server.py
index 1fcfc1dc0c973e576ca52eea20f82f2a65fe8534..ce50876b11129537c8442fc0a13eb76507db9b7e
--- a/foo/server.py
+++ b/heuristics/distillable/server.py
@@ -1,8 +1,14 @@
+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
 import argparse
 import collections
 import cherrypy
 import json
 import os
+import sys
 import time
 
 genDelta = [
@@ -57,7 +63,8 @@ def saveData(service):
   currentTime = time.time()
   path = os.path.join(service.getArchivePath(), 'archive-%s.json' % time.strftime(timeFormat))
   with open(path, 'w') as outfile:
-    json.dump(service.getData(), outfile)
+    json.dump(service.getData(), outfile, indent=2)
+    print 'saved to %s' % (path)
   lastGeneration.append({
     'lastId': lastUpdate,
     'path': path,
@@ -83,11 +90,23 @@ class Service(object):
       self.data = json.load(inf)
       self.initIdxMap()
 
+    archives = [os.path.join(self.archive_path, f) for f in os.listdir(self.archive_path)]
+    archives = [f for f in archives if os.path.isfile(f)]
+    for archive in archives:
+      with open(archive) as inf:
+        last = json.load(inf)
+        for i in last:
+          url = i['url']
+          if url in self.idxMap and 'good' in i:
+            assert self.data[self.idxMap[url]]['index'] == i['index']
+            self.data[self.idxMap[url]]['good'] = i['good']
+            print "%d good = %s" % (self.idxMap[url], i['good'])
+
     self.archives = []
     for i in range(genCount):
       self.archives.append([])
 
-    self.saver = cherrypy.process.plugins.BackgroundTask(2.0 * 60, saveData, [self])
+    self.saver = cherrypy.process.plugins.BackgroundTask(1.0 * 60, saveData, [self])
     self.saver.start()
 
   def initIdxMap(self):
@@ -174,14 +193,15 @@ if __name__ == '__main__':
     '/': {
       'tools.response_headers.on': True,
       'tools.response_headers.headers': [('Content-Type', 'text/plain')],
-    },
-    '/foo': {
       'tools.staticdir.on': True,
-      'tools.staticdir.dir': '/usr/local/google/code/dom_distiller/foo',
+      'tools.staticdir.dir': os.getcwd(),
+      'tools.staticdir.index': 'index.html',
     },
     '/images': {
       'tools.staticdir.on': True,
       'tools.staticdir.dir': os.path.join(os.getcwd(), options.data_dir),
+      'tools.expires.on': True,
+      'tools.expires.secs': 60,
     }
   }
-  cherrypy.quickstart(service, '/', conf)
+  cherrypy.quickstart(service, '', conf)