Add callstats.py run --refresh flag

tools/callstats.py

 #!/usr/bin/env python
 # Copyright 2016 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 '''
 Usage: runtime-call-stats.py [-h] <command> ...
 
 Optional arguments:
   -h, --help  show this help message and exit
 
@@ skipping 31 matching lines @@
     elif ' ' in arg:
       arg = "'{}'".format(arg)
     return arg
   print " ".join(map(fix_for_printing, cmd_args))
 
 
 def start_replay_server(args, sites):
   with tempfile.NamedTemporaryFile(prefix='callstats-inject-', suffix='.js',
                                    mode='wt', delete=False) as f:
     injection = f.name
-    generate_injection(f, sites)
+    generate_injection(f, sites, args.refresh)
   cmd_args = [
       args.replay_bin,
       "--port=4080",
       "--ssl_port=4443",
       "--no-dns_forwarding",
       "--use_closest_match",
       "--no-diff_unknown_requests",
       "--inject_scripts=deterministic.js,{}".format(injection),
       args.replay_wpr,
   ]
   print "=" * 80
   print_command(cmd_args)
   with open(os.devnull, 'w') as null:
     server = subprocess.Popen(cmd_args, stdout=null, stderr=null)
   print "RUNNING REPLAY SERVER: %s with PID=%s" % (args.replay_bin, server.pid)
   print "=" * 80
   return {'process': server, 'injection': injection}
 
 
 def stop_replay_server(server):
   print("SHUTTING DOWN REPLAY SERVER %s" % server['process'].pid)
   server['process'].terminate()
   os.remove(server['injection'])
 
 
-def generate_injection(f, sites):
+def generate_injection(f, sites, refreshes=0):
   print >> f, """\
 (function() {
+  let s = window.sessionStorage.getItem("refreshCounter");
+  let refreshTotal = """, refreshes, """;
+  let refreshCounter = s ? parseInt(s) : refreshTotal;
+  let refreshId = refreshTotal - refreshCounter;
+  if (refreshCounter > 0) {
+    window.sessionStorage.setItem("refreshCounter", refreshCounter-1);
+  }
+
   function match(url, item) {
     if ('regexp' in item) return url.match(item.regexp) !== null;
     let url_wanted = item.url;
     // Allow automatic redirections from http to https.
     if (url_wanted.startsWith("http://") && url.startsWith("https://")) {
       url_wanted = "https://" + url_wanted.substr(7);
     }
     return url.startsWith(url_wanted);
   };
 
   function onLoad(e) {
     let url = e.target.URL;
     for (let item of sites) {
       if (!match(url, item)) continue;
-      let timeout = 'timeline' in item ? 2500 * item.timeline + 3000
+      let timeout = 'timeline' in item ? 2500 * item.timeline
                   : 'timeout'  in item ? 1000 * (item.timeout - 3)
                   : 10000;
       console.log("Setting time out of " + timeout + " for: " + url);
-      window.setTimeout(function () {
+      window.setTimeout(function() {
         console.log("Time is out for: " + url);
-        %GetAndResetRuntimeCallStats(1);
+        let msg = "STATS: (" + refreshId + ") " + url;
+        %GetAndResetRuntimeCallStats(1, msg);
+        if (refreshCounter > 0) {
+          console.log("Refresh counter is " + refreshCounter + ", refreshing: " + url);
+          window.location.reload();
+        }
       }, timeout);
       return;
     }
     console.log("Ignoring: " + url);
   };
 
   let sites =
     """, json.dumps(sites), """;
 
   console.log("Event listenner added for: " + window.location.href);
   window.addEventListener("load", onLoad);
 })();"""
 
 
 def run_site(site, domain, args, timeout=None):
   print "="*80
   print "RUNNING DOMAIN %s" % domain
   print "="*80
   result_template = "{domain}#{count}.txt" if args.repeat else "{domain}.txt"
   count = 0
   if timeout is None: timeout = args.timeout
-  if args.replay_wpr: timeout += 1
+  if args.replay_wpr:
+    timeout *= 1 + args.refresh
+    timeout += 1
   while count == 0 or args.repeat is not None and count < args.repeat:
     count += 1
     result = result_template.format(domain=domain, count=count)
     retries = 0
     while args.retries is None or retries < args.retries:
       retries += 1
       try:
         if args.user_data_dir:
           user_data_dir = args.user_data_dir
         else:
           user_data_dir = tempfile.mkdtemp(prefix="chr_")
         js_flags = "--runtime-call-stats"
         if args.replay_wpr: js_flags += " --allow-natives-syntax"
         if args.js_flags: js_flags += " " + args.js_flags
         chrome_flags = [
             "--no-default-browser-check",
             "--disable-translate",
-            "--disable-seccomp-sandbox",
             "--js-flags={}".format(js_flags),
             "--no-first-run",
             "--user-data-dir={}".format(user_data_dir),
         ]
         if args.replay_wpr:
           chrome_flags += [
               "--host-resolver-rules=MAP *:80 localhost:4080, "  \
                                     "MAP *:443 localhost:4443, " \
                                     "EXCLUDE localhost",
               "--ignore-certificate-errors",
+              "--disable-seccomp-sandbox",
               "--disable-web-security",
               "--reduce-security-for-testing",
               "--allow-insecure-localhost",
           ]
         else:
           chrome_flags += [
-            "--single-process",
+              "--single-process",
           ]
         if args.chrome_flags:
           chrome_flags += args.chrome_flags.split()
         cmd_args = [
             "timeout", str(timeout),
             args.with_chrome
         ] + chrome_flags + [ site ]
         print "- " * 40
         print_command(cmd_args)
         print "- " * 40
         with open(result, "wt") as f:
           status = subprocess.call(cmd_args, stdout=f)
         # 124 means timeout killed chrome, 0 means the user was bored first!
         # If none of these two happened, then chrome apparently crashed, so
         # it must be called again.
         if status != 124 and status != 0:
           print("CHROME CRASHED, REPEATING RUN");
           continue
         # If the stats file is empty, chrome must be called again.
         if os.path.isfile(result) and os.path.getsize(result) > 0:
           if args.print_url:
             with open(result, "at") as f:
               print >> f
               print >> f, "URL: {}".format(site)
           break
-        if retries <= 5: timeout += 1
+        if retries <= 6: timeout += 2 ** (retries-1)
         print("EMPTY RESULT, REPEATING RUN");
       finally:
         if not args.user_data_dir:
           shutil.rmtree(user_data_dir)
 
 
 def read_sites_file(args):
   try:
     sites = []
     try:
@@ skipping 91 matching lines @@
     # Process the whole file and sum repeating entries.
     D = { 'Sum': {'time': 0, 'count': 0} }
     for line in f:
       line = line.strip()
       # Discard headers and footers.
       if not line: continue
       if line.startswith("Runtime Function"): continue
       if line.startswith("===="): continue
       if line.startswith("----"): continue
       if line.startswith("URL:"): continue
+      if line.startswith("STATS:"): continue
       # We have a regular line.
       fields = line.split()
       key = fields[0]
       time = float(fields[1].replace("ms", ""))
       count = int(fields[3])
       if key not in D: D[key] = { 'time': 0, 'count': 0 }
       D[key]['time'] += time
       D[key]['count'] += count
       # We calculate the sum, if it's not the "total" line.
       if key != "Total":
@@ skipping 87 matching lines @@
 
 def do_json(args):
   J = {}
   for path in args.logdirs:
     if os.path.isdir(path):
       for root, dirs, files in os.walk(path):
         version = os.path.basename(root)
         if version not in J: J[version] = {}
         for filename in files:
           if filename.endswith(".txt"):
-            m = re.match(r'^([^#]+)(#.*)?$', filename)
+            m = re.match(r'^([^#]+)(#.*)?\.txt$', filename)
             domain = m.group(1)
             if domain not in J[version]: J[version][domain] = {}
             read_stats(os.path.join(root, filename), J[version][domain])
   for version, T in J.items():
     for domain, S in T.items():
       A = []
       for name, value in S.items():
         # We don't want the calculated sum in the JSON file.
         if name == "Sum": continue
         entry = [name]
@@ skipping 41 matching lines @@
   subparsers["run"].add_argument(
       "--js-flags", type=str, default="",
       help="specify additional V8 flags")
   subparsers["run"].add_argument(
       "--no-url", dest="print_url", action="store_false", default=True,
       help="do not include url in statistics file")
   subparsers["run"].add_argument(
       "-n", "--repeat", type=int, metavar="<num>",
       help="specify iterations for each website (default: once)")
   subparsers["run"].add_argument(
+      "-k", "--refresh", type=int, metavar="<num>", default=0,
+      help="specify refreshes for each iteration (default: 0)")
+  subparsers["run"].add_argument(
       "--replay-wpr", type=str, metavar="<path>",
       help="use the specified web page replay (.wpr) archive")
   subparsers["run"].add_argument(
       "--replay-bin", type=str, metavar="<path>",
       help="specify the replay.py script typically located in " \
            "$CHROMIUM/src/third_party/webpagereplay/replay.py")
   subparsers["run"].add_argument(
       "-r", "--retries", type=int, metavar="<num>",
       help="specify retries if website is down (default: forever)")
   subparsers["run"].add_argument(
@@ skipping 53 matching lines @@
     args.error("use either option --sites-file or site URLs")
     sys.exit(1)
   elif args.command == "run" and not coexist(args.replay_wpr, args.replay_bin):
     args.error("options --replay-wpr and --replay-bin must be used together")
     sys.exit(1)
   else:
     args.func(args)
 
 if __name__ == "__main__":
   sys.exit(main())