[tools] Add support to launch the replay server separately for callstats.py

tools/callstats.py

 #!/usr/bin/env python
 # Copyright 2016 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 '''
 Usage: callstats.py [-h] <command> ...
 
 Optional arguments:
   -h, --help  show this help message and exit
 
@@ skipping 105 matching lines @@
       }, timeout);
       return;
     }
     console.log("Ignoring: " + url);
   };
   var sites =
     """, json.dumps(sites), """;
   onLoad(window.location.href);
 })();"""
 
+def get_chrome_flags(js_flags="'--runtime-calls-stats --allow-natives-syntax'",
+                     user_data_dir="/var/tmp/`date +%S`"):

[nickie, 2016/09/28 09:08:09]

The default parameter here is probably not recommended as a temp directory,
unless somebody (the bots?) want it like this.  Lines 171-174 and the cleaning
up in lines 214-216 were used for this purpose.  As far as I can see, there are
no other call sites of this function, except for the one in line 178 and the
usage message.

[Camillo Bruni, 2016/09/28 10:36:34]

Will push the tmp dir flag to there.

+  return [
+      "--no-default-browser-check",
+      "--no-sandbox",
+      "--disable-translate",
+      "--enable-benchmarking",
+      "--js-flags={}".format(js_flags),
+      "--no-first-run",
+      "--user-data-dir={}".format(user_data_dir),
+    ]
+
+def get_chrome_replay_args(args):
+  http_port = 4080 + args.port_offset
+  https_port = 4443 + args.port_offset
+  return [
+      "--host-resolver-rules=MAP *:80 localhost:%s, "  \
+                            "MAP *:443 localhost:%s, " \
+                            "EXCLUDE localhost" % (
+                                http_port, https_port),
+      "--ignore-certificate-errors",
+      "--disable-seccomp-sandbox",
+      "--disable-web-security",
+      "--reduce-security-for-testing",
+      "--allow-insecure-localhost",
+    ]
 
 def run_site(site, domain, args, timeout=None):
   print "="*80
   print "RUNNING DOMAIN %s" % domain
   print "="*80
   result_template = "{domain}#{count}.txt" if args.repeat else "{domain}.txt"
   count = 0
   if timeout is None: timeout = args.timeout
   if args.replay_wpr:
     timeout *= 1 + args.refresh
     timeout += 1
   retries_since_good_run = 0
   while count == 0 or args.repeat is not None and count < args.repeat:
     count += 1
     result = result_template.format(domain=domain, count=count)
     retries = 0
     while args.retries is None or retries < args.retries:
       retries += 1
       try:
         if args.user_data_dir:
           user_data_dir = args.user_data_dir
         else:
           user_data_dir = tempfile.mkdtemp(prefix="chr_")
         js_flags = "--runtime-call-stats"
         if args.replay_wpr: js_flags += " --allow-natives-syntax"
         if args.js_flags: js_flags += " " + args.js_flags
-        chrome_flags = [
-            "--no-default-browser-check",
-            "--no-sandbox",
-            "--disable-translate",
-            "--enable-benchmarking",
-            "--js-flags={}".format(js_flags),
-            "--no-first-run",
-            "--user-data-dir={}".format(user_data_dir),
-        ]
+        chrome_flags = get_chrome_flags(js_flags, user_data_dir)
         if args.replay_wpr:
-          http_port = 4080 + args.port_offset
-          https_port = 4443 + args.port_offset
-          chrome_flags += [
-              "--host-resolver-rules=MAP *:80 localhost:%s, "  \
-                                    "MAP *:443 localhost:%s, " \
-                                    "EXCLUDE localhost" % (
-                                        http_port, https_port),
-              "--ignore-certificate-errors",
-              "--disable-seccomp-sandbox",
-              "--disable-web-security",
-              "--reduce-security-for-testing",
-              "--allow-insecure-localhost",
-          ]
+          chrome_flags += get_chrome_replay_flags(args.port_offset)
         else:
-          chrome_flags += [
-              "--single-process",
-          ]
+          chrome_flags += [ "--single-process", ]
         if args.chrome_flags:
           chrome_flags += args.chrome_flags.split()
         cmd_args = [
             "timeout", str(timeout),
             args.with_chrome
         ] + chrome_flags + [ site ]
         print "- " * 40
         print_command(cmd_args)
         print "- " * 40
         with open(result, "wt") as f:
@@ skipping 39 matching lines @@
         for line in f:
           line = line.strip()
           if not line or line.startswith('#'): continue
           sites.append({'url': line, 'timeout': args.timeout})
     return sites
   except IOError as e:
     args.error("Cannot read from {}. {}.".format(args.sites_file, e.strerror))
     sys.exit(1)
 
 
-def do_run(args):
+def read_sites(args):
   # Determine the websites to benchmark.
   if args.sites_file:
-    sites = read_sites_file(args)
-  else:
-    sites = [{'url': site, 'timeout': args.timeout} for site in args.sites]
+    return read_sites_file(args)
+  return [{'url': site, 'timeout': args.timeout} for site in args.sites]
+
+def do_run(args):
+  sites = read_sites(args)
+  replay_server = start_replay_server(args, sites) if args.replay_wpr else None
   # Disambiguate domains, if needed.
   L = []
   domains = {}
   for item in sites:
     site = item['url']
     domain = None
     if args.domain:
       domain = args.domain
     elif 'domain' in item:
       domain = item['domain']
     else:
       m = re.match(r'^(https?://)?([^/]+)(/.*)?$', site)
       if not m:
         args.error("Invalid URL {}.".format(site))
         continue
       domain = m.group(2)
     entry = [site, domain, None, item['timeout']]
     if domain not in domains:
       domains[domain] = entry
     else:
       if not isinstance(domains[domain], int):
         domains[domain][2] = 1
         domains[domain] = 1
       domains[domain] += 1
       entry[2] = domains[domain]
     L.append(entry)
-  replay_server = start_replay_server(args, sites) if args.replay_wpr else None
   try:
     # Run them.
     for site, domain, count, timeout in L:
       if count is not None: domain = "{}%{}".format(domain, count)
-      print site, domain, timeout
+      print(site, domain, timeout)
       run_site(site, domain, args, timeout)
   finally:
     if replay_server:
       stop_replay_server(replay_server)
 
 
+def do_run_replay_server(args):
+  sites = read_sites(args)
+  print("- " * 40)
+  print("Launch chromium with the following commands for debugging:")
+  print("    $CHROMIUM_DIR/out/Release/chomium")
+  print("        " +
+        (" \\\n        ".join(get_chrome_flags()+get_chrome_replay_args(args))) +

[nickie, 2016/09/28 09:08:09]

Here the users will see the /var/tmp/`date` temp directory, which I suppose is
what you want...

[Camillo Bruni, 2016/09/28 10:36:34]

right, made this more obvious by pushing down the parameters to here.

+        " \\")
+  print("        <URL>")
+  print("- " * 40)
+  replay_server = start_replay_server(args, sites)
+  try:
+    replay_server['process'].wait()
+  finally:
+   stop_replay_server(replay_server)
+
+
 # Calculate statistics.
 
 def statistics(data):
   N = len(data)
   average = numpy.average(data)
   median = numpy.median(data)
   low = numpy.min(data)
   high= numpy.max(data)
   if N > 1:
     # evaluate sample variance by setting delta degrees of freedom (ddof) to
@@ skipping 236 matching lines @@
   given = sum(1 for x in l if x)
   return given == 0 or given == len(l)
 
 def main():
   parser = argparse.ArgumentParser()
   subparser_adder = parser.add_subparsers(title="commands", dest="command",
                                           metavar="<command>")
   subparsers = {}
   # Command: run.
   subparsers["run"] = subparser_adder.add_parser(
-      "run", help="run --help")
+      "run", help="Replay websites and collect runtime stats data.")
   subparsers["run"].set_defaults(
       func=do_run, error=subparsers["run"].error)
   subparsers["run"].add_argument(
       "--chrome-flags", type=str, default="",
       help="specify additional chrome flags")
   subparsers["run"].add_argument(
       "--js-flags", type=str, default="",
       help="specify additional V8 flags")
   subparsers["run"].add_argument(
-      "--domain", type=str, default="",
-      help="specify the output file domain name")
-  subparsers["run"].add_argument(
-      "--no-url", dest="print_url", action="store_false", default=True,
-      help="do not include url in statistics file")
-  subparsers["run"].add_argument(
-      "-n", "--repeat", type=int, metavar="<num>",
-      help="specify iterations for each website (default: once)")
-  subparsers["run"].add_argument(
-      "-k", "--refresh", type=int, metavar="<num>", default=0,
-      help="specify refreshes for each iteration (default: 0)")
-  subparsers["run"].add_argument(
-      "--replay-wpr", type=str, metavar="<path>",
-      help="use the specified web page replay (.wpr) archive")
-  subparsers["run"].add_argument(
-      "--replay-bin", type=str, metavar="<path>",
-      help="specify the replay.py script typically located in " \
-           "$CHROMIUM/src/third_party/webpagereplay/replay.py")
-  subparsers["run"].add_argument(
-      "-r", "--retries", type=int, metavar="<num>",
-      help="specify retries if website is down (default: forever)")
-  subparsers["run"].add_argument(
-      "-f", "--sites-file", type=str, metavar="<path>",
-      help="specify file containing benchmark websites")
-  subparsers["run"].add_argument(
-      "-t", "--timeout", type=int, metavar="<seconds>", default=60,
-      help="specify seconds before chrome is killed")
-  subparsers["run"].add_argument(
-      "-p", "--port-offset", type=int, metavar="<offset>", default=0,
-      help="specify the offset for the replay server's default ports")
-  subparsers["run"].add_argument(
       "-u", "--user-data-dir", type=str, metavar="<path>",
       help="specify user data dir (default is temporary)")
   subparsers["run"].add_argument(
       "-c", "--with-chrome", type=str, metavar="<path>",
       default="/usr/bin/google-chrome",
       help="specify chrome executable to use")
   subparsers["run"].add_argument(
-      "-l", "--log-stderr", type=str, metavar="<path>",
-      help="specify where chrome's stderr should go (default: /dev/null)")
+      "-r", "--retries", type=int, metavar="<num>",
+      help="specify retries if website is down (default: forever)")
   subparsers["run"].add_argument(
-      "sites", type=str, metavar="<URL>", nargs="*",
-      help="specify benchmark website")
+      "--no-url", dest="print_url", action="store_false", default=True,
+      help="do not include url in statistics file")
+  subparsers["run"].add_argument(
+      "--domain", type=str, default="",
+      help="specify the output file domain name")
+  subparsers["run"].add_argument(
+      "-n", "--repeat", type=int, metavar="<num>",
+      help="specify iterations for each website (default: once)")
+
+  def add_replay_args(subparser):
+    subparser.add_argument(
+        "-k", "--refresh", type=int, metavar="<num>", default=0,
+        help="specify refreshes for each iteration (default: 0)")
+    subparser.add_argument(
+        "--replay-wpr", type=str, metavar="<path>",
+        help="use the specified web page replay (.wpr) archive")
+    subparser.add_argument(
+        "--replay-bin", type=str, metavar="<path>",
+        help="specify the replay.py script typically located in " \
+             "$CHROMIUM/src/third_party/webpagereplay/replay.py")
+    subparser.add_argument(
+        "-f", "--sites-file", type=str, metavar="<path>",
+        help="specify file containing benchmark websites")
+    subparser.add_argument(
+        "-t", "--timeout", type=int, metavar="<seconds>", default=60,
+        help="specify seconds before chrome is killed")
+    subparser.add_argument(
+        "-p", "--port-offset", type=int, metavar="<offset>", default=0,
+        help="specify the offset for the replay server's default ports")
+    subparser.add_argument(
+        "-l", "--log-stderr", type=str, metavar="<path>",
+        help="specify where chrome's stderr should go (default: /dev/null)")
+    subparser.add_argument(
+        "sites", type=str, metavar="<URL>", nargs="*",
+        help="specify benchmark website")
+  add_replay_args(subparsers["run"])
+
+  # Command: replay-server
+  subparsers["replay"] = subparser_adder.add_parser(
+      "replay", help="Run the replay server for debugging purposes")
+  subparsers["replay"].set_defaults(
+      func=do_run_replay_server, error=subparsers["replay"].error)
+  add_replay_args(subparsers["replay"])
+
   # Command: stats.
   subparsers["stats"] = subparser_adder.add_parser(
-      "stats", help="stats --help")
+      "stats", help="Analize the results file create by the 'run' command.")
   subparsers["stats"].set_defaults(
       func=do_stats, error=subparsers["stats"].error)
   subparsers["stats"].add_argument(
       "-l", "--limit", type=int, metavar="<num>", default=0,
       help="limit how many items to print (default: none)")
   subparsers["stats"].add_argument(
       "-s", "--sort", choices=["asc", "desc"], default="asc",
       help="specify sorting order (default: ascending)")
   subparsers["stats"].add_argument(
       "-n", "--no-total", dest="totals", action="store_false", default=True,
       help="do not print totals")
   subparsers["stats"].add_argument(
       "logfiles", type=str, metavar="<logfile>", nargs="*",
       help="specify log files to parse")
   subparsers["stats"].add_argument(
       "--aggregate", dest="aggregate", action="store_true", default=False,
-      help="Create aggregated entries. Adds Group-* entries at the toplevel. " +
+      help="Create aggregated entries. Adds Group-* entries at the toplevel. " \
       "Additionally creates a Total page with all entries.")
+
   # Command: json.
   subparsers["json"] = subparser_adder.add_parser(
-      "json", help="json --help")
+      "json", help="Collect results file created by the 'run' command into" \
+          "a single json file.")
   subparsers["json"].set_defaults(
       func=do_json, error=subparsers["json"].error)
   subparsers["json"].add_argument(
       "logdirs", type=str, metavar="<logdir>", nargs="*",
       help="specify directories with log files to parse")
   subparsers["json"].add_argument(
       "--aggregate", dest="aggregate", action="store_true", default=False,
-      help="Create aggregated entries. Adds Group-* entries at the toplevel. " +
+      help="Create aggregated entries. Adds Group-* entries at the toplevel. " \
       "Additionally creates a Total page with all entries.")
+
   # Command: help.
   subparsers["help"] = subparser_adder.add_parser(
       "help", help="help information")
   subparsers["help"].set_defaults(
       func=lambda args: do_help(parser, subparsers, args),
       error=subparsers["help"].error)
   subparsers["help"].add_argument(
       "help_cmd", type=str, metavar="<command>", nargs="?",
       help="command for which to display help")
+
   # Execute the command.
   args = parser.parse_args()
   setattr(args, 'script_path', os.path.dirname(sys.argv[0]))
   if args.command == "run" and coexist(args.sites_file, args.sites):
     args.error("use either option --sites-file or site URLs")
     sys.exit(1)
   elif args.command == "run" and not coexist(args.replay_wpr, args.replay_bin):
     args.error("options --replay-wpr and --replay-bin must be used together")
     sys.exit(1)
   else:
     args.func(args)
 
 if __name__ == "__main__":
   sys.exit(main())