sandwich: Implement SandwichTaskBuilder

tools/android/loading/sandwich.py

 #! /usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Instructs Chrome to load series of web pages and reports results.
 
 When running Chrome is sandwiched between preprocessed disk caches and
 WepPageReplay serving all connections.
 
@@ skipping 17 matching lines @@
 from pylib import constants
 import devil_chromium
 
 import chrome_cache
 import common_util
 import emulation
 import options
 import sandwich_metrics
 import sandwich_misc
 from sandwich_runner import SandwichRunner
+import sandwich_tasks
+import task_manager
 from trace_test.webserver_test import WebServer
 
 
 # Use options layer to access constants.
 OPTIONS = options.OPTIONS
 
 
 def _ArgumentParser():
   """Build a command line argument's parser."""
   # Command parser when dealing with jobs.
   common_job_parser = argparse.ArgumentParser(add_help=False)
   common_job_parser.add_argument('--job', required=True,
                                  help='JSON file with job description.')
 
+  orchestra_parser = task_manager.CommandLineParser()
+
   # Plumbing parser to configure OPTIONS.
   plumbing_parser = OPTIONS.GetParentParser('plumbing options')
 
   # Main parser
   parser = argparse.ArgumentParser(parents=[plumbing_parser])
   subparsers = parser.add_subparsers(dest='subcommand', help='subcommand line')
 
   # Record WPR subcommand.
   record_wpr = subparsers.add_parser('record-wpr', parents=[common_job_parser],
                                      help='Record WPR from sandwich job.')
@@ skipping 97 matching lines @@
       help='Record a test trace using the trace_test.webserver_test.')
   record_trace_parser.add_argument('--source-dir', type=str, required=True,
                                    help='Base path where the files are opened'
                                         'by the web server.')
   record_trace_parser.add_argument('--page', type=str, required=True,
                                    help='Source page in source-dir to navigate '
                                         'to.')
   record_trace_parser.add_argument('-o', '--output', type=str, required=True,
                                    help='Output path of the generated trace.')
 
+  # Run all subcommand.
+  run_all = subparsers.add_parser('run-all',
+                                  parents=[common_job_parser, orchestra_parser],
+                                  help='Run the sandwich orchestra.')

[pasko, 2016/04/11 14:54:08]

'Run all steps using the task manager infrastructure.

[gabadie, 2016/04/13 09:53:44]

Done.

+  run_all.add_argument('-g', '--gen-full', action='store_true',
+                       help='Generate the full orchestra.')

[pasko, 2016/04/11 14:54:08]

there is no more orchestra. For now we can say:
'Generate the full graph with all possible network simulation types.'

[gabadie, 2016/04/13 09:53:44]

Done.

+  run_all.add_argument('--wpr-archive', default=None, type=str,
+                       dest='wpr_archive_path',
+                       help='Web page replay archive to load job\'s urls '

[pasko, 2016/04/11 14:54:08]

help='WebPageReplay archive to use, instead of generating one'

[gabadie, 2016/04/13 09:53:44]

Done.

+                            'from.')
+  run_all.add_argument('--url-repeat', default=1, type=int,
+                       help='How many times to repeat the urls.')
+
   return parser
 
 
 def _RecordWprMain(args):
   sandwich_runner = SandwichRunner()
   sandwich_runner.LoadJob(args.job)
   sandwich_runner.PullConfigFromArgs(args)
   sandwich_runner.wpr_record = True
   sandwich_runner.PrintConfig()
   if not os.path.isdir(os.path.dirname(args.wpr_archive_path)):
@@ skipping 58 matching lines @@
     sandwich_runner.trace_output_directory = os.path.join(out_path, 'run')
     with WebServer.Context(
         source_dir=args.source_dir, communication_dir=out_path) as server:
       address = server.Address()
       sandwich_runner.urls = ['http://%s/%s' % (address, args.page)]
       sandwich_runner.Run()
     shutil.copy(os.path.join(out_path, 'run', '0', 'trace.json'), args.output)
   return 0
 
 
+def _RunAllMain(args):
+  runner_modifiers = {
+    '2g': sandwich_tasks.EmulateNetworkModifier('Regular2G'),

[pasko, 2016/04/11 14:54:08]

Let's rename it to something like NetworkSimulationTransformer

we can actually avoid this dict of runner_modifiers and deal with '2g' ->
'Regular2G' transition in the callback:

...

for type in sandwich_tasks.SUPPORTED_NETWORK_TYPES:
  transformer = sandwich_tasks.CreateNetworkSimulationTrasformer(type)
  builder.PopulateClearCacheLoadBenchmark(
      benchmark_name='clearcache-' + type,
      sandwich_runner_transformer=transformer)
  builder.PopulateNoStatePrefetchLoadBenchmark(
      ...
      ...)

I think 'transformer' more explicitly suggests that it would take the sandwich
runner as parameter and would change its state, while a 'modifier' could be
interpreted as a 'small addition' .. like those regexp modifiers.

[gabadie, 2016/04/13 09:53:44]

Done.

+    '3g': sandwich_tasks.EmulateNetworkModifier('Regular3G'),
+    '4g': sandwich_tasks.EmulateNetworkModifier('Regular4G')
+  }
+
+  builder = sandwich_tasks.SandwichTaskBuilder(
+      output_directory=args.output,
+      job_path=args.job,
+      url_repeat=args.url_repeat)
+  if args.wpr_archive_path:
+    builder.SetOriginalWprPath(args.wpr_archive_path)
+  else:
+    builder.PopulateWPRRecordingTask()
+  builder.PopulateCommonPipelines()
+  builder.PopulateFullCacheLoadBenchmark()
+  builder.PopulateClearCacheLoadBenchmark()
+  builder.PopulateNoStatePrefetchLoadBenchmark()
+
+  if args.gen_full:
+    for key, runner_modifier in runner_modifiers.iteritems():
+      builder.PopulateClearCacheLoadBenchmark(
+          benchmark_name='clearcache-' + key,
+          runner_modifier=runner_modifier)
+      builder.PopulateNoStatePrefetchLoadBenchmark(
+          benchmark_name='prefetch-' + key,
+          runner_modifier=runner_modifier)
+
+  return task_manager.ExecuteWithCommandLine(
+      args, builder.tasks.values(), builder.default_final_tasks)
+
+
 def main(command_line_args):
   logging.basicConfig(level=logging.INFO)
   devil_chromium.Initialize()
 
   args = _ArgumentParser().parse_args(command_line_args)
   OPTIONS.SetParsedArgs(args)
 
   if args.subcommand == 'record-wpr':
     return _RecordWprMain(args)
   if args.subcommand == 'patch-wpr':
     sandwich_misc.PatchWpr(args.wpr_archive_path)
     return 0
   if args.subcommand == 'create-cache':
     return _CreateCacheMain(args)
   if args.subcommand == 'run':
     return _RunJobMain(args)
   if args.subcommand == 'extract-metrics':
     return _ExtractMetricsMain(args)
   if args.subcommand == 'filter-cache':
     return _FilterCacheMain(args)
   if args.subcommand == 'record-test-trace':
     return _RecordWebServerTestTrace(args)
+  if args.subcommand == 'run-all':
+    return _RunAllMain(args)
   assert False
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))