Added TodoMVC startup benchmarks.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import math
 import optparse
 import os
@@ skipping 60 matching lines @@
         out.seek(0, os.SEEK_END)
     p = subprocess.Popen(cmd_list, stdout = out, stderr=subprocess.PIPE,
                          stdin=subprocess.PIPE, shell=self.has_shell)
     output, stderr = p.communicate(std_in)
     if output:
       print output
     if stderr:
       print stderr
     return output, stderr
 
+  def RunBrowserPerfRunnerCmd(self, browser, url_path, file_path_to_test_code,
+      trace_file, code_root=''):
+    command_list = [os.path.join(DART_REPO_LOC, utils.GetBuildRoot(
+        utils.GuessOS(), 'release', 'ia32'), 'dart-sdk', 'bin', 'dart'),
+        '--package-root=%s' % os.path.join(file_path_to_test_code, 'packages'),
+        os.path.join(file_path_to_test_code, 'packages',
+        'browser_controller', 'browser_perf_testing.dart'), '--browser',
+        browser, '--test_path=%s' % url_path]
+    if code_root != '':
+      command_list += ['--code_root=%s' % code_root]
+    self.RunCmd(command_list, trace_file, append=True)
+
   def TimeCmd(self, cmd):
     """Determine the amount of (real) time it takes to execute a given
     command."""
     start = time.time()
     self.RunCmd(cmd)
     return time.time() - start
 
   def ClearOutUnversionedFiles(self):
     """Remove all files that are unversioned by svn."""
     if os.path.exists(DART_REPO_LOC):
@@ skipping 78 matching lines @@
         or 'InvalidUriError' in stderr or "Couldn't download" in stdout or
         'Unable to download' in stdout):
       # Couldn't find the SDK on Google Storage. Build it locally.
 
       # TODO(efortuna): Currently always building ia32 architecture because we
       # don't have test statistics for what's passing on x64. Eliminate arch
       # specification when we have tests running on x64, too.
       shutil.rmtree(os.path.join(os.getcwd(),
                     utils.GetBuildRoot(utils.GuessOS())),
                     onerror=TestRunner._OnRmError)
-      lines = self.RunCmd([os.path.join('.', 'tools', 'build.py'), '-m',
+      lines = self.RunCmd(['python', os.path.join('tools', 'build.py'), '-m',
                             'release', '--arch=ia32', 'create_sdk'])
 
       for line in lines:
         if 'BUILD FAILED' in line:
           # Someone checked in a broken build! Stop trying to make it work
           # and wait to try again.
           print 'Broken Build'
           return 1
     return 0
 
@@ skipping 524 matching lines @@
 
   class DromaeoPerfTester(DromaeoTester):
     def RunTests(self):
       """Run dromaeo in the browser."""
       success, _, _ = self.test.test_runner.GetArchive('dartium')
       if not success:
         # Unable to download dartium. Try later.
         return
 
       # Build tests.
-      # TODO(efortuna): Make the pub functionality a separate function.
       current_path = os.getcwd()
       os.chdir(os.path.join(DART_REPO_LOC, 'samples', 'third_party',
           'dromaeo'))
-      self.test.test_runner.RunCmd([os.path.join(DART_REPO_LOC,
-          utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'),
-          'dart-sdk', 'bin', 'pub'), 'install']) # TODO: pub upgrade?
       stdout, _ = self.test.test_runner.RunCmd([os.path.join(DART_REPO_LOC,
           utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'),
-          'dart-sdk', 'bin', 'pub'), 'build'])
+          'dart-sdk', 'bin', 'pub'), 'build', '--mode=debug'])

[Jennifer Messerly, 2014/03/19 19:27:57]

should this be debug? Might be worth a comment either way :)
we don't do anything especially slow in Polymer.dart for mode=debug, but we
might start someday :D

[Emily Fortuna, 2014/03/24 19:45:56]

Yes, this is debug on purpose. Siggi and I discussed this at length last week. I
can fill you in in person for the complete explanation, but I'll also add a
little comment here, too.

       os.chdir(current_path)
       if 'failed' in stdout:
         return
 
       versions = DromaeoTester.GetDromaeoVersions()
 
       for browser in BrowserTester.GetBrowsers():
         for version_name in versions:
           if not self.test.IsValidCombination(browser, version_name):
             continue
           version = DromaeoTest.DromaeoPerfTester.GetDromaeoUrlQuery(
               browser, version_name)
           self.test.trace_file = os.path.join(TOP_LEVEL_DIR,
               'tools', 'testing', 'perf_testing', self.test.result_folder_name,
               'dromaeo-%s-%s-%s' % (self.test.cur_time, browser, version_name))
           self.AddSvnRevisionToTrace(self.test.trace_file, browser)
           url_path = '/'.join(['/code_root', 'build', 'web', 'index%s.html?%s'%(
               '-dart' if version_name == 'dart_html' else '-js',
               version)])
 
-          self.test.test_runner.RunCmd(
-              [os.path.join(DART_REPO_LOC, utils.GetBuildRoot(
-               utils.GuessOS(), 'release', 'ia32'), 'dart-sdk', 'bin', 'dart'),
-               '--package-root=%s' % os.path.join(DART_REPO_LOC, 'samples',
-               'third_party', 'dromaeo', 'packages'), os.path.join(
-               DART_REPO_LOC, 'samples', 'third_party', 'dromaeo', 'packages',
-               'browser_controller', 'browser_perf_testing.dart'),
-               '--browser', browser,
-               '--test_path', url_path], self.test.trace_file,
-               append=True)
+          self.test.test_runner.RunBrowserPerfRunnerCmd(browser, url_path,
+              os.path.join(DART_REPO_LOC, 'samples', 'third_party', 'dromaeo'),
+              self.test.trace_file)
 
     @staticmethod
     def GetDromaeoUrlQuery(browser, version):
       version = version.replace('_','AND')
       tags = DromaeoTester.GetValidDromaeoTags()
       return 'OR'.join([ '%sAND%s' % (version, tag) for tag in tags])
 
 
   class DromaeoFileProcessor(Processor):
     def ProcessFile(self, afile, should_post_file):
@@ skipping 36 matching lines @@
                   upload_success = upload_success and self.ReportResults(
                       name, score, browser, version, revision_num,
                       self.GetScoreType(name))
                 else:
                   upload_success = False
 
       f.close()
       self.CalculateGeometricMean(browser, version, revision_num)
       return upload_success
 
+class TodoMVCTester(BrowserTester):
+    @staticmethod
+    def GetVersions():
+      return ['js', 'dart2js_html', 'dart_html']
+
+    @staticmethod
+    def GetBenchmarks():
+      return ['TodoMVCstartup']
+
+class TodoMVCStartupTest(RuntimePerformanceTest):
+  """Start up TodoMVC and see how long it takes to start."""
+  def __init__(self, test_runner):
+    super(TodoMVCStartupTest, self).__init__(
+        self.Name(),
+        BrowserTester.GetBrowsers(True),
+        'browser',
+        TodoMVCTester.GetVersions(),
+        TodoMVCTester.GetBenchmarks(), test_runner,
+        self.TodoMVCStartupTester(self),
+        self.TodoMVCFileProcessor(self))
+
+  @staticmethod
+  def Name():
+    return 'todoMvcStartup'
+
+  class TodoMVCStartupTester(BrowserTester):
+    def RunTests(self):
+      """Run dromaeo in the browser."""
+      success, _, _ = self.test.test_runner.GetArchive('dartium')
+      if not success:
+        # Unable to download dartium. Try later.
+        return
+
+      dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo')

[Jennifer Messerly, 2014/03/19 19:27:57]

future TODO: maybe we should move both of these out of "samples"? I seem to
recall a request a while back to move dromaeo if possible. Maybe under
tests/perf/... or something along those lines.

[Emily Fortuna, 2014/03/24 19:45:56]

I'm going to leave this as is for now because of the License stuff, but I'm open
to moving it in the future.

+      current_path = os.getcwd()
+
+      os.chdir(os.path.join(DART_REPO_LOC, 'samples', 'third_party',
+          'todomvc_performance'))

[Jennifer Messerly, 2014/03/19 19:27:57]

fyi -- I was thinking of moving TodoMVC out of third_party
(https://code.google.com/p/dart/issues/detail?id=17599), not sure if that
impacts the path here. Then again since it includes the JS code maybe
"third_party" still makes sense. Polymer-JS uses the same license as Dart but I
think the transitive closure includes things like "flatiron"

+      self.test.test_runner.RunCmd([os.path.join(DART_REPO_LOC,
+          utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'),
+          'dart-sdk', 'bin', 'pub'), 'build', '--mode=debug'])
+      os.chdir('js_todomvc');
+      self.test.test_runner.RunCmd([os.path.join(DART_REPO_LOC,
+          utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'),
+          'dart-sdk', 'bin', 'pub'), 'get'])
+
+      versions = TodoMVCTester.GetVersions()
+
+      for browser in BrowserTester.GetBrowsers():
+        for version_name in versions:
+          if not self.test.IsValidCombination(browser, version_name):
+            continue
+          self.test.trace_file = os.path.join(TOP_LEVEL_DIR,
+              'tools', 'testing', 'perf_testing', self.test.result_folder_name,
+              'todoMvcStartup-%s-%s-%s' % (self.test.cur_time, browser,
+              version_name))
+          self.AddSvnRevisionToTrace(self.test.trace_file, browser)
+
+          if version_name == 'js':
+            code_root = os.path.join(DART_REPO_LOC, 'samples', 'third_party',
+                'todomvc_performance', 'js_todomvc')
+            self.test.test_runner.RunBrowserPerfRunnerCmd(browser,
+                '/code_root/index.html', code_root, self.test.trace_file,
+                code_root)
+          else:
+            self.test.test_runner.RunBrowserPerfRunnerCmd(browser,
+                '/code_root/build/web/startup-performance.html', os.path.join(
+                DART_REPO_LOC, 'samples', 'third_party', 'todomvc_performance'),
+                self.test.trace_file)
+
+  class TodoMVCFileProcessor(Processor):
+    def ProcessFile(self, afile, should_post_file):
+      """Comb through the html to find the performance results.
+      Returns: True if we successfully posted our data to storage."""
+      parts = afile.split('-')
+      browser = parts[2]
+      version = parts[3]
+
+      bench_dict = self.test.values_dict[browser][version]
+
+      f = self.OpenTraceFile(afile, should_post_file)
+      lines = f.readlines()
+      i = 0
+      revision_num = 0
+      revision_pattern = r'Revision: (\d+)'
+      result_pattern = r'The startup time is (\d+)'
+
+      upload_success = True
+      for line in lines:
+        rev = re.match(revision_pattern, line.strip().replace('"', ''))
+        if rev:
+          revision_num = int(rev.group(1))
+          continue
+
+        results = re.search(result_pattern, line)
+        if results:
+          score = float(results.group(1))
+          name = TodoMVCTester.GetBenchmarks()[0]
+          bench_dict[name] += [float(score)]
+          self.test.revision_dict[browser][version][name] += \
+              [revision_num]
+          if not self.test.test_runner.no_upload and should_post_file:
+            upload_success = upload_success and self.ReportResults(
+                name, score, browser, version, revision_num,
+                self.GetScoreType(name))
+
+      f.close()
+      self.CalculateGeometricMean(browser, version, revision_num)
+      return upload_success
+
+
 class TestBuilder(object):
   """Construct the desired test object."""
   available_suites = dict((suite.Name(), suite) for suite in [
-      DromaeoTest])
+      DromaeoTest, TodoMVCStartupTest])
 
   @staticmethod
   def MakeTest(test_name, test_runner):
     return TestBuilder.available_suites[test_name](test_runner)
 
   @staticmethod
   def AvailableSuiteNames():
     return TestBuilder.available_suites.keys()
 
 
@@ skipping 115 matching lines @@
       else:
         if runner.backfill:
           results_set = FillInBackHistory(results_set, runner)
         else:
           time.sleep(200)
   else:
     runner.RunTestSequence()
 
 if __name__ == '__main__':
   main()