Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(190)

Side by Side Diff: tools/run_perf.py

Issue 1696293002: Revert of [tools] add --pretty switch to run_perf.py (Closed) Base URL: https://chromium.googlesource.com/v8/v8.git@master
Patch Set: Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | tools/unittests/run_perf_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 the V8 project authors. All rights reserved. 2 # Copyright 2014 the V8 project authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """ 6 """
7 Performance runner for d8. 7 Performance runner for d8.
8 8
9 Call e.g. with tools/run-perf.py --arch ia32 some_suite.json 9 Call e.g. with tools/run-perf.py --arch ia32 some_suite.json
10 10
(...skipping 387 matching lines...) Expand 10 before | Expand all | Expand 10 after
398 self.timeout = suite.get("timeout_%s" % arch, self.timeout) 398 self.timeout = suite.get("timeout_%s" % arch, self.timeout)
399 self.units = suite.get("units", parent.units) 399 self.units = suite.get("units", parent.units)
400 self.total = suite.get("total", parent.total) 400 self.total = suite.get("total", parent.total)
401 401
402 # A regular expression for results. If the parent graph provides a 402 # A regular expression for results. If the parent graph provides a
403 # regexp and the current suite has none, a string place holder for the 403 # regexp and the current suite has none, a string place holder for the
404 # suite name is expected. 404 # suite name is expected.
405 # TODO(machenbach): Currently that makes only sense for the leaf level. 405 # TODO(machenbach): Currently that makes only sense for the leaf level.
406 # Multiple place holders for multiple levels are not supported. 406 # Multiple place holders for multiple levels are not supported.
407 if parent.results_regexp: 407 if parent.results_regexp:
408 try: 408 regexp_default = parent.results_regexp % re.escape(suite["name"])
409 regexp_default = parent.results_regexp % re.escape(suite["name"])
410 except TypeError:
411 regexp_default = parent.results_regexp
412 else: 409 else:
413 regexp_default = None 410 regexp_default = None
414 self.results_regexp = suite.get("results_regexp", regexp_default) 411 self.results_regexp = suite.get("results_regexp", regexp_default)
415 412
416 # A similar regular expression for the standard deviation (optional). 413 # A similar regular expression for the standard deviation (optional).
417 if parent.stddev_regexp: 414 if parent.stddev_regexp:
418 stddev_default = parent.stddev_regexp % re.escape(suite["name"]) 415 stddev_default = parent.stddev_regexp % re.escape(suite["name"])
419 else: 416 else:
420 stddev_default = None 417 stddev_default = None
421 self.stddev_regexp = suite.get("stddev_regexp", stddev_default) 418 self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
(...skipping 161 matching lines...) Expand 10 before | Expand all | Expand 10 after
583 self.shell_dir_no_patch = options.shell_dir_no_patch 580 self.shell_dir_no_patch = options.shell_dir_no_patch
584 self.extra_flags = options.extra_flags.split() 581 self.extra_flags = options.extra_flags.split()
585 582
586 @staticmethod 583 @staticmethod
587 def GetPlatform(options): 584 def GetPlatform(options):
588 if options.android_build_tools: 585 if options.android_build_tools:
589 return AndroidPlatform(options) 586 return AndroidPlatform(options)
590 else: 587 else:
591 return DesktopPlatform(options) 588 return DesktopPlatform(options)
592 589
593 def GetPrettyFormatted(self, options):
594 return self
595
596 def PreExecution(self):
597 pass
598
599 def PostExecution(self):
600 pass
601
602 def PreTests(self, node, path):
603 pass
604
605 def PrintResult(self, result):
606 pass
607
608 def _PrintStdout(self, title, output):
609 print title % "Stdout"
610 print output.stdout
611
612 def _Run(self, runnable, count, no_patch=False): 590 def _Run(self, runnable, count, no_patch=False):
613 raise NotImplementedError() # pragma: no cover 591 raise NotImplementedError() # pragma: no cover
614 592
615 def Run(self, runnable, count): 593 def Run(self, runnable, count):
616 """Execute the benchmark's main file. 594 """Execute the benchmark's main file.
617 595
618 If options.shell_dir_no_patch is specified, the benchmark is run once with 596 If options.shell_dir_no_patch is specified, the benchmark is run once with
619 and once without patch. 597 and once without patch.
620 Args: 598 Args:
621 runnable: A Runnable benchmark instance. 599 runnable: A Runnable benchmark instance.
622 count: The number of this (repeated) run. 600 count: The number of this (repeated) run.
623 Returns: A tuple with the benchmark outputs with and without patch. The 601 Returns: A tuple with the benchmark outputs with and without patch. The
624 latter will be None if options.shell_dir_no_patch was not 602 latter will be None if options.shell_dir_no_patch was not
625 specified. 603 specified.
626 """ 604 """
627 stdout = self._Run(runnable, count, no_patch=False) 605 stdout = self._Run(runnable, count, no_patch=False)
628 if self.shell_dir_no_patch: 606 if self.shell_dir_no_patch:
629 return stdout, self._Run(runnable, count, no_patch=True) 607 return stdout, self._Run(runnable, count, no_patch=True)
630 else: 608 else:
631 return stdout, None 609 return stdout, None
632 610
633 611
634 class PlatformFormattedMixin(object):
635 """
636 Helper mixin that adds formatted output used when running benchmarks
637 with the --pretty flag.
638 """
639
640 def _PrintStdout(self, title, output):
641 sys.stdout.write("\r")
642 if output.exit_code != 0:
643 print output.stdout
644 return
645 # Assume the time is on the last line
646 result_line = output.stdout.splitlines()[-1].strip()
647 sys.stdout.write(result_line)
648 # Fill with spaces up to 80 characters.
649 sys.stdout.write(' '*max(0, 80-len(result_line)))
650 sys.stdout.flush()
651
652 def _GetMean(self, trace):
653 results = trace['results']
654 if len(results) == 0:
655 return 0
656 # If the tests provided a stddev the results consists of one single average
657 # value, so return that instead.
658 if trace['stddev']:
659 return results[0]
660 # For a non-zero length results list calculate the average here.
661 return sum([float(x) for x in results]) / len(results)
662
663 def _GetDeviation(self, trace):
664 # If the benchmark provided a stddev use that directly.
665 stddev = trace['stddev']
666 if stddev:
667 return stddev
668 # If no stddev was provided calculate it from the results.
669 results = trace['results']
670 if len(results) == 0:
671 return 0
672 mean = self._GetMean(trace)
673 square_deviation = sum((float(x)-mean)**2 for x in results)
674 return (square_deviation / len(results)) ** 0.5
675
676 def PrintResult(self, result):
677 if result.errors:
678 print "\r:Errors:"
679 print "\n".join(set(result.errors))
680 else:
681 trace = result.traces[0]
682 average = self._GetMean(trace)
683 stdev = self._GetDeviation(trace)
684 stdev_percentage = 100 * stdev / average if average != 0 else 0
685 result_string = "\r %s +/- %3.2f%% %s" % (
686 average, stdev_percentage, trace['units'])
687 sys.stdout.write(result_string)
688 # Fill with spaces up to 80 characters.
689 sys.stdout.write(' '*max(0, 80-len(result_string)))
690 sys.stdout.write("\n")
691 sys.stdout.flush()
692
693
694 class DesktopPlatform(Platform): 612 class DesktopPlatform(Platform):
695 def __init__(self, options): 613 def __init__(self, options):
696 super(DesktopPlatform, self).__init__(options) 614 super(DesktopPlatform, self).__init__(options)
697 615
698 def GetPrettyFormatted(self, options): 616 def PreExecution(self):
699 return PrettyFormattedDesktopPlatform(options) 617 pass
618
619 def PostExecution(self):
620 pass
700 621
701 def PreTests(self, node, path): 622 def PreTests(self, node, path):
702 if isinstance(node, RunnableConfig): 623 if isinstance(node, RunnableConfig):
703 node.ChangeCWD(path) 624 node.ChangeCWD(path)
704 625
705 def _Run(self, runnable, count, no_patch=False): 626 def _Run(self, runnable, count, no_patch=False):
706 suffix = ' - without patch' if no_patch else '' 627 suffix = ' - without patch' if no_patch else ''
707 shell_dir = self.shell_dir_no_patch if no_patch else self.shell_dir 628 shell_dir = self.shell_dir_no_patch if no_patch else self.shell_dir
708 title = ">>> %%s (#%d)%s:" % ((count + 1), suffix) 629 title = ">>> %%s (#%d)%s:" % ((count + 1), suffix)
709 try: 630 try:
710 output = commands.Execute( 631 output = commands.Execute(
711 runnable.GetCommand(shell_dir, self.extra_flags), 632 runnable.GetCommand(shell_dir, self.extra_flags),
712 timeout=runnable.timeout, 633 timeout=runnable.timeout,
713 ) 634 )
714 except OSError as e: # pragma: no cover 635 except OSError as e: # pragma: no cover
715 print title % "OSError" 636 print title % "OSError"
716 print e 637 print e
717 return "" 638 return ""
718 self._PrintStdout(title, output) 639 print title % "Stdout"
640 print output.stdout
719 if output.stderr: # pragma: no cover 641 if output.stderr: # pragma: no cover
720 # Print stderr for debugging. 642 # Print stderr for debugging.
721 print title % "Stderr" 643 print title % "Stderr"
722 print output.stderr 644 print output.stderr
723 if output.timed_out: 645 if output.timed_out:
724 print ">>> Test timed out after %ss." % runnable.timeout 646 print ">>> Test timed out after %ss." % runnable.timeout
725 if '--prof' in self.extra_flags: 647 if '--prof' in self.extra_flags:
726 os_prefix = {"linux": "linux", "macos": "mac"}.get(utils.GuessOS()) 648 os_prefix = {"linux": "linux", "macos": "mac"}.get(utils.GuessOS())
727 if os_prefix: 649 if os_prefix:
728 tick_tools = os.path.join(TOOLS_BASE, "%s-tick-processor" % os_prefix) 650 tick_tools = os.path.join(TOOLS_BASE, "%s-tick-processor" % os_prefix)
729 subprocess.check_call(tick_tools + " --only-summary", shell=True) 651 subprocess.check_call(tick_tools + " --only-summary", shell=True)
730 else: # pragma: no cover 652 else: # pragma: no cover
731 print "Profiler option currently supported on Linux and Mac OS." 653 print "Profiler option currently supported on Linux and Mac OS."
732 return output.stdout 654 return output.stdout
733 655
734 656
735 class PrettyFormattedDesktopPlatform(PlatformFormattedMixin, DesktopPlatform):
736 pass
737
738
739 class AndroidPlatform(Platform): # pragma: no cover 657 class AndroidPlatform(Platform): # pragma: no cover
740 DEVICE_DIR = "/data/local/tmp/v8/" 658 DEVICE_DIR = "/data/local/tmp/v8/"
741 659
742 def __init__(self, options): 660 def __init__(self, options):
743 super(AndroidPlatform, self).__init__(options) 661 super(AndroidPlatform, self).__init__(options)
744 LoadAndroidBuildTools(options.android_build_tools) 662 LoadAndroidBuildTools(options.android_build_tools)
745 663
746 if not options.device: 664 if not options.device:
747 # Detect attached device if not specified. 665 # Detect attached device if not specified.
748 devices = adb_wrapper.AdbWrapper.Devices() 666 devices = adb_wrapper.AdbWrapper.Devices()
749 assert devices and len(devices) == 1, ( 667 assert devices and len(devices) == 1, (
750 "None or multiple devices detected. Please specify the device on " 668 "None or multiple devices detected. Please specify the device on "
751 "the command-line with --device") 669 "the command-line with --device")
752 options.device = str(devices[0]) 670 options.device = str(devices[0])
753 self.adb_wrapper = adb_wrapper.AdbWrapper(options.device) 671 self.adb_wrapper = adb_wrapper.AdbWrapper(options.device)
754 self.device = device_utils.DeviceUtils(self.adb_wrapper) 672 self.device = device_utils.DeviceUtils(self.adb_wrapper)
755 673
756 def GetPrettyFormatted(self, options):
757 return PrettyFormattedAndroidPlatform(options)
758
759 def PreExecution(self): 674 def PreExecution(self):
760 perf = perf_control.PerfControl(self.device) 675 perf = perf_control.PerfControl(self.device)
761 perf.SetHighPerfMode() 676 perf.SetHighPerfMode()
762 677
763 # Remember what we have already pushed to the device. 678 # Remember what we have already pushed to the device.
764 self.pushed = set() 679 self.pushed = set()
765 680
766 def PostExecution(self): 681 def PostExecution(self):
767 perf = perf_control.PerfControl(self.device) 682 perf = perf_control.PerfControl(self.device)
768 perf.SetDefaultPerfMode() 683 perf.SetDefaultPerfMode()
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after
835 self._PushExecutable(self.shell_dir, "bin", node.binary) 750 self._PushExecutable(self.shell_dir, "bin", node.binary)
836 if self.shell_dir_no_patch: 751 if self.shell_dir_no_patch:
837 self._PushExecutable( 752 self._PushExecutable(
838 self.shell_dir_no_patch, "bin_no_patch", node.binary) 753 self.shell_dir_no_patch, "bin_no_patch", node.binary)
839 754
840 if isinstance(node, RunnableConfig): 755 if isinstance(node, RunnableConfig):
841 self._PushFile(bench_abs, node.main, bench_rel) 756 self._PushFile(bench_abs, node.main, bench_rel)
842 for resource in node.resources: 757 for resource in node.resources:
843 self._PushFile(bench_abs, resource, bench_rel) 758 self._PushFile(bench_abs, resource, bench_rel)
844 759
845 def _PrintStdout(self, title, output):
846 print title % "Stdout"
847 print "\n".join(output.stdout)
848
849 def _Run(self, runnable, count, no_patch=False): 760 def _Run(self, runnable, count, no_patch=False):
850 suffix = ' - without patch' if no_patch else '' 761 suffix = ' - without patch' if no_patch else ''
851 target_dir = "bin_no_patch" if no_patch else "bin" 762 target_dir = "bin_no_patch" if no_patch else "bin"
852 title = ">>> %%s (#%d)%s:" % ((count + 1), suffix) 763 title = ">>> %%s (#%d)%s:" % ((count + 1), suffix)
853 cache = cache_control.CacheControl(self.device) 764 cache = cache_control.CacheControl(self.device)
854 cache.DropRamCaches() 765 cache.DropRamCaches()
855 binary_on_device = os.path.join( 766 binary_on_device = os.path.join(
856 AndroidPlatform.DEVICE_DIR, target_dir, runnable.binary) 767 AndroidPlatform.DEVICE_DIR, target_dir, runnable.binary)
857 cmd = [binary_on_device] + runnable.GetCommandFlags(self.extra_flags) 768 cmd = [binary_on_device] + runnable.GetCommandFlags(self.extra_flags)
858 769
859 # Relative path to benchmark directory. 770 # Relative path to benchmark directory.
860 if runnable.path: 771 if runnable.path:
861 bench_rel = os.path.normpath(os.path.join(*runnable.path)) 772 bench_rel = os.path.normpath(os.path.join(*runnable.path))
862 else: 773 else:
863 bench_rel = "." 774 bench_rel = "."
864 775
865 try: 776 try:
866 output = self.device.RunShellCommand( 777 output = self.device.RunShellCommand(
867 cmd, 778 cmd,
868 cwd=os.path.join(AndroidPlatform.DEVICE_DIR, bench_rel), 779 cwd=os.path.join(AndroidPlatform.DEVICE_DIR, bench_rel),
869 timeout=runnable.timeout, 780 timeout=runnable.timeout,
870 retries=0, 781 retries=0,
871 ) 782 )
872 self._PrintStdout(title, output) 783 stdout = "\n".join(output)
784 print title % "Stdout"
785 print stdout
873 except device_errors.CommandTimeoutError: 786 except device_errors.CommandTimeoutError:
874 print ">>> Test timed out after %ss." % runnable.timeout 787 print ">>> Test timed out after %ss." % runnable.timeout
875 stdout = "" 788 stdout = ""
876 return stdout 789 return stdout
877 790
878 791
879 class PrettyFormattedAndroidPlatform(PlatformFormattedMixin, AndroidPlatform):
880 pass
881
882
883 # TODO: Implement results_processor. 792 # TODO: Implement results_processor.
884 def Main(args): 793 def Main(args):
885 logging.getLogger().setLevel(logging.INFO) 794 logging.getLogger().setLevel(logging.INFO)
886 parser = optparse.OptionParser() 795 parser = optparse.OptionParser()
887 parser.add_option("--android-build-tools", 796 parser.add_option("--android-build-tools",
888 help="Path to chromium's build/android. Specifying this " 797 help="Path to chromium's build/android. Specifying this "
889 "option will run tests using android platform.") 798 "option will run tests using android platform.")
890 parser.add_option("--arch", 799 parser.add_option("--arch",
891 help=("The architecture to run tests for, " 800 help=("The architecture to run tests for, "
892 "'auto' or 'native' for auto-detect"), 801 "'auto' or 'native' for auto-detect"),
893 default="x64") 802 default="x64")
894 parser.add_option("--buildbot", 803 parser.add_option("--buildbot",
895 help="Adapt to path structure used on buildbots", 804 help="Adapt to path structure used on buildbots",
896 default=False, action="store_true") 805 default=False, action="store_true")
897 parser.add_option("--device", 806 parser.add_option("--device",
898 help="The device ID to run Android tests on. If not given " 807 help="The device ID to run Android tests on. If not given "
899 "it will be autodetected.") 808 "it will be autodetected.")
900 parser.add_option("--extra-flags", 809 parser.add_option("--extra-flags",
901 help="Additional flags to pass to the test executable", 810 help="Additional flags to pass to the test executable",
902 default="") 811 default="")
903 parser.add_option("--json-test-results", 812 parser.add_option("--json-test-results",
904 help="Path to a file for storing json results.") 813 help="Path to a file for storing json results.")
905 parser.add_option("--json-test-results-no-patch", 814 parser.add_option("--json-test-results-no-patch",
906 help="Path to a file for storing json results from run " 815 help="Path to a file for storing json results from run "
907 "without patch.") 816 "without patch.")
908 parser.add_option("--outdir", help="Base directory with compile output", 817 parser.add_option("--outdir", help="Base directory with compile output",
909 default="out") 818 default="out")
910 parser.add_option("--outdir-no-patch", 819 parser.add_option("--outdir-no-patch",
911 help="Base directory with compile output without patch") 820 help="Base directory with compile output without patch")
912 parser.add_option("--pretty",
913 help="Print human readable output",
914 default=False, action="store_true")
915 parser.add_option("--binary-override-path", 821 parser.add_option("--binary-override-path",
916 help="JavaScript engine binary. By default, d8 under " 822 help="JavaScript engine binary. By default, d8 under "
917 "architecture-specific build dir. " 823 "architecture-specific build dir. "
918 "Not supported in conjunction with outdir-no-patch.") 824 "Not supported in conjunction with outdir-no-patch.")
919 825
920 (options, args) = parser.parse_args(args) 826 (options, args) = parser.parse_args(args)
921 827
922 if len(args) == 0: # pragma: no cover 828 if len(args) == 0: # pragma: no cover
923 parser.print_help() 829 parser.print_help()
924 return 1 830 return 1
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
960 options.shell_dir = os.path.dirname(options.binary_override_path) 866 options.shell_dir = os.path.dirname(options.binary_override_path)
961 default_binary_name = os.path.basename(options.binary_override_path) 867 default_binary_name = os.path.basename(options.binary_override_path)
962 868
963 if options.outdir_no_patch: 869 if options.outdir_no_patch:
964 options.shell_dir_no_patch = os.path.join( 870 options.shell_dir_no_patch = os.path.join(
965 workspace, options.outdir_no_patch, build_config) 871 workspace, options.outdir_no_patch, build_config)
966 else: 872 else:
967 options.shell_dir_no_patch = None 873 options.shell_dir_no_patch = None
968 874
969 platform = Platform.GetPlatform(options) 875 platform = Platform.GetPlatform(options)
970 if options.pretty:
971 platform = platform.GetPrettyFormatted(options)
972 876
973 results = Results() 877 results = Results()
974 results_no_patch = Results() 878 results_no_patch = Results()
975 for path in args: 879 for path in args:
976 path = os.path.abspath(path) 880 path = os.path.abspath(path)
977 881
978 if not os.path.exists(path): # pragma: no cover 882 if not os.path.exists(path): # pragma: no cover
979 results.errors.append("Configuration file %s does not exist." % path) 883 results.errors.append("Configuration file %s does not exist." % path)
980 continue 884 continue
981 885
(...skipping 21 matching lines...) Expand all
1003 def Runner(): 907 def Runner():
1004 """Output generator that reruns several times.""" 908 """Output generator that reruns several times."""
1005 for i in xrange(0, max(1, runnable.run_count)): 909 for i in xrange(0, max(1, runnable.run_count)):
1006 # TODO(machenbach): Allow timeout per arch like with run_count per 910 # TODO(machenbach): Allow timeout per arch like with run_count per
1007 # arch. 911 # arch.
1008 yield platform.Run(runnable, i) 912 yield platform.Run(runnable, i)
1009 913
1010 # Let runnable iterate over all runs and handle output. 914 # Let runnable iterate over all runs and handle output.
1011 result, result_no_patch = runnable.Run( 915 result, result_no_patch = runnable.Run(
1012 Runner, trybot=options.shell_dir_no_patch) 916 Runner, trybot=options.shell_dir_no_patch)
1013 platform.PrintResult(result)
1014 results += result 917 results += result
1015 results_no_patch += result_no_patch 918 results_no_patch += result_no_patch
1016 platform.PostExecution() 919 platform.PostExecution()
1017 920
1018 if options.json_test_results: 921 if options.json_test_results:
1019 results.WriteToFile(options.json_test_results) 922 results.WriteToFile(options.json_test_results)
1020 else: # pragma: no cover 923 else: # pragma: no cover
1021 if not options.pretty: 924 print results
1022 print results
1023 925
1024 if options.json_test_results_no_patch: 926 if options.json_test_results_no_patch:
1025 results_no_patch.WriteToFile(options.json_test_results_no_patch) 927 results_no_patch.WriteToFile(options.json_test_results_no_patch)
1026 else: # pragma: no cover 928 else: # pragma: no cover
1027 if not options.pretty: 929 print results_no_patch
1028 print results_no_patch
1029 930
1030 return min(1, len(results.errors)) 931 return min(1, len(results.errors))
1031 932
1032 if __name__ == "__main__": # pragma: no cover 933 if __name__ == "__main__": # pragma: no cover
1033 sys.exit(Main(sys.argv[1:])) 934 sys.exit(Main(sys.argv[1:]))
OLDNEW
« no previous file with comments | « no previous file | tools/unittests/run_perf_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698