OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright (c) 2011 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2011 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Basic pyauto performance tests. | 6 """Basic pyauto performance tests. |
7 | 7 |
8 For tests that need to be run for multiple iterations (e.g., so that average | 8 For tests that need to be run for multiple iterations (e.g., so that average |
9 and standard deviation values can be reported), the default number of iterations | 9 and standard deviation values can be reported), the default number of iterations |
10 run for each of these tests is specified by |_DEFAULT_NUM_ITERATIONS|. | 10 run for each of these tests is specified by |_DEFAULT_NUM_ITERATIONS|. |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
105 if values: | 105 if values: |
106 avg = float(sum(values)) / len(values) | 106 avg = float(sum(values)) / len(values) |
107 if len(values) > 1: | 107 if len(values) > 1: |
108 temp_vals = [math.pow(x - avg, 2) for x in values] | 108 temp_vals = [math.pow(x - avg, 2) for x in values] |
109 std_dev = math.sqrt(sum(temp_vals) / (len(temp_vals) - 1)) | 109 std_dev = math.sqrt(sum(temp_vals) / (len(temp_vals) - 1)) |
110 return avg, std_dev | 110 return avg, std_dev |
111 | 111 |
112 def _OutputPerfGraphValue(self, description, value): | 112 def _OutputPerfGraphValue(self, description, value): |
113 """Outputs a performance value to have it graphed on the performance bots. | 113 """Outputs a performance value to have it graphed on the performance bots. |
114 | 114 |
115 Only used for ChromeOS. | 115 Only used for ChromeOS. The performance bots have a 30-character limit on |
| 116 the length of the description for a performance value. Any characters |
| 117 beyond that are truncated before results are stored in the autotest |
| 118 database. |
116 | 119 |
117 Args: | 120 Args: |
118 description: A string description of the performance value. | 121 description: A string description of the performance value. This will |
| 122 be truncated to 30 characters when stored in the autotest |
| 123 database. |
119 value: A numeric value representing a single performance measurement. | 124 value: A numeric value representing a single performance measurement. |
120 """ | 125 """ |
121 if self.IsChromeOS(): | 126 if self.IsChromeOS(): |
| 127 if len(description) > 30: |
| 128 logging.warning('The description "%s" will be truncated to "%s" ' |
| 129 '(length 30) when added to the autotest database.', |
| 130 description, description[:30]) |
122 print '\n%s(\'%s\', %.2f)%s' % (self._PERF_OUTPUT_MARKER_PRE, description, | 131 print '\n%s(\'%s\', %.2f)%s' % (self._PERF_OUTPUT_MARKER_PRE, description, |
123 value, self._PERF_OUTPUT_MARKER_POST) | 132 value, self._PERF_OUTPUT_MARKER_POST) |
124 | 133 |
125 def _PrintSummaryResults(self, description, values, units): | 134 def _PrintSummaryResults(self, description, values, units): |
126 """Logs summary measurement information. | 135 """Logs summary measurement information. |
127 | 136 |
| 137 This function computes and outputs the average and standard deviation of |
| 138 the specified list of value measurements. It also invokes |
| 139 _OutputPerfGraphValue() with the computed *average* value, to ensure the |
| 140 average value can be plotted in a performance graph. |
| 141 |
128 Args: | 142 Args: |
129 description: A string description for the specified results. | 143 description: A string description for the specified results. |
130 values: A list of numeric value measurements. | 144 values: A list of numeric value measurements. |
131 units: A string specifying the units for the specified measurements. | 145 units: A string specifying the units for the specified measurements. |
132 """ | 146 """ |
133 logging.info('Results for: ' + description) | 147 logging.info('Results for: ' + description) |
134 if values: | 148 if values: |
135 avg, std_dev = self._AvgAndStdDev(values) | 149 avg, std_dev = self._AvgAndStdDev(values) |
136 logging.info('Number of iterations: %d', len(values)) | 150 logging.info('Number of iterations: %d', len(values)) |
137 for val in values: | 151 for val in values: |
(...skipping 450 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
588 # Wait until the final result is computed, then retrieve and output it. | 602 # Wait until the final result is computed, then retrieve and output it. |
589 # TODO(dennisjeffrey): Have all tests in this class use JSON.stringify() | 603 # TODO(dennisjeffrey): Have all tests in this class use JSON.stringify() |
590 # to send results back from the Javascript. | 604 # to send results back from the Javascript. |
591 js = 'window.domAutomationController.send("" + final_average_fps);' | 605 js = 'window.domAutomationController.send("" + final_average_fps);' |
592 self.assertTrue( | 606 self.assertTrue( |
593 self.WaitUntil( | 607 self.WaitUntil( |
594 lambda: self.ExecuteJavascript(js, tab_index=1) != '-1', | 608 lambda: self.ExecuteJavascript(js, tab_index=1) != '-1', |
595 timeout=300, expect_retval=True, retry_sleep=0.25), | 609 timeout=300, expect_retval=True, retry_sleep=0.25), |
596 msg='Timed out when waiting for test result.') | 610 msg='Timed out when waiting for test result.') |
597 result = float(self.ExecuteJavascript(js, tab_index=1)) | 611 result = float(self.ExecuteJavascript(js, tab_index=1)) |
598 logging.info('Result for %s: %.2f FPS (average)' % (description, result)) | 612 logging.info('Result for %s: %.2f FPS (average)', description, result) |
599 self._OutputPerfGraphValue('%s_%s' % ('FPS', description), result) | 613 self._OutputPerfGraphValue('%s_%s' % ('FPS', description), result) |
600 | 614 |
601 def testFlashGaming(self): | 615 def testFlashGaming(self): |
602 """Runs a simple flash gaming benchmark test.""" | 616 """Runs a simple flash gaming benchmark test.""" |
603 webpage_url = self.GetHttpURLForDataPath('pyauto_private', 'flash', | 617 webpage_url = self.GetHttpURLForDataPath('pyauto_private', 'flash', |
604 'FlashGamingTest2.html') | 618 'FlashGamingTest2.html') |
605 self._RunFlashTestForAverageFPS(webpage_url, 'FlashGaming') | 619 self._RunFlashTestForAverageFPS(webpage_url, 'FlashGaming') |
606 | 620 |
607 def testFlashText(self): | 621 def testFlashText(self): |
608 """Runs a simple flash text benchmark test.""" | 622 """Runs a simple flash text benchmark test.""" |
(...skipping 27 matching lines...) Expand all Loading... |
636 json_result += "}"; | 650 json_result += "}"; |
637 window.domAutomationController.send(json_result); | 651 window.domAutomationController.send(json_result); |
638 """ | 652 """ |
639 result = eval(self.ExecuteJavascript(js_result, tab_index=1)) | 653 result = eval(self.ExecuteJavascript(js_result, tab_index=1)) |
640 for benchmark in result: | 654 for benchmark in result: |
641 mflops = result[benchmark][0] | 655 mflops = result[benchmark][0] |
642 mem = result[benchmark][1] | 656 mem = result[benchmark][1] |
643 if benchmark.endswith('_mflops'): | 657 if benchmark.endswith('_mflops'): |
644 benchmark = benchmark[:benchmark.find('_mflops')] | 658 benchmark = benchmark[:benchmark.find('_mflops')] |
645 logging.info('Results for ScimarkGui_' + benchmark + ':') | 659 logging.info('Results for ScimarkGui_' + benchmark + ':') |
646 logging.info(' %.2f MFLOPS' % mflops) | 660 logging.info(' %.2f MFLOPS', mflops) |
647 logging.info(' %.2f MB' % mem) | 661 logging.info(' %.2f MB', mem) |
648 self._OutputPerfGraphValue( | 662 self._OutputPerfGraphValue( |
649 '%s_ScimarkGui-%s-MFLOPS' % ('MFLOPS', benchmark), mflops) | 663 '%s_ScimarkGui-%s-MFLOPS' % ('MFLOPS', benchmark), mflops) |
650 self._OutputPerfGraphValue( | 664 self._OutputPerfGraphValue( |
651 '%s_ScimarkGui-%s-Mem' % ('MB', benchmark), mem) | 665 '%s_ScimarkGui-%s-Mem' % ('MB', benchmark), mem) |
652 | 666 |
653 | 667 |
654 class PerfTestServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): | 668 class PerfTestServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): |
655 """Request handler for the local performance test server.""" | 669 """Request handler for the local performance test server.""" |
656 | 670 |
657 def _IgnoreHandler(self, unused_args): | 671 def _IgnoreHandler(self, unused_args): |
(...skipping 13 matching lines...) Expand all Loading... |
671 args: A dictionary of arguments for the current GET request. Must | 685 args: A dictionary of arguments for the current GET request. Must |
672 contain 'filename' and 'mb' keys that refer to the name of the | 686 contain 'filename' and 'mb' keys that refer to the name of the |
673 file to create and its desired size, respectively. | 687 file to create and its desired size, respectively. |
674 """ | 688 """ |
675 megabytes = None | 689 megabytes = None |
676 filename = None | 690 filename = None |
677 try: | 691 try: |
678 megabytes = int(args['mb'][0]) | 692 megabytes = int(args['mb'][0]) |
679 filename = args['filename'][0] | 693 filename = args['filename'][0] |
680 except (ValueError, KeyError, IndexError), e: | 694 except (ValueError, KeyError, IndexError), e: |
681 logging.exception('Server error creating file: %s' % e) | 695 logging.exception('Server error creating file: %s', e) |
682 assert megabytes and filename | 696 assert megabytes and filename |
683 with open(os.path.join(self.server.docroot, filename), 'wb') as f: | 697 with open(os.path.join(self.server.docroot, filename), 'wb') as f: |
684 f.write('X' * 1024 * 1024 * megabytes) | 698 f.write('X' * 1024 * 1024 * megabytes) |
685 self.send_response(200) | 699 self.send_response(200) |
686 self.end_headers() | 700 self.end_headers() |
687 | 701 |
688 def _DeleteFileHandler(self, args): | 702 def _DeleteFileHandler(self, args): |
689 """A GET handler that deletes the specified local file. | 703 """A GET handler that deletes the specified local file. |
690 | 704 |
691 Args: | 705 Args: |
692 args: A dictionary of arguments for the current GET request. Must | 706 args: A dictionary of arguments for the current GET request. Must |
693 contain a 'filename' key that refers to the name of the file | 707 contain a 'filename' key that refers to the name of the file |
694 to delete, relative to the server's document root. | 708 to delete, relative to the server's document root. |
695 """ | 709 """ |
696 filename = None | 710 filename = None |
697 try: | 711 try: |
698 filename = args['filename'][0] | 712 filename = args['filename'][0] |
699 except (KeyError, IndexError), e: | 713 except (KeyError, IndexError), e: |
700 logging.exception('Server error deleting file: %s' % e) | 714 logging.exception('Server error deleting file: %s', e) |
701 assert filename | 715 assert filename |
702 try: | 716 try: |
703 os.remove(os.path.join(self.server.docroot, filename)) | 717 os.remove(os.path.join(self.server.docroot, filename)) |
704 except OSError, e: | 718 except OSError, e: |
705 logging.warning('OS error removing file: %s' % e) | 719 logging.warning('OS error removing file: %s', e) |
706 self.send_response(200) | 720 self.send_response(200) |
707 self.end_headers() | 721 self.end_headers() |
708 | 722 |
709 def _StartUploadHandler(self, args): | 723 def _StartUploadHandler(self, args): |
710 """A GET handler to serve a page that uploads the given amount of data. | 724 """A GET handler to serve a page that uploads the given amount of data. |
711 | 725 |
712 When the page loads, the specified amount of data is automatically | 726 When the page loads, the specified amount of data is automatically |
713 uploaded to the same local server that is handling the current request. | 727 uploaded to the same local server that is handling the current request. |
714 | 728 |
715 Args: | 729 Args: |
716 args: A dictionary of arguments for the current GET request. Must | 730 args: A dictionary of arguments for the current GET request. Must |
717 contain an 'mb' key that refers to the size of the data to | 731 contain an 'mb' key that refers to the size of the data to |
718 upload. | 732 upload. |
719 """ | 733 """ |
720 megabytes = None | 734 megabytes = None |
721 try: | 735 try: |
722 megabytes = int(args['mb'][0]) | 736 megabytes = int(args['mb'][0]) |
723 except (ValueError, KeyError, IndexError), e: | 737 except (ValueError, KeyError, IndexError), e: |
724 logging.exception('Server error starting upload: %s' % e) | 738 logging.exception('Server error starting upload: %s', e) |
725 assert megabytes | 739 assert megabytes |
726 script = """ | 740 script = """ |
727 <html> | 741 <html> |
728 <head> | 742 <head> |
729 <script type='text/javascript'> | 743 <script type='text/javascript'> |
730 function startUpload() { | 744 function startUpload() { |
731 var megabytes = %s; | 745 var megabytes = %s; |
732 var data = Array((1024 * 1024 * megabytes) + 1).join('X'); | 746 var data = Array((1024 * 1024 * megabytes) + 1).join('X'); |
733 var boundary = '***BOUNDARY***'; | 747 var boundary = '***BOUNDARY***'; |
734 var xhr = new XMLHttpRequest(); | 748 var xhr = new XMLHttpRequest(); |
(...skipping 142 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
877 """Identifies the port number to which the server is currently bound. | 891 """Identifies the port number to which the server is currently bound. |
878 | 892 |
879 Returns: | 893 Returns: |
880 The numeric port number to which the server is currently bound. | 894 The numeric port number to which the server is currently bound. |
881 """ | 895 """ |
882 return self._server.server_address[1] | 896 return self._server.server_address[1] |
883 | 897 |
884 | 898 |
885 if __name__ == '__main__': | 899 if __name__ == '__main__': |
886 pyauto_functional.Main() | 900 pyauto_functional.Main() |
OLD | NEW |