OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Performance tests for Chrome Endure (long-running perf tests on Chrome). | 6 """Performance tests for Chrome Endure (long-running perf tests on Chrome). |
7 | 7 |
8 This module accepts the following environment variable inputs: | 8 This module accepts the following environment variable inputs: |
9 TEST_LENGTH: The number of seconds in which to run each test. | 9 TEST_LENGTH: The number of seconds in which to run each test. |
10 PERF_STATS_INTERVAL: The number of seconds to wait in-between each sampling | 10 PERF_STATS_INTERVAL: The number of seconds to wait in-between each sampling |
11 of performance/memory statistics. | 11 of performance/memory statistics. |
12 | 12 |
13 The following variables are related to the Deep Memory Profiler. | 13 The following variables are related to the Deep Memory Profiler. |
14 DEEP_MEMORY_PROFILE: Enable the Deep Memory Profiler if it's set to 'True'. | 14 DEEP_MEMORY_PROFILE: Enable the Deep Memory Profiler if it's set to 'True'. |
15 DEEP_MEMORY_PROFILE_SAVE: Don't clean up dump files if it's set to 'True'. | 15 DEEP_MEMORY_PROFILE_SAVE: Don't clean up dump files if it's set to 'True'. |
16 DEEP_MEMORY_PROFILE_UPLOAD: Upload dumped files if the variable has a Google | 16 DEEP_MEMORY_PROFILE_UPLOAD: Upload dumped files if the variable has a Google |
17 Storage bucket like gs://chromium-endure/. The 'gsutil' script in $PATH | 17 Storage bucket like gs://chromium-endure/. The 'gsutil' script in $PATH |
18 is used by default, or set a variable 'GSUTIL' to specify a path to the | 18 is used by default, or set a variable 'GSUTIL' to specify a path to the |
19 'gsutil' script. A variable 'REVISION' (or 'BUILDBOT_GOT_REVISION') is | 19 'gsutil' script. A variable 'REVISION' (or 'BUILDBOT_GOT_REVISION') is |
20 used as a subdirectory in the destination if it is set. | 20 used as a subdirectory in the destination if it is set. |
21 GSUTIL: A path to the 'gsutil' script. Not mandatory. | 21 GSUTIL: A path to the 'gsutil' script. Not mandatory. |
22 REVISION: A string that represents the revision or some build configuration. | 22 REVISION: A string that represents the revision or some build configuration. |
23 Not mandatory. | 23 Not mandatory. |
24 BUILDBOT_GOT_REVISION: Similar to 'REVISION', but checked only if 'REVISION' | 24 BUILDBOT_GOT_REVISION: Similar to 'REVISION', but checked only if 'REVISION' |
25 is not specified. Not mandatory. | 25 is not specified. Not mandatory. |
26 | |
27 ENDURE_NO_WPR: Run tests without Web Page Replay if it's set. | |
28 WPR_RECORD: Run tests in record mode. If you want to make a fresh | |
29 archive, make sure to delete the old one, otherwise | |
30 it will append to the old one. | |
31 WPR_ARCHIVE_PATH: an alternative archive file to use. | |
32 """ | 26 """ |
33 | 27 |
34 from datetime import datetime | 28 from datetime import datetime |
35 import json | 29 import json |
36 import logging | 30 import logging |
37 import os | 31 import os |
38 import re | 32 import re |
39 import subprocess | 33 import subprocess |
40 import tempfile | 34 import tempfile |
41 import time | 35 import time |
42 | 36 |
43 import perf | 37 import perf |
44 import pyauto_functional # Must be imported before pyauto. | 38 import pyauto_functional # Must be imported before pyauto. |
45 import pyauto | 39 import pyauto |
46 import pyauto_errors | 40 import pyauto_errors |
47 import pyauto_utils | 41 import pyauto_utils |
48 import remote_inspector_client | 42 import remote_inspector_client |
49 import selenium.common.exceptions | 43 import selenium.common.exceptions |
50 from selenium.webdriver.support.ui import WebDriverWait | 44 from selenium.webdriver.support.ui import WebDriverWait |
51 import webpagereplay | |
52 | 45 |
53 | 46 |
54 class NotSupportedEnvironmentError(RuntimeError): | 47 class NotSupportedEnvironmentError(RuntimeError): |
55 """Represent an error raised since the environment (OS) is not supported.""" | 48 """Represent an error raised since the environment (OS) is not supported.""" |
56 pass | 49 pass |
57 | 50 |
58 | 51 |
59 class DeepMemoryProfiler(object): | 52 class DeepMemoryProfiler(object): |
60 """Controls Deep Memory Profiler (dmprof) for endurance tests.""" | 53 """Controls Deep Memory Profiler (dmprof) for endurance tests.""" |
61 DEEP_MEMORY_PROFILE = False | 54 DEEP_MEMORY_PROFILE = False |
(...skipping 233 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
295 """ | 288 """ |
296 | 289 |
297 _DEFAULT_TEST_LENGTH_SEC = 60 * 60 * 6 # Tests run for 6 hours. | 290 _DEFAULT_TEST_LENGTH_SEC = 60 * 60 * 6 # Tests run for 6 hours. |
298 _GET_PERF_STATS_INTERVAL = 60 * 5 # Measure perf stats every 5 minutes. | 291 _GET_PERF_STATS_INTERVAL = 60 * 5 # Measure perf stats every 5 minutes. |
299 # TODO(dennisjeffrey): Do we still need to tolerate errors? | 292 # TODO(dennisjeffrey): Do we still need to tolerate errors? |
300 _ERROR_COUNT_THRESHOLD = 50 # Number of errors to tolerate. | 293 _ERROR_COUNT_THRESHOLD = 50 # Number of errors to tolerate. |
301 _REVISION = '' | 294 _REVISION = '' |
302 _GSUTIL = 'gsutil' | 295 _GSUTIL = 'gsutil' |
303 | 296 |
304 def setUp(self): | 297 def setUp(self): |
305 # The Web Page Replay environment variables must be parsed before | |
306 # perf.BasePerfTest.setUp() | |
307 self._ParseReplayEnv() | |
308 # The environment variables for the Deep Memory Profiler must be set | 298 # The environment variables for the Deep Memory Profiler must be set |
309 # before perf.BasePerfTest.setUp() to inherit them to Chrome. | 299 # before perf.BasePerfTest.setUp() to inherit them to Chrome. |
310 self._dmprof = DeepMemoryProfiler() | 300 self._dmprof = DeepMemoryProfiler() |
311 self._revision = str(os.environ.get('REVISION', self._REVISION)) | 301 self._revision = str(os.environ.get('REVISION', self._REVISION)) |
312 if not self._revision: | 302 if not self._revision: |
313 self._revision = str(os.environ.get('BUILDBOT_GOT_REVISION', | 303 self._revision = str(os.environ.get('BUILDBOT_GOT_REVISION', |
314 self._REVISION)) | 304 self._REVISION)) |
315 self._gsutil = str(os.environ.get('GSUTIL', self._GSUTIL)) | 305 self._gsutil = str(os.environ.get('GSUTIL', self._GSUTIL)) |
316 if self._dmprof: | 306 if self._dmprof: |
317 self._dmprof.SetUp(self.IsLinux(), self._revision, self._gsutil) | 307 self._dmprof.SetUp(self.IsLinux(), self._revision, self._gsutil) |
(...skipping 14 matching lines...) Expand all Loading... |
332 | 322 |
333 # Set up a remote inspector client associated with tab 0. | 323 # Set up a remote inspector client associated with tab 0. |
334 logging.info('Setting up connection to remote inspector...') | 324 logging.info('Setting up connection to remote inspector...') |
335 self._remote_inspector_client = ( | 325 self._remote_inspector_client = ( |
336 remote_inspector_client.RemoteInspectorClient()) | 326 remote_inspector_client.RemoteInspectorClient()) |
337 logging.info('Connection to remote inspector set up successfully.') | 327 logging.info('Connection to remote inspector set up successfully.') |
338 | 328 |
339 self._test_start_time = 0 | 329 self._test_start_time = 0 |
340 self._num_errors = 0 | 330 self._num_errors = 0 |
341 self._events_to_output = [] | 331 self._events_to_output = [] |
342 self._StartReplayServerIfNecessary() | |
343 | 332 |
344 def tearDown(self): | 333 def tearDown(self): |
345 logging.info('Terminating connection to remote inspector...') | 334 logging.info('Terminating connection to remote inspector...') |
346 self._remote_inspector_client.Stop() | 335 self._remote_inspector_client.Stop() |
347 logging.info('Connection to remote inspector terminated.') | 336 logging.info('Connection to remote inspector terminated.') |
348 | 337 |
349 # Must be done at end of this function except for post-cleaning after | 338 # Must be done at end of this function except for post-cleaning after |
350 # Chrome finishes. | 339 # Chrome finishes. |
351 perf.BasePerfTest.tearDown(self) | 340 perf.BasePerfTest.tearDown(self) |
352 | 341 |
353 # Must be done after perf.BasePerfTest.tearDown() | 342 # Must be done after perf.BasePerfTest.tearDown() |
354 self._StopReplayServerIfNecessary() | |
355 if self._dmprof: | 343 if self._dmprof: |
356 self._dmprof.TearDown() | 344 self._dmprof.TearDown() |
357 | 345 |
358 def _GetArchiveName(self): | |
359 """Return the Web Page Replay archive name that corresponds to a test. | |
360 | |
361 Override this function to return the name of an archive that | |
362 corresponds to the test, e.g "ChromeEndureGmailTest.wpr". | |
363 | |
364 Returns: | |
365 None, by default no archive name is provided. | |
366 """ | |
367 return None | |
368 | |
369 def _ParseReplayEnv(self): | |
370 """Parse Web Page Replay related envrionment variables.""" | |
371 if 'ENDURE_NO_WPR' in os.environ: | |
372 self._use_wpr = False | |
373 logging.info('Skipping Web Page Replay since ENDURE_NO_WPR is set.') | |
374 else: | |
375 self._archive_path = None | |
376 if 'WPR_ARCHIVE_PATH' in os.environ: | |
377 self._archive_path = os.environ.get('WPR_ARCHIVE_PATH') | |
378 else: | |
379 if self._GetArchiveName(): | |
380 self._archive_path = ChromeEndureReplay.Path( | |
381 'archive', archive_name=self._GetArchiveName()) | |
382 self._is_record_mode = 'WPR_RECORD' in os.environ | |
383 if self._is_record_mode: | |
384 if self._archive_path: | |
385 self._use_wpr = True | |
386 else: | |
387 self._use_wpr = False | |
388 logging.info('Fail to record since a valid archive path can not ' + | |
389 'be generated. Did you implement ' + | |
390 '_GetArchiveName() in your test?') | |
391 else: | |
392 if self._archive_path and os.path.exists(self._archive_path): | |
393 self._use_wpr = True | |
394 else: | |
395 self._use_wpr = False | |
396 logging.info( | |
397 'Skipping Web Page Replay since archive file %sdoes not exist.', | |
398 self._archive_path + ' ' if self._archive_path else '') | |
399 | |
400 def ExtraChromeFlags(self): | 346 def ExtraChromeFlags(self): |
401 """Ensures Chrome is launched with custom flags. | 347 """Ensures Chrome is launched with custom flags. |
402 | 348 |
403 Returns: | 349 Returns: |
404 A list of extra flags to pass to Chrome when it is launched. | 350 A list of extra flags to pass to Chrome when it is launched. |
405 """ | 351 """ |
406 # The same with setUp, but need to fetch the environment variable since | 352 # The same with setUp, but need to fetch the environment variable since |
407 # ExtraChromeFlags is called before setUp. | 353 # ExtraChromeFlags is called before setUp. |
408 deep_memory_profile = DeepMemoryProfiler.GetEnvironmentVariable( | 354 deep_memory_profile = DeepMemoryProfiler.GetEnvironmentVariable( |
409 'DEEP_MEMORY_PROFILE', bool, DeepMemoryProfiler.DEEP_MEMORY_PROFILE) | 355 'DEEP_MEMORY_PROFILE', bool, DeepMemoryProfiler.DEEP_MEMORY_PROFILE) |
410 | 356 |
411 # Ensure Chrome enables remote debugging on port 9222. This is required to | 357 # Ensure Chrome enables remote debugging on port 9222. This is required to |
412 # interact with Chrome's remote inspector. | 358 # interact with Chrome's remote inspector. |
413 # Also, enable the memory benchmarking V8 extension for heap dumps. | 359 # Also, enable the memory benchmarking V8 extension for heap dumps. |
414 extra_flags = ['--remote-debugging-port=9222', | 360 extra_flags = ['--remote-debugging-port=9222', |
415 '--enable-memory-benchmarking'] | 361 '--enable-memory-benchmarking'] |
416 if deep_memory_profile: | 362 if deep_memory_profile: |
417 extra_flags.append('--no-sandbox') | 363 extra_flags.append('--no-sandbox') |
418 if self._use_wpr: | |
419 extra_flags.extend(ChromeEndureReplay.CHROME_FLAGS) | |
420 return perf.BasePerfTest.ExtraChromeFlags(self) + extra_flags | 364 return perf.BasePerfTest.ExtraChromeFlags(self) + extra_flags |
421 | 365 |
422 def _OnTimelineEvent(self, event_info): | 366 def _OnTimelineEvent(self, event_info): |
423 """Invoked by the Remote Inspector Client when a timeline event occurs. | 367 """Invoked by the Remote Inspector Client when a timeline event occurs. |
424 | 368 |
425 Args: | 369 Args: |
426 event_info: A dictionary containing raw information associated with a | 370 event_info: A dictionary containing raw information associated with a |
427 timeline event received from Chrome's remote inspector. Refer to | 371 timeline event received from Chrome's remote inspector. Refer to |
428 chrome/src/third_party/WebKit/Source/WebCore/inspector/Inspector.json | 372 chrome/src/third_party/WebKit/Source/WebCore/inspector/Inspector.json |
429 for the format of this dictionary. | 373 for the format of this dictionary. |
(...skipping 281 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
711 try: | 655 try: |
712 element = self._GetElement(driver.find_element_by_xpath, xpath) | 656 element = self._GetElement(driver.find_element_by_xpath, xpath) |
713 element.click() | 657 element.click() |
714 except (selenium.common.exceptions.StaleElementReferenceException, | 658 except (selenium.common.exceptions.StaleElementReferenceException, |
715 selenium.common.exceptions.TimeoutException) as e: | 659 selenium.common.exceptions.TimeoutException) as e: |
716 logging.exception('WebDriver exception: %s' % e) | 660 logging.exception('WebDriver exception: %s' % e) |
717 return False | 661 return False |
718 | 662 |
719 return True | 663 return True |
720 | 664 |
721 def _StartReplayServerIfNecessary(self): | |
722 """Start replay server if necessary.""" | |
723 if self._use_wpr: | |
724 mode = 'record' if self._is_record_mode else 'replay' | |
725 self._wpr_server = ChromeEndureReplay.ReplayServer(self._archive_path) | |
726 self._wpr_server.StartServer() | |
727 logging.info('Web Page Replay server has started in %s mode.', mode) | |
728 | |
729 def _StopReplayServerIfNecessary(self): | |
730 """Stop the Web Page Replay server if necessary. | |
731 | |
732 This method has to be called AFTER all network connections which go | |
733 through Web Page Replay server have shut down. Otherwise the | |
734 Web Page Replay server will hang to wait for them. A good | |
735 place is to call it at the end of the teardown process. | |
736 """ | |
737 if self._use_wpr: | |
738 self._wpr_server.StopServer() | |
739 logging.info('The Web Page Replay server stopped.') | |
740 | |
741 | 665 |
742 class ChromeEndureControlTest(ChromeEndureBaseTest): | 666 class ChromeEndureControlTest(ChromeEndureBaseTest): |
743 """Control tests for Chrome Endure.""" | 667 """Control tests for Chrome Endure.""" |
744 | 668 |
745 _WEBAPP_NAME = 'Control' | 669 _WEBAPP_NAME = 'Control' |
746 _TAB_TITLE_SUBSTRING = 'Chrome Endure Control Test' | 670 _TAB_TITLE_SUBSTRING = 'Chrome Endure Control Test' |
747 | 671 |
748 def testControlAttachDetachDOMTree(self): | 672 def testControlAttachDetachDOMTree(self): |
749 """Continually attach and detach a DOM tree from a basic document.""" | 673 """Continually attach and detach a DOM tree from a basic document.""" |
750 test_description = 'AttachDetachDOMTree' | 674 test_description = 'AttachDetachDOMTree' |
(...skipping 30 matching lines...) Expand all Loading... |
781 # the DOM tree from the document, wait half a second. | 705 # the DOM tree from the document, wait half a second. |
782 self._ClickElementByXpath(driver, 'id("attach")') | 706 self._ClickElementByXpath(driver, 'id("attach")') |
783 time.sleep(0.5) | 707 time.sleep(0.5) |
784 self._ClickElementByXpath(driver, 'id("detach")') | 708 self._ClickElementByXpath(driver, 'id("detach")') |
785 time.sleep(0.5) | 709 time.sleep(0.5) |
786 | 710 |
787 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, | 711 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, |
788 test_description, lambda: scenario(driver)) | 712 test_description, lambda: scenario(driver)) |
789 | 713 |
790 | 714 |
791 # TODO(dennisjeffrey): Make new WPR recordings of the Gmail tests so that we | |
792 # can remove the special handling for when self._use_wpr is True. | |
793 class ChromeEndureGmailTest(ChromeEndureBaseTest): | |
794 """Long-running performance tests for Chrome using Gmail.""" | |
795 | |
796 _WEBAPP_NAME = 'Gmail' | |
797 _TAB_TITLE_SUBSTRING = 'Gmail' | |
798 _FRAME_XPATH = 'id("canvas_frame")' | |
799 | |
800 def setUp(self): | |
801 ChromeEndureBaseTest.setUp(self) | |
802 | |
803 self._FRAME_XPATH = self._FRAME_XPATH if self._use_wpr else '' | |
804 | |
805 # Log into a test Google account and open up Gmail. | |
806 self._LoginToGoogleAccount(account_key='test_google_account_gmail') | |
807 self.NavigateToURL(self._GetConfig().get('gmail_url')) | |
808 self.assertTrue( | |
809 self.WaitUntil(lambda: self._TAB_TITLE_SUBSTRING in | |
810 self.GetActiveTabTitle(), | |
811 timeout=60, expect_retval=True, retry_sleep=1), | |
812 msg='Timed out waiting for Gmail to load. Tab title is: %s' % | |
813 self.GetActiveTabTitle()) | |
814 | |
815 self._driver = self.NewWebDriver() | |
816 # Any call to wait.until() will raise an exception if the timeout is hit. | |
817 # TODO(dennisjeffrey): Remove the need for webdriver's wait using the new | |
818 # DOM mutation observer mechanism. | |
819 self._wait = WebDriverWait(self._driver, timeout=60) | |
820 | |
821 | |
822 if self._use_wpr: | |
823 # Wait until Gmail's 'canvas_frame' loads and the 'Inbox' link is present. | |
824 # TODO(dennisjeffrey): Check with the Gmail team to see if there's a | |
825 # better way to tell when the webpage is ready for user interaction. | |
826 self._wait.until( | |
827 self._SwitchToCanvasFrame) # Raises exception if the timeout is hit. | |
828 | |
829 # Wait for the inbox to appear. | |
830 self.WaitForDomNode('//a[starts-with(@title, "Inbox")]', | |
831 frame_xpath=self._FRAME_XPATH) | |
832 | |
833 # Test whether latency dom element is available. | |
834 try: | |
835 self._GetLatencyDomElement(5000) | |
836 self._has_latency = True | |
837 except pyauto_errors.JSONInterfaceError: | |
838 logging.info('Skip recording latency as latency ' + | |
839 'dom element is not available.') | |
840 self._has_latency = False | |
841 | |
842 def _GetArchiveName(self): | |
843 """Return Web Page Replay archive name.""" | |
844 return 'ChromeEndureGmailTest.wpr' | |
845 | |
846 def _SwitchToCanvasFrame(self, driver): | |
847 """Switch the WebDriver to Gmail's 'canvas_frame', if it's available. | |
848 | |
849 Args: | |
850 driver: A selenium.webdriver.remote.webdriver.WebDriver object. | |
851 | |
852 Returns: | |
853 True, if the switch to Gmail's 'canvas_frame' is successful, or | |
854 False if not. | |
855 """ | |
856 try: | |
857 driver.switch_to_frame('canvas_frame') | |
858 return True | |
859 except selenium.common.exceptions.NoSuchFrameException: | |
860 return False | |
861 | |
862 def _GetLatencyDomElement(self, timeout=-1): | |
863 """Returns a reference to the latency info element in the Gmail DOM. | |
864 | |
865 Args: | |
866 timeout: The maximum amount of time (in milliseconds) to wait for | |
867 the latency dom element to appear, defaults to the | |
868 default automation timeout. | |
869 Returns: | |
870 A latency dom element. | |
871 """ | |
872 latency_xpath = ( | |
873 '//span[starts-with(text(), "Why was the last action slow?")]') | |
874 self.WaitForDomNode(latency_xpath, timeout=timeout, | |
875 frame_xpath=self._FRAME_XPATH) | |
876 return self._GetElement(self._driver.find_element_by_xpath, latency_xpath) | |
877 | |
878 def _WaitUntilDomElementRemoved(self, dom_element): | |
879 """Waits until the given element is no longer attached to the DOM. | |
880 | |
881 Args: | |
882 dom_element: A selenium.webdriver.remote.WebElement object. | |
883 """ | |
884 def _IsElementStale(): | |
885 try: | |
886 dom_element.tag_name | |
887 except selenium.common.exceptions.StaleElementReferenceException: | |
888 return True | |
889 return False | |
890 | |
891 self.WaitUntil(_IsElementStale, timeout=60, expect_retval=True) | |
892 | |
893 def _ClickElementAndRecordLatency(self, element, test_description, | |
894 action_description): | |
895 """Clicks a DOM element and records the latency associated with that action. | |
896 | |
897 To account for scenario warm-up time, latency values during the first | |
898 minute of test execution are not recorded. | |
899 | |
900 Args: | |
901 element: A selenium.webdriver.remote.WebElement object to click. | |
902 test_description: A string description of what the test does, used for | |
903 outputting results to be graphed. Should not contain spaces. For | |
904 example, 'ComposeDiscard' for Gmail. | |
905 action_description: A string description of what action is being | |
906 performed. Should not contain spaces. For example, 'Compose'. | |
907 """ | |
908 if not self._has_latency: | |
909 element.click() | |
910 return | |
911 latency_dom_element = self._GetLatencyDomElement() | |
912 element.click() | |
913 # Wait for the old latency value to be removed, before getting the new one. | |
914 self._WaitUntilDomElementRemoved(latency_dom_element) | |
915 | |
916 latency_dom_element = self._GetLatencyDomElement() | |
917 match = re.search(r'\[(\d+) ms\]', latency_dom_element.text) | |
918 if match: | |
919 latency = int(match.group(1)) | |
920 elapsed_time = int(round(time.time() - self._test_start_time)) | |
921 if elapsed_time > 60: # Ignore the first minute of latency measurements. | |
922 self._OutputPerfGraphValue( | |
923 '%sLatency' % action_description, [(elapsed_time, latency)], 'msec', | |
924 graph_name='%s%s-%sLatency' % (self._WEBAPP_NAME, test_description, | |
925 action_description), | |
926 units_x='seconds') | |
927 else: | |
928 logging.warning('Could not identify latency value.') | |
929 | |
930 def testGmailComposeDiscard(self): | |
931 """Continuously composes/discards an e-mail before sending. | |
932 | |
933 This test continually composes/discards an e-mail using Gmail, and | |
934 periodically gathers performance stats that may reveal memory bloat. | |
935 """ | |
936 test_description = 'ComposeDiscard' | |
937 | |
938 def scenario_wpr(): | |
939 # Click the "Compose" button, enter some text into the "To" field, enter | |
940 # some text into the "Subject" field, then click the "Discard" button to | |
941 # discard the message. | |
942 compose_xpath = '//div[text()="COMPOSE"]' | |
943 self.WaitForDomNode(compose_xpath, frame_xpath=self._FRAME_XPATH) | |
944 compose_button = self._GetElement(self._driver.find_element_by_xpath, | |
945 compose_xpath) | |
946 self._ClickElementAndRecordLatency( | |
947 compose_button, test_description, 'Compose') | |
948 | |
949 to_xpath = '//textarea[@name="to"]' | |
950 self.WaitForDomNode(to_xpath, frame_xpath=self._FRAME_XPATH) | |
951 to_field = self._GetElement(self._driver.find_element_by_xpath, to_xpath) | |
952 to_field.send_keys('nobody@nowhere.com') | |
953 | |
954 subject_xpath = '//input[@name="subject"]' | |
955 self.WaitForDomNode(subject_xpath, frame_xpath=self._FRAME_XPATH) | |
956 subject_field = self._GetElement(self._driver.find_element_by_xpath, | |
957 subject_xpath) | |
958 subject_field.send_keys('This message is about to be discarded') | |
959 | |
960 discard_xpath = '//div[text()="Discard"]' | |
961 self.WaitForDomNode(discard_xpath, frame_xpath=self._FRAME_XPATH) | |
962 discard_button = self._GetElement(self._driver.find_element_by_xpath, | |
963 discard_xpath) | |
964 discard_button.click() | |
965 | |
966 # Wait for the message to be discarded, assumed to be true after the | |
967 # "To" field is removed from the webpage DOM. | |
968 self._wait.until(lambda _: not self._GetElement( | |
969 self._driver.find_element_by_name, 'to')) | |
970 | |
971 def scenario_live(): | |
972 compose_xpath = '//div[text()="COMPOSE"]' | |
973 self.WaitForDomNode(compose_xpath, frame_xpath=self._FRAME_XPATH) | |
974 compose_button = self._GetElement(self._driver.find_element_by_xpath, | |
975 compose_xpath) | |
976 self._ClickElementAndRecordLatency( | |
977 compose_button, test_description, 'Compose') | |
978 | |
979 to_xpath = '//textarea[@name="to"]' | |
980 self.WaitForDomNode(to_xpath, frame_xpath=self._FRAME_XPATH) | |
981 to_field = self._GetElement(self._driver.find_element_by_xpath, to_xpath) | |
982 to_field.send_keys('nobody@nowhere.com') | |
983 | |
984 subject_xpath = '//input[@name="subjectbox"]' | |
985 self.WaitForDomNode(subject_xpath, frame_xpath=self._FRAME_XPATH) | |
986 subject_field = self._GetElement(self._driver.find_element_by_xpath, | |
987 subject_xpath) | |
988 subject_field.send_keys('This message is about to be discarded') | |
989 | |
990 discard_xpath = '//div[@aria-label="Discard draft"]' | |
991 self.WaitForDomNode(discard_xpath, frame_xpath=self._FRAME_XPATH) | |
992 discard_button = self._GetElement(self._driver.find_element_by_xpath, | |
993 discard_xpath) | |
994 discard_button.click() | |
995 | |
996 # Wait for the message to be discarded, assumed to be true after the | |
997 # "To" element is removed from the webpage DOM. | |
998 self._wait.until(lambda _: not self._GetElement( | |
999 self._driver.find_element_by_name, 'to')) | |
1000 | |
1001 scenario = scenario_wpr if self._use_wpr else scenario_live | |
1002 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, | |
1003 test_description, scenario, | |
1004 frame_xpath=self._FRAME_XPATH) | |
1005 | |
1006 def testGmailAlternateThreadlistConversation(self): | |
1007 """Alternates between threadlist view and conversation view. | |
1008 | |
1009 This test continually clicks between the threadlist (Inbox) and the | |
1010 conversation view (e-mail message view), and periodically gathers | |
1011 performance stats that may reveal memory bloat. | |
1012 """ | |
1013 test_description = 'ThreadConversation' | |
1014 | |
1015 def scenario(): | |
1016 # Click an e-mail to see the conversation view, wait 1 second, click the | |
1017 # "Inbox" link to see the threadlist, wait 1 second. | |
1018 | |
1019 # Find the first thread (e-mail) identified by a "span" tag that contains | |
1020 # an "email" attribute. Then click it and wait for the conversation view | |
1021 # to appear (assumed to be visible when a particular div exists on the | |
1022 # page). | |
1023 thread_xpath = '//span[@email]' | |
1024 self.WaitForDomNode(thread_xpath, frame_xpath=self._FRAME_XPATH) | |
1025 thread = self._GetElement(self._driver.find_element_by_xpath, | |
1026 thread_xpath) | |
1027 self._ClickElementAndRecordLatency( | |
1028 thread, test_description, 'Conversation') | |
1029 self.WaitForDomNode('//div[text()="Click here to "]', | |
1030 frame_xpath=self._FRAME_XPATH) | |
1031 time.sleep(1) | |
1032 | |
1033 # Find the inbox link and click it. Then wait for the inbox to be shown | |
1034 # (assumed to be true when the particular div from the conversation view | |
1035 # no longer appears on the page). | |
1036 inbox_xpath = '//a[starts-with(text(), "Inbox")]' | |
1037 self.WaitForDomNode(inbox_xpath, frame_xpath=self._FRAME_XPATH) | |
1038 inbox = self._GetElement(self._driver.find_element_by_xpath, inbox_xpath) | |
1039 self._ClickElementAndRecordLatency(inbox, test_description, 'Threadlist') | |
1040 self._wait.until( | |
1041 lambda _: not self._GetElement( | |
1042 self._driver.find_element_by_xpath, | |
1043 '//div[text()="Click here to "]')) | |
1044 time.sleep(1) | |
1045 | |
1046 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, | |
1047 test_description, scenario, | |
1048 frame_xpath=self._FRAME_XPATH) | |
1049 | |
1050 def testGmailAlternateTwoLabels(self): | |
1051 """Continuously alternates between two labels. | |
1052 | |
1053 This test continually clicks between the "Inbox" and "Sent Mail" labels, | |
1054 and periodically gathers performance stats that may reveal memory bloat. | |
1055 """ | |
1056 test_description = 'AlternateLabels' | |
1057 | |
1058 def scenario(): | |
1059 # Click the "Sent Mail" label, wait for 1 second, click the "Inbox" label, | |
1060 # wait for 1 second. | |
1061 | |
1062 # Click the "Sent Mail" label, then wait for the tab title to be updated | |
1063 # with the substring "sent". | |
1064 sent_xpath = '//a[starts-with(text(), "Sent Mail")]' | |
1065 self.WaitForDomNode(sent_xpath, frame_xpath=self._FRAME_XPATH) | |
1066 sent = self._GetElement(self._driver.find_element_by_xpath, sent_xpath) | |
1067 self._ClickElementAndRecordLatency(sent, test_description, 'SentMail') | |
1068 self.assertTrue( | |
1069 self.WaitUntil(lambda: 'Sent Mail' in self.GetActiveTabTitle(), | |
1070 timeout=60, expect_retval=True, retry_sleep=1), | |
1071 msg='Timed out waiting for Sent Mail to appear.') | |
1072 time.sleep(1) | |
1073 | |
1074 # Click the "Inbox" label, then wait for the tab title to be updated with | |
1075 # the substring "inbox". | |
1076 inbox_xpath = '//a[starts-with(text(), "Inbox")]' | |
1077 self.WaitForDomNode(inbox_xpath, frame_xpath=self._FRAME_XPATH) | |
1078 inbox = self._GetElement(self._driver.find_element_by_xpath, inbox_xpath) | |
1079 self._ClickElementAndRecordLatency(inbox, test_description, 'Inbox') | |
1080 self.assertTrue( | |
1081 self.WaitUntil(lambda: 'Inbox' in self.GetActiveTabTitle(), | |
1082 timeout=60, expect_retval=True, retry_sleep=1), | |
1083 msg='Timed out waiting for Inbox to appear.') | |
1084 time.sleep(1) | |
1085 | |
1086 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, | |
1087 test_description, scenario, | |
1088 frame_xpath=self._FRAME_XPATH) | |
1089 | |
1090 def testGmailExpandCollapseConversation(self): | |
1091 """Continuously expands/collapses all messages in a conversation. | |
1092 | |
1093 This test opens up a conversation (e-mail thread) with several messages, | |
1094 then continually alternates between the "Expand all" and "Collapse all" | |
1095 views, while periodically gathering performance stats that may reveal memory | |
1096 bloat. | |
1097 """ | |
1098 test_description = 'ExpandCollapse' | |
1099 | |
1100 # Enter conversation view for a particular thread. | |
1101 thread_xpath = '//span[@email]' | |
1102 self.WaitForDomNode(thread_xpath, frame_xpath=self._FRAME_XPATH) | |
1103 thread = self._GetElement(self._driver.find_element_by_xpath, thread_xpath) | |
1104 thread.click() | |
1105 self.WaitForDomNode('//div[text()="Click here to "]', | |
1106 frame_xpath=self._FRAME_XPATH) | |
1107 | |
1108 def scenario(): | |
1109 # Click on the "Expand all" icon, wait for 1 second, click on the | |
1110 # "Collapse all" icon, wait for 1 second. | |
1111 | |
1112 # Click the "Expand all" icon, then wait for that icon to be removed from | |
1113 # the page. | |
1114 expand_xpath = '//img[@alt="Expand all"]' | |
1115 self.WaitForDomNode(expand_xpath, frame_xpath=self._FRAME_XPATH) | |
1116 expand = self._GetElement(self._driver.find_element_by_xpath, | |
1117 expand_xpath) | |
1118 self._ClickElementAndRecordLatency(expand, test_description, 'ExpandAll') | |
1119 self.WaitForDomNode( | |
1120 '//img[@alt="Expand all"]/parent::*/parent::*/parent::*' | |
1121 '[@style="display: none;"]', | |
1122 frame_xpath=self._FRAME_XPATH) | |
1123 time.sleep(1) | |
1124 | |
1125 # Click the "Collapse all" icon, then wait for that icon to be removed | |
1126 # from the page. | |
1127 collapse_xpath = '//img[@alt="Collapse all"]' | |
1128 self.WaitForDomNode(collapse_xpath, frame_xpath=self._FRAME_XPATH) | |
1129 collapse = self._GetElement(self._driver.find_element_by_xpath, | |
1130 collapse_xpath) | |
1131 collapse.click() | |
1132 self.WaitForDomNode( | |
1133 '//img[@alt="Collapse all"]/parent::*/parent::*/parent::*' | |
1134 '[@style="display: none;"]', | |
1135 frame_xpath=self._FRAME_XPATH) | |
1136 time.sleep(1) | |
1137 | |
1138 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, | |
1139 test_description, scenario, | |
1140 frame_xpath=self._FRAME_XPATH) | |
1141 | |
1142 | |
1143 class ChromeEndureDocsTest(ChromeEndureBaseTest): | |
1144 """Long-running performance tests for Chrome using Google Docs.""" | |
1145 | |
1146 _WEBAPP_NAME = 'Docs' | |
1147 _TAB_TITLE_SUBSTRING = 'Google Drive' | |
1148 | |
1149 def setUp(self): | |
1150 ChromeEndureBaseTest.setUp(self) | |
1151 | |
1152 # Log into a test Google account and open up Google Docs. | |
1153 self._LoginToGoogleAccount() | |
1154 self.NavigateToURL(self._GetConfig().get('docs_url')) | |
1155 self.assertTrue( | |
1156 self.WaitUntil(lambda: self._TAB_TITLE_SUBSTRING in | |
1157 self.GetActiveTabTitle(), | |
1158 timeout=60, expect_retval=True, retry_sleep=1), | |
1159 msg='Timed out waiting for Docs to load. Tab title is: %s' % | |
1160 self.GetActiveTabTitle()) | |
1161 | |
1162 self._driver = self.NewWebDriver() | |
1163 | |
1164 def _GetArchiveName(self): | |
1165 """Return Web Page Replay archive name.""" | |
1166 return 'ChromeEndureDocsTest.wpr' | |
1167 | |
1168 def testDocsAlternatelyClickLists(self): | |
1169 """Alternates between two different document lists. | |
1170 | |
1171 This test alternately clicks the "Shared with me" and "My Drive" buttons in | |
1172 Google Docs, and periodically gathers performance stats that may reveal | |
1173 memory bloat. | |
1174 """ | |
1175 test_description = 'AlternateLists' | |
1176 | |
1177 def sort_menu_setup(): | |
1178 # Open and close the "Sort" menu to get some DOM nodes to appear that are | |
1179 # used by the scenario in this test. | |
1180 sort_xpath = '//div[text()="Sort"]' | |
1181 self.WaitForDomNode(sort_xpath) | |
1182 sort_button = self._GetElement(self._driver.find_element_by_xpath, | |
1183 sort_xpath) | |
1184 sort_button.click() | |
1185 sort_button.click() | |
1186 sort_button.click() | |
1187 | |
1188 def scenario(): | |
1189 # Click the "Shared with me" button, wait for 1 second, click the | |
1190 # "My Drive" button, wait for 1 second. | |
1191 | |
1192 # Click the "Shared with me" button and wait for a div to appear. | |
1193 if not self._ClickElementByXpath( | |
1194 self._driver, '//div[text()="Shared with me"]'): | |
1195 self._num_errors += 1 | |
1196 logging.warning('Logging an automation error: click "shared with me".') | |
1197 try: | |
1198 self.WaitForDomNode('//div[text()="Share date"]') | |
1199 except pyauto_errors.JSONInterfaceError: | |
1200 # This case can occur when the page reloads; set things up again. | |
1201 sort_menu_setup() | |
1202 time.sleep(1) | |
1203 | |
1204 # Click the "My Drive" button and wait for a resulting div to appear. | |
1205 if not self._ClickElementByXpath( | |
1206 self._driver, '//span[starts-with(text(), "My Drive")]'): | |
1207 self._num_errors += 1 | |
1208 logging.warning('Logging an automation error: click "my drive".') | |
1209 try: | |
1210 self.WaitForDomNode('//div[text()="Quota used"]') | |
1211 except pyauto_errors.JSONInterfaceError: | |
1212 # This case can occur when the page reloads; set things up again. | |
1213 sort_menu_setup() | |
1214 time.sleep(1) | |
1215 | |
1216 sort_menu_setup() | |
1217 | |
1218 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, | |
1219 test_description, scenario) | |
1220 | |
1221 | |
1222 class ChromeEndurePlusTest(ChromeEndureBaseTest): | |
1223 """Long-running performance tests for Chrome using Google Plus.""" | |
1224 | |
1225 _WEBAPP_NAME = 'Plus' | |
1226 _TAB_TITLE_SUBSTRING = 'Google+' | |
1227 | |
1228 def setUp(self): | |
1229 ChromeEndureBaseTest.setUp(self) | |
1230 | |
1231 # Log into a test Google account and open up Google Plus. | |
1232 self._LoginToGoogleAccount() | |
1233 self.NavigateToURL(self._GetConfig().get('plus_url')) | |
1234 loaded_tab_title = self.GetActiveTabTitle() | |
1235 self.assertTrue(self._TAB_TITLE_SUBSTRING in loaded_tab_title, | |
1236 msg='Loaded tab title does not contain "%s": "%s"' % | |
1237 (self._TAB_TITLE_SUBSTRING, loaded_tab_title)) | |
1238 | |
1239 self._driver = self.NewWebDriver() | |
1240 | |
1241 def _GetArchiveName(self): | |
1242 """Return Web Page Replay archive name.""" | |
1243 return 'ChromeEndurePlusTest.wpr' | |
1244 | |
1245 def testPlusAlternatelyClickStreams(self): | |
1246 """Alternates between two different streams. | |
1247 | |
1248 This test alternately clicks the "Friends" and "Family" buttons using | |
1249 Google Plus, and periodically gathers performance stats that may reveal | |
1250 memory bloat. | |
1251 """ | |
1252 test_description = 'AlternateStreams' | |
1253 | |
1254 def scenario(): | |
1255 # Click the "Friends" button, wait for 1 second, click the "Family" | |
1256 # button, wait for 1 second. | |
1257 | |
1258 # Click the "Friends" button and wait for a resulting div to appear. | |
1259 if not self._ClickElementByXpath( | |
1260 self._driver, | |
1261 '//div[text()="Friends" and ' | |
1262 'starts-with(@data-dest, "stream/circles")]'): | |
1263 self._num_errors += 1 | |
1264 logging.warning('Logging an automation error: click "Friends" button.') | |
1265 | |
1266 try: | |
1267 self.WaitForDomNode('//span[contains(., "in Friends")]') | |
1268 except (pyauto_errors.JSONInterfaceError, | |
1269 pyauto_errors.JavascriptRuntimeError): | |
1270 self._num_errors += 1 | |
1271 logging.warning('Logging an automation error: wait for "in Friends".') | |
1272 | |
1273 time.sleep(1) | |
1274 | |
1275 # Click the "Family" button and wait for a resulting div to appear. | |
1276 if not self._ClickElementByXpath( | |
1277 self._driver, | |
1278 '//div[text()="Family" and ' | |
1279 'starts-with(@data-dest, "stream/circles")]'): | |
1280 self._num_errors += 1 | |
1281 logging.warning('Logging an automation error: click "Family" button.') | |
1282 | |
1283 try: | |
1284 self.WaitForDomNode('//span[contains(., "in Family")]') | |
1285 except (pyauto_errors.JSONInterfaceError, | |
1286 pyauto_errors.JavascriptRuntimeError): | |
1287 self._num_errors += 1 | |
1288 logging.warning('Logging an automation error: wait for "in Family".') | |
1289 | |
1290 time.sleep(1) | |
1291 | |
1292 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, | |
1293 test_description, scenario) | |
1294 | |
1295 | |
1296 class IndexedDBOfflineTest(ChromeEndureBaseTest): | 715 class IndexedDBOfflineTest(ChromeEndureBaseTest): |
1297 """Long-running performance tests for IndexedDB, modeling offline usage.""" | 716 """Long-running performance tests for IndexedDB, modeling offline usage.""" |
1298 | 717 |
1299 _WEBAPP_NAME = 'IndexedDBOffline' | 718 _WEBAPP_NAME = 'IndexedDBOffline' |
1300 _TAB_TITLE_SUBSTRING = 'IndexedDB Offline' | 719 _TAB_TITLE_SUBSTRING = 'IndexedDB Offline' |
1301 | 720 |
1302 def setUp(self): | 721 def setUp(self): |
1303 ChromeEndureBaseTest.setUp(self) | 722 ChromeEndureBaseTest.setUp(self) |
1304 | 723 |
1305 url = self.GetHttpURLForDataPath('indexeddb', 'endure', 'app.html') | 724 url = self.GetHttpURLForDataPath('indexeddb', 'endure', 'app.html') |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1346 pyauto_errors.JavascriptRuntimeError): | 765 pyauto_errors.JavascriptRuntimeError): |
1347 self._num_errors += 1 | 766 self._num_errors += 1 |
1348 logging.warning('Logging an automation error: wait for "offline".') | 767 logging.warning('Logging an automation error: wait for "offline".') |
1349 | 768 |
1350 time.sleep(1) | 769 time.sleep(1) |
1351 | 770 |
1352 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, | 771 self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING, |
1353 test_description, scenario) | 772 test_description, scenario) |
1354 | 773 |
1355 | 774 |
1356 class ChromeEndureReplay(object): | |
1357 """Run Chrome Endure tests with network simulation via Web Page Replay.""" | |
1358 | |
1359 _PATHS = { | |
1360 'archive': | |
1361 'src/chrome/test/data/pyauto_private/webpagereplay/{archive_name}', | |
1362 'scripts': | |
1363 'src/chrome/test/data/chrome_endure/webpagereplay/wpr_deterministic.js', | |
1364 } | |
1365 | |
1366 WEBPAGEREPLAY_HOST = '127.0.0.1' | |
1367 WEBPAGEREPLAY_HTTP_PORT = 8080 | |
1368 WEBPAGEREPLAY_HTTPS_PORT = 8413 | |
1369 | |
1370 CHROME_FLAGS = webpagereplay.GetChromeFlags( | |
1371 WEBPAGEREPLAY_HOST, | |
1372 WEBPAGEREPLAY_HTTP_PORT, | |
1373 WEBPAGEREPLAY_HTTPS_PORT) | |
1374 | |
1375 @classmethod | |
1376 def Path(cls, key, **kwargs): | |
1377 return perf.FormatChromePath(cls._PATHS[key], **kwargs) | |
1378 | |
1379 @classmethod | |
1380 def ReplayServer(cls, archive_path): | |
1381 """Create a replay server.""" | |
1382 # Inject customized scripts for Google webapps. | |
1383 # See the javascript file for details. | |
1384 scripts = cls.Path('scripts') | |
1385 if not os.path.exists(scripts): | |
1386 raise IOError('Injected scripts %s not found.' % scripts) | |
1387 replay_options = ['--inject_scripts', scripts] | |
1388 if 'WPR_RECORD' in os.environ: | |
1389 replay_options.append('--append') | |
1390 return webpagereplay.ReplayServer(archive_path, | |
1391 cls.WEBPAGEREPLAY_HOST, 0, | |
1392 cls.WEBPAGEREPLAY_HTTP_PORT, | |
1393 cls.WEBPAGEREPLAY_HTTPS_PORT, | |
1394 replay_options) | |
1395 | |
1396 | |
1397 if __name__ == '__main__': | 775 if __name__ == '__main__': |
1398 pyauto_functional.Main() | 776 pyauto_functional.Main() |
OLD | NEW |