Using multi-tab story in TabSwitching Benchmark

tools/perf/measurements/tab_switching_unittest.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import contextlib
 from telemetry.internal.results import page_test_results
 from telemetry.testing import page_test_test_case
 
 from measurements import tab_switching
 
@@ skipping 45 matching lines @@
 
     # Set up a browser with two tabs open
     browser = BrowserForTest()
     tab_0 = self.MakeTabForTest()
     browser.AddTab(tab_0)
     tab_1 = self.MakeTabForTest()
     browser.AddTab(tab_1)
 
     # Mock histogram result to test _IsDone really works.
     expected_histogram = [
-        # To get first_histogram for last tab (tab_1).
+        # DidNavigateToPage() calls GetHistogram() once
         '{"count": 0, "buckets": []}',
-        # First _IsDone check for tab_0. Retry.

[vovoy, 2017/03/23 09:37:49]

_IsDone() check loop is removed in the written tab_switching measurement

-        '{"count": 0, "buckets": []}',
-        # Second _IsDone check for tab_0. Retry.
-        '{"count": 0, "buckets": []}',
-        # Third _IsDone check for tab_0. Pass.
-        '{"count": 1, "buckets": [{"low": 1, "high": 2, "count": 1}]}',
-        # To get prev_histogram. End of tab_0 loop.
-        '{"count": 1, "buckets": [{"low": 1, "high": 2, "count": 1}]}',
-        # First _IsDone check for tab_1. Retry.
-        '{"count": 1, "buckets": [{"low": 1, "high": 2, "count": 1}]}',
-        # Second _IsDone check for tab_1. Pass.
-        '{"count": 2, "buckets": [{"low": 1, "high": 2, "count": 1},'
-        '{"low": 2, "high": 3, "count": 1}]}',
-        # To get prev_histogram. End of tab_1 loop.
+        # ValidateAndMeasurePage() calls GetHistogram() once
         '{"count": 2, "buckets": [{"low": 1, "high": 2, "count": 1},'
         '{"low": 2, "high": 3, "count": 1}]}',
         ]
     mock_get_histogram = mock.MagicMock(side_effect=expected_histogram)

[nednguyen, 2017/03/23 13:15:10]

I really like to get away with using mock here because it won't let we know when
the histogram in the actual Chrome binary break. Can you just use a real test
mulitab story instead?

[vovoy, 2017/03/23 16:55:58]

I think unittest shall test one thing at a time.
in measurements/tab_switching_unittest.py, shall only test code in
measurements/tab_switching.py .
it's unittest's advantage to test independently from a Chrome binary.

[nednguyen, 2017/03/23 23:49:06]

from my experience, these mock tests are mostly ineffective against catching
breakage & end up being guard changing test. 

You can keep a few unittest for logic that you think tricky, but I would
strongly encourage you to use real Chrome & add integration test that  assert
the metrics you expect to see are there (see:
https://cs.chromium.org/chromium/src/tools/perf/measurements/smoothness_unitt...)

[vovoy, 2017/03/24 00:25:51]

OK, I will check how smoothness_unittest works.

[vovoy, 2017/03/24 16:35:30]

Done.

 
     with contextlib.nested(
         mock.patch('telemetry.value.histogram_util.GetHistogram',
                    mock_get_histogram),
         mock.patch('metrics.keychain_metric.KeychainMetric')):
+      measure.DidNavigateToPage(story_set.stories[0], browser.tabs[-1])
       measure.ValidateAndMeasurePage(story_set.stories[0], browser.tabs[-1],
                                      page_test_results.PageTestResults())
       self.assertEqual(len(expected_histogram),
                        len(mock_get_histogram.mock_calls))
+      # The last tab is passed to DidNavigateToPage() and
+      # ValidateAndMeasurePage()
       expected_calls = [mock.call(mock.ANY, mock.ANY, t) for t in
-                        [tab_1] + [tab_0] * 4 + [tab_1] * 3]
+                        [browser.tabs[-1]] * 2]
       self.assertEqual(expected_calls, mock_get_histogram.mock_calls)