Limit the time spent on a single iteration of PhishingDOMFeatureExtractor.

chrome/renderer/safe_browsing/phishing_dom_feature_extractor_browsertest.cc

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Note that although this is not a "browser" test, it runs as part of
 // browser_tests.  This is because WebKit does not work properly if it is
 // shutdown and re-initialized.  Since browser_tests runs each test in a
 // new process, this avoids the problem.
 
 #include "chrome/renderer/safe_browsing/phishing_dom_feature_extractor.h"
 
 #include <string.h>  // for memcpy()
 #include <map>
 #include <string>
 
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/message_loop.h"
 #include "base/process.h"
 #include "base/string_util.h"
+#include "base/time.h"
 #include "chrome/common/main_function_params.h"
 #include "chrome/common/render_messages.h"
 #include "chrome/common/sandbox_init_wrapper.h"
 #include "chrome/renderer/mock_render_process.h"
 #include "chrome/renderer/render_thread.h"
 #include "chrome/renderer/render_view.h"
 #include "chrome/renderer/render_view_visitor.h"
 #include "chrome/renderer/renderer_main_platform_delegate.h"
+#include "chrome/renderer/safe_browsing/feature_extractor_clock.h"
 #include "chrome/renderer/safe_browsing/features.h"
 #include "googleurl/src/gurl.h"
 #include "ipc/ipc_channel.h"
 #include "net/http/http_response_headers.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/WebKit/WebKit/chromium/public/WebFrame.h"
 #include "third_party/WebKit/WebKit/chromium/public/WebURLRequest.h"
 #include "third_party/WebKit/WebKit/chromium/public/WebView.h"
 #include "webkit/glue/webkit_glue.h"
 
 using ::testing::ContainerEq;
+using ::testing::Return;
 
 namespace safe_browsing {
 
 class PhishingDOMFeatureExtractorTest : public ::testing::Test,
                                         public IPC::Channel::Listener,
                                         public RenderViewVisitor {
  public:
   // IPC::Channel::Listener implementation.
   virtual void OnMessageReceived(const IPC::Message& message) {
     IPC_BEGIN_MESSAGE_MAP(PhishingDOMFeatureExtractorTest, message)
       IPC_MESSAGE_HANDLER(ViewHostMsg_RenderViewReady, OnRenderViewReady)
       IPC_MESSAGE_HANDLER(ViewHostMsg_DidStopLoading, OnDidStopLoading)
       IPC_MESSAGE_HANDLER(ViewHostMsg_RequestResource, OnRequestResource)
     IPC_END_MESSAGE_MAP()
   }
 
   // RenderViewVisitor implementation.
   virtual bool Visit(RenderView* render_view) {
     view_ = render_view;
     return false;
   }
 
  protected:
+  class MockClock : public FeatureExtractorClock {
+   public:
+    MOCK_METHOD0(Now, base::TimeTicks());
+  };
+
   virtual void SetUp() {
     // Set up the renderer.  This code is largely adapted from
     // render_view_test.cc and renderer_main.cc.  Note that we use a
     // MockRenderProcess (because we don't need to use IPC for painting),
     // but we use a real RenderThread so that we can use the ResourceDispatcher
     // to fetch network resources.  These are then served canned content
     // in OnRequestResource().
     sandbox_init_wrapper_.reset(new SandboxInitWrapper);
     command_line_.reset(new CommandLine(CommandLine::ARGUMENTS_ONLY));
     params_.reset(new MainFunctionParams(*command_line_,
@@ skipping 15 matching lines @@
     // We can't call View::Create() directly here or else we won't get
     // RenderProcess's lazy initialization of WebKit.
     view_ = NULL;
     ViewMsg_New_Params params;
     params.parent_window = 0;
     params.view_id = kViewId;
     params.session_storage_namespace_id = kInvalidSessionStorageNamespaceId;
     ASSERT_TRUE(channel_->Send(new ViewMsg_New(params)));
     msg_loop_.Run();
 
-    extractor_.reset(new PhishingDOMFeatureExtractor(view_));
+    clock_ = new MockClock();
+    extractor_.reset(new PhishingDOMFeatureExtractor(view_, clock_));
   }
 
   virtual void TearDown() {
     // Try very hard to collect garbage before shutting down.
     GetMainFrame()->collectGarbage();
     GetMainFrame()->collectGarbage();
 
     ASSERT_TRUE(channel_->Send(new ViewMsg_Close(kViewId)));
     do {
       msg_loop_.RunAllPending();
@@ skipping 115 matching lines @@
   scoped_ptr<IPC::Channel> channel_;
   RenderThread* render_thread_;  // owned by mock_process_
   scoped_ptr<MockRenderProcess> mock_process_;
   RenderView* view_;  // not owned, deletes itself on close
   scoped_ptr<RendererMainPlatformDelegate> platform_;
   scoped_ptr<MainFunctionParams> params_;
   scoped_ptr<CommandLine> command_line_;
   scoped_ptr<SandboxInitWrapper> sandbox_init_wrapper_;
 
   scoped_ptr<PhishingDOMFeatureExtractor> extractor_;
+  MockClock* clock_;  // owned by extractor_
   // Map of URL -> response body for network requests from the renderer.
   // Any URLs not in this map are served a 404 error.
   std::map<std::string, std::string> responses_;
   bool success_;  // holds the success value from ExtractFeatures
 };
 
 int PhishingDOMFeatureExtractorTest::next_thread_id_ = 0;
 
 TEST_F(PhishingDOMFeatureExtractorTest, FormFeatures) {
+  // This test doesn't exercise the extraction timing.
+  EXPECT_CALL(*clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now()));
   responses_["http://host.com/"] =
       "<html><head><body>"
       "<form action=\"query\"><input type=text><input type=checkbox></form>"
       "<form action=\"http://cgi.host.com/submit\"></form>"
       "<form action=\"http://other.com/\"></form>"
       "<form action=\"query\"></form>"
       "<form></form></body></html>";
 
   FeatureMap expected_features;
   expected_features.AddBooleanFeature(features::kPageHasForms);
@@ skipping 36 matching lines @@
   expected_features.Clear();
   expected_features.AddBooleanFeature(features::kPageHasTextInputs);
 
   features.Clear();
   LoadURL("http://host.com/");
   ASSERT_TRUE(ExtractFeatures(&features));
   EXPECT_THAT(features.features(), ContainerEq(expected_features.features()));
 }
 
 TEST_F(PhishingDOMFeatureExtractorTest, LinkFeatures) {
+  // This test doesn't exercise the extraction timing.
+  EXPECT_CALL(*clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now()));
   responses_["http://www.host.com/"] =
       "<html><head><body>"
       "<a href=\"http://www2.host.com/abc\">link</a>"
       "<a name=page_anchor></a>"
       "<a href=\"http://www.chromium.org/\">chromium</a>"
       "</body></html";
 
   FeatureMap expected_features;
   expected_features.AddRealFeature(features::kPageExternalLinksFreq, 0.5);
   expected_features.AddRealFeature(features::kPageSecureLinksFreq, 0.0);
@@ skipping 20 matching lines @@
   expected_features.AddBooleanFeature(features::kPageLinkDomain +
                                       std::string("chromium.org"));
 
   features.Clear();
   LoadURL("https://www.host.com/");
   ASSERT_TRUE(ExtractFeatures(&features));
   EXPECT_THAT(features.features(), ContainerEq(expected_features.features()));
 }
 
 TEST_F(PhishingDOMFeatureExtractorTest, ScriptAndImageFeatures) {
+  // This test doesn't exercise the extraction timing.
+  EXPECT_CALL(*clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now()));
   responses_["http://host.com/"] =
       "<html><head><script></script><script></script></head></html>";
 
   FeatureMap expected_features;
   expected_features.AddBooleanFeature(features::kPageNumScriptTagsGTOne);
 
   FeatureMap features;
   LoadURL("http://host.com/");
   ASSERT_TRUE(ExtractFeatures(&features));
   EXPECT_THAT(features.features(), ContainerEq(expected_features.features()));
 
   responses_["http://host.com/"] =
       "<html><head><script></script><script></script><script></script>"
       "<script></script><script></script><script></script><script></script>"
       "</head><body><img src=\"blah.gif\">"
       "<img src=\"http://host2.com/blah.gif\"></body></html>";
 
   expected_features.Clear();
   expected_features.AddBooleanFeature(features::kPageNumScriptTagsGTOne);
   expected_features.AddBooleanFeature(features::kPageNumScriptTagsGTSix);
   expected_features.AddRealFeature(features::kPageImgOtherDomainFreq, 0.5);
 
   features.Clear();
   LoadURL("http://host.com/");
   ASSERT_TRUE(ExtractFeatures(&features));
   EXPECT_THAT(features.features(), ContainerEq(expected_features.features()));
 }
 
 TEST_F(PhishingDOMFeatureExtractorTest, SubFrames) {
+  // This test doesn't exercise the extraction timing.
+  EXPECT_CALL(*clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now()));
+
   // Test that features are aggregated across all frames.
   responses_["http://host.com/"] =
       "<html><body><input type=text><a href=\"info.html\">link</a>"
       "<iframe src=\"http://host2.com/\"></iframe>"
       "<iframe src=\"http://host3.com/\"></iframe>"
       "</body></html>";
 
   responses_["http://host2.com/"] =
       "<html><head><script></script><body>"
       "<form action=\"http://host4.com/\"><input type=checkbox></form>"
@@ skipping 28 matching lines @@
   expected_features.AddRealFeature(features::kPageSecureLinksFreq, 0.25);
   expected_features.AddBooleanFeature(features::kPageNumScriptTagsGTOne);
   expected_features.AddRealFeature(features::kPageImgOtherDomainFreq, 1.0);
 
   FeatureMap features;
   LoadURL("http://host.com/");
   ASSERT_TRUE(ExtractFeatures(&features));
   EXPECT_THAT(features.features(), ContainerEq(expected_features.features()));
 }
 
-// TODO(bryner): Test extraction with multiple passes, including the case where
-// the node we stopped on is removed from the document.
+TEST_F(PhishingDOMFeatureExtractorTest, Continuation) {
+  // For this test, we'll cause the feature extraction to run multiple
+  // iterations by incrementing the clock.
+
+  // This page has a total of 50 elements.  For the external forms feature to
+  // be computed correctly, the extractor has to examine the whole document.
+  // Note: the empty HEAD is important -- WebKit will synthesize a HEAD if
+  // there isn't one present, which can be confusing for the element counts.
+  std::string response = "<html><head></head><body>"
+      "<form action=\"ondomain\"></form>";
+  for (int i = 0; i < 45; ++i) {
+    response.append("<p>");
+  }
+  response.append("<form action=\"http://host2.com/\"></form></body></html>");
+  responses_["http://host.com/"] = response;
+
+  // Advance the clock 30 ms every 10 elements processed, 10 ms between chunks.
+  // Note that this assumes kClockCheckGranularity = 10 and
+  // kMaxTimePerChunkMs = 50.
+  base::TimeTicks now = base::TimeTicks::Now();
+  EXPECT_CALL(*clock_, Now())
+      // Time check at the start of extraction.
+      .WillOnce(Return(now))
+      // Time check at the start of the first chunk of work.
+      .WillOnce(Return(now))
+      // Time check after the first 10 elements.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(30)))
+      // Time check after the next 10 elements.  This is over the chunk
+      // time limit, so a continuation task will be posted.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(60)))
+      // Time check at the start of the second chunk of work.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(70)))
+      // Time check after resuming iteration for the second chunk.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(72)))
+      // Time check after the next 10 elements.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(100)))
+      // Time check after the next 10 elements.  This will trigger another
+      // continuation task.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(130)))
+      // Time check at the start of the third chunk of work.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(140)))
+      // Time check after resuming iteration for the third chunk.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(142)))
+      // Time check after the last 10 elements.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(170)))
+      // A final time check for the histograms.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(180)));
+
+  FeatureMap expected_features;
+  expected_features.AddBooleanFeature(features::kPageHasForms);
+  expected_features.AddRealFeature(features::kPageActionOtherDomainFreq, 0.5);
+
+  FeatureMap features;
+  LoadURL("http://host.com/");
+  ASSERT_TRUE(ExtractFeatures(&features));
+  EXPECT_THAT(features.features(), ContainerEq(expected_features.features()));
+  // Make sure none of the mock expectations carry over to the next test.
+  ::testing::Mock::VerifyAndClearExpectations(clock_);
+
+  // Now repeat the test with the same page, but advance the clock faster so
+  // that the extraction time exceeds the maximum total time for the feature
+  // extractor.  Extraction should fail.  Note that this assumes
+  // kMaxTotalTimeMs = 500.
+  EXPECT_CALL(*clock_, Now())
+      // Time check at the start of extraction.
+      .WillOnce(Return(now))
+      // Time check at the start of the first chunk of work.
+      .WillOnce(Return(now))
+      // Time check after the first 10 elements.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(300)))
+      // Time check at the start of the second chunk of work.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(350)))
+      // Time check after resuming iteration for the second chunk.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(360)))
+      // Time check after the next 10 elements.  This is over the limit.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(600)))
+      // A final time check for the histograms.
+      .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(620)));
+
+  features.Clear();
+  EXPECT_FALSE(ExtractFeatures(&features));
+}
 
 }  // namespace safe_browsing