[Offline Pages] Add basic evaluation tests and related changes.

chrome/android/javatests/src/org/chromium/chrome/browser/offlinepages/OfflinePageManualEvaluationTest.java

Index: chrome/android/javatests/src/org/chromium/chrome/browser/offlinepages/OfflinePageManualEvaluationTest.java
diff --git a/chrome/android/javatests/src/org/chromium/chrome/browser/offlinepages/OfflinePageManualEvaluationTest.java b/chrome/android/javatests/src/org/chromium/chrome/browser/offlinepages/OfflinePageManualEvaluationTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..9659ac0b1c9a892a42c26776b424ae0edad3fe71
--- /dev/null
+++ b/chrome/android/javatests/src/org/chromium/chrome/browser/offlinepages/OfflinePageManualEvaluationTest.java
@@ -0,0 +1,348 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.chrome.browser.offlinepages;
+
+import android.app.NotificationManager;
+import android.content.Context;
+import android.os.Environment;
+import android.text.TextUtils;
+import android.util.LongSparseArray;
+
+import org.chromium.base.Callback;
+import org.chromium.base.ContextUtils;
+import org.chromium.base.Log;
+import org.chromium.base.ThreadUtils;
+import org.chromium.base.test.util.Manual;
+import org.chromium.chrome.browser.ChromeActivity;
+import org.chromium.chrome.browser.offlinepages.evaluation.OfflinePageEvaluationBridge;
+import org.chromium.chrome.browser.offlinepages.evaluation.OfflinePageEvaluationBridge.OfflinePageEvaluationObserver;
+import org.chromium.chrome.browser.profiles.Profile;
+import org.chromium.chrome.test.ChromeActivityTestCaseBase;
+import org.chromium.components.offlinepages.BackgroundSavePageResult;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Manual evaluation tests for OfflinePageBridge.SavePageLater.

[dougarnett, 2016/10/19 16:46:25]

What is "manual" about this? Really more of a batched set of tests, right?
Maybe:
  * Tests OfflinePageBridge.SavePageLater over a batch of urls. ...  ?

[romax, 2016/10/19 19:32:42]

Sure... I was thinking this test can only be tested manually so I typed "manual"
:)
Will fix some comments and naming related. Thanks!

+ * Testing against a list of top EM urls, try to call SavePageLater on each of the url. And also
+ * record metrics (failure rate, time elapsed etc.) by writing metrics to a file on external
+ * storage.
+ * */
+public class OfflinePageManualEvaluationTest extends ChromeActivityTestCaseBase<ChromeActivity> {

[dougarnett, 2016/10/19 16:46:25]

OfflinePageSavePageLaterEvaluationTest ?

[romax, 2016/10/19 19:32:42]

Done.

+    class TimeDelta {
+        public void setStartTime(Long startTime) {
+            mStartTime = startTime;
+        }
+        public void setEndTime(Long endTime) {
+            mEndTime = endTime;
+        }
+        // Return time delta in milliseconds.
+        public Long getTimeDelta() {
+            return mEndTime - mStartTime;
+        }
+
+        private Long mStartTime, mEndTime;
+    }
+
+    private static final String TAG = "OPManualEvaluation";
+    private static final String NAMESPACE = "async_loading";
+    private static final String NEW_LINE = System.getProperty("line.separator");
+    private static final File INPUT_FILE =
+            new File(Environment.getExternalStorageDirectory(), "paquete/top_em_urls.txt");
+    private static final File OUTPUT_FILE =
+            new File(Environment.getExternalStorageDirectory(), "paquete/offline_eval_results.txt");
+    private static final int TIMEOUT_MS = 5000;
+    private static final String DELIMITER = ";";
+
+    private OfflinePageEvaluationBridge mBridge;
+    private OfflinePageEvaluationObserver mObserver;
+
+    private Semaphore mSem;
+    private List<String> mUrls;
+    private int mCount;
+    private boolean mIsUserRequested;
+
+    private LongSparseArray<OfflinePageItem> mPages;
+    private LongSparseArray<Integer> mRequestStatus;
+    private LongSparseArray<TimeDelta> mRequestTimeDelta;
+    private LongSparseArray<String> mRequestIdToUrl;
+    private OutputStreamWriter mOutput;
+    private Long mTimeoutPerUrlInSeconds = 0L;
+    private File mInputFile = INPUT_FILE;
+    private File mOutputFile = OUTPUT_FILE;
+
+    public OfflinePageManualEvaluationTest() {
+        super(ChromeActivity.class);
+    }
+
+    private void initializeBridgeForProfile(final boolean shortcut) throws InterruptedException {
+        final Semaphore semaphore = new Semaphore(0);
+        ThreadUtils.runOnUiThread(new Runnable() {
+            @Override
+            public void run() {
+                Profile profile = Profile.getLastUsedProfile();
+                mBridge = OfflinePageEvaluationBridge.getForProfile(profile, shortcut);
+                if (mBridge == null || mBridge.isOfflinePageModelLoaded()) {
+                    semaphore.release();
+                    return;
+                }
+                mBridge.addObserver(new OfflinePageEvaluationObserver() {
+                    @Override
+                    public void offlinePageModelLoaded() {
+                        semaphore.release();
+                        mBridge.removeObserver(this);
+                    }
+                });
+            }
+        });
+        assertTrue(semaphore.tryAcquire(TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    }
+
+    @Override
+    protected void setUp() throws Exception {
+        super.setUp();
+        mPages = new LongSparseArray<OfflinePageItem>();
+        mRequestStatus = new LongSparseArray<Integer>();
+        mRequestTimeDelta = new LongSparseArray<TimeDelta>();
+        mRequestIdToUrl = new LongSparseArray<String>();
+        mCount = 0;
+        mIsUserRequested = false;
+        getUrls();
+        assertTrue("URLs weren't loaded.", mUrls != null);
+        assertFalse("No valid URLs in the input file.", mUrls.size() == 0);
+    }
+
+    @Override
+    protected void tearDown() throws Exception {
+        super.tearDown();
+        NotificationManager nm =
+                (NotificationManager) ContextUtils.getApplicationContext().getSystemService(
+                        Context.NOTIFICATION_SERVICE);
+        nm.cancelAll();
+    }
+
+    @Override
+    public void startMainActivity() throws InterruptedException {
+        startMainActivityOnBlankPage();
+    }
+
+    private BufferedReader getInputStream(File inputFile) throws FileNotFoundException {
+        FileReader fileReader = new FileReader(inputFile);
+        BufferedReader bufferedReader = new BufferedReader(fileReader);
+        return bufferedReader;
+    }
+
+    private OutputStreamWriter getOutputStream(File outputFile) throws IOException {
+        return new FileWriter(outputFile);
+    }
+
+    protected void setUpBridge(final boolean shortcut) throws InterruptedException {
+        initializeBridgeForProfile(shortcut);
+        mObserver = new OfflinePageEvaluationObserver() {
+            public void savePageRequestAdded(SavePageRequest request) {
+                mRequestStatus.put(request.getRequestId(), -1);
+                mRequestIdToUrl.put(request.getRequestId(), request.getUrl());
+
+                TimeDelta delta = new TimeDelta();
+                delta.setStartTime(System.currentTimeMillis());
+                mRequestTimeDelta.put(request.getRequestId(), delta);
+            }
+            public void savePageRequestCompleted(SavePageRequest request, int status) {
+                mRequestTimeDelta.get(request.getRequestId())
+                        .setEndTime(System.currentTimeMillis());
+                if (mRequestStatus.get(request.getRequestId()) == -1) {
+                    mCount++;
+                }
+                mRequestStatus.put(request.getRequestId(), status);
+                if (mCount == mUrls.size()) {
+                    mSem.release();
+                    return;
+                }
+            }
+            public void savePageRequestChanged(SavePageRequest request) {}
+        };
+        mBridge.addObserver(mObserver);
+    }
+
+    private void savePageLater(final String url, final String namespace)
+            throws InterruptedException {
+        ThreadUtils.runOnUiThread(new Runnable() {
+            @Override
+            public void run() {
+                mBridge.savePageLater(url, namespace, mIsUserRequested);
+            }
+        });
+    }
+
+    private void pushRequestCoordinator() throws InterruptedException {
+        ThreadUtils.runOnUiThread(new Runnable() {
+            @Override
+            public void run() {
+                boolean res = BackgroundSchedulerBridge.startProcessing(
+                        OfflinePageUtils.getDeviceConditions(getActivity().getBaseContext()),
+                        new Callback<Boolean>() {
+                            @Override
+                            public void onResult(Boolean result) {}
+                        });
+            }
+        });
+    }
+
+    private void runThroughURLs(List<String> urls) throws InterruptedException, IOException {
+        mSem = new Semaphore(0);
+        // If no timeout value is given, set 30 seconds for each url as default.
+        if (mTimeoutPerUrlInSeconds == 0) {
+            mTimeoutPerUrlInSeconds = 30L;
+        }
+        for (String url : mUrls) {
+            savePageLater(url, NAMESPACE);
+        }
+
+        if (!mSem.tryAcquire(urls.size() * mTimeoutPerUrlInSeconds, TimeUnit.SECONDS)) {
+            writeResults(false);
+        } else {
+            packData();
+            writeResults(true);
+        }
+    }
+
+    private void getUrls() throws IOException, InterruptedException {
+        mUrls = new ArrayList<String>();
+        try {
+            BufferedReader bufferedReader = getInputStream(mInputFile);
+            try {
+                String url;
+                while ((url = bufferedReader.readLine()) != null) {
+                    if (!TextUtils.isEmpty(url)) {
+                        mUrls.add(url);
+                    }
+                }
+            } finally {
+                if (bufferedReader != null) {
+                    bufferedReader.close();
+                }
+            }
+        } catch (FileNotFoundException e) {
+            Log.e(TAG, e.getMessage(), e);
+            fail(String.format("URL file %s is not found.", mInputFile));
+        }
+    }
+
+    // Translate the int value of status to BackgroundSavePageResult.
+    private String statusToString(int status) {
+        switch (status) {
+            case BackgroundSavePageResult.SUCCESS:
+                return "SUCCESS";
+            case BackgroundSavePageResult.PRERENDER_FAILURE:
+                return "PRERENDER_FAILURE";
+            case BackgroundSavePageResult.PRERENDER_CANCELED:
+                return "PRERENDER_CANCELED";
+            case BackgroundSavePageResult.FOREGROUND_CANCELED:
+                return "FOREGROUND_CANCELED";
+            case BackgroundSavePageResult.SAVE_FAILED:
+                return "SAVE_FAILED";
+            case BackgroundSavePageResult.EXPIRED:
+                return "EXPIRED";
+            case BackgroundSavePageResult.RETRY_COUNT_EXCEEDED:
+                return "RETRY_COUNT_EXCEEDED";
+            case BackgroundSavePageResult.START_COUNT_EXCEEDED:
+                return "START_COUNT_EXCEEDED";
+            case BackgroundSavePageResult.REMOVED:
+                return "REMOVED";
+            case -1:
+                return "NOT COMPLETED";
+            default:
+                return "UNDEFINED STATUS";
+        }
+    }
+
+    private void packData() throws InterruptedException {
+        final Semaphore semaphore = new Semaphore(0);
+        ThreadUtils.runOnUiThread(new Runnable() {
+            @Override
+            public void run() {
+                mBridge.getAllPages(new Callback<List<OfflinePageItem>>() {
+                    @Override
+                    public void onResult(List<OfflinePageItem> pages) {
+                        for (OfflinePageItem page : pages) {
+                            mPages.put(page.getOfflineId(), page);
+                        }
+                        semaphore.release();
+                    }
+                });
+            }
+        });
+        assertTrue(semaphore.tryAcquire(TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    }
+
+    private void writeResults(boolean completed) throws IOException, InterruptedException {
+        mOutput = getOutputStream(mOutputFile);
+        packData();
+        try {
+            if (!completed) {
+                mOutput.write("Timed out, incompleted results!" + NEW_LINE);
+            }
+            for (int i = 0; i < mRequestStatus.size(); i++) {
+                long requestId = mRequestStatus.keyAt(i);
+                int status = mRequestStatus.valueAt(i);
+                OfflinePageItem page = mPages.get(requestId);
+                if (page == null) {
+                    mOutput.write(mRequestIdToUrl.get(requestId) + DELIMITER
+                            + statusToString(status) + NEW_LINE);
+                    continue;
+                }
+                mOutput.write(page.getUrl() + DELIMITER + statusToString(status) + DELIMITER
+                        + page.getFileSize() / 1000 + " KB" + DELIMITER
+                        + mRequestTimeDelta.get(requestId).getTimeDelta() + NEW_LINE);
+            }
+        } catch (FileNotFoundException e) {
+            Log.e(TAG, e.getMessage(), e);
+        } finally {
+            if (mOutput != null) {
+                mOutput.close();
+            }
+        }
+    }
+
+    /**
+     * The test would terminate after #urls * timeoutPerUrl even if some urls are being processed.
+     */
+    @Manual
+    public void testFailureRateWithTimeout() throws IOException, InterruptedException {
+        mTimeoutPerUrlInSeconds = (long) (90); // Timeout for a page is 90 seconds.
+        mIsUserRequested = true;
+        mOutputFile = new File(Environment.getExternalStorageDirectory(), "paquete/results1.txt");
+        setUpBridge(true);
+        runThroughURLs(mUrls);
+    }
+
+    @Manual
+    public void testFailureRate() throws IOException, InterruptedException {
+        mTimeoutPerUrlInSeconds = (long) (24 * 60 * 60); // Timeout after 1 day.
+        mOutputFile = new File(Environment.getExternalStorageDirectory(), "paquete/results2.txt");
+        setUpBridge(true);
+        runThroughURLs(mUrls);
+    }
+
+    @Manual
+    public void testFailureRateAuto() throws IOException, InterruptedException {
+        mTimeoutPerUrlInSeconds = (long) (24 * 60 * 60); // Timeout after 1 day.
+        mOutputFile =
+                new File(Environment.getExternalStorageDirectory(), "paquete/results_auto.txt");
+        setUpBridge(false);
+        runThroughURLs(mUrls);
+    }
+}