Move content deduplication into the scan process.

ui/file_manager/file_manager/background/js/media_import_handler.js

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Namespace
 var importer = importer || {};
 
 /**
  * Interface providing access to information about active import processes.
  *
@@ skipping 14 matching lines @@
 
 /**
  * Handler for importing media from removable devices into the user's Drive.
  *
  * @constructor
  * @implements {importer.ImportRunner}
  * @struct
  *
  * @param {!ProgressCenter} progressCenter
  * @param {!importer.HistoryLoader} historyLoader
- * @param {!importer.DuplicateFinder.Factory} duplicateFinderFactory
  * @param {!analytics.Tracker} tracker
  */
 importer.MediaImportHandler =

[mtomasz, 2015/03/05 04:36:53]

nit: Will it fit in one line? If not, then we break usually at function(\n.

[Steve McKay, 2015/03/05 21:39:03]

Done.

-    function(progressCenter, historyLoader, duplicateFinderFactory, tracker) {
+    function(progressCenter, historyLoader, tracker) {
   /** @private {!ProgressCenter} */
   this.progressCenter_ = progressCenter;
 
   /** @private {!importer.HistoryLoader} */
   this.historyLoader_ = historyLoader;
 
   /** @private {!importer.TaskQueue} */
   this.queue_ = new importer.TaskQueue();
 
   // Prevent the system from sleeping while imports are active.
   this.queue_.setActiveCallback(function() {
     chrome.power.requestKeepAwake('system');
   });
   this.queue_.setIdleCallback(function() {
     chrome.power.releaseKeepAwake();
   });
 
-  /** @private {!importer.DuplicateFinder.Factory} */
-  this.duplicateFinderFactory_ = duplicateFinderFactory;
-
   /** @private {!analytics.Tracker} */
   this.tracker_ = tracker;
 
   /** @private {number} */
   this.nextTaskId_ = 0;
 };
 
 /** @override */
 importer.MediaImportHandler.prototype.importFromScanResult =
     function(scanResult, destination, directoryPromise) {
 
   var task = new importer.MediaImportHandler.ImportTask(
       this.generateTaskId_(),
       this.historyLoader_,
       scanResult,
       directoryPromise,
-      this.duplicateFinderFactory_.create(),
       destination,
       this.tracker_);
 
   task.addObserver(this.onTaskProgress_.bind(this, task));
 
   this.queue_.queueTask(task);
 
   return task;
 };
 
@@ skipping 61 matching lines @@
  * FileOperationManager.CopyTask) but this is a temporary state of affairs.
  *
  * @constructor
  * @extends {importer.TaskQueue.BaseTask}
  * @struct
  *
  * @param {string} taskId
  * @param {!importer.HistoryLoader} historyLoader
  * @param {!importer.ScanResult} scanResult
  * @param {!Promise<!DirectoryEntry>} directoryPromise
- * @param {!importer.DuplicateFinder} duplicateFinder A duplicate-finder linked
- *     to the import destination, that will be used to deduplicate imports.
  * @param {!importer.Destination} destination The logical destination.
  * @param {!analytics.Tracker} tracker
  */
 importer.MediaImportHandler.ImportTask = function(
     taskId,
     historyLoader,
     scanResult,
     directoryPromise,
-    duplicateFinder,
     destination,
     tracker) {
 
   importer.TaskQueue.BaseTask.call(this, taskId);
   /** @private {string} */
   this.taskId_ = taskId;
 
   /** @private {!importer.Destination} */
   this.destination_ = destination;
 
   /** @private {!Promise<!DirectoryEntry>} */
   this.directoryPromise_ = directoryPromise;
 
-  /** @private {!importer.DuplicateFinder} */
-  this.deduplicator_ = duplicateFinder;
-
   /** @private {!importer.ScanResult} */
   this.scanResult_ = scanResult;
 
   /** @private {!importer.HistoryLoader} */
   this.historyLoader_ = historyLoader;
 
   /** @private {!analytics.Tracker} */
   this.tracker_ = tracker;
 
   /** @private {number} */
   this.totalBytes_ = 0;
 
   /** @private {number} */
   this.processedBytes_ = 0;
 
   /** @private {number} */
   this.remainingFilesCount_ = 0;
 
   /** @private {?function()} */
   this.cancelCallback_ = null;
 
   /** @private {boolean} Indicates whether this task was canceled. */
   this.canceled_ = false;
 
-  /** @private {number} Number of files deduped by content dedupe. */
-  this.dedupeCount_ = 0;
-
   /** @private {number} */
   this.errorCount_ = 0;
 };
 
 /** @struct */
 importer.MediaImportHandler.ImportTask.prototype = {
   /** @return {number} Number of imported bytes */
   get processedBytes() { return this.processedBytes_; },
 
   /** @return {number} Total number of bytes to import */
@@ skipping 48 matching lines @@
     // (including calling #requestCancel again).
     var cancelCallback = this.cancelCallback_;
     this.cancelCallback_ = null;
     cancelCallback();
   }
 };
 
 /** @private */
 importer.MediaImportHandler.ImportTask.prototype.initialize_ = function() {
   var stats = this.scanResult_.getStatistics();
-
   this.remainingFilesCount_ = stats.newFileCount;
   this.totalBytes_ = stats.sizeBytes;
   this.notify(importer.TaskQueue.UpdateType.PROGRESS);
 
   this.tracker_.send(metrics.ImportEvents.STARTED);
-  this.tracker_.send(metrics.ImportEvents.HISTORY_DEDUPE_COUNT
-                     .value(stats.duplicateFileCount));
 };
 
 /**
  * Initiates an import to the given location.  This should only be called once
  * the scan result indicates that it is ready.
  *
  * @private
  */
 importer.MediaImportHandler.ImportTask.prototype.importScanEntries_ =
     function() {
@@ skipping 11 matching lines @@
  * @param {!DirectoryEntry} destinationDirectory
  * @param {function()} completionCallback Called after this operation is
  *     complete.
  * @param {!FileEntry} entry The entry to import.
  * @private
  */
 importer.MediaImportHandler.ImportTask.prototype.importOne_ =
     function(destinationDirectory, completionCallback, entry) {
   if (this.canceled_) {
     this.notify(importer.TaskQueue.UpdateType.CANCELED);
-    this.tracker_.send(metrics.ImportEvents.CANCELLED);
-    this.sendImportStats_();
+    this.sendImportStats_(false);
     return;
   }
 
-  this.deduplicator_.checkDuplicate(entry)
-      .then(
-          /** @param {boolean} isDuplicate */
-          function(isDuplicate) {
-            if (isDuplicate) {
-              // If the given file is a duplicate, don't import it again.  Just
-              // update the progress indicator.
-              this.dedupeCount_++;
-              this.markAsImported_(entry);
-              this.processedBytes_ += entry.size;
-              this.notify(importer.TaskQueue.UpdateType.PROGRESS);
-              return Promise.resolve();
-            } else {
-              return this.copy_(entry, destinationDirectory);
-            }
-          }.bind(this))
+  this.copy_(entry, destinationDirectory)
       // Regardless of the result of this copy, push on to the next file.
       .then(completionCallback)
       .catch(
           /** @param {*} error */
           function(error) {
             importer.getLogger().catcher('import-task-import-one')(error);
             completionCallback();
           });
 };
 
@@ skipping 122 matching lines @@
       /** @param {!importer.ImportHistory} history */
       function(history) {
         history.markImported(entry, this.destination_);
       }.bind(this))
       .catch(importer.getLogger().catcher('import-task-mark-as-imported'));
 };
 
 /** @private */
 importer.MediaImportHandler.ImportTask.prototype.onSuccess_ = function() {
   this.notify(importer.TaskQueue.UpdateType.COMPLETE);
-  this.tracker_.send(metrics.ImportEvents.ENDED);
-  this.sendImportStats_();
+  this.sendImportStats_(true);

[mtomasz, 2015/03/05 04:36:53]

nit: enum or /* arg_name */ for this true? It's impossible to guess what it
stands for.

[Steve McKay, 2015/03/05 21:39:03]

Done.

 };
 
 /**
  * Sends import statistics to analytics.
+ *
+ * @param {boolean} completedSuccessfully True if task

[Ben Kwa, 2015/03/05 20:05:10]

suggestion: You could reuse the TaskQueue.UpdateType instead of a boolean.

[Steve McKay, 2015/03/05 21:39:03]

Done.

+ *     wasn't cancelled or err'd out.
  */
-importer.MediaImportHandler.ImportTask.prototype.sendImportStats_ = function() {
+importer.MediaImportHandler.ImportTask.prototype.sendImportStats_ =
+    function(completedSuccessfully) {
+  var scanStats = this.scanResult_.getStatistics();
+
+  if (!completedSuccessfully) {
+    this.tracker_.send(metrics.ImportEvents.CANCELLED);
+  } else {
+    // Ideally we'd report bytes imported for cancelled imports too.
+    // We just don't have the information needed to do this accurately.

[Ben Kwa, 2015/03/05 20:05:10]

The processedBytes_ counter should be a fairly accurate count of number of bytes
imported for cancelled tasks.

[Steve McKay, 2015/03/05 21:39:03]

Perfect!

+    var megaBytes = Math.floor(scanStats.sizeBytes / (1024 * 1024));
+    this.tracker_.send(

[Ben Kwa, 2015/03/05 20:05:10]

We also need to do this.tracker_.send(metrics.ImportEvents.ENDED);

[Steve McKay, 2015/03/05 21:39:03]

Removed that as there are plenty of surrogate events.

+        metrics.ImportEvents.MEGABYTES_IMPORTED.value(megaBytes));
+  }
+
+  // FYI, if the task was cancelled we substract the remaining files
+  // from the number of files in the scanStats.
+  var importFileCount = scanStats.newFileCount - this.remainingFilesCount_;
   this.tracker_.send(
-      metrics.ImportEvents.CONTENT_DEDUPE_COUNT
-          .value(this.dedupeCount_));
-  // TODO(kenobi): Send correct import byte counts.
-  var importFileCount = this.scanResult_.getStatistics().newFileCount -
-      (this.dedupeCount_ + this.remainingFilesCount_);
+      metrics.ImportEvents.FILES_IMPORTED.value(importFileCount));
+
+  if (this.errorCount_ > 0) {
+    this.tracker_.send(metrics.ImportEvents.ERRORS.value(this.errorCount_));
+  }
+
+  // Finally we want to report on the number of duplicates
+  // that were identified during scanning.
+  var totalDeduped = 0;
+  Object.keys(scanStats.duplicates).forEach(
+      /**
+       * @param {!importer.Disposition} disposition
+       * @this {importer.MediaImportHandler.ImportTask}
+       */
+      function(disposition) {
+        var count = scanStats.duplicates[disposition];
+        totalDeduped += count;
+        this.tracker_.send(
+            metrics.ImportEvents.FILES_DEDUPLICATED
+                .label(disposition)
+                .value(count));
+      }.bind(this));
+
   this.tracker_.send(
-      metrics.ImportEvents.FILE_COUNT
-          .value(importFileCount));
-
-  this.tracker_.send(metrics.ImportEvents.ERROR.value(this.errorCount_));
-
-  // Send aggregate deduplication timings, to avoid flooding analytics with one
-  // timing per file.
-  var deduplicatorStats = this.deduplicator_.getStatistics();
-  this.tracker_.sendTiming(
-      metrics.Categories.ACQUISITION,
-      metrics.timing.Variables.COMPUTE_HASH,
-      deduplicatorStats.computeHashTime,
-      'In Place');
-  this.tracker_.sendTiming(
-      metrics.Categories.ACQUISITION,
-      metrics.timing.Variables.SEARCH_BY_HASH,
-      deduplicatorStats.searchHashTime);
-
+      metrics.ImportEvents.FILES_DEDUPLICATED
+          .label('all-duplicates')
+          .value(totalDeduped));
 };