Desktop chrome tab capture-chooseDesktopMedia()

chrome/browser/media/native_desktop_media_list.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/media/native_desktop_media_list.h"
 
 #include <map>
 #include <set>
 #include <sstream>
 
 #include "base/hash.h"
 #include "base/logging.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/threading/sequenced_worker_pool.h"
+#include "chrome/browser/mac/bluetooth_utility.h"
 #include "chrome/browser/media/desktop_media_list_observer.h"
+#include "chrome/browser/profiles/profile_manager.h"
+#include "chrome/browser/ui/browser_finder.h"
+#include "chrome/browser/ui/host_desktop.h"
+#include "chrome/browser/ui/tabs/tab_strip_model.h"
 #include "chrome/grit/generated_resources.h"
+#include "components/favicon/content/content_favicon_driver.h"
 #include "content/public/browser/browser_thread.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/render_process_host.h"
 #include "media/base/video_util.h"
 #include "third_party/libyuv/include/libyuv/scale_argb.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
 #include "third_party/webrtc/modules/desktop_capture/screen_capturer.h"
 #include "third_party/webrtc/modules/desktop_capture/window_capturer.h"
+#include "ui/aura/window.h"
 #include "ui/base/l10n/l10n_util.h"
 #include "ui/gfx/skia_util.h"
+#include "ui/snapshot/snapshot.h"
 
 using content::BrowserThread;
 using content::DesktopMediaID;
 
 namespace {
 
 // Update the list every second.
 const int kDefaultUpdatePeriod = 1000;
 
 // Returns a hash of a DesktopFrame content to detect when image for a desktop
@@ skipping 35 matching lines @@
   result.unlockPixels();
 
   return gfx::ImageSkia::CreateFrom1xBitmap(result);
 }
 
 }  // namespace
 
 NativeDesktopMediaList::SourceDescription::SourceDescription(
     DesktopMediaID id,
     const base::string16& name)
-    : id(id),
-      name(name) {
-}
+    : id(id), name(name) {}
 
 class NativeDesktopMediaList::Worker
     : public webrtc::DesktopCapturer::Callback {
  public:
   Worker(base::WeakPtr<NativeDesktopMediaList> media_list,
          scoped_ptr<webrtc::ScreenCapturer> screen_capturer,
          scoped_ptr<webrtc::WindowCapturer> window_capturer);
   ~Worker() override;
 
   void Refresh(const gfx::Size& thumbnail_size,
-               content::DesktopMediaID::Id view_dialog_id);
+               content::DesktopMediaID::Id view_dialog_id,
+               std::vector<SourceDescription> tab_sources);
 
  private:
   typedef std::map<DesktopMediaID, uint32> ImageHashesMap;
 
   // webrtc::DesktopCapturer::Callback interface.
   webrtc::SharedMemory* CreateSharedMemory(size_t size) override;
   void OnCaptureCompleted(webrtc::DesktopFrame* frame) override;
 
   base::WeakPtr<NativeDesktopMediaList> media_list_;
 
@@ skipping 17 matching lines @@
   if (screen_capturer_)
     screen_capturer_->Start(this);
   if (window_capturer_)
     window_capturer_->Start(this);
 }
 
 NativeDesktopMediaList::Worker::~Worker() {}
 
 void NativeDesktopMediaList::Worker::Refresh(
     const gfx::Size& thumbnail_size,
-    content::DesktopMediaID::Id view_dialog_id) {
+    content::DesktopMediaID::Id view_dialog_id,
+    std::vector<SourceDescription> tab_sources) {
   std::vector<SourceDescription> sources;
 
   if (screen_capturer_) {
     webrtc::ScreenCapturer::ScreenList screens;
     if (screen_capturer_->GetScreenList(&screens)) {
       bool mutiple_screens = screens.size() > 1;
       base::string16 title;
       for (size_t i = 0; i < screens.size(); ++i) {
         if (mutiple_screens) {
           title = l10n_util::GetStringFUTF16Int(
@@ skipping 16 matching lines @@
            it != windows.end(); ++it) {
         // Skip the picker dialog window.
         if (it->id != view_dialog_id) {
           sources.push_back(SourceDescription(
               DesktopMediaID(DesktopMediaID::TYPE_WINDOW, it->id),
               base::UTF8ToUTF16(it->title)));
         }
       }
     }
   }
+
+  // Add tab sources to all media sources
+  sources.insert(sources.end(), tab_sources.begin(), tab_sources.end());
+
   // Update list of windows before updating thumbnails.
   BrowserThread::PostTask(
       BrowserThread::UI, FROM_HERE,
       base::Bind(&NativeDesktopMediaList::OnSourcesList,
                  media_list_, sources));
 
   ImageHashesMap new_image_hashes;
 
   // Get a thumbnail for each source.
   for (size_t i = 0; i < sources.size(); ++i) {
     SourceDescription& source = sources[i];
     switch (source.id.type) {
       case DesktopMediaID::TYPE_SCREEN:
         if (!screen_capturer_->SelectScreen(source.id.id))
           continue;
         screen_capturer_->Capture(webrtc::DesktopRegion());
         break;
 
       case DesktopMediaID::TYPE_WINDOW:
         if (!window_capturer_->SelectWindow(source.id.id))
           continue;
         window_capturer_->Capture(webrtc::DesktopRegion());
         break;
+      case DesktopMediaID::TYPE_TAB:
+        // Favicon has already captured for tabs.

[qiangchen, 2015/12/07 22:45:56]

Consider moving the Favicon "capturing" here.

[miu, 2015/12/08 01:54:38]

IIUC, I think that has to be done on the UI thread, not here.

[GeorgeZ, 2015/12/09 19:36:37]

This is a no UI thread. Favicon capture has to to done in UI thread.

+        break;
 
       default:
         NOTREACHED();
     }
 
+    // Use captured tab favicon as thumbnail.
+    if (source.id.type == DesktopMediaID::TYPE_TAB) {
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::Bind(&NativeDesktopMediaList::OnSourceTabThumbnail, media_list_,
+                     source));
+      continue;
+    }
+
     // Expect that DesktopCapturer to always captures frames synchronously.
     // |current_frame_| may be NULL if capture failed (e.g. because window has
     // been closed).
     if (current_frame_) {
       uint32 frame_hash = GetFrameHash(current_frame_.get());
       new_image_hashes[source.id] = frame_hash;
 
       // Scale the image only if it has changed.
       ImageHashesMap::iterator it = image_hashes_.find(source.id);
       if (it == image_hashes_.end() || it->second != frame_hash) {
         gfx::ImageSkia thumbnail =
             ScaleDesktopFrame(current_frame_.Pass(), thumbnail_size);
         BrowserThread::PostTask(
             BrowserThread::UI, FROM_HERE,
-            base::Bind(&NativeDesktopMediaList::OnSourceThumbnail,
-                        media_list_, i, thumbnail));
+            base::Bind(&NativeDesktopMediaList::OnSourceThumbnail, media_list_,
+                       i, thumbnail));
       }
     }
   }
 
   image_hashes_.swap(new_image_hashes);
 
   BrowserThread::PostTask(
       BrowserThread::UI, FROM_HERE,
       base::Bind(&NativeDesktopMediaList::OnRefreshFinished, media_list_));
 }
 
 webrtc::SharedMemory* NativeDesktopMediaList::Worker::CreateSharedMemory(
     size_t size) {
   return NULL;
 }
 
 void NativeDesktopMediaList::Worker::OnCaptureCompleted(
     webrtc::DesktopFrame* frame) {
   current_frame_.reset(frame);
 }
 
 NativeDesktopMediaList::NativeDesktopMediaList(
     scoped_ptr<webrtc::ScreenCapturer> screen_capturer,
-    scoped_ptr<webrtc::WindowCapturer> window_capturer)
+    scoped_ptr<webrtc::WindowCapturer> window_capturer,
+    bool tab_capture_enabled)
     : screen_capturer_(screen_capturer.Pass()),
       window_capturer_(window_capturer.Pass()),
+      tab_capture_enabled_(tab_capture_enabled),
       update_period_(base::TimeDelta::FromMilliseconds(kDefaultUpdatePeriod)),
       thumbnail_size_(100, 100),
       view_dialog_id_(content::DesktopMediaID::TYPE_NONE, -1),
       observer_(NULL),
       weak_factory_(this) {
   base::SequencedWorkerPool* worker_pool = BrowserThread::GetBlockingPool();
   capture_task_runner_ = worker_pool->GetSequencedTaskRunner(
       worker_pool->GetSequenceToken());
 }
 
@@ skipping 11 matching lines @@
   thumbnail_size_ = thumbnail_size;
 }
 
 void NativeDesktopMediaList::SetViewDialogWindowId(
     content::DesktopMediaID dialog_id) {
   view_dialog_id_ = dialog_id;
 }
 
 void NativeDesktopMediaList::StartUpdating(DesktopMediaListObserver* observer) {
   DCHECK(!observer_);
-  DCHECK(screen_capturer_ || window_capturer_);
+  DCHECK(screen_capturer_ || window_capturer_ || tab_capture_enabled_);
 
   observer_ = observer;
 
   worker_.reset(new Worker(weak_factory_.GetWeakPtr(),
                            screen_capturer_.Pass(), window_capturer_.Pass()));
   Refresh();
 }
 
 int NativeDesktopMediaList::GetSourceCount() const {
   return sources_.size();
 }
 
 const DesktopMediaList::Source& NativeDesktopMediaList::GetSource(
     int index) const {
   return sources_[index];
 }
 
 void NativeDesktopMediaList::Refresh() {
+  // Get tab source list

[qiangchen, 2015/12/07 22:45:56]

Consider moving this block to Worker::Refresh.

[GeorgeZ, 2015/12/09 19:36:37]

Many work has to be done in UI thread here. worker is in a no UI thread.

+  std::vector<SourceDescription> tab_sources;
+  if (tab_capture_enabled_) {
+    if (!GetTabSourceDescription(tab_sources)) {
+      tab_sources.clear();
+      LOG(ERROR) << "Failed to get tab sources description";

[miu, 2015/12/08 01:54:38]

Please remove this logging statement.  This operation can't fail (see comment
below).

[GeorgeZ, 2015/12/09 19:36:37]

Done.

+    }
+  }
+
   capture_task_runner_->PostTask(
       FROM_HERE, base::Bind(&Worker::Refresh, base::Unretained(worker_.get()),
-                            thumbnail_size_, view_dialog_id_.id));
+                            thumbnail_size_, view_dialog_id_.id, tab_sources));
+}
+
+bool NativeDesktopMediaList::GetTabSourceDescription(

[qiangchen, 2015/12/07 22:45:56]

GetTabSourceDescriptions is better.

[miu, 2015/12/08 01:54:38]

This method should not return anything because it will always succeed.  (Note
that you never "return false" anywhere.)

[GeorgeZ, 2015/12/09 19:36:37]

Done.

[GeorgeZ, 2015/12/09 19:36:37]

Done.

+    std::vector<SourceDescription>& tab_sources) {
+// Only support mac, linux, windows and chrome OS, the last three use aura.
+#if !defined(OS_MACOSX) && !defined(USE_AURA)
+  return true;
+#endif
+  // Clear old favicon map.
+  tab_icon_map_.clear();
+
+  // Use to sort tabs based on their last active time stamps.
+  std::map<base::TimeTicks, SourceDescription> tab_map;
+
+  Profile* profile = ProfileManager::GetLastUsedProfileAllowedByPolicy();
+  std::vector<Browser*> browsers = FindAllTabbedBrowsersWithProfile(
+      profile, chrome::HOST_DESKTOP_TYPE_NATIVE);
+
+  for (auto browser : browsers) {
+    TabStripModel* tab_strip_model = browser->tab_strip_model();
+    DCHECK(tab_strip_model);
+
+    for (int i = 0; i < tab_strip_model->count(); i++) {
+      // Get DesktopMediaID.
+      content::WebContents* contents = tab_strip_model->GetWebContentsAt(i);
+      DesktopMediaID media_id;
+      media_id.type = DesktopMediaID::TYPE_TAB;
+      content::RenderFrameHost* const main_frame = contents->GetMainFrame();
+      DCHECK(main_frame);
+      media_id.render_process_id = main_frame->GetProcess()->GetID();
+      media_id.main_render_frame_id = main_frame->GetRoutingID();
+
+      // Get tab title.
+      base::string16 title =
+          base::UTF8ToUTF16("Chrome tab_") + contents->GetTitle();

[miu, 2015/12/08 01:54:38]

ditto: Make this a string resource.

[GeorgeZ, 2015/12/09 19:36:37]

Done.

+
+      // Get tab favicon.
+      favicon::FaviconDriver* favicon_driver =
+          favicon::ContentFaviconDriver::FromWebContents(contents);
+      if (favicon_driver) {
+        gfx::Image tab_icon = favicon_driver->GetFavicon();
+        int window_id =
+            (media_id.main_render_frame_id << 12) + media_id.render_process_id;
+        tab_icon_map_[window_id] =
+            CreateImageEncloseFavicon(thumbnail_size_, tab_icon);
+      }
+
+      // Get tab's last active time stamp.
+      base::TimeTicks t = contents->GetLastActiveTime();
+      tab_map.insert(std::make_pair(t, SourceDescription(media_id, title)));
+    }
+  }
+
+  // Add timely sorted tab sources into vector. Most recent one first.
+  tab_sources.clear();
+  for (auto it = tab_map.rbegin(); it != tab_map.rend(); ++it) {
+    tab_sources.push_back(it->second);
+  }
+
+  return true;
 }
 
 void NativeDesktopMediaList::OnSourcesList(
     const std::vector<SourceDescription>& new_sources) {
   typedef std::set<content::DesktopMediaID> SourceSet;
   SourceSet new_source_set;
   for (size_t i = 0; i < new_sources.size(); ++i) {
     new_source_set.insert(new_sources[i].id);
   }
   // Iterate through the old sources to find the removed sources.
@@ skipping 52 matching lines @@
 }
 
 void NativeDesktopMediaList::OnSourceThumbnail(
     int index,
     const gfx::ImageSkia& image) {
   DCHECK_LT(index, static_cast<int>(sources_.size()));
   sources_[index].thumbnail = image;
   observer_->OnSourceThumbnailChanged(index);
 }
 
+void NativeDesktopMediaList::OnSourceTabThumbnail(SourceDescription source) {

[qiangchen, 2015/12/07 22:45:56]

Is it possible to remove this function, and merge functionality into
OnSourceThumbNail?

[GeorgeZ, 2015/12/09 19:36:37]

Favicon is capture and stored in a UI thead. While desktop snapshot is capture
in none UI Thread. It can be awkward to try to merge those two functions into
one, since I have to pass necessary info from one thread to another.

+  DCHECK(source.id.type == DesktopMediaID::TYPE_TAB);
+
+  for (size_t i = 0; i < sources_.size(); ++i) {
+    if (sources_[i].id == source.id) {
+      int window_id =
+          (source.id.main_render_frame_id << 12) + source.id.render_process_id;
+      // Use favicon as thumbnail for tab.
+      if (tab_icon_map_.count(window_id)) {
+        // Only do update when there is a change to avoid flickering.
+        if (sources_[i].thumbnail.size() != tab_icon_map_[window_id].size() ||
+            sources_[i].name != source.name) {
+          sources_[i].thumbnail = tab_icon_map_[window_id];
+          observer_->OnSourceThumbnailChanged(i);
+        }
+      }
+      break;
+    }
+  }
+}
+
 void NativeDesktopMediaList::OnRefreshFinished() {
   BrowserThread::PostDelayedTask(
       BrowserThread::UI, FROM_HERE,
       base::Bind(&NativeDesktopMediaList::Refresh,
                  weak_factory_.GetWeakPtr()),
       update_period_);
 }