Desktop chrome tab capture-chooseDesktopMedia()

chrome/browser/media/native_desktop_media_list.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/media/native_desktop_media_list.h"
 
 #include <map>
 #include <set>
 #include <sstream>
 
 #include "base/hash.h"
 #include "base/logging.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/threading/sequenced_worker_pool.h"
+#include "chrome/browser/mac/bluetooth_utility.h"
 #include "chrome/browser/media/desktop_media_list_observer.h"
+#include "chrome/browser/profiles/profile_manager.h"
+#include "chrome/browser/ui/browser_finder.h"
+#include "chrome/browser/ui/host_desktop.h"
+#include "chrome/browser/ui/tabs/tab_strip_model.h"
 #include "chrome/grit/generated_resources.h"
+#include "components/favicon/content/content_favicon_driver.h"
 #include "content/public/browser/browser_thread.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/render_process_host.h"
 #include "media/base/video_util.h"
 #include "third_party/libyuv/include/libyuv/scale_argb.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
 #include "third_party/webrtc/modules/desktop_capture/screen_capturer.h"
 #include "third_party/webrtc/modules/desktop_capture/window_capturer.h"
+#include "ui/aura/window.h"
 #include "ui/base/l10n/l10n_util.h"
 #include "ui/gfx/skia_util.h"
+#include "ui/snapshot/snapshot.h"
 
 using content::BrowserThread;
 using content::DesktopMediaID;
 
 namespace {
 
 // Update the list every second.
 const int kDefaultUpdatePeriod = 1000;
 
 // Returns a hash of a DesktopFrame content to detect when image for a desktop
@@ skipping 35 matching lines @@
   result.unlockPixels();
 
   return gfx::ImageSkia::CreateFrom1xBitmap(result);
 }
 
 }  // namespace
 
 NativeDesktopMediaList::SourceDescription::SourceDescription(
     DesktopMediaID id,
     const base::string16& name)
-    : id(id),
-      name(name) {
-}
+    : id(id), name(name) {}
 
 class NativeDesktopMediaList::Worker
     : public webrtc::DesktopCapturer::Callback {
  public:
   Worker(base::WeakPtr<NativeDesktopMediaList> media_list,
          scoped_ptr<webrtc::ScreenCapturer> screen_capturer,
          scoped_ptr<webrtc::WindowCapturer> window_capturer);
   ~Worker() override;
 
   void Refresh(const gfx::Size& thumbnail_size,
-               content::DesktopMediaID::Id view_dialog_id);
+               content::DesktopMediaID::Id view_dialog_id,
+               const std::vector<SourceDescription>& tab_sources);
 
  private:
   typedef std::map<DesktopMediaID, uint32> ImageHashesMap;
 
   // webrtc::DesktopCapturer::Callback interface.
   webrtc::SharedMemory* CreateSharedMemory(size_t size) override;
   void OnCaptureCompleted(webrtc::DesktopFrame* frame) override;
 
   base::WeakPtr<NativeDesktopMediaList> media_list_;
 
@@ skipping 17 matching lines @@
   if (screen_capturer_)
     screen_capturer_->Start(this);
   if (window_capturer_)
     window_capturer_->Start(this);
 }
 
 NativeDesktopMediaList::Worker::~Worker() {}
 
 void NativeDesktopMediaList::Worker::Refresh(
     const gfx::Size& thumbnail_size,
-    content::DesktopMediaID::Id view_dialog_id) {
+    content::DesktopMediaID::Id view_dialog_id,
+    const std::vector<SourceDescription>& tab_sources) {
   std::vector<SourceDescription> sources;
 
   if (screen_capturer_) {
     webrtc::ScreenCapturer::ScreenList screens;
     if (screen_capturer_->GetScreenList(&screens)) {
       bool mutiple_screens = screens.size() > 1;
       base::string16 title;
       for (size_t i = 0; i < screens.size(); ++i) {
         if (mutiple_screens) {
           title = l10n_util::GetStringFUTF16Int(
@@ skipping 16 matching lines @@
            it != windows.end(); ++it) {
         // Skip the picker dialog window.
         if (it->id != view_dialog_id) {
           sources.push_back(SourceDescription(
               DesktopMediaID(DesktopMediaID::TYPE_WINDOW, it->id),
               base::UTF8ToUTF16(it->title)));
         }
       }
     }
   }
+
+  // Add tab sources to all media sources
+  sources.insert(sources.end(), tab_sources.begin(), tab_sources.end());
+
   // Update list of windows before updating thumbnails.
   BrowserThread::PostTask(
       BrowserThread::UI, FROM_HERE,
       base::Bind(&NativeDesktopMediaList::OnSourcesList,
                  media_list_, sources));
 
   ImageHashesMap new_image_hashes;
 
   // Get a thumbnail for each source.
   for (size_t i = 0; i < sources.size(); ++i) {
     SourceDescription& source = sources[i];
     switch (source.id.type) {
       case DesktopMediaID::TYPE_SCREEN:
         if (!screen_capturer_->SelectScreen(source.id.id))
           continue;
         screen_capturer_->Capture(webrtc::DesktopRegion());
         break;
 
       case DesktopMediaID::TYPE_WINDOW:
         if (!window_capturer_->SelectWindow(source.id.id))
           continue;
         window_capturer_->Capture(webrtc::DesktopRegion());
         break;
+      case DesktopMediaID::TYPE_TAB:
+        // Favicon has already captured for tabs.
+        break;
 
       default:
         NOTREACHED();
     }
 
+    // Use captured tab favicon as thumbnail.
+    if (source.id.type == DesktopMediaID::TYPE_TAB) {
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::Bind(&NativeDesktopMediaList::OnSourceTabThumbnail, media_list_,
+                     source));
+      continue;
+    }
+
     // Expect that DesktopCapturer to always captures frames synchronously.
     // |current_frame_| may be NULL if capture failed (e.g. because window has
     // been closed).
     if (current_frame_) {
       uint32 frame_hash = GetFrameHash(current_frame_.get());
       new_image_hashes[source.id] = frame_hash;
 
       // Scale the image only if it has changed.
       ImageHashesMap::iterator it = image_hashes_.find(source.id);
       if (it == image_hashes_.end() || it->second != frame_hash) {
         gfx::ImageSkia thumbnail =
             ScaleDesktopFrame(current_frame_.Pass(), thumbnail_size);
         BrowserThread::PostTask(
             BrowserThread::UI, FROM_HERE,
-            base::Bind(&NativeDesktopMediaList::OnSourceThumbnail,
-                        media_list_, i, thumbnail));
+            base::Bind(&NativeDesktopMediaList::OnSourceThumbnail, media_list_,
+                       i, thumbnail));
       }
     }
   }
 
   image_hashes_.swap(new_image_hashes);
 
   BrowserThread::PostTask(
       BrowserThread::UI, FROM_HERE,
       base::Bind(&NativeDesktopMediaList::OnRefreshFinished, media_list_));
 }
 
 webrtc::SharedMemory* NativeDesktopMediaList::Worker::CreateSharedMemory(
     size_t size) {
   return NULL;
 }
 
 void NativeDesktopMediaList::Worker::OnCaptureCompleted(
     webrtc::DesktopFrame* frame) {
   current_frame_.reset(frame);
 }
 
 NativeDesktopMediaList::NativeDesktopMediaList(
     scoped_ptr<webrtc::ScreenCapturer> screen_capturer,
-    scoped_ptr<webrtc::WindowCapturer> window_capturer)
+    scoped_ptr<webrtc::WindowCapturer> window_capturer,
+    bool tab_capture_enabled)
     : screen_capturer_(screen_capturer.Pass()),
       window_capturer_(window_capturer.Pass()),
+      tab_capture_enabled_(tab_capture_enabled),
       update_period_(base::TimeDelta::FromMilliseconds(kDefaultUpdatePeriod)),
       thumbnail_size_(100, 100),
       view_dialog_id_(content::DesktopMediaID::TYPE_NONE, -1),
       observer_(NULL),
       weak_factory_(this) {
   base::SequencedWorkerPool* worker_pool = BrowserThread::GetBlockingPool();
   capture_task_runner_ = worker_pool->GetSequencedTaskRunner(
       worker_pool->GetSequenceToken());
 }
 
@@ skipping 11 matching lines @@
   thumbnail_size_ = thumbnail_size;
 }
 
 void NativeDesktopMediaList::SetViewDialogWindowId(
     content::DesktopMediaID dialog_id) {
   view_dialog_id_ = dialog_id;
 }
 
 void NativeDesktopMediaList::StartUpdating(DesktopMediaListObserver* observer) {
   DCHECK(!observer_);
-  DCHECK(screen_capturer_ || window_capturer_);
+  DCHECK(screen_capturer_ || window_capturer_ || tab_capture_enabled_);
 
   observer_ = observer;
 
   worker_.reset(new Worker(weak_factory_.GetWeakPtr(),
                            screen_capturer_.Pass(), window_capturer_.Pass()));
   Refresh();
 }
 
 int NativeDesktopMediaList::GetSourceCount() const {
   return sources_.size();
 }
 
 const DesktopMediaList::Source& NativeDesktopMediaList::GetSource(
     int index) const {
   return sources_[index];
 }
 
 void NativeDesktopMediaList::Refresh() {
+  // Get tab source list
+  std::vector<SourceDescription> tab_sources;
+  if (tab_capture_enabled_)
+    GetTabSourceDescriptions(&tab_sources);
+
   capture_task_runner_->PostTask(
       FROM_HERE, base::Bind(&Worker::Refresh, base::Unretained(worker_.get()),
-                            thumbnail_size_, view_dialog_id_.id));
+                            thumbnail_size_, view_dialog_id_.id, tab_sources));
+}
+
+void NativeDesktopMediaList::GetTabSourceDescriptions(
+    std::vector<SourceDescription>* tab_sources) {
+// Only support mac, linux, windows and chrome OS, the last three use aura.

[Sergey Ulanov, 2015/12/18 19:45:05]

Are there any other platforms where this code is compiled? I don't think you
need this ifdef. Even if this file was compiled on other platforms (e.g.
android) it would would with this change because you have unreachable code after
return.

[GeorgeZ, 2015/12/18 21:57:26]

This file is compiled by Android platform. Without ifdef, the android try bolt
will fail because some associated libs are not available. Do you have any better
solution for this case? 

[Sergey Ulanov, 2016/01/04 18:59:29]

Exclude this file on android? There is no reason to compile it there.
It's likely it will still fail to compile even with this ifdef.

[GeorgeZ, 2016/01/06 22:41:39]

Android builds in git cl try passed with "#if !define ...". I just modified
gypi. to avoid those file to be built for android instead of using "#if !defined
..."

+#if !defined(OS_MACOSX) && !defined(USE_AURA)
+  return;
+#endif
+  // Clear old favicon map.
+  tab_icon_map_.clear();
+
+  // Use to sort tabs based on their last active time stamps.
+  std::map<base::TimeTicks, SourceDescription> tab_map;
+
+  Profile* profile = ProfileManager::GetLastUsedProfileAllowedByPolicy();
+  std::vector<Browser*> browsers = FindAllTabbedBrowsersWithProfile(
+      profile, chrome::HOST_DESKTOP_TYPE_NATIVE);
+
+  for (auto browser : browsers) {

[mark a. foltz, 2016/01/04 20:11:41]

Is the Browser* guaranteed to stay alive while this loop executes?

[GeorgeZ, 2016/01/06 22:41:39]

Thanks to point out this issue. I modified the code that I immediately enumerate
and register all webcontents* after I get Browser*, so that Browser* only need
be alive for a very short period of time. I added a WebContentsRegistry class
that will register webcontents * into a integer id. The webcontents* can be
gotten from registered id if webcontents hasn't been destroyed. Otherwise a
nullptr will be return. This will largely avoid visiting a destroyed
webcontents*.

[GeorgeZ, 2016/01/08 19:39:02]

According to Avi@, the browser* and Webcontents* will be alive in the lifetime
of this function, since this function, creation and destroy of browser* and
Webcontents* are in the same thread. 

+    TabStripModel* tab_strip_model = browser->tab_strip_model();
+    DCHECK(tab_strip_model);
+
+    for (int i = 0; i < tab_strip_model->count(); i++) {
+      // Get DesktopMediaID.
+      content::WebContents* contents = tab_strip_model->GetWebContentsAt(i);
+      DesktopMediaID media_id;
+      media_id.type = DesktopMediaID::TYPE_TAB;
+      content::RenderFrameHost* const main_frame = contents->GetMainFrame();
+      DCHECK(main_frame);
+      media_id.tab_id.render_process_id = main_frame->GetProcess()->GetID();
+      media_id.tab_id.main_render_frame_id = main_frame->GetRoutingID();
+
+      // Get tab title.
+      base::string16 title = l10n_util::GetStringFUTF16(
+          IDS_DESKTOP_MEDIA_PICKER_CHROME_TAB_TITLE, contents->GetTitle());
+
+      // Create thumbnail based on favicon for tab.
+      favicon::FaviconDriver* favicon_driver =
+          favicon::ContentFaviconDriver::FromWebContents(contents);
+      if (favicon_driver) {
+        gfx::Image tab_icon = favicon_driver->GetFavicon();
+        tab_icon_map_[std::pair<int, int>(
+            media_id.tab_id.render_process_id,
+            media_id.tab_id.main_render_frame_id)] =
+            CreateEnlargedFaviconImage(thumbnail_size_, tab_icon);
+      }
+
+      // Get tab's last active time stamp.
+      base::TimeTicks t = contents->GetLastActiveTime();
+      tab_map.insert(std::make_pair(t, SourceDescription(media_id, title)));
+    }
+  }
+
+  // Add timely sorted tab sources into vector. Most recent one first.
+  tab_sources->clear();
+  for (auto it = tab_map.rbegin(); it != tab_map.rend(); ++it) {
+    tab_sources->push_back(it->second);
+  }
 }
 
 void NativeDesktopMediaList::OnSourcesList(
     const std::vector<SourceDescription>& new_sources) {
   typedef std::set<content::DesktopMediaID> SourceSet;
   SourceSet new_source_set;
   for (size_t i = 0; i < new_sources.size(); ++i) {
     new_source_set.insert(new_sources[i].id);
   }
   // Iterate through the old sources to find the removed sources.
@@ skipping 52 matching lines @@
 }
 
 void NativeDesktopMediaList::OnSourceThumbnail(
     int index,
     const gfx::ImageSkia& image) {
   DCHECK_LT(index, static_cast<int>(sources_.size()));
   sources_[index].thumbnail = image;
   observer_->OnSourceThumbnailChanged(index);
 }
 
+void NativeDesktopMediaList::OnSourceTabThumbnail(SourceDescription source) {
+  DCHECK(source.id.type == DesktopMediaID::TYPE_TAB);
+
+  for (size_t i = 0; i < sources_.size(); ++i) {
+    if (sources_[i].id == source.id) {
+      // Use favicon as thumbnail for tab.
+      std::pair<int, int> window_id(source.id.tab_id.render_process_id,
+                                    source.id.tab_id.main_render_frame_id);
+      if (tab_icon_map_.count(window_id)) {
+        // Only do update when there is a change to avoid flickering.
+        if (sources_[i].thumbnail.size() != tab_icon_map_[window_id].size() ||
+            sources_[i].name != source.name) {
+          sources_[i].thumbnail = tab_icon_map_[window_id];
+          observer_->OnSourceThumbnailChanged(i);
+        }
+      }
+      break;
+    }
+  }
+}
+
 void NativeDesktopMediaList::OnRefreshFinished() {
   BrowserThread::PostDelayedTask(
       BrowserThread::UI, FROM_HERE,
       base::Bind(&NativeDesktopMediaList::Refresh,
                  weak_factory_.GetWeakPtr()),
       update_period_);
 }