Decouple capture timestamp and reference time

content/renderer/media/video_capture_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Notes about usage of this object by VideoCaptureImplManager.
 //
 // VideoCaptureImplManager access this object by using a Unretained()
 // binding and tasks on the IO thread. It is then important that
 // VideoCaptureImpl never post task to itself. All operations must be
 // synchronous.
@@ skipping 173 matching lines @@
       if (state_ == VIDEO_CAPTURE_STATE_STARTED)
         return;
       params_ = params;
       if (params_.requested_format.frame_rate >
           media::limits::kMaxFramesPerSecond) {
         params_.requested_format.frame_rate =
             media::limits::kMaxFramesPerSecond;
       }
       DVLOG(1) << "StartCapture: starting with first resolution "
                << params_.requested_format.frame_size.ToString();
-      first_frame_timestamp_ = base::TimeTicks();
       StartCaptureInternal();
     }
   }
 }
 
 void VideoCaptureImpl::StopCapture(int client_id) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   // A client ID can be in only one client list.
   // If this ID is in any client list, we can just remove it from
   // that client list and don't have to run the other following RemoveClient().
@@ skipping 92 matching lines @@
     if (cb2_iter != client_buffer2s_.end()) {
       DCHECK(!cb2_iter->second.get() || cb2_iter->second->HasOneRef())
           << "Instructed to delete buffer we are still using.";
       client_buffer2s_.erase(cb2_iter);
     }
   }
 }
 
 void VideoCaptureImpl::OnBufferReceived(
     int buffer_id,
-    base::TimeTicks timestamp,
+    base::TimeDelta timestamp,
     const base::DictionaryValue& metadata,
     media::VideoPixelFormat pixel_format,
     media::VideoFrame::StorageType storage_type,
     const gfx::Size& coded_size,
     const gfx::Rect& visible_rect) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   if (state_ != VIDEO_CAPTURE_STATE_STARTED || suspended_ ||
       pixel_format != media::PIXEL_FORMAT_I420 ||
       (storage_type != media::VideoFrame::STORAGE_SHMEM &&
        storage_type != media::VideoFrame::STORAGE_GPU_MEMORY_BUFFERS)) {
     // Crash in debug builds since the host should not have provided a buffer
     // with an unsupported pixel format or storage type.
     DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
     DCHECK(storage_type == media::VideoFrame::STORAGE_SHMEM ||
            storage_type == media::VideoFrame::STORAGE_GPU_MEMORY_BUFFERS);
     Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id,
                                              gpu::SyncToken(), -1.0));
     return;
   }
-  if (first_frame_timestamp_.is_null())
-    first_frame_timestamp_ = timestamp;
 
+  base::TimeTicks reference_time;
+  media::VideoFrameMetadata frame_metadata;
+  frame_metadata.MergeInternalValuesFrom(metadata);
+  const bool success = frame_metadata.GetTimeTicks(
+      media::VideoFrameMetadata::REFERENCE_TIME, &reference_time);
+  DCHECK(success);
+
+  if (first_frame_ref_time_.is_null())
+    first_frame_ref_time_ = reference_time;
+
+  // If the timestamp is not prepared, we use reference time to make a rough

[miu, 2016/06/08 19:24:44]

This wasn't in PS1. Looks like there are some capturers that don't set the
VideoFrame::timestamp?

Actually, when I look up the code for tab capture, I see this is the case. Could
you add a "TODO(miu): Fix upstream capturers to always set timestamp and
reference time." here, with a link to this crbug: http://crbug.com/618407

[qiangchen, 2016/06/08 20:08:54]

Yeah, when I change the signature of OnIncomingCapturedVideoFrame,
I found ThreadSafeCaptureOracle::DidCaptureFrame does not prepare timestamp. I
do not know what is that, and it looks like that the code path is deep.

+  // estimate.
+  if (timestamp.is_zero())
+    timestamp = reference_time - first_frame_ref_time_;
+
+  // TODO(qiangchen): Change the metric name to "reference_time" and
+  // "timestamp", so that we have consistent naming everywhere.
   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
   TRACE_EVENT_INSTANT2("cast_perf_test", "OnBufferReceived",
                        TRACE_EVENT_SCOPE_THREAD, "timestamp",
-                       timestamp.ToInternalValue(), "time_delta",
-                       (timestamp - first_frame_timestamp_).ToInternalValue());
+                       (reference_time - base::TimeTicks()).InMicroseconds(),
+                       "time_delta", timestamp.InMicroseconds());
 
   scoped_refptr<media::VideoFrame> frame;
   BufferFinishedCallback buffer_finished_callback;
   std::unique_ptr<gpu::SyncToken> release_sync_token(new gpu::SyncToken);
   switch (storage_type) {
     case media::VideoFrame::STORAGE_GPU_MEMORY_BUFFERS: {
       const auto& iter = client_buffer2s_.find(buffer_id);
       DCHECK(iter != client_buffer2s_.end());
       scoped_refptr<ClientBuffer2> buffer = iter->second;
       const auto& handles = buffer->gpu_memory_buffer_handles();
       frame = media::VideoFrame::WrapExternalYuvGpuMemoryBuffers(
-          media::PIXEL_FORMAT_I420,
-          coded_size,
-          gfx::Rect(coded_size),
-          coded_size,
-          buffer->stride(media::VideoFrame::kYPlane),
+          media::PIXEL_FORMAT_I420, coded_size, gfx::Rect(coded_size),
+          coded_size, buffer->stride(media::VideoFrame::kYPlane),
           buffer->stride(media::VideoFrame::kUPlane),
           buffer->stride(media::VideoFrame::kVPlane),
           buffer->data(media::VideoFrame::kYPlane),
           buffer->data(media::VideoFrame::kUPlane),
           buffer->data(media::VideoFrame::kVPlane),
           handles[media::VideoFrame::kYPlane],
           handles[media::VideoFrame::kUPlane],
-          handles[media::VideoFrame::kVPlane],
-          timestamp - first_frame_timestamp_);
+          handles[media::VideoFrame::kVPlane], timestamp);
       buffer_finished_callback = media::BindToCurrentLoop(
           base::Bind(&VideoCaptureImpl::OnClientBufferFinished2,
                      weak_factory_.GetWeakPtr(), buffer_id, buffer));
       break;
     }
     case media::VideoFrame::STORAGE_SHMEM: {
       const auto& iter = client_buffers_.find(buffer_id);
       DCHECK(iter != client_buffers_.end());
       const scoped_refptr<ClientBuffer> buffer = iter->second;
       frame = media::VideoFrame::WrapExternalSharedMemory(
           pixel_format, coded_size, visible_rect,
           gfx::Size(visible_rect.width(), visible_rect.height()),
           reinterpret_cast<uint8_t*>(buffer->buffer()->memory()),
           buffer->buffer_size(), buffer->buffer()->handle(),
-          0 /* shared_memory_offset */, timestamp - first_frame_timestamp_);
+          0 /* shared_memory_offset */, timestamp);
       buffer_finished_callback = media::BindToCurrentLoop(
           base::Bind(&VideoCaptureImpl::OnClientBufferFinished,
                      weak_factory_.GetWeakPtr(), buffer_id, buffer));
       break;
     }
     default:
       NOTREACHED();
       break;
   }
   if (!frame) {
     Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id,
                                              gpu::SyncToken(), -1.0));
     return;
   }
 
-  frame->metadata()->SetTimeTicks(media::VideoFrameMetadata::REFERENCE_TIME,
-                                  timestamp);
   frame->AddDestructionObserver(
       base::Bind(&VideoCaptureImpl::DidFinishConsumingFrame, frame->metadata(),
                  base::Passed(&release_sync_token), buffer_finished_callback));
 
   frame->metadata()->MergeInternalValuesFrom(metadata);
 
+  // TODO(qiangchen): Dive into the full code path to let frame metadata hold
+  // reference time rather than using an extra parameter.

[miu, 2016/06/08 19:24:44]

Yep, this "deliver callback" API was developed before VideoFrameMetadata even
existed.

[qiangchen, 2016/06/08 20:08:54]

Acknowledged.

   for (const auto& client : clients_)
-    client.second.deliver_frame_cb.Run(frame, timestamp);
+    client.second.deliver_frame_cb.Run(frame, reference_time);
 }
 
 void VideoCaptureImpl::OnClientBufferFinished(
     int buffer_id,
     const scoped_refptr<ClientBuffer>& /* ignored_buffer */,
     const gpu::SyncToken& release_sync_token,
     double consumer_resource_utilization) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id,
                                            release_sync_token,
@@ skipping 148 matching lines @@
   double consumer_resource_utilization = -1.0;
   if (!metadata->GetDouble(media::VideoFrameMetadata::RESOURCE_UTILIZATION,
                            &consumer_resource_utilization)) {
     consumer_resource_utilization = -1.0;
   }
 
   callback_to_io_thread.Run(*release_sync_token, consumer_resource_utilization);
 }
 
 }  // namespace content