Web MIDI backend for Windows 10

media/midi/midi_manager_winrt.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/midi/midi_manager_winrt.h"
+
+#include <robuffer.h>
+#include <windows.devices.enumeration.h>
+#include <windows.devices.midi.h>
+#include <wrl/event.h>
+
+#include "base/bind.h"
+#include "base/containers/hash_tables.h"
+#include "base/strings/string16.h"
+#include "base/strings/utf_string_conversions.h"
+#include "base/timer/timer.h"
+#include "base/win/windows_version.h"
+
+namespace media {
+namespace midi {
+namespace {
+
+namespace WRL = Microsoft::WRL;
+
+using namespace ABI::Windows::Devices::Enumeration;
+using namespace ABI::Windows::Devices::Midi;
+using namespace ABI::Windows::Foundation;
+using namespace ABI::Windows::Storage::Streams;
+
+// Factory functions that activate and create WinRT components. The caller takes
+// ownership of the returning ComPtr.
+template <typename InterfaceType, base::char16 const* runtime_class_id>
+WRL::ComPtr<InterfaceType> WrlStaticsFactory() {

[Shao-Chuan Lee, 2016/08/18 08:57:50]

Missing TODO: try replacing WRL::ComPtr with base::win::ScopedComPtr.

+  WRL::ComPtr<InterfaceType> com_ptr;
+
+  HRESULT hr = GetActivationFactory(
+      WRL::Wrappers::HStringReference(runtime_class_id).Get(), &com_ptr);
+  DCHECK(SUCCEEDED(hr));
+
+  return std::move(com_ptr);

[Shao-Chuan Lee, 2016/08/22 06:53:17]

std::move not required, NRVO applies here.

[Shao-Chuan Lee, 2016/08/23 04:29:04]

Done.

+}
+
+WRL::ComPtr<IBufferFactory> GetBufferFactory() {
+  return std::move(

[Shao-Chuan Lee, 2016/08/22 06:53:17]

ditto

[Shao-Chuan Lee, 2016/08/23 04:29:04]

Done.

+      WrlStaticsFactory<IBufferFactory,
+                        RuntimeClass_Windows_Storage_Streams_Buffer>());
+}
+
+WRL::ComPtr<IDeviceInformationStatics> GetDeviceInformationStatics() {
+  return std::move(

[Shao-Chuan Lee, 2016/08/22 06:53:17]

ditto

[Shao-Chuan Lee, 2016/08/23 04:29:04]

Done.

+      WrlStaticsFactory<
+          IDeviceInformationStatics,
+          RuntimeClass_Windows_Devices_Enumeration_DeviceInformation>());
+}
+
+template <typename T, HRESULT (T::*method)(HSTRING*)>
+std::string GetStringFromObjectMethod(T* obj) {
+  HSTRING result;
+  HRESULT hr = (obj->*method)(&result);
+  DCHECK(SUCCEEDED(hr));
+
+  const base::char16* buffer = WindowsGetStringRawBuffer(result, nullptr);
+  if (buffer) {

[Takashi Toyoshima, 2016/08/18 11:09:22]

You don't need "{}" for one line.

[Shao-Chuan Lee, 2016/08/22 06:53:17]

Done.

+    return base::WideToUTF8(buffer);
+  }
+  return "";

[Takashi Toyoshima, 2016/08/18 11:09:22]

returning std::string() is recommended to return an empty string in chromium.

[Shao-Chuan Lee, 2016/08/22 06:53:17]

Done.

+}
+
+template <typename T>
+std::string GetIdString(T* obj) {
+  return GetStringFromObjectMethod<T, &T::get_Id>(obj);
+}
+
+template <typename T>
+std::string GetDeviceIdString(T* obj) {
+  return GetStringFromObjectMethod<T, &T::get_DeviceId>(obj);
+}
+
+std::string GetNameString(IDeviceInformation* info) {
+  return GetStringFromObjectMethod<IDeviceInformation,
+                                   &IDeviceInformation::get_Name>(info);
+}
+
+uint8_t* GetPointerToBufferData(IBuffer* buffer) {
+  WRL::ComPtr<IInspectable> inspectable(
+      reinterpret_cast<IInspectable*>(buffer));
+  WRL::ComPtr<Windows::Storage::Streams::IBufferByteAccess> buffer_byte_access;
+
+  HRESULT hr = inspectable.As(&buffer_byte_access);
+  DCHECK(SUCCEEDED(hr));
+
+  uint8_t* ptr = nullptr;
+  hr = buffer_byte_access->Buffer(&ptr);
+  DCHECK(SUCCEEDED(hr));
+
+  // Lifetime of the pointing buffer is controlled by the buffer object.
+  return ptr;
+}
+
+template <typename InterfaceType>
+struct MidiPort {
+  MidiPort() = default;
+
+  uint32_t index;
+  WRL::ComPtr<InterfaceType> handle;
+  base::TimeTicks start_time;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(MidiPort);
+};
+
+template <typename InterfaceType,
+          typename RuntimeType,
+          typename StaticsInterfaceType,
+          base::char16 const* runtime_class_id>
+class MidiPortManager {
+ public:
+  MidiPortManager() = default;
+
+  void StartWatcher() {
+    HRESULT hr;
+
+    HSTRING device_selector = nullptr;
+    hr = WrlStaticsFactory<StaticsInterfaceType, runtime_class_id>()

[Shao-Chuan Lee, 2016/08/18 08:57:50]

Retrieve object from WrlStaticsFactory at constructor for reuse?

[Shao-Chuan Lee, 2016/08/23 04:29:04]

Done.

+             ->GetDeviceSelector(&device_selector);
+    DCHECK(SUCCEEDED(hr));
+
+    hr = GetDeviceInformationStatics()->CreateWatcherAqsFilter(device_selector,
+                                                               &watcher_);
+    DCHECK(SUCCEEDED(hr));
+
+    hr = watcher_->add_Added(
+        WRL::Callback<ITypedEventHandler<DeviceWatcher*, DeviceInformation*>>(
+            this, &MidiPortManager::OnAdded)
+            .Get(),
+        &token_Added_);
+    DCHECK(SUCCEEDED(hr));
+
+    hr = watcher_->add_EnumerationCompleted(
+        WRL::Callback<ITypedEventHandler<DeviceWatcher*, IInspectable*>>(
+            this, &MidiPortManager::OnEnumerationCompleted)
+            .Get(),
+        &token_EnumerationCompleted_);
+    DCHECK(SUCCEEDED(hr));
+
+    hr = watcher_->add_Removed(
+        WRL::Callback<
+            ITypedEventHandler<DeviceWatcher*, DeviceInformationUpdate*>>(
+            this, &MidiPortManager::OnRemoved)
+            .Get(),
+        &token_Removed_);
+    DCHECK(SUCCEEDED(hr));
+
+    hr = watcher_->add_Stopped(
+        WRL::Callback<ITypedEventHandler<DeviceWatcher*, IInspectable*>>(
+            this, &MidiPortManager::OnStopped)
+            .Get(),
+        &token_Stopped_);
+    DCHECK(SUCCEEDED(hr));
+
+    hr = watcher_->add_Updated(
+        WRL::Callback<
+            ITypedEventHandler<DeviceWatcher*, DeviceInformationUpdate*>>(
+            this, &MidiPortManager::OnUpdated)
+            .Get(),
+        &token_Updated_);
+    DCHECK(SUCCEEDED(hr));
+
+    hr = watcher_->Start();
+    DCHECK(SUCCEEDED(hr));
+  }
+
+  MidiPort<InterfaceType>* GetPortByDeviceId(std::string dev_id) {
+    base::AutoLock auto_lock(ports_lock_);
+    auto it = ports_.find(dev_id);
+    if (it == ports_.end())
+      return nullptr;
+    return it->second.get();
+  }
+
+  MidiPort<InterfaceType>* GetPortByIndex(uint32_t port_index) {
+    base::AutoLock auto_lock(ports_lock_);
+    auto it = ports_.find(port_ids_[port_index]);
+    if (it == ports_.end())
+      return nullptr;
+    return it->second.get();
+  }
+
+  // DeviceWatcher callbacks:
+  HRESULT OnAdded(IDeviceWatcher* watcher, IDeviceInformation* info) {
+    // TODO(shaochuan): Disable Microsoft GS Wavetable Synth due to security
+    // reasons. http://crbug.com/499279
+
+    {
+      base::AutoLock auto_lock(ports_lock_);
+      port_names_[GetIdString(info)] = GetNameString(info);
+    }
+
+    HSTRING dev_id_hstring;
+    HRESULT hr = info->get_Id(&dev_id_hstring);
+    DCHECK(SUCCEEDED(hr));
+
+    WRL::ComPtr<IAsyncOperation<RuntimeType*>> async_op;
+
+    hr = WrlStaticsFactory<StaticsInterfaceType, runtime_class_id>()
+             ->FromIdAsync(dev_id_hstring, &async_op);
+    DCHECK(SUCCEEDED(hr));
+
+    hr = async_op->put_Completed(
+        WRL::Callback<IAsyncOperationCompletedHandler<RuntimeType*>>(
+            this, &MidiPortManager::OnCompletedGetPortFromIdAsync)
+            .Get());
+    DCHECK(SUCCEEDED(hr));
+
+    {
+      base::AutoLock auto_lock(async_ops_lock_);
+      async_ops_.insert(std::move(async_op));
+    }
+
+    return S_OK;
+  }
+
+  HRESULT OnEnumerationCompleted(IDeviceWatcher* watcher, IInspectable* insp) {
+    // TODO(shaochuan)
+    return S_OK;
+  }
+
+  HRESULT OnRemoved(IDeviceWatcher* watcher, IDeviceInformationUpdate* update) {
+    MidiPort<InterfaceType>* port = GetPortByDeviceId(GetIdString(update));
+    DCHECK(port != nullptr);
+
+    SetPortState(port->index, MIDI_PORT_DISCONNECTED);
+
+    port->handle = nullptr;
+
+    return S_OK;
+  }
+
+  HRESULT OnStopped(IDeviceWatcher* watcher, IInspectable* insp) {
+    return S_OK;
+  }
+
+  HRESULT OnUpdated(IDeviceWatcher* watcher, IDeviceInformationUpdate* update) {
+    // TODO(shaochuan): Check for fields to be updated here.
+    return S_OK;
+  }
+
+ private:
+  HRESULT OnCompletedGetPortFromIdAsync(IAsyncOperation<RuntimeType*>* async_op,
+                                        AsyncStatus status) {
+    // TODO(shaochuan): Check if port open time is accurate.
+    const auto now = base::TimeTicks::Now();
+
+    InterfaceType* handle;
+    HRESULT hr = async_op->GetResults(&handle);
+    DCHECK(SUCCEEDED(hr));
+
+    RegisterOnMessageReceived(handle);
+
+    std::string dev_id = GetDeviceIdString(handle);
+
+    MidiPort<InterfaceType>* port = GetPortByDeviceId(dev_id);
+
+    if (port == nullptr) {
+      base::AutoLock auto_lock(ports_lock_);
+
+      // TODO(shaochuan): Fill in manufacturer and driver version.
+      AddPort(MidiPortInfo(dev_id, std::string("Manufacturer"),
+                           port_names_[dev_id], std::string("DriverVersion"),
+                           MIDI_PORT_OPENED));
+
+      port = new MidiPort<InterfaceType>;
+      port->index = static_cast<uint32_t>(port_ids_.size());
+
+      ports_[dev_id].reset(port);
+      port_ids_.push_back(dev_id);
+    } else {
+      SetPortState(port->index, MIDI_PORT_CONNECTED);
+    }
+
+    port->handle = handle;
+    port->start_time = now;
+
+    {
+      base::AutoLock auto_lock(async_ops_lock_);
+      auto it = async_ops_.find(async_op);
+      DCHECK(it != async_ops_.end());
+      async_ops_.erase(it);
+    }
+
+    return S_OK;
+  }
+
+  virtual void RegisterOnMessageReceived(InterfaceType* handle) {}
+
+  // Calls midi_manager_->Add{Input,Output}Port.
+  virtual void AddPort(MidiPortInfo info) = 0;
+
+  // Calls midi_manager_->Set{Input,Output}PortState.
+  virtual void SetPortState(uint32_t port_index, MidiPortState state) = 0;
+
+  WRL::ComPtr<IDeviceWatcher> watcher_;
+  EventRegistrationToken token_Added_, token_EnumerationCompleted_,
+      token_Removed_, token_Stopped_, token_Updated_;
+
+  // Locks required since fields are being manipulated by callbacks from WinRT
+  // on the OS callback thread.
+  base::Lock ports_lock_;
+  base::hash_map<std::string, std::unique_ptr<MidiPort<InterfaceType>>>
+      ports_;                          // GUARDED_BY(ports_lock_)
+  std::vector<std::string> port_ids_;  // GUARDED_BY(ports_lock_)
+  base::hash_map<std::string, std::string>
+      port_names_;  // GUARDED_BY(ports_lock_)
+
+  // Keeps AsyncOperation objects before the operation completes.
+  base::Lock async_ops_lock_;
+  std::set<WRL::ComPtr<IAsyncOperation<RuntimeType*>>>
+      async_ops_;  // GUARDED_BY(async_ops_lock_)
+};
+
+}  // namespace
+
+class MidiManagerWinrt::MidiInPortManager final
+    : public MidiPortManager<IMidiInPort,
+                             MidiInPort,
+                             IMidiInPortStatics,
+                             RuntimeClass_Windows_Devices_Midi_MidiInPort> {
+ public:
+  MidiInPortManager(MidiManagerWinrt* midi_manager)
+      : midi_manager_(midi_manager) {}
+
+ private:
+  // MidiPortManager overrides:
+  void RegisterOnMessageReceived(IMidiInPort* handle) override {
+    base::AutoLock auto_lock(tokens_lock_);
+    EventRegistrationToken& token = tokens_[GetDeviceIdString(handle)];
+
+    handle->add_MessageReceived(
+        WRL::Callback<
+            ITypedEventHandler<MidiInPort*, MidiMessageReceivedEventArgs*>>(
+            this, &MidiInPortManager::OnMessageReceived)
+            .Get(),
+        &token);
+  }
+
+  void AddPort(MidiPortInfo info) { midi_manager_->AddInputPort(info); }
+
+  void SetPortState(uint32_t port_index, MidiPortState state) {
+    midi_manager_->SetInputPortState(port_index, state);
+  }
+
+  // Callback on receiving MIDI input message.
+  HRESULT OnMessageReceived(IMidiInPort* handle,
+                            IMidiMessageReceivedEventArgs* args) {
+    MidiPort<IMidiInPort>* port = GetPortByDeviceId(GetDeviceIdString(handle));
+    DCHECK(port != nullptr);
+
+    WRL::ComPtr<IMidiMessage> message;
+    HRESULT hr = args->get_Message(&message);
+    DCHECK(SUCCEEDED(hr));
+
+    WRL::ComPtr<IBuffer> buffer;
+    hr = message->get_RawData(&buffer);
+    DCHECK(SUCCEEDED(hr));
+
+    uint8_t* p_buffer_data = GetPointerToBufferData(buffer.Get());
+
+    uint32_t data_length;
+    hr = buffer->get_Length(&data_length);
+    DCHECK(SUCCEEDED(hr));
+    DCHECK(data_length != 0);
+
+    std::vector<uint8_t> data(p_buffer_data, p_buffer_data + data_length);
+
+    // Time since port opened in 100-nanosecond units.
+    TimeSpan time_span;
+    hr = message->get_Timestamp(&time_span);
+    DCHECK(SUCCEEDED(hr));
+
+    midi_manager_->ReceiveMidiData(
+        port->index, &data[0], data.size(),
+        port->start_time +
+            base::TimeDelta::FromMicroseconds(time_span.Duration / 10));
+
+    return S_OK;
+  }
+
+  MidiManagerWinrt* midi_manager_;
+
+  base::Lock tokens_lock_;
+  base::hash_map<std::string, EventRegistrationToken>
+      tokens_;  // GUARDED_BY(tokens_lock_)
+
+  DISALLOW_COPY_AND_ASSIGN(MidiInPortManager);
+};
+
+class MidiManagerWinrt::MidiOutPortManager final
+    : public MidiPortManager<IMidiOutPort,
+                             IMidiOutPort,
+                             IMidiOutPortStatics,
+                             RuntimeClass_Windows_Devices_Midi_MidiOutPort> {
+ public:
+  MidiOutPortManager(MidiManagerWinrt* midi_manager)
+      : midi_manager_(midi_manager) {}
+
+ private:
+  // MidiPortManager overrides:
+  void AddPort(MidiPortInfo info) { midi_manager_->AddOutputPort(info); }
+
+  void SetPortState(uint32_t port_index, MidiPortState state) {
+    midi_manager_->SetOutputPortState(port_index, state);
+  }
+
+  MidiManagerWinrt* midi_manager_;
+
+  DISALLOW_COPY_AND_ASSIGN(MidiOutPortManager);
+};
+
+MidiManagerWinrt::MidiManagerWinrt() : com_thread_("Windows MIDI COM Thread") {}
+
+MidiManagerWinrt::~MidiManagerWinrt() {}
+
+void MidiManagerWinrt::StartInitialization() {

[Takashi Toyoshima, 2016/08/18 11:09:22]

You need a lock object to access class members to ensure modifications are
synchronized between the main thread and the I/O thread. MidiManagerWinrt is
constructed and destructed on the main thread, and StartInitialization() and
Finalize() run on the I/O thread. Otherwise, double-free-ish crashes may happen
randomly inside the destructor.

Note: If you are not so familiar with multi-threading specific problems, please
let me know. I can have an offline chat.

+  DCHECK(base::win::GetVersion() >= base::win::VERSION_WIN10);
+
+  scheduler_.reset(new MidiScheduler(this));

[Takashi Toyoshima, 2016/08/18 11:09:21]

Now scheduler_ is constructed on the I/O thread, and used on the com thread too.
This causes race issues.

As I discussed offline, can you modify MidiScheduler to take a task runner that
runs the posted task on requested time?

[Shao-Chuan Lee, 2016/08/22 06:53:17]

Changes in separate CL https://codereview.chromium.org/2262043002/.

+
+  com_thread_.init_com_with_mta(true);
+  com_thread_.Start();
+
+  com_thread_.message_loop()->PostTask(

[Takashi Toyoshima, 2016/08/18 11:09:22]

Instead of message_loop(), task_runner() is recommended to be used for new code.
See crbug.com/616447 for details.

[Shao-Chuan Lee, 2016/08/22 06:53:17]

Done.

+      FROM_HERE, base::Bind(&MidiManagerWinrt::InitializeOnComThread,
+                            base::Unretained(this)));
+
+  CompleteInitialization(Result::OK);

[Takashi Toyoshima, 2016/08/18 11:09:22]

As I commented for the previous patch set, it's fine to move this call to the
place there code runs after the all initialization finishes.

TODO is fine for this CL.

[Shao-Chuan Lee, 2016/08/23 04:29:04]

Now called when both MidiPortManagers are ready (OnEnumerationComplete).

+}
+
+void MidiManagerWinrt::Finalize() {
+  // TODO(shaochuan): Check if everything is destructed gracefully.

[Takashi Toyoshima, 2016/08/18 11:09:21]

As I commented for StartInitialization(), we need a lock here.

+  port_manager_in_.reset();

[Takashi Toyoshima, 2016/08/18 11:09:22]

Since these are initialized on the com thread, these should be destructed on the
com thread, too.

To solve this, you can post a finalize task to the com thread, then call Stop().
It runs all posted tasks on the target thread, and the caller blocks until tasks
finish.

[Shao-Chuan Lee, 2016/08/23 04:29:04]

Done.

+  port_manager_out_.reset();
+
+  scheduler_.reset();
+
+  com_thread_.Stop();
+}
+
+void MidiManagerWinrt::DispatchSendMidiData(MidiManagerClient* client,
+                                            uint32_t port_index,
+                                            const std::vector<uint8_t>& data,
+                                            double timestamp) {
+  com_thread_.message_loop()->PostTask(

[Takashi Toyoshima, 2016/08/18 11:09:21]

task_runner

[Shao-Chuan Lee, 2016/08/22 06:53:17]

Done.

+      FROM_HERE,
+      base::Bind(&MidiManagerWinrt::PostSendDataTaskOnComThread,
+                 base::Unretained(this), client, port_index, data, timestamp));
+}
+
+void MidiManagerWinrt::AssertOnComThread() {
+  DCHECK_EQ(com_thread_.GetThreadId(), base::PlatformThread::CurrentId());

[Takashi Toyoshima, 2016/08/18 11:09:22]

Use base/threading/therad_checker instead of having this method?

[Shao-Chuan Lee, 2016/08/22 06:53:17]

Done.

+}
+
+void MidiManagerWinrt::InitializeOnComThread() {
+  AssertOnComThread();
+
+  port_manager_in_.reset(new MidiInPortManager(this));
+  port_manager_out_.reset(new MidiOutPortManager(this));
+
+  port_manager_in_->StartWatcher();
+  port_manager_out_->StartWatcher();
+}
+
+void MidiManagerWinrt::PostSendDataTaskOnComThread(
+    MidiManagerClient* client,
+    uint32_t port_index,
+    const std::vector<uint8_t>& data,
+    double timestamp) {
+  AssertOnComThread();
+
+  scheduler_->PostSendDataTask(
+      client, data.size(), timestamp,
+      base::Bind(&MidiManagerWinrt::SendOnComThread, base::Unretained(this),
+                 port_index, data));
+}
+
+void MidiManagerWinrt::SendOnComThread(uint32_t port_index,
+                                       const std::vector<uint8_t>& data) {
+  AssertOnComThread();
+
+  WRL::ComPtr<IBuffer> buffer;
+  HRESULT hr =
+      GetBufferFactory()->Create(static_cast<UINT32>(data.size()), &buffer);
+  DCHECK(SUCCEEDED(hr));
+

[Takashi Toyoshima, 2016/08/18 11:09:22]

Is it fine to continue following steps when HRESULT contains an error result?
DCHECK runs only on debug build, and production will continue to run even on the
condition the check fails.

I'm afraid that buffer related operation's failure could result in a security
issue.

[Shao-Chuan Lee, 2016/08/23 04:29:04]

I guess buffer allocation may fail on cases such as OOM. We should confirm which
DCHECKs should be replaced with CHECKs.

+  hr = buffer->put_Length(static_cast<UINT32>(data.size()));
+  DCHECK(SUCCEEDED(hr));
+
+  uint8_t* p_buffer_data = GetPointerToBufferData(buffer.Get());
+
+  std::copy(data.begin(), data.end(), p_buffer_data);
+
+  MidiPort<IMidiOutPort>* port = port_manager_out_->GetPortByIndex(port_index);
+  DCHECK(port != nullptr);
+
+  hr = port->handle->SendBuffer(buffer.Get());
+  DCHECK(SUCCEEDED(hr));
+}
+
+MidiManager* MidiManager::Create() {
+  return new MidiManagerWinrt();
+}
+
+}  // namespace midi
+}  // namespace media