cubeb - re-enable and polish IAudioClient3 to achieve lower latencies

Author: Filoppi

src/cubeb_wasapi.cpp

@@ -7,6 +7,24 @@
 #define _WIN32_WINNT 0x0603
 #define NOMINMAX
 
+// Makes sure cubeb doesn't try to initialize an IAudioClient3 with unsupported formats, as it forces its own (e.g. unsupported bit depths or sample rates).
+// I'm not 100% sure this is right as IAudioClient::IsFormatSupported() might not be relevant with IAudioClient3, but it probably is.
+#define CHECK_MIXER_FORMAT_SUPPORT 1
+// Force raw output (skips some processing) with IAudioClient3
+#define USE_RAW_OUTPUT 0
+// Forces the output format to match our mixer one
+#define FORCE_MATCHING_OUTPUT_FORMAT 0
+// "Forces" (?) a IAudioClient3 to resample (in high quality) our stream to the device output sample rate
+#define CONVERT_STREAM 0
+// If we are targeting IAudioClient3, we don't want to limit the latency to the default latency (likely 10ms), but to the minimum one.
+// Though it seems that IAudioClient::GetDevicePeriod() returns lower latencies than GetSharedModeEnginePeriod() allows, so that's weird.
+#define ALLOW_MIN_LATENCY 0
+// Can we go lower than the user/client requested latency? This isn't particularly necessary as dolphin should produce enough audio samples to go with latencies < 10ms (which is what it asks for),
+// and the user requested latency is just a suggestion.
+#define FORCE_RESPECT_USER_LATENCY 0
+// IAudioClient3::GetSharedModeEnginePeriod() doesn't seem to work as well as IAudioClient::GetDevicePeriod() and often return a min and max latency of 10ms, even if lower ones would be supported
+#define USE_AUDIO_CLIENT_3_LATENCY 0
+
 #include <algorithm>
 #include <atomic>
 #include <audioclient.h>
@@ -1867,6 +1885,45 @@ wasapi_get_min_latency(cubeb * ctx, cubeb_stream_params params,
     return CUBEB_ERROR;
   }
 
+#if USE_AUDIO_CLIENT_3_LATENCY
+  //TODO: this is likely unreliable as we can't know the actual mixer format cubeb will ask for later on (we'd need to calculate it) (and the min latency could change based on that)
+  com_ptr<IAudioClient3> client3;
+  hr = device->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER, NULL,
+                        client3.receive_vpp());
+  if (SUCCEEDED(hr)) {
+    WAVEFORMATEX * mix_format = nullptr;
+    hr = client3->GetMixFormat(&mix_format);
+
+    if (SUCCEEDED(hr)) {
+      uint32_t default_period = 0, fundamental_period = 0, min_period = 0,
+               max_period = 0;
+      hr = client3->GetSharedModeEnginePeriod(
+        mix_format, &default_period, &fundamental_period, &min_period,
+        &max_period);
+
+      auto sample_rate = mix_format->nSamplesPerSec;
+      CoTaskMemFree(mix_format);
+      if (SUCCEEDED(hr)) {
+        // Keep values in the same format as IAudioDevice::GetDevicePeriod()
+        REFERENCE_TIME min_period_rt(frames_to_hns(sample_rate, min_period));
+        REFERENCE_TIME default_period_rt(frames_to_hns(sample_rate, default_period));
+        LOG("default device period: %I64d, minimum device period: %I64d",
+            default_period_rt, min_period_rt);
+
+  #if ALLOW_MIN_LATENCY
+        *latency_frames = min_period;
+  #else
+        *latency_frames = default_period; // This can be 0
+  #endif
+
+        LOG("Minimum latency in frames: %u", *latency_frames);
+
+        return CUBEB_OK;
+      }
+    }
+  }
+#endif
+
   com_ptr<IAudioClient> client;
   hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
                         client.receive_vpp());
@@ -1891,10 +1948,8 @@ wasapi_get_min_latency(cubeb * ctx, cubeb_stream_params params,
      synchronizing the stream and the engine.
      http://msdn.microsoft.com/en-us/library/windows/desktop/dd370871%28v=vs.85%29.aspx
    */
-
-  // #ifdef _WIN32_WINNT_WIN10
-#if 0
-     *latency_frames = hns_to_frames(params.rate, minimum_period);
+#if defined(_WIN32_WINNT_WIN10) && ALLOW_MIN_LATENCY
+  *latency_frames = hns_to_frames(params.rate, minimum_period);
 #else
   *latency_frames = hns_to_frames(params.rate, default_period);
 #endif
@@ -1987,7 +2042,10 @@ handle_channel_layout(cubeb_stream * stm, EDataFlow direction,
   if (hr == S_FALSE) {
     /* Channel layout not supported, but WASAPI gives us a suggestion. Use it,
        and handle the eventual upmix/downmix ourselves. Ignore the subformat of
-       the suggestion, since it seems to always be IEEE_FLOAT. */
+       the suggestion, since it seems to always be IEEE_FLOAT.
+       This fallback doesn't update the bit depth, so if a device
+       only supported bit depths cubeb doesn't support, we will need to
+       make sure the stream is compatible with it (e.g. IAudioClient3 is not) */
     LOG("Using WASAPI suggested format: channels: %d", closest->nChannels);
     XASSERT(closest->wFormatTag == WAVE_FORMAT_EXTENSIBLE);
     WAVEFORMATEXTENSIBLE * closest_pcm =
@@ -2031,12 +2089,12 @@ initialize_iaudioclient2(com_ptr<IAudioClient> & audio_client)
   return CUBEB_OK;
 }
 
-#if 0
 bool
 initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
                          cubeb_stream * stm,
                          const com_heap_ptr<WAVEFORMATEX> & mix_format,
-                         DWORD flags, EDataFlow direction)
+                         DWORD flags, EDataFlow direction,
+                         REFERENCE_TIME latency_hns)
 {
   com_ptr<IAudioClient3> audio_client3;
   audio_client->QueryInterface<IAudioClient3>(audio_client3.receive());
@@ -2072,37 +2130,98 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
   //   would do this, then stop and use IAudioClient instead.
 
   HRESULT hr;
+
+#if CHECK_MIXER_FORMAT_SUPPORT
+  WAVEFORMATEX * tmp = nullptr;
+  // The mixer format here might have bit depths "forced" by cubeb even if the output device doesn't directly support it.
+  // This doesn't work with IAudioClient3 as there it applies no resampling.
+  hr = audio_client3->IsFormatSupported(AUDCLNT_SHAREMODE_SHARED, mix_format.get(), &tmp);
+  if (FAILED(hr)) {
+    LOG("IAudioClient3 attempted format is not supported: error: %lx", hr);
+    return false;
+  }
+  CoTaskMemFree(tmp);
+#endif
+
   uint32_t default_period = 0, fundamental_period = 0, min_period = 0,
            max_period = 0;
+  //TODO: review... this returns the same default/min/max latency every time on my PC (10ms, which is high),
+  //independently of the device and its settings (with no other apps running). Not sure if it means that lag would be bigger than with IAudioDevice(1).
   hr = audio_client3->GetSharedModeEnginePeriod(
       mix_format.get(), &default_period, &fundamental_period, &min_period,
       &max_period);
   if (FAILED(hr)) {
     LOG("Could not get shared mode engine period: error: %lx", hr);
     return false;
   }
-  uint32_t requested_latency = stm->latency;
-  if (requested_latency >= default_period) {
-    LOG("Requested latency %i greater than default latency %i, not using "
+  uint32_t requested_latency = hns_to_frames(mix_format->nSamplesPerSec, latency_hns);
+#if FORCE_RESPECT_USER_LATENCY
+  if (requested_latency > max_period) {
+    // Fallback to IAudioClient(1) as it's less restrictive towards bigger latencies
+    LOG("Requested latency %i greater than max latency %i, not using "
         "IAudioClient3",
-        requested_latency, default_period);
+        requested_latency, max_period);
     return false;
   }
+#endif
   LOG("Got shared mode engine period: default=%i fundamental=%i min=%i max=%i",
       default_period, fundamental_period, min_period, max_period);
   // Snap requested latency to a valid value
   uint32_t old_requested_latency = requested_latency;
+  // The period is required to be a multiple of the fundamental period (and >= min and <= max, which should still be true)
+  requested_latency -= requested_latency % fundamental_period;
   if (requested_latency < min_period) {
     requested_latency = min_period;
   }
-  requested_latency -= (requested_latency - min_period) % fundamental_period;
+  // This is likely unnecessary, but it won't hurt
+  if (requested_latency > max_period) {
+    requested_latency = max_period;
+  }
   if (requested_latency != old_requested_latency) {
     LOG("Requested latency %i was adjusted to %i", old_requested_latency,
         requested_latency);
   }
 
-  hr = audio_client3->InitializeSharedAudioStream(flags, requested_latency,
+#if USE_RAW_OUTPUT || FORCE_MATCHING_OUTPUT_FORMAT
+  AudioClientProperties properties = {0};
+  properties.cbSize = sizeof(AudioClientProperties);
+  properties.bIsOffload = false; //TODO: review this (take from chromium)
+#ifndef __MINGW32__
+#if USE_RAW_OUTPUT
+  // Raw audio streams skip some kinds of processing, like AEC and AGC.
+  // Do this independent from CUBEB_STREAM_PREF_RAW, we force this with IAudioClient3, to hopefully get the lowest latency/quality and support.
+  properties.Options |= AUDCLNT_STREAMOPTIONS_RAW;
+#endif
+#if FORCE_MATCHING_OUTPUT_FORMAT
+  properties.Options |= AUDCLNT_STREAMOPTIONS_MATCH_FORMAT;
+#endif
+#endif
+  com_ptr<IAudioClient2> audio_client2;
+  hr = audio_client->QueryInterface<IAudioClient2>(audio_client2.receive());
+  if (audio_client2) {
+    hr = audio_client2->SetClientProperties(&properties);
+  }
+  if (FAILED(hr)) {
+    LOG("Could not set IAudioClient2 properties: error: %lx", hr);
+    // This is not fatal, it should work anyway (AUDCLNT_STREAMOPTIONS_RAW might not be supported)
+  }
+#endif
+
+  DWORD new_flags = flags;
+#if CONVERT_STREAM
+  // Always add these flags to IAudioClient3 (they can help if the stream doesn't have the same format as the output)
+  new_flags |= AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM;
+  new_flags |= AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY;
+#endif
+
+  hr = audio_client3->InitializeSharedAudioStream(new_flags, requested_latency,
                                                   mix_format.get(), NULL);
+  if (hr == AUDCLNT_E_INVALID_STREAM_FLAG && CONVERT_STREAM) {
+    LOG("Got AUDCLNT_E_INVALID_STREAM_FLAG, removing some flag");
+    hr = audio_client3->InitializeSharedAudioStream(flags, requested_latency,
+                                                    mix_format.get(), NULL);
+  }
+
   if (SUCCEEDED(hr)) {
     return true;
   } else if (hr == AUDCLNT_E_ENGINE_PERIODICITY_LOCKED) {
@@ -2114,22 +2233,48 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
   }
 
   uint32_t current_period = 0;
-  WAVEFORMATEX * current_format = nullptr;
+  WAVEFORMATEX * current_format_ptr = nullptr;
   // We have to pass a valid WAVEFORMATEX** and not nullptr, otherwise
   // GetCurrentSharedModeEnginePeriod will return E_POINTER
-  hr = audio_client3->GetCurrentSharedModeEnginePeriod(&current_format,
+  hr = audio_client3->GetCurrentSharedModeEnginePeriod(&current_format_ptr,
                                                        &current_period);
-  CoTaskMemFree(current_format);
   if (FAILED(hr)) {
     LOG("Could not get current shared mode engine period: error: %lx", hr);
     return false;
   }
+  com_heap_ptr<WAVEFORMATEX> current_format(current_format_ptr);
+#if 1
+  // Unless some other external app locked the shared mode engine period within our audio initialization,
+  // this likely shouldn't happen
+  if (current_format->nSamplesPerSec != mix_format->nSamplesPerSec)
+  {
+    hr = audio_client3->GetSharedModeEnginePeriod(
+        current_format.get(), &default_period, &fundamental_period, &min_period,
+        &max_period);
+    if (FAILED(hr)) {
+      LOG("IAudioClient3::GetCurrentSharedModeEnginePeriod() returned a different mixer format (nSamplesPerSec) from IAudioClient::GetMixFormat(); not using IAudioClient3");
+      return false;
+    }
+    LOG("IAudioClient3::GetCurrentSharedModeEnginePeriod() returned a different mixer format (nSamplesPerSec) from IAudioClient::GetMixFormat(); attempting a matching latency");
+    REFERENCE_TIME current_period_hns = frames_to_hns(current_format->nSamplesPerSec, current_period);
+    current_period = hns_to_frames(mix_format->nSamplesPerSec, current_period_hns);
+    current_period -= current_period % fundamental_period;
+    // Note: the follow up IAudioClient3::InitializeSharedAudioStream() is likely to fail anyway,
+    // given that the period was already locked and we changed it.
+    // The only right way to proceed would be to change the "mix_format->nSamplesPerSec" to "current_format->nSamplesPerSec",
+    // but we can't do that for now.
+  }
+#endif
 
-  if (current_period >= default_period) {
-    LOG("Current shared mode engine period %i too high, not using IAudioClient",
-        current_period);
+#if FORCE_RESPECT_USER_LATENCY
+  // This used to return false if current_period was >= default_latency, but that didn't seem to make sense
+  if (old_requested_latency > current_period) {
+    LOG("Requested latency %i greater than currently locked shared mode latency %i, not using "
+        "IAudioClient3",
+        old_requested_latency, current_period);
     return false;
   }
+#endif
 
   hr = audio_client3->InitializeSharedAudioStream(flags, current_period,
                                                   mix_format.get(), NULL);
@@ -2142,7 +2287,6 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
   LOG("Could not initialize shared stream with IAudioClient3: error: %lx", hr);
   return false;
 }
-#endif
 
 #define DIRECTION_NAME (direction == eCapture ? "capture" : "render")
 
@@ -2166,6 +2310,8 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
     return CUBEB_ERROR;
   }
 
+  const bool has_capture = direction == eCapture || direction == eAll;
+
   stm->stream_reset_lock.assert_current_thread_owns();
   // If user doesn't specify a particular device, we can choose another one when
   // the given devid is unavailable.
@@ -2202,17 +2348,17 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
 
     /* Get a client. We will get all other interfaces we need from
      * this pointer. */
-#if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
-    hr = device->Activate(__uuidof(IAudioClient3),
-                          CLSCTX_INPROC_SERVER,
-                          NULL, audio_client.receive_vpp());
-    if (hr == E_NOINTERFACE) {
-#endif
-    hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
-                          audio_client.receive_vpp());
-#if 0
+    if (!has_capture) {
+      hr = device->Activate(__uuidof(IAudioClient3),
+                        CLSCTX_INPROC_SERVER,
+                        NULL, audio_client.receive_vpp());
+    }
+    // IAudioClient3 has problems with capture sessions:
+    // https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
+    if (has_capture || hr == E_NOINTERFACE) {
+      hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
+                            audio_client.receive_vpp());
     }
-#endif
 
     if (FAILED(hr)) {
       LOG("Could not activate the device to get an audio"
@@ -2341,16 +2487,12 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
     }
   }
 
-#if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
-  if (initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction)) {
+  if (!has_capture && initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction, latency_hns)) {
     LOG("Initialized with IAudioClient3");
   } else {
-#endif
-  hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, latency_hns, 0,
-                                mix_format.get(), NULL);
-#if 0
+    hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, latency_hns, 0, mix_format.get(), NULL);
   }
-#endif
+
   if (FAILED(hr)) {
     LOG("Unable to initialize audio client for %s: %lx.", DIRECTION_NAME, hr);
     return CUBEB_ERROR;
@@ -3310,6 +3452,7 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
                                              CUBEB_DEVICE_FMT_S16NE);
   ret.default_format = CUBEB_DEVICE_FMT_F32NE;
   prop_variant fmtvar;
+  WAVEFORMATEX* wfx = NULL;
   hr = propstore->GetValue(PKEY_AudioEngine_DeviceFormat, &fmtvar);
   if (SUCCEEDED(hr) && fmtvar.vt == VT_BLOB) {
     if (fmtvar.blob.cbSize == sizeof(PCMWAVEFORMAT)) {
@@ -3319,8 +3462,7 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
       ret.max_rate = ret.min_rate = ret.default_rate = pcm->wf.nSamplesPerSec;
       ret.max_channels = pcm->wf.nChannels;
     } else if (fmtvar.blob.cbSize >= sizeof(WAVEFORMATEX)) {
-      WAVEFORMATEX * wfx =
-          reinterpret_cast<WAVEFORMATEX *>(fmtvar.blob.pBlobData);
+      wfx = reinterpret_cast<WAVEFORMATEX *>(fmtvar.blob.pBlobData);
 
       if (fmtvar.blob.cbSize >= sizeof(WAVEFORMATEX) + wfx->cbSize ||
           wfx->wFormatTag == WAVE_FORMAT_PCM) {
@@ -3330,6 +3472,16 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
     }
   }
 
+#if USE_AUDIO_CLIENT_3_LATENCY
+  // Here we guess that an IAudioClient3 stream will successfully be initialized later (it might fail).
+  com_ptr<IAudioClient3> client3;
+  uint32_t def, fun, min, max;
+  if (wfx && SUCCEEDED(dev->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER, NULL, client3.receive_vpp()))
+   && SUCCEEDED(client3->GetSharedModeEnginePeriod(wfx, &def, &fun, &min, &max))) {
+	  ret.latency_lo = min;
+	  ret.latency_hi = def;
+  } else
+#endif
   if (SUCCEEDED(dev->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER,
                               NULL, client.receive_vpp())) &&
       SUCCEEDED(client->GetDevicePeriod(&def_period, &min_period))) {