d3d12: Improve GPU resource gc

Author: halx99

axmol/rhi/CMakeLists.txt

@@ -136,7 +136,6 @@ elseif(AX_RENDER_API STREQUAL "d3d12")
     rhi/d3d12/RenderPipeline12.h
     rhi/d3d12/ShaderModule12.h
     rhi/d3d12/Texture12.h
-    rhi/d3d12/Utils12.h
     rhi/d3d12/Program12.h
     rhi/d3d12/VertexLayout12.h
     rhi/d3d12/DescriptorHeapAllocator12.h
@@ -153,7 +152,6 @@ elseif(AX_RENDER_API STREQUAL "d3d12")
     rhi/d3d12/RenderPipeline12.cpp
     rhi/d3d12/ShaderModule12.cpp
     rhi/d3d12/Texture12.cpp
-    rhi/d3d12/Utils12.cpp
     rhi/d3d12/Program12.cpp
     rhi/d3d12/VertexLayout12.cpp
     rhi/d3d12/DescriptorHeapAllocator12.cpp

axmol/rhi/DXUtils.cpp

@@ -1,4 +1,5 @@
 #include "axmol/rhi/DXUtils.h"
+#include "axmol/platform/Common.h"
 
 namespace ax::rhi::dxutils
 {
@@ -175,4 +176,15 @@ DXGI_FORMAT getUAVCompatibleFormat(DXGI_FORMAT format)
     }
 }
 
+void fatalError(std::string_view op, HRESULT hr)
+{
+    auto msg = fmt::format("{}: 0x{:08X}", op, static_cast<unsigned>(hr));
+#if AX_RENDER_API == AX_RENDER_API_D3D12
+    showAlert(msg, "axmol: D3D12: Fatal Error", AlertStyle::IconError | AlertStyle::RequireSync);
+#else
+    showAlert(msg, "axmol: D3D11: Fatal Error", AlertStyle::IconError | AlertStyle::RequireSync);
+#endif
+    utils::killCurrentProcess();  // kill current process, don't cause crash when driver issue.
+}
+
 }  // namespace ax::rhi::dxutils

axmol/rhi/DXUtils.h

@@ -31,5 +31,15 @@ int evalulateMaxTexSize(D3D_FEATURE_LEVEL fl);
 
 DXGI_FORMAT getUAVCompatibleFormat(DXGI_FORMAT format);
 
+void fatalError(std::string_view op, HRESULT hr);
+
 }  // namespace ax::rhi::dxutils
 /** @} */
+
+#define _AXASSERT_HR(expr)                                          \
+    do                                                              \
+    {                                                               \
+        HRESULT _hr = (expr);                                       \
+        if (FAILED(_hr))                                            \
+            ::ax::rhi::dxutils::fatalError(#expr " failed"sv, _hr); \
+    } while (0)

axmol/rhi/d3d12/Driver12.cpp

@@ -32,7 +32,6 @@
 #include "axmol/rhi/d3d12/DepthStencilState12.h"
 #include "axmol/rhi/d3d12/VertexLayout12.h"
 #include "axmol/rhi/d3d12/Sampler12.h"
-#include "axmol/rhi/d3d12/Utils12.h"
 #include "axmol/base/Logging.h"
 #include "axmol/rhi/RHIUtils.h"
 #include "axmol/rhi/SamplerCache.h"
@@ -416,15 +415,17 @@ void DriverImpl::initializeDevice()
 
     ComPtr<IDXGIAdapter1> adapter;
     ComPtr<IDXGIFactory6> factory6;
-    uint32_t adapterIndex{0};
     hr = _dxgiFactory->QueryInterface(IID_PPV_ARGS(&factory6));
     if (SUCCEEDED(hr))
     {
         // IDXGIFactory6 is not availablee on all versions of windows 10, If it is available, use it
         // to enumerate the adapters based on the desired power preference.
-        while ((hr = factory6->EnumAdapterByGpuPreference(
-                    adapterIndex, gpuPref, IID_PPV_ARGS(adapter.ReleaseAndGetAddressOf()))) != DXGI_ERROR_NOT_FOUND)
+        for (uint32_t adapterIndex = 0;; ++adapterIndex)
         {
+            hr = factory6->EnumAdapterByGpuPreference(adapterIndex, gpuPref,
+                                                      IID_PPV_ARGS(adapter.ReleaseAndGetAddressOf()));
+            if (FAILED(hr))  // when start with Visual Studio graphics debugging, will fail
+                break;
             hr = D3D12CreateDevice(adapter.Get(), D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(&_device));
             if (SUCCEEDED(hr))
             {
@@ -434,10 +435,15 @@ void DriverImpl::initializeDevice()
         }
     }
     else
-    {
+    {  // fallback: win10 lower versions
         std::vector<std::pair<int, ComPtr<IDXGIAdapter1>>> adapters;
-        while (_dxgiFactory->EnumAdapters1(adapterIndex++, adapter.ReleaseAndGetAddressOf()) != DXGI_ERROR_NOT_FOUND)
+        uint32_t adapterIndex{0};
+        for (uint32_t adapterIndex = 0;; ++adapterIndex)
         {
+            hr = _dxgiFactory->EnumAdapters1(adapterIndex, adapter.ReleaseAndGetAddressOf());
+            if (FAILED(hr))
+                break;
+
             DXGI_ADAPTER_DESC desc;
             adapter->GetDesc(&desc);
 
@@ -459,30 +465,36 @@ void DriverImpl::initializeDevice()
             adapters.emplace_back(score, adapter);
         }
 
-        if (powerPreferrence == PowerPreference::HighPerformance)
-        {
-            std::stable_sort(adapters.begin(), adapters.end(),
-                             [](auto& lhs, auto& rhs) { return lhs.first > rhs.first; });
-        }
-        else if (powerPreferrence == PowerPreference::LowPower)
+        if (!adapters.empty())
         {
-            std::stable_sort(adapters.begin(), adapters.end(),
-                             [](auto& lhs, auto& rhs) { return lhs.first < rhs.first; });
-        }
+            if (powerPreferrence == PowerPreference::HighPerformance)
+            {
+                std::stable_sort(adapters.begin(), adapters.end(),
+                                 [](auto& lhs, auto& rhs) { return lhs.first > rhs.first; });
+            }
+            else if (powerPreferrence == PowerPreference::LowPower)
+            {
+                std::stable_sort(adapters.begin(), adapters.end(),
+                                 [](auto& lhs, auto& rhs) { return lhs.first < rhs.first; });
+            }
 
-        for (auto& [_, adapter] : adapters)
-        {
-            hr = D3D12CreateDevice(adapter.Get(), D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(&_device));
-            if (SUCCEEDED(hr))
+            for (auto& [_, adapter] : adapters)
             {
-                _adapter = std::move(adapter);
-                break;
+                hr = D3D12CreateDevice(adapter.Get(), D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(&_device));
+                if (SUCCEEDED(hr))
+                {
+                    _adapter = std::move(adapter);
+                    break;
+                }
             }
         }
     }
 
     if (!_adapter)
-        AX_D3D_FAST_FAIL(hr);
+    {
+        _AXASSERT_HR(D3D12CreateDevice(nullptr, D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(&_device)));
+        _AXASSERT_HR(_dxgiFactory->EnumAdapterByLuid(_device->GetAdapterLuid(), IID_PPV_ARGS(&_adapter)));
+    }
 
     // Create graphics queue
     D3D12_COMMAND_QUEUE_DESC qdesc{};
@@ -803,11 +815,11 @@ void DriverImpl::processDisposalQueue(uint64_t completeFence)
         for (size_t i = 0; i < _disposalQueue.size();)
         {
             auto& res = _disposalQueue[i];
-            if (res.fenceValue < completeFence)
+            if (res.fenceValue <= completeFence)
             {
                 if (res.type == DisposableResource::Type::Resource)
                 {
-                    res.resource->Release();
+                    SafeRelease(res.resource);
                 }
                 else
                 {
@@ -827,7 +839,7 @@ void DriverImpl::processDisposalQueue(uint64_t completeFence)
 
 void DriverImpl::cleanPendingResources()
 {
-    waitDeviceIdle();
+    waitForGPU();
     processDisposalQueue(UINT64_MAX);
 }
 
@@ -1041,9 +1053,9 @@ bool DriverImpl::generateMipmaps(ID3D12GraphicsCommandList* cmd, ID3D12Resource*
     {
         CD3DX12_HEAP_PROPERTIES heapProps(D3D12_HEAP_TYPE_UPLOAD);
         auto bufDesc = CD3DX12_RESOURCE_DESC::Buffer(totalConstSize);
-        if (!CheckHR(getDevice()->CreateCommittedResource(&heapProps, D3D12_HEAP_FLAG_NONE, &bufDesc,
-                                                          D3D12_RESOURCE_STATE_GENERIC_READ, nullptr,
-                                                          IID_PPV_ARGS(&constUpload)),
+        if (!CheckHR(_device->CreateCommittedResource(&heapProps, D3D12_HEAP_FLAG_NONE, &bufDesc,
+                                                      D3D12_RESOURCE_STATE_GENERIC_READ, nullptr,
+                                                      IID_PPV_ARGS(&constUpload)),
                      "Create constant upload buffer"))
             return false;
     }
@@ -1220,7 +1232,7 @@ D3D12BlobHandle DriverImpl::compileMipmapCS(bool isArray)
     return csBlob;
 }
 
-void DriverImpl::waitDeviceIdle()
+void DriverImpl::waitForGPU()
 {
     if (_idleFence && _idleEvent && _gfxQueue)
     {

axmol/rhi/d3d12/Driver12.h

@@ -176,7 +176,7 @@ class DriverImpl : public DriverBase
 
     void processDisposalQueue(uint64_t completeFence);
 
-    void waitDeviceIdle();
+    void waitForGPU() override;
 
 protected:
     void queueDisposalInternal(DisposableResource&& res);

axmol/rhi/d3d12/RenderContext12.cpp

@@ -167,6 +167,18 @@ static HRESULT runOnUIThread(const ComPtr<ICoreDispatcher>& dispatcher, _Fty&& f
 }
 #endif
 
+uint64_t GPUFence::wait() const
+{
+    const auto completeFenceValue = this->handle->GetCompletedValue();
+    if (completeFenceValue < this->value)
+    {
+        this->handle->SetEventOnCompletion(this->value, this->event);
+        WaitForSingleObject(this->event, INFINITE);
+        return this->value;
+    }
+    return completeFenceValue;
+}
+
 RenderContextImpl::RenderContextImpl(DriverImpl* driver, void* surfaceContext) : _driver(driver)
 {
     _device        = driver->getDevice();
@@ -342,17 +354,22 @@ RenderContextImpl::RenderContextImpl(DriverImpl* driver, void* surfaceContext) :
 
 RenderContextImpl::~RenderContextImpl()
 {
-    _driver->waitDeviceIdle();
+    _driver->waitForGPU();
 
     AX_SAFE_RELEASE_NULL(_screenRT);
     AX_SAFE_RELEASE_NULL(_renderPipeline);
 
     for (int i = 0; i < MAX_FRAMES_IN_FLIGHT; ++i)
     {
-        if (_fenceEvents[i])
+        if (_inflightFences[i].event)
+        {
+            CloseHandle(_inflightFences[i].event);
+            _inflightFences[i].event = nullptr;
+        }
+
+        if (_inflightFences[i].handle)
         {
-            CloseHandle(_fenceEvents[i]);
-            _fenceEvents[i] = nullptr;
+            SafeRelease(_inflightFences[i].handle);
         }
 
         if (_srvHeaps[i])
@@ -380,11 +397,10 @@ void RenderContextImpl::createCommandObjects()
         _commandLists[i]->Close();
 
         // Fence + event
-        hr = _device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&_fences[i]));
+        hr = _device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&_inflightFences[i].handle));
         AXASSERT(SUCCEEDED(hr), "CreateFence failed");
-        _fenceValues[i] = 0;
-        _fenceEvents[i] = CreateEvent(nullptr, FALSE, FALSE, nullptr);
-        AXASSERT(_fenceEvents[i] != nullptr, "CreateEvent failed");
+        _inflightFences[i].event = CreateEvent(nullptr, FALSE, FALSE, nullptr);
+        AXASSERT(!!_inflightFences[i], "CreateEvent failed");
     }
 }
 
@@ -428,32 +444,24 @@ void RenderContextImpl::setRenderPipeline(RenderPipeline* renderPipeline)
 
 uint64_t RenderContextImpl::getCompletedFenceValue() const
 {
-    return _fenceValues[_currentFrame];
+    return _completedFenceValue;
 }
 
 bool RenderContextImpl::beginFrame()
 {
     // Wait fence of current frame
-    auto fence = _fences[_currentFrame];
-
-    const auto completeFenceValue = fence->GetCompletedValue();
-    const auto frameFenceValue    = _fenceValues[_currentFrame];
-    if (completeFenceValue < frameFenceValue)
-    {
-        fence->SetEventOnCompletion(frameFenceValue, _fenceEvents[_currentFrame]);
-        WaitForSingleObject(_fenceEvents[_currentFrame], INFINITE);
-    }
-
-    _advanceFenceValues[_currentFrame] = frameFenceValue + 1;
+    auto& currentFence   = _inflightFences[_currentFrame];
+    _completedFenceValue = currentFence.wait();
+    _driver->processDisposalQueue(_completedFenceValue);
 
-    if (!_fenceCompletionOps.empty())
+    if (!_frameCompletionOps.empty())
     {
-        for (auto& op : _fenceCompletionOps)
-            op(completeFenceValue);
-        _fenceCompletionOps.clear();
+        for (auto&& op : _frameCompletionOps)
+            op(_completedFenceValue);
+        _frameCompletionOps.clear();
     }
 
-    _driver->processDisposalQueue(completeFenceValue);
+    currentFence.value = ++_frameFenceValue;
 
     if (_swapchainDirty)
     {
@@ -520,7 +528,7 @@ void RenderContextImpl::beginRenderPass(RenderTarget* renderTarget, const Render
     // Bind RTV/DSV and clear according to flags
     rtImpl->beginRenderPass(_currentCmdList, descriptor, _renderTargetWidth, _renderTargetHeight, _currentImageIndex);
 
-    rtImpl->setLastFenceValue(_advanceFenceValues[_currentFrame]);
+    rtImpl->setLastFenceValue(_frameFenceValue);
 }
 
 void RenderContextImpl::endRenderPass()
@@ -563,8 +571,8 @@ void RenderContextImpl::endFrame()
 #endif
 
     // Signal fence for this frame
-    _graphicsQueue->Signal(_fences[_currentFrame].Get(), _advanceFenceValues[_currentFrame]);
-    _fenceValues[_currentFrame] = _advanceFenceValues[_currentFrame];
+    auto& currentFence = _inflightFences[_currentFrame];
+    _graphicsQueue->Signal(currentFence.handle, currentFence.value);
 
     // Next frame index
     _currentFrame = (_currentFrame + 1) % MAX_FRAMES_IN_FLIGHT;
@@ -849,8 +857,7 @@ void RenderContextImpl::prepareDrawing(ID3D12GraphicsCommandList* cmd)
 
     applyPendingDynamicStates();
 
-    const auto advanceFenceValue = _advanceFenceValues[_currentFrame];
-    _vertexBuffer->setLastFenceValue(advanceFenceValue);
+    _vertexBuffer->setLastFenceValue(_frameFenceValue);
 
     // bind vertex buffers
     if (!_instanceBuffer)
@@ -863,7 +870,7 @@ void RenderContextImpl::prepareDrawing(ID3D12GraphicsCommandList* cmd)
     }
     else
     {
-        _instanceBuffer->setLastFenceValue(advanceFenceValue);
+        _instanceBuffer->setLastFenceValue(_frameFenceValue);
         D3D12_VERTEX_BUFFER_VIEW views[2]{};
         views[0].BufferLocation = _vertexBuffer->internalResource()->GetGPUVirtualAddress();
         views[0].SizeInBytes    = static_cast<UINT>(_vertexBuffer->getSize());
@@ -919,7 +926,7 @@ void RenderContextImpl::prepareDrawing(ID3D12GraphicsCommandList* cmd)
                     maxSlot = slot;
 
                 auto textureImpl = static_cast<TextureImpl*>(bindingSet.texs[i]);
-                textureImpl->setLastFenceValue(advanceFenceValue);
+                textureImpl->setLastFenceValue(_frameFenceValue);
                 auto srvHandle = textureImpl->internalHandle().srv;
                 assert(!!srvHandle);
 
@@ -1057,7 +1064,7 @@ void RenderContextImpl::readPixels(RenderTarget* rt,
     }
     rt->retain();
 
-    _fenceCompletionOps.emplace_back([this, rt, preserveAxisHint, callback = std::move(callback)](uint64_t) mutable {
+    _frameCompletionOps.emplace_back([this, rt, preserveAxisHint, callback = std::move(callback)](uint64_t) mutable {
         readPixelsInternal(rt, preserveAxisHint, callback);
 
         rt->release();