When the renderer fails, try to fall back to D2D + WARP; retry changes (#19636)

This commit also ups the number of render failures that are permissible
to 6 (one try plus 5 retries), and moves us to use an exponential
backoff rather than a simple geometric one.

It also suppresses the dialog box in case of present failures for Stable
users. I feel like the warning dialog should be used for something that
the user can actually do something about...

Closes #15601
Closes #18198

src/cascadia/TerminalControl/ControlCore.cpp

@@ -152,12 +152,27 @@ namespace winrt::Microsoft::Terminal::Control::implementation
 
             _renderer->SetBackgroundColorChangedCallback([this]() { _rendererBackgroundColorChanged(); });
             _renderer->SetFrameColorChangedCallback([this]() { _rendererTabColorChanged(); });
-            _renderer->SetRendererEnteredErrorStateCallback([this]() { RendererEnteredErrorState.raise(nullptr, nullptr); });
+            _renderer->SetRendererEnteredErrorStateCallback([this]() { _rendererEnteredErrorState(); });
         }
 
         UpdateSettings(settings, unfocusedAppearance);
     }
 
+    void ControlCore::_rendererEnteredErrorState()
+    {
+        // The first time the renderer fails out (after all of its own retries), switch it to D2D and WARP
+        // and force it to try again. If it _still_ fails, we can let it halt.
+        if (_renderFailures++ == 0)
+        {
+            const auto lock = _terminal->LockForWriting();
+            _renderEngine->SetGraphicsAPI(parseGraphicsAPI(GraphicsAPI::Direct2D));
+            _renderEngine->SetSoftwareRendering(true);
+            _renderer->EnablePainting();
+            return;
+        }
+        RendererEnteredErrorState.raise(nullptr, nullptr);
+    }
+
     void ControlCore::_setupDispatcherAndCallbacks()
     {
         // Get our dispatcher. If we're hosted in-proc with XAML, this will get
@@ -917,6 +932,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         _renderEngine->SetSoftwareRendering(_settings.SoftwareRendering());
         // Inform the renderer of our opacity
         _renderEngine->EnableTransparentBackground(_isBackgroundTransparent());
+        _renderFailures = 0; // We may have changed the engine; reset the failure counter.
 
         // Trigger a redraw to repaint the window background and tab colors.
         _renderer->TriggerRedrawAll(true, true);
@@ -1983,6 +1999,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
     {
         // The lock must be held, because it calls into IRenderData which is shared state.
         const auto lock = _terminal->LockForWriting();
+        _renderFailures = 0;
         _renderer->EnablePainting();
     }
 

src/cascadia/TerminalControl/ControlCore.h

@@ -344,6 +344,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         safe_void_coroutine _renderEngineSwapChainChanged(const HANDLE handle);
         void _rendererBackgroundColorChanged();
         void _rendererTabColorChanged();
+        void _rendererEnteredErrorState();
 #pragma endregion
 
         void _raiseReadOnlyWarning();
@@ -398,6 +399,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         float _panelWidth{ 0 };
         float _panelHeight{ 0 };
         float _compositionScale{ 0 };
+        uint8_t _renderFailures{ 0 };
         bool _forceCursorVisible = false;
 
         // Audio stuff.

src/features.xml

@@ -68,6 +68,13 @@
         </alwaysEnabledBrandingTokens>
     </feature>
 
+    <feature>
+        <name>Feature_AtlasEngineLoudErrors</name>
+        <description>Atlas Engine can optionally support signaling every presentation failure to its consumer. For now, we only want that to happen in non-Release builds.</description>
+        <stage>AlwaysEnabled</stage>
+        <alwaysDisabledReleaseTokens/>
+    </feature>
+
     <feature>
         <name>Feature_NearbyFontLoading</name>
         <description>Controls whether fonts in the same directory as the binary are used during rendering. Disabled for conhost so that it doesn't iterate the entire system32 directory.</description>

src/renderer/atlas/AtlasEngine.r.cpp

@@ -61,13 +61,18 @@ catch (const wil::ResultException& exception)
         return E_PENDING;
     }
 
-    if (_p.warningCallback)
+    if constexpr (Feature_AtlasEngineLoudErrors::IsEnabled())
     {
-        try
+        // We may fail to present repeatedly, e.g. if there's a short-term device failure.
+        // We should not bombard the consumer with repeated warning callbacks (where they may present a dialog to the user).
+        if (_p.warningCallback)
         {
-            _p.warningCallback(hr, {});
+            try
+            {
+                _p.warningCallback(hr, {});
+            }
+            CATCH_LOG()
         }
-        CATCH_LOG()
     }
 
     _b.reset();

src/renderer/base/renderer.cpp

@@ -12,9 +12,10 @@ using namespace Microsoft::Console::Types;
 using PointTree = interval_tree::IntervalTree<til::point, size_t>;
 
 static constexpr TimerRepr TimerReprMax = std::numeric_limits<TimerRepr>::max();
-static constexpr DWORD maxRetriesForRenderEngine = 3;
+// We want there to be five retry periods; after the last one, we will mark the render as failed.
-// The renderer will wait this number of milliseconds * how many tries have elapsed before trying again.
+static constexpr unsigned int maxRetriesForRenderEngine = 5;
-static constexpr DWORD renderBackoffBaseTimeMilliseconds = 150;
+// The renderer will wait this number of milliseconds * 2^tries before trying again.
+static constexpr DWORD renderBackoffBaseTimeMilliseconds = 100;
 
 // Routine Description:
 // - Creates a new renderer controller for a console.
@@ -326,40 +327,44 @@ DWORD Renderer::_timerToMillis(TimerRepr t) noexcept
 // - HRESULT S_OK, GDI error, Safe Math error, or state/argument errors.
 [[nodiscard]] HRESULT Renderer::PaintFrame()
 {
-    auto tries = maxRetriesForRenderEngine;
+    HRESULT hr{ S_FALSE };
-    while (tries > 0)
+    // Attempt zero doesn't count as a retry. We should try maxRetries + 1 times.
+    for (unsigned int attempt = 0u; attempt <= maxRetriesForRenderEngine; ++attempt)
     {
+        if (attempt > 0) [[unlikely]]
+        {
+            // Add a bit of backoff.
+            // Sleep 100, 200, 400, 600, 800ms, 1600ms before failing out and disabling the renderer.
+            Sleep(renderBackoffBaseTimeMilliseconds * (1 << (attempt - 1)));
+        }
+
         // BODGY: Optimally we would want to retry per engine, but that causes different
         // problems (intermittent inconsistent states between text renderer and UIA output,
         // not being able to lock the cursor location, etc.).
-        const auto hr = _PaintFrame();
+        hr = _PaintFrame();
         if (SUCCEEDED(hr))
         {
             break;
         }
 
         LOG_HR_IF(hr, hr != E_PENDING);
-
-        if (--tries == 0)
-        {
-            // Stop trying.
-            _disablePainting();
-            if (_pfnRendererEnteredErrorState)
-            {
-                _pfnRendererEnteredErrorState();
-            }
-            // If there's no callback, we still don't want to FAIL_FAST: the renderer going black
-            // isn't near as bad as the entire application aborting. We're a component. We shouldn't
-            // abort applications that host us.
-            return S_FALSE;
-        }
-
-        // Add a bit of backoff.
-        // Sleep 150ms, 300ms, 450ms before failing out and disabling the renderer.
-        Sleep(renderBackoffBaseTimeMilliseconds * (maxRetriesForRenderEngine - tries));
     }
 
-    return S_OK;
+    if (FAILED(hr))
+    {
+        // Stop trying.
+        _disablePainting();
+        if (_pfnRendererEnteredErrorState)
+        {
+            _pfnRendererEnteredErrorState();
+        }
+        // If there's no callback, we still don't want to FAIL_FAST: the renderer going black
+        // isn't near as bad as the entire application aborting. We're a component. We shouldn't
+        // abort applications that host us.
+        hr = S_FALSE;
+    }
+
+    return hr;
 }
 
 [[nodiscard]] HRESULT Renderer::_PaintFrame() noexcept