Make sure the inverted cursor is always readable (#3647) (#13748)

Currently "Inverse Cursor" is actually simply bitwise inversed.
It works fine, except when it does not, namely in the middle of the spectrum:
Anything close enough to dark grey (index 8, RGB(128, 128, 128) in the
classic palette) will for obvious reasons become almost the same dark grey
again after the inversion.

The issue is addressed by additionally `XOR`ing the inverted color with
RGB(63, 63, 63). This distorts the result enough to avoid collisions
in the middle. Ultimately this restores the behavior that was in
Windows Console since the Middle Ages (and still exists in ConhostV1.dll).

## PR Checklist
* [x] Closes #3647
* [x] CLA signed
* [x] Tests added/passed
* [x] I've discussed this with core contributors already. If not checked, I'm ready to accept this work might be rejected in favor of a different grand plan. Issue number where discussion took place: #3647

## Validation Steps Performed
1. Open OpenConsole.
2. Properties, Terminal, Cursor Colors, choose Inverse Color.
3. Optionally set the RGB value of the 8th color in the standard palette to RGB(128, 128, 128) on the Colors tab, but the default one will do too.
4. Type `color 80` to see black text on dark grey background.
5. Make sure the inverted cursor is visible.
6. Repeat in WT with both default and experimental renderers.

.github/actions/spelling/allow/apis.txt

@@ -103,6 +103,7 @@ lround
 Lsa
 lsass
 LSHIFT
+LTGRAY
 MAINWINDOW
 memchr
 memicmp
@@ -146,6 +147,7 @@ OUTLINETEXTMETRICW
 overridable
 PACL
 PAGESCROLL
+PATINVERT
 PEXPLICIT
 PICKFOLDERS
 pmr

src/renderer/atlas/shader_ps.hlsl

@@ -101,13 +101,27 @@ float4 main(float4 pos: SV_Position): SV_Target
     // Colored cursors are drawn "in between" the background color and the text of a cell.
     [branch] if (cell.flags & CellFlags_Cursor)
     {
-        [flatten] if (cursorColor != INVALID_COLOR)
+        [branch] if (cursorColor != INVALID_COLOR)
         {
             // The cursor texture is stored at the top-left-most glyph cell.
             // Cursor pixels are either entirely transparent or opaque.
             // --> We can just use .a as a mask to flip cursor pixels on or off.
             color = alphaBlendPremultiplied(color, decodeRGBA(cursorColor) * glyphs[cellPos].a);
         }
+        else if (glyphs[cellPos].a != 0)
+        {
+            // Make sure the cursor is always readable (see gh-3647)
+            // If we imagine the two colors to be in 0-255 instead of 0-1,
+            // this effectively XORs them with 63. This avoids a situation
+            // where a gray background color (0.5) gets inverted to the
+            // same gray making the cursor invisible.
+            float2x4 colors = { color, fg };
+            float2x4 ip; // integral part
+            float2x4 frac = modf(colors * (255.0f / 64.0f), ip);
+            colors = (3.0f - ip + frac) * (64.0f / 255.0f);
+            color = float4(colors[0].rgb, 1);
+            fg = float4(colors[1].rgb, 1);
+        }
     }
 
     // Layer 2:
@@ -146,7 +160,7 @@ float4 main(float4 pos: SV_Position): SV_Target
         // What a nice coincidence that we have exactly 8 flags to handle right now!
         // `mask` will mask away any positive results from checks we don't want.
         // (I.e. even if we're in an underline, it doesn't matter if we don't want an underline.)
-        bool4x2 mask = {
+        bool2x4 mask = {
             cell.flags & CellFlags_BorderLeft,
             cell.flags & CellFlags_BorderTop,
             cell.flags & CellFlags_BorderRight,
@@ -159,7 +173,7 @@ float4 main(float4 pos: SV_Position): SV_Target
         // The following <lineWidth checks rely on underflow turning the
         // uint into a way larger number than any reasonable lineWidth.
         // That way we don't need to write `y >= lo && y < hi`.
-        bool4x2 checks = {
+        bool2x4 checks = {
             // These 2 expand to 4 bools, because cellPos is a
             // uint2 vector which results in a bool2 result each.
             cellPos < thinLineWidth,
@@ -176,16 +190,6 @@ float4 main(float4 pos: SV_Position): SV_Target
         }
     }
 
-    // Layer 3 (optional):
-    // Uncolored cursors are used as a mask that inverts the cells color.
-    [branch] if (cell.flags & CellFlags_Cursor)
-    {
-        [flatten] if (cursorColor == INVALID_COLOR && glyphs[cellPos].a != 0)
-        {
-            color = float4(1 - color.rgb, 1);
-        }
-    }
-
     // Layer 4:
     // The current selection is drawn semi-transparent on top.
     [branch] if (cell.flags & CellFlags_Selected)

src/renderer/dx/CustomTextRenderer.cpp

@@ -395,8 +395,8 @@ try
         if (firstPass)
         {
             // Draw a backplate behind the cursor in the *background* color so that we can invert it later.
-            // We're going to draw the exact same color as the background behind the cursor
-            const til::color color{ drawingContext.backgroundBrush->GetColor() };
+            // Make sure the cursor is always readable (see gh-3647)
+            const til::color color{ til::color{ drawingContext.backgroundBrush->GetColor() } ^ RGB(63, 63, 63) };
             RETURN_IF_FAILED(d2dContext->CreateSolidColorBrush(color.with_alpha(255),
                                                                &brush));
         }

src/renderer/gdi/paint.cpp

@@ -733,7 +733,11 @@ bool GdiEngine::FontHasWesternScript(HDC hdc)
 
         for (auto r : cursorInvertRects)
         {
-            RETURN_HR_IF(E_FAIL, !(InvertRect(_hdcMemoryContext, &r)));
+            // Make sure the cursor is always readable (see gh-3647)
+            const auto PrevObject = SelectObject(_hdcMemoryContext, GetStockObject(LTGRAY_BRUSH));
+            const auto Result = PatBlt(_hdcMemoryContext, r.left, r.top, r.right - r.left, r.bottom - r.top, PATINVERT);
+            SelectObject(_hdcMemoryContext, PrevObject);
+            RETURN_HR_IF(E_FAIL, !Result);
         }
     }