Warn when using old MacType versions (#17369)

This adds a check for whether MacType is injected and whether it's
a known bad version (pre-2023). In that case we avoid calling the
known faulty `ID2D1Device4` interface. We could avoid it in general to
fix the issue without a warning (it's only a very mild optimization),
but on the other hand, the bug that MacType has is a very serious one
and it's probably better overall to suggest users to update.

See: https://github.com/snowie2000/mactype/pull/938

## Validation Steps Performed
* MacType 2021.1-RC1 results in a warning and no crash ✅
* MacType 2023.5.31 results in no warning ✅

---------

Co-authored-by: Dustin L. Howett <duhowett@microsoft.com>

src/cascadia/TerminalControl/Resources/en-US/Resources.resw

@@ -217,6 +217,10 @@ Please either install the missing font or choose another one.</value>
     <value>Unable to find the following fonts: {0}. Please either install them or choose different fonts.</value>
     <comment>{Locked="{0}"} This is a warning dialog shown when the user selects a font that isn't installed.</comment>
   </data>
+  <data name="RendererErrorMacType" xml:space="preserve">
+    <value>Your version of MacType is incompatible with this application. Please update to version 2023.5.31 or later.</value>
+    <comment>{Locked="2023.5.31","MacType"}</comment>
+  </data>
   <data name="RendererErrorOther" xml:space="preserve">
     <value>Renderer encountered an unexpected error: {0:#010x} {1}</value>
     <comment>{Locked="{0:#010x}","{1}"} {0:#010x} is a placeholder for a Windows error code (e.g. 0x88985002). {1} is the corresponding message.</comment>

src/cascadia/TerminalControl/TermControl.cpp

@@ -1143,6 +1143,9 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         case DWRITE_E_NOFONT:
             message = winrt::hstring{ fmt::format(std::wstring_view{ RS_(L"RendererErrorFontNotFound") }, parameter) };
             break;
+        case ATLAS_ENGINE_ERROR_MAC_TYPE:
+            message = RS_(L"RendererErrorMacType");
+            break;
         default:
         {
             wchar_t buf[512];

src/renderer/atlas/BackendD3D.cpp

@@ -37,6 +37,7 @@ TIL_FAST_MATH_BEGIN
 #pragma warning(disable : 26459) // You called an STL function '...' with a raw pointer parameter at position '...' that may be unsafe [...].
 #pragma warning(disable : 26481) // Don't use pointer arithmetic. Use span instead (bounds.1).
 #pragma warning(disable : 26482) // Only index into arrays using constant expressions (bounds.2).
+#pragma warning(disable : 26490) // Don't use reinterpret_cast (type.1).
 
 // Initializing large arrays can be very costly compared to how cheap some of these functions are.
 #define ALLOW_UNINITIALIZED_BEGIN _Pragma("warning(push)") _Pragma("warning(disable : 26494)")
@@ -797,6 +798,13 @@ void BackendD3D::_resetGlyphAtlas(const RenderingPayload& p)
 
 void BackendD3D::_resizeGlyphAtlas(const RenderingPayload& p, const u16 u, const u16 v)
 {
+#if defined(_M_X64) || defined(_M_IX86)
+    static const auto faultyMacTypeVersion = _checkMacTypeVersion(p);
+#else
+    // The affected versions of MacType are unavailable on ARM.
+    static constexpr auto faultyMacTypeVersion = false;
+#endif
+
     _d2dRenderTarget.reset();
     _d2dRenderTarget4.reset();
     _glyphAtlas.reset();
@@ -842,9 +850,13 @@ void BackendD3D::_resizeGlyphAtlas(const RenderingPayload& p, const u16 u, const
         _d2dRenderTarget4->GetDevice(device.addressof());
 
         device->SetMaximumTextureMemory(0);
-        if (const auto device4 = device.try_query<ID2D1Device4>())
+
+        if (!faultyMacTypeVersion)
         {
-            device4->SetMaximumColorGlyphCacheMemory(0);
+            if (const auto device4 = device.try_query<ID2D1Device4>())
+            {
+                device4->SetMaximumColorGlyphCacheMemory(0);
+            }
         }
     }
 
@@ -859,6 +871,62 @@ void BackendD3D::_resizeGlyphAtlas(const RenderingPayload& p, const u16 u, const
     _rectPackerData = Buffer<stbrp_node>{ u };
 }
 
+// MacType is a popular 3rd party system to give the font rendering on Windows a softer look.
+// It's particularly popular in China. Unfortunately, it hooks ID2D1Device4 incorrectly:
+//   https://github.com/snowie2000/mactype/pull/938
+// This results in crashes. Not a lot of them, but enough to constantly show up.
+// The issue was fixed in the MacType v1.2023.5.31 release, the only one in 2023.
+//
+// Please feel free to remove this check in a few years.
+bool BackendD3D::_checkMacTypeVersion(const RenderingPayload& p)
+{
+#ifdef _WIN64
+    static constexpr auto name = L"MacType64.Core.dll";
+#else
+    static constexpr auto name = L"MacType.Core.dll";
+#endif
+
+    wil::unique_hmodule handle;
+    if (!GetModuleHandleExW(0, name, handle.addressof()))
+    {
+        return false;
+    }
+
+    const auto resource = FindResourceW(handle.get(), MAKEINTRESOURCE(VS_VERSION_INFO), RT_VERSION);
+    if (!resource)
+    {
+        return false;
+    }
+
+    const auto dataHandle = LoadResource(handle.get(), resource);
+    if (!dataHandle)
+    {
+        return false;
+    }
+
+    const auto data = LockResource(dataHandle);
+    if (!data)
+    {
+        return false;
+    }
+
+    VS_FIXEDFILEINFO* info;
+    UINT varLen = 0;
+    if (!VerQueryValueW(data, L"\\", reinterpret_cast<void**>(&info), &varLen))
+    {
+        return false;
+    }
+
+    const auto faulty = info->dwFileVersionMS < (1 << 16 | 2023);
+
+    if (faulty && p.warningCallback)
+    {
+        p.warningCallback(ATLAS_ENGINE_ERROR_MAC_TYPE, {});
+    }
+
+    return faulty;
+}
+
 BackendD3D::QuadInstance& BackendD3D::_getLastQuad() noexcept
 {
     assert(_instancesCount != 0);

src/renderer/atlas/BackendD3D.h

@@ -204,6 +204,7 @@ namespace Microsoft::Console::Render::Atlas
         void _d2dEndDrawing();
         ATLAS_ATTR_COLD void _resetGlyphAtlas(const RenderingPayload& p);
         ATLAS_ATTR_COLD void _resizeGlyphAtlas(const RenderingPayload& p, u16 u, u16 v);
+        static bool _checkMacTypeVersion(const RenderingPayload& p);
         QuadInstance& _getLastQuad() noexcept;
         QuadInstance& _appendQuad();
         ATLAS_ATTR_COLD void _bumpInstancesSize();

src/renderer/atlas/common.h

@@ -53,6 +53,8 @@ namespace Microsoft::Console::Render::Atlas
     // My best effort of replicating __attribute__((cold)) from gcc/clang.
 #define ATLAS_ATTR_COLD __declspec(noinline)
 
+#define ATLAS_ENGINE_ERROR_MAC_TYPE MAKE_HRESULT(SEVERITY_SUCCESS, FACILITY_ITF, 'MT')
+
     template<typename T>
     struct vec2
     {