Implement WSLA GPU support (#13400)

* Implement WSLA GPU support

* Spelling

* Fix remaining typos

* More typos

* Merge

* Remove extra logline

src/windows/common/helpers.cpp

@@ -24,6 +24,7 @@ Abstract:
 #include <gslhelpers.h>
 #include "registry.hpp"
 #include "versionhelpers.h"
+#include "hcs.hpp"
 #include <regstr.h>
 
 // Version numbers for various functionality that was backported.
@@ -460,6 +461,14 @@ std::filesystem::path wsl::windows::common::helpers::GetWslConfigPath(_In_opt_ H
     return wsl::windows::common::helpers::GetUserProfilePath(userToken) / L".wslconfig";
 }
 
+bool wsl::windows::common::helpers::IsDisableVgpuSettingsSupported()
+{
+    static constexpr std::pair<uint32_t, uint32_t> c_schemaVersionNickel{2, 7};
+
+    // See if the Windows version has the required platform change.
+    return ((wsl::windows::common::hcs::GetSchemaVersion() >= c_schemaVersionNickel) && (GetWindowsVersion().BuildNumber >= 22545));
+}
+
 bool wsl::windows::common::helpers::IsPackageInstalled(_In_ LPCWSTR PackageFamilyName)
 {
     UINT32 packageCount = 0;

src/windows/common/helpers.hpp

@@ -150,6 +150,8 @@ std::filesystem::path GetUserProfilePath(_In_opt_ HANDLE userToken = nullptr);
 
 std::filesystem::path GetWslConfigPath(_In_opt_ HANDLE userToken = nullptr);
 
+bool IsDisableVgpuSettingsSupported();
+
 bool IsPackageInstalled(_In_ LPCWSTR PackageFamilyName);
 
 bool IsServicePresent(_In_ LPCWSTR ServiceName);

src/windows/lswclient/DllMain.cpp

@@ -88,6 +88,7 @@ try
     settings.EnableEarlyBootDmesg = UserSettings->Options.EnableEarlyBootDmesg;
     settings.NetworkingMode = UserSettings->Networking.Mode;
     settings.EnableDnsTunneling = UserSettings->Networking.DnsTunneling;
+    settings.EnableGPU = UserSettings->GPU.Enable;
 
     THROW_IF_FAILED(session->CreateVirtualMachine(&settings, &virtualMachineInstance));
     ConfigureComSecurity(virtualMachineInstance.get());
@@ -429,4 +430,11 @@ try
 {
     return reinterpret_cast<ILSWVirtualMachine*>(VirtualMachine)->UnmountWindowsFolder(Target);
 }
+CATCH_RETURN();
+
+HRESULT WslMountGpuLibraries(LSWVirtualMachineHandle VirtualMachine, const char* LibrariesMountPoint, const char* DriversMountpoint)
+try
+{
+    return reinterpret_cast<ILSWVirtualMachine*>(VirtualMachine)->MountGpuLibraries(LibrariesMountPoint, DriversMountpoint);
+}
 CATCH_RETURN();

src/windows/lswclient/LSWApi.h

@@ -71,13 +71,19 @@ struct Networking
     bool DnsTunneling; // Not implemented yet.
 };
 
+struct GPU
+{
+    bool Enable;
+};
+
 struct VirtualMachineSettings
 {
-    LPCWSTR DisplayName;  // Not implemented yet
-    struct Memory Memory; // Not implemented yet
-    struct CPU CPU;       // Not implemented yet
+    LPCWSTR DisplayName;
+    struct Memory Memory;
+    struct CPU CPU;
     struct Options Options;
     struct Networking Networking;
+    struct GPU GPU;
 };
 
 typedef void* LSWVirtualMachineHandle;
@@ -209,6 +215,8 @@ HRESULT WslMountWindowsFolder(LSWVirtualMachineHandle VirtualMachine, LPCWSTR Wi
 
 HRESULT WslUnmountWindowsFolder(LSWVirtualMachineHandle VirtualMachine, const char* LinuxPath);
 
+HRESULT WslMountGpuLibraries(LSWVirtualMachineHandle VirtualMachine, const char* LibrariesMountPoint, const char* DriversMountpoint);
+
 #ifdef __cplusplus
 }
 #endif

src/windows/lswclient/lswclient.def

@@ -21,3 +21,4 @@ EXPORTS
     WslSetPackageUrl
     WslMountWindowsFolder
     WslUnmountWindowsFolder
+    WslMountGpuLibraries

src/windows/service/exe/LSWVirtualMachine.cpp

@@ -272,6 +272,23 @@ void LSWVirtualMachine::Start()
     m_initChannel = wsl::shared::SocketChannel{std::move(socket), "mini_init", m_vmTerminatingEvent.get()};
 
     ConfigureNetworking();
+
+    // Configure GPU if requested.
+    if (m_settings.EnableGPU)
+    {
+        hcs::ModifySettingRequest<hcs::GpuConfiguration> gpuRequest{};
+        gpuRequest.ResourcePath = L"VirtualMachine/ComputeTopology/Gpu";
+        gpuRequest.RequestType = hcs::ModifyRequestType::Update;
+        gpuRequest.Settings.AssignmentMode = hcs::GpuAssignmentMode::Mirror;
+        gpuRequest.Settings.AllowVendorExtension = true;
+        if (wsl::windows::common::helpers::IsDisableVgpuSettingsSupported())
+        {
+            gpuRequest.Settings.DisableGdiAcceleration = true;
+            gpuRequest.Settings.DisablePresentation = true;
+        }
+
+        wsl::windows::common::hcs::ModifyComputeSystem(m_computeSystem.get(), wsl::shared::ToJsonW(gpuRequest).c_str());
+    }
 }
 
 void LSWVirtualMachine::ConfigureNetworking()
@@ -921,4 +938,52 @@ try
 
     return S_OK;
 }
+CATCH_RETURN();
+
+HRESULT LSWVirtualMachine::MountGpuLibraries(_In_ LPCSTR LibrariesMountPoint, _In_ LPCSTR DriversMountpoint)
+try
+{
+    RETURN_HR_IF(HRESULT_FROM_WIN32(ERROR_INVALID_CONFIG_VALUE), !m_settings.EnableGPU);
+
+    auto [channel, _, __] = Fork(LSW_FORK::Thread);
+
+    auto windowsPath = wil::GetWindowsDirectoryW<std::wstring>();
+
+    // Mount drivers.
+    RETURN_IF_FAILED(MountWindowsFolder(std::format(L"{}\\System32\\DriverStore\\FileRepository", windowsPath).c_str(), DriversMountpoint, true));
+
+    // Mount the inbox libraries.
+    auto inboxLibPath = std::format(L"{}\\System32\\lxss\\lib", windowsPath);
+    std::optional<std::string> inboxLibMountPoint;
+    if (std::filesystem::is_directory(inboxLibPath))
+    {
+        inboxLibMountPoint = std::format("{}/inbox", LibrariesMountPoint);
+        RETURN_IF_FAILED(MountWindowsFolder(inboxLibPath.c_str(), inboxLibMountPoint->c_str(), true));
+    }
+
+    // Mount the packaged libraries.
+
+#ifdef WSL_GPU_LIB_PATH
+
+    auto packagedLibPath = std::filesystem::path(TEXT(WSL_GPU_LIB_PATH));
+
+#else
+
+    auto packagedLibPath = wslutil::GetBasePath() / L"lib";
+
+#endif
+
+    auto packagedLibMountPoint = std::format("{}/packaged", LibrariesMountPoint);
+    RETURN_IF_FAILED(MountWindowsFolder(packagedLibPath.c_str(), packagedLibMountPoint.c_str(), true));
+
+    // Mount an overlay containing both inbox and packaged libraries (the packaged mount takes precedence).
+    std::string options = "lowerdir=" + packagedLibMountPoint;
+    if (inboxLibMountPoint.has_value())
+    {
+        options += ":" + inboxLibMountPoint.value();
+    }
+
+    RETURN_IF_FAILED(Mount("none", LibrariesMountPoint, "overlay", options.c_str(), 0));
+    return S_OK;
+}
 CATCH_RETURN();

src/windows/service/exe/LSWVirtualMachine.h

@@ -46,6 +46,7 @@ public:
     IFACEMETHOD(DetachDisk(_In_ ULONG Lun)) override;
     IFACEMETHOD(MountWindowsFolder(_In_ LPCWSTR WindowsPath, _In_ LPCSTR LinuxPath, _In_ BOOL ReadOnly)) override;
     IFACEMETHOD(UnmountWindowsFolder(_In_ LPCSTR LinuxPath)) override;
+    IFACEMETHOD(MountGpuLibraries(_In_ LPCSTR LibrariesMountPoint, _In_ LPCSTR DriversMountpoint)) override;
 
 private:
     static int32_t MountImpl(wsl::shared::SocketChannel& Channel, LPCSTR Source, _In_ LPCSTR Target, _In_ LPCSTR Type, _In_ LPCSTR Options, _In_ ULONG Flags);

src/windows/service/exe/WslCoreVm.cpp

@@ -56,7 +56,6 @@ DEFINE_GUID(VIRTIO_PMEM_DEVICE_ID, 0xEDBB24BB, 0x5E19, 0x40F4, 0x8A, 0x0F, 0x82,
 
 static constexpr size_t c_bootEntropy = 0x1000;
 static constexpr auto c_localDevicesKey = L"SOFTWARE\\Microsoft\\Terminal Server Client\\LocalDevices";
-static constexpr std::pair<uint32_t, uint32_t> c_schemaVersionNickel{2, 7};
 
 // {ABB755FC-1B86-4255-83E2-E5787ABCF6C2}
 static constexpr GUID c_pmemClassId = {0xABB755FC, 0x1B86, 0x4255, {0x83, 0xe2, 0xe5, 0x78, 0x7a, 0xbc, 0xf6, 0xc2}};
@@ -369,7 +368,7 @@ void WslCoreVm::Initialize(const GUID& VmId, const wil::shared_handle& UserToken
         gpuRequest.RequestType = hcs::ModifyRequestType::Update;
         gpuRequest.Settings.AssignmentMode = hcs::GpuAssignmentMode::Mirror;
         gpuRequest.Settings.AllowVendorExtension = true;
-        if (IsDisableVgpuSettingsSupported())
+        if (wsl::windows::common::helpers::IsDisableVgpuSettingsSupported())
         {
             gpuRequest.Settings.DisableGdiAcceleration = true;
             gpuRequest.Settings.DisablePresentation = true;
@@ -921,12 +920,6 @@ void WslCoreVm::AddDrvFsShare(_In_ bool Admin, _In_ HANDLE UserToken)
     }
 }
 
-bool WslCoreVm::IsDisableVgpuSettingsSupported() const
-{
-    // See if the Windows version has the required platform change.
-    return ((wsl::windows::common::hcs::GetSchemaVersion() >= c_schemaVersionNickel) && (m_windowsVersion.BuildNumber >= 22545));
-}
-
 _Requires_lock_held_(m_guestDeviceLock)
 void WslCoreVm::AddPlan9Share(
     _In_ PCWSTR AccessName, _In_ PCWSTR Path, [[maybe_unused]] _In_ UINT32 Port, _In_ hcs::Plan9ShareFlags Flags, _In_ HANDLE UserToken, _In_opt_ PCWSTR VirtIoTag)

src/windows/service/exe/WslCoreVm.h

@@ -267,8 +267,6 @@ private:
 
     bool IsDnsTunnelingSupported() const;
 
-    bool IsDisableVgpuSettingsSupported() const;
-
     _Requires_lock_held_(m_lock)
     DiskMountResult MountDiskLockHeld(
         _In_ PCWSTR Disk, _In_ DiskType MountDiskType, _In_ ULONG PartitionIndex, _In_opt_ PCWSTR Name, _In_opt_ PCWSTR Type, _In_opt_ PCWSTR Options);

src/windows/service/inc/wslservice.idl

@@ -463,6 +463,7 @@ interface ILSWVirtualMachine : IUnknown
     HRESULT DetachDisk([in] ULONG Lun);
     HRESULT MountWindowsFolder([in] LPCWSTR WindowsPath, [in] LPCSTR LinuxPath, [in] BOOL ReadOnly);
     HRESULT UnmountWindowsFolder([in] LPCSTR LinuxPath);
+    HRESULT MountGpuLibraries([in] LPCSTR LibrariesMountPoint, [in] LPCSTR DriversMountpoint);
 }
 
 typedef
@@ -476,6 +477,7 @@ struct _VIRTUAL_MACHINE_SETTINGS {
     BOOL EnableDnsTunneling;
     BOOL EnableDebugShell;
     BOOL EnableEarlyBootDmesg;
+    BOOL EnableGPU;
 } VIRTUAL_MACHINE_SETTINGS;
 
 

test/windows/LSWTests.cpp

@@ -936,4 +936,74 @@ class LSWTests
             VERIFY_FAIL();
         }
     }
+
+    TEST_METHOD(GPU)
+    {
+        WSL2_TEST_ONLY();
+
+        VirtualMachineSettings settings{};
+        settings.CPU.CpuCount = 4;
+        settings.DisplayName = L"LSW";
+        settings.Memory.MemoryMb = 2048;
+        settings.Options.BootTimeoutMs = 30 * 1000;
+        settings.Networking.Mode = NetworkingModeNAT;
+        settings.GPU.Enable = true;
+
+        auto vm = CreateVm(&settings);
+
+        // Validate that the GPU device is available.
+        VERIFY_ARE_EQUAL(RunCommand(vm.get(), {"/bin/bash", "-c", "test -c /dev/dxg"}), 0);
+
+        // Validate GPU mounts
+        VERIFY_SUCCEEDED(WslMountGpuLibraries(vm.get(), "/usr/lib/wsl/lib", "/usr/lib/wsl/drivers"));
+
+        std::vector<const char*> commandLine{"/bin/sh", nullptr};
+
+        std::vector<ProcessFileDescriptorSettings> fds(2);
+        fds[0].Number = 0;
+        fds[0].Type = TerminalInput;
+        fds[1].Number = 1;
+        fds[1].Type = TerminalOutput;
+
+        CreateProcessSettings createProcessSettings{};
+        createProcessSettings.Executable = "/bin/sh";
+        createProcessSettings.Arguments = commandLine.data();
+        createProcessSettings.FileDescriptors = fds.data();
+        createProcessSettings.FdCount = static_cast<ULONG>(fds.size());
+
+        auto expectMount = [&](const std::string& target, const std::optional<std::string>& options) {
+            auto cmd = std::format("set -o pipefail ; findmnt '{}' | tail  -n 1", target);
+            auto [pid, in, out, err] = LaunchCommand(vm.get(), {"/bin/bash", "-c", cmd.c_str()});
+
+            auto output = ReadToString((SOCKET)out.get());
+            auto error = ReadToString((SOCKET)err.get());
+
+            WaitResult result{};
+            VERIFY_SUCCEEDED(WslWaitForLinuxProcess(vm.get(), pid, INFINITE, &result));
+            if (result.Code != (options.has_value() ? 0 : 1))
+            {
+                LogError("%hs failed. code=%i, output: %hs, error: %hs", cmd.c_str(), result.Code, output.c_str(), error.c_str());
+                VERIFY_FAIL();
+            }
+
+            if (options.has_value() && !PathMatchSpecA(output.c_str(), options->c_str()))
+            {
+                std::wstring message = std::format(L"Output: '{}' didn't match pattern: '{}'", output, options.value());
+                VERIFY_FAIL(message.c_str());
+            }
+        };
+
+        expectMount(
+            "/usr/lib/wsl/drivers",
+            "/usr/lib/wsl/drivers*9p*relatime,aname=*,cache=5,access=client,msize=65536,trans=fd,rfd=*,wfd=*");
+        expectMount("/usr/lib/wsl/lib", "/usr/lib/wsl/lib none*overlay ro,relatime,lowerdir=/usr/lib/wsl/lib/packaged*");
+
+        // Validate that trying to mount the shared with GPU support disabled fails.
+        {
+            settings.GPU.Enable = false;
+            auto vm = CreateVm(&settings);
+
+            VERIFY_ARE_EQUAL(WslMountGpuLibraries(vm.get(), "/usr/lib/wsl/lib", "/usr/lib/wsl/drivers"), HRESULT_FROM_WIN32(ERROR_INVALID_CONFIG_VALUE));
+        }
+    }
 };