iOS/Sources/App/WebView/WebRTC/WebRTCViewPlayerViewModel.swift
Copilot cd86bc3ade
Add audio support to WebRTC video player with mute/unmute controls (#4074)
## Summary

Enables audio playback in the native WebRTC video player and adds
mute/unmute controls. The previous implementation had audio disabled via
`useManualAudio = true` to avoid microphone permission requests.

**Key changes:**

- **Audio session configuration**: Uses
`AVAudioSession.Category.playback` for receive-only audio (no microphone
access required)
- **Remote audio track management**: Properly extracts audio track from
peer connection transceivers after remote SDP is set
- **Mute/unmute UI**: Added button at bottom-left of player using SF
Symbols (`speaker.wave.fill` / `speaker.slash.fill`)
- **State synchronization**: ViewModel mute state always reflects actual
audio track state

**Technical details:**

```swift
// Audio session configured for playback only
audioSession.setCategory(AVAudioSession.Category.playback.rawValue)
audioSession.setMode(AVAudioSession.Mode.spokenAudio.rawValue)

// Remote audio track extracted after SDP negotiation
remoteAudioTrack = peerConnection.transceivers
    .first(where: { $0.mediaType == .audio })?
    .receiver.track as? RTCAudioTrack
```

The implementation removes microphone-related warnings from the
experimental disclaimer.

## Screenshots

N/A - Audio feature, no visual changes beyond the mute button icon which
follows existing control patterns.

## Link to pull request in Documentation repository

Documentation: home-assistant/companion.home-assistant#

## Any other notes

Audio mode uses `.spokenAudio` for optimal voice/camera audio playback.
The mute button follows the same auto-hide behavior as other player
controls (3-second timeout).

<!-- START COPILOT CODING AGENT SUFFIX -->



<!-- START COPILOT ORIGINAL PROMPT -->



<details>

<summary>Original prompt</summary>

> Let's start supporting audio on WebRTCVideoPlayerView, including
control mute/unmute


</details>



<!-- START COPILOT CODING AGENT TIPS -->
---

 Let Copilot coding agent [set things up for
you](https://github.com/home-assistant/iOS/issues/new?title=+Set+up+Copilot+instructions&body=Configure%20instructions%20for%20this%20repository%20as%20documented%20in%20%5BBest%20practices%20for%20Copilot%20coding%20agent%20in%20your%20repository%5D%28https://gh.io/copilot-coding-agent-tips%29%2E%0A%0A%3COnboard%20this%20repo%3E&assignees=copilot)
— coding agent works faster and does higher quality work when set up for
your repo.

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: bgoncal <5808343+bgoncal@users.noreply.github.com>
2025-12-10 16:34:30 +00:00

208 lines
6.9 KiB
Swift

import Foundation
import HAKit
import Shared
import WebRTC
enum WebRTCSignalType: String {
case session
case answer
case candidate
case unknown
init(_ raw: String) {
self = WebRTCSignalType(rawValue: raw) ?? .unknown
}
}
final class WebRTCViewPlayerViewModel: ObservableObject {
enum Constants: String {
case offer = "camera/webrtc/offer"
case cadidate = "camera/webrtc/candidate"
}
var webRTCClient: WebRTCClient?
private var sessionId: String?
private var pendingCandidates: [RTCIceCandidate] = []
private let server: Server
private let cameraEntityId: String
@Published var failureReason: String?
@Published var showLoader: Bool = true
@Published var controlsVisible: Bool = true
@Published var isMuted: Bool = true
var hideControlsWorkItem: DispatchWorkItem?
init(server: Server, cameraEntityId: String) {
self.server = server
self.cameraEntityId = cameraEntityId
}
// MARK: - UI
func showControlsTemporarily() {
controlsVisible = true
hideControlsWorkItem?.cancel()
let workItem = DispatchWorkItem { [weak self] in
self?.controlsVisible = false
}
hideControlsWorkItem = workItem
DispatchQueue.main.asyncAfter(deadline: .now() + 3, execute: workItem)
}
func toggleMute() {
guard let webRTCClient else { return }
if webRTCClient.isAudioMuted() {
webRTCClient.unmuteAudio()
} else {
webRTCClient.muteAudio()
}
// Always get the final state from the client to ensure consistency
isMuted = webRTCClient.isAudioMuted()
}
// MARK: - WebRTC
func start() {
webRTCClient = nil
webRTCClient = WebRTCClient(iceServers: AppConstants.WebRTC.iceServers)
guard let webRTCClient else {
assertionFailure("WebRTCClient initialization failed")
return
}
webRTCClient.delegate = self
webRTCClient.offer { [weak self] sdp in
guard let self else {
assertionFailure("Self is nil in WebRTCViewPlayerViewModel.start")
return
}
guard let api = Current.api(for: server) else {
assertionFailure("API for server is nil")
return
}
api.connection.subscribe(to: .init(type: .webSocket(Constants.offer.rawValue), data: [
"entity_id": cameraEntityId,
"offer": sdp.sdp,
])) { [weak self] result in
switch result {
case let .success(data):
Current.Log.verbose("WebRTC offer sent successfully: \(data)")
case let .failure(error):
Current.Log.error("Failed to send WebRTC offer: \(error.localizedDescription)")
self?.showLoader = false
self?.failureReason = error.localizedDescription
}
} handler: { [weak self] _, data in
guard let self else { return }
guard let typeString: String = try? data.decode("type") else {
assertionFailure("Failed to decode type from data")
return
}
let type = WebRTCSignalType(typeString)
switch type {
case .session:
handleSession(data)
case .answer:
handleAnswer(data)
case .candidate:
handleCandidate(data)
case .unknown:
debugPrint("Unknown type: \(typeString)")
}
}
}
}
private func handleSession(_ data: HAData) {
guard let sessionId: String = try? data.decode("session_id") else {
assertionFailure("Failed to decode session_id from data")
return
}
self.sessionId = sessionId
for candidate in pendingCandidates {
sendCandidate(candidate)
}
pendingCandidates.removeAll()
}
private func handleAnswer(_ data: HAData) {
guard let answerSDP: String = try? data.decode("answer") else {
assertionFailure("Failed to decode answer from data")
return
}
let sdp = RTCSessionDescription(type: .answer, sdp: answerSDP)
webRTCClient?.set(remoteSdp: sdp) { error in
if let error {
Current.Log.error("Failed to set remote SDP: \(error.localizedDescription)")
}
}
}
private func handleCandidate(_ data: HAData) {
guard let candidateDict: [String: Any] = try? data.decode("candidate") else {
assertionFailure("Failed to decode candidate from data")
return
}
guard let candidateStr = candidateDict["candidate"] as? String else {
assertionFailure("Missing candidate string in candidateDict")
return
}
let sdpMLineIndex = candidateDict["sdpMLineIndex"] as? Int32 ?? 0
let sdpMid = candidateDict["sdpMid"] as? String
let candidate = RTCIceCandidate(
sdp: candidateStr,
sdpMLineIndex: sdpMLineIndex,
sdpMid: sdpMid
)
webRTCClient?.set(remoteCandidate: candidate) { error in
if let error {
print("Failed to add remote candidate: \(error)")
}
}
}
private func sendCandidate(_ candidate: RTCIceCandidate) {
guard let sessionId else {
// No session yet, store for later
pendingCandidates.append(candidate)
return
}
guard let api = Current.api(for: server) else {
assertionFailure("API for server is nil")
return
}
// Send candidate to backend
api.connection.send(.init(type: .webSocket(Constants.cadidate.rawValue), data: [
"entity_id": cameraEntityId,
"session_id": sessionId,
"candidate": [
"candidate": candidate.sdp,
"sdpMid": candidate.sdpMid ?? "0",
"sdpMLineIndex": candidate.sdpMLineIndex,
],
])).promise.pipe { result in
switch result {
case let .fulfilled(data):
Current.Log.verbose("Sent candidate: \(data)")
case let .rejected(error):
Current.Log.error("Failed to send candidate: \(error.localizedDescription)")
}
}
}
}
extension WebRTCViewPlayerViewModel: WebRTCClientDelegate {
func webRTCClient(_ client: WebRTCClient, didDiscoverLocalCandidate candidate: RTCIceCandidate) {
sendCandidate(candidate)
}
func webRTCClient(_ client: WebRTCClient, didChangeConnectionState state: RTCIceConnectionState) {
debugPrint(state)
}
func webRTCClient(_ client: WebRTCClient, didReceiveData data: Data) {
debugPrint(data)
}
}