From f113487937b8722789ab002db88b291876dc3479 Mon Sep 17 00:00:00 2001 From: Adam Gastineau Date: Fri, 15 Aug 2025 10:22:29 -0700 Subject: [PATCH] API for accessing hand gestures --- .../com/penumbraos/bridge/BridgeService.kt | 7 + .../aidl/com/penumbraos/bridge/IBridge.aidl | 4 +- .../bridge/IHandGestureProvider.aidl | 8 ++ .../bridge/callback/IHandGestureCallback.aidl | 6 + .../penumbraos/bridge_system/Entrypoint.kt | 2 + .../provider/HandGestureProvider.kt | 128 ++++++++++++++++++ .../provider/TouchpadProvider.kt | 21 +-- .../penumbraos/bridge_system/util/input.kt | 21 +++ .../java/com/penumbraos/sdk/PenumbraClient.kt | 10 +- .../penumbraos/sdk/api/HandGestureClient.kt | 32 +++++ .../sdk/api/types/HandGestureReceiver.kt | 6 + 11 files changed, 227 insertions(+), 18 deletions(-) create mode 100644 bridge-shared/aidl/com/penumbraos/bridge/IHandGestureProvider.aidl create mode 100644 bridge-shared/aidl/com/penumbraos/bridge/callback/IHandGestureCallback.aidl create mode 100644 bridge-system/src/main/java/com/penumbraos/bridge_system/provider/HandGestureProvider.kt create mode 100644 bridge-system/src/main/java/com/penumbraos/bridge_system/util/input.kt create mode 100644 sdk/src/main/java/com/penumbraos/sdk/api/HandGestureClient.kt create mode 100644 sdk/src/main/java/com/penumbraos/sdk/api/types/HandGestureReceiver.kt diff --git a/bridge-core/src/main/java/com/penumbraos/bridge/BridgeService.kt b/bridge-core/src/main/java/com/penumbraos/bridge/BridgeService.kt index d556053..f0461cd 100644 --- a/bridge-core/src/main/java/com/penumbraos/bridge/BridgeService.kt +++ b/bridge-core/src/main/java/com/penumbraos/bridge/BridgeService.kt @@ -16,6 +16,7 @@ class BridgeService { private var touchpadProvider: ITouchpadProvider? = null private var ledProvider: ILedProvider? = null + private var handGestureProvider: IHandGestureProvider? = null private var handTrackingProvider: IHandTrackingProvider? = null private var esimProvider: IEsimProvider? = null @@ -48,6 +49,10 @@ class BridgeService { return this@BridgeService.ledProvider?.asBinder() } + override fun getHandGestureProvider(): IBinder? { + return this@BridgeService.handGestureProvider?.asBinder() + } + override fun getHandTrackingProvider(): IBinder? { return this@BridgeService.handTrackingProvider?.asBinder() } @@ -71,6 +76,7 @@ class BridgeService { sttProvider: ISttProvider?, touchpadProvider: ITouchpadProvider?, ledProvider: ILedProvider?, + handGestureProvider: IHandGestureProvider?, handTrackingProvider: IHandTrackingProvider?, esimProvider: IEsimProvider? ) { @@ -83,6 +89,7 @@ class BridgeService { this@BridgeService.touchpadProvider = touchpadProvider this@BridgeService.ledProvider = ledProvider + this@BridgeService.handGestureProvider = handGestureProvider this@BridgeService.handTrackingProvider = handTrackingProvider this@BridgeService.esimProvider = esimProvider diff --git a/bridge-shared/aidl/com/penumbraos/bridge/IBridge.aidl b/bridge-shared/aidl/com/penumbraos/bridge/IBridge.aidl index bbb449c..3a7a1da 100644 --- a/bridge-shared/aidl/com/penumbraos/bridge/IBridge.aidl +++ b/bridge-shared/aidl/com/penumbraos/bridge/IBridge.aidl @@ -6,6 +6,7 @@ import com.penumbraos.bridge.IDnsProvider; import com.penumbraos.bridge.ISttProvider; import com.penumbraos.bridge.ITouchpadProvider; import com.penumbraos.bridge.ILedProvider; +import com.penumbraos.bridge.IHandGestureProvider; import com.penumbraos.bridge.IHandTrackingProvider; import com.penumbraos.bridge.IEsimProvider; import com.penumbraos.bridge.ISettingsProvider; @@ -20,13 +21,14 @@ interface IBridge { IBinder getTouchpadProvider(); IBinder getLedProvider(); + IBinder getHandGestureProvider(); IBinder getHandTrackingProvider(); IBinder getEsimProvider(); IBinder getSettingsProvider(); IBinder getShellProvider(); - void registerSystemService(IHttpProvider httpProvider, IWebSocketProvider webSocketProvider, IDnsProvider dnsProvider, ISttProvider sttProvider, ITouchpadProvider touchpadProvider, ILedProvider ledProvider, IHandTrackingProvider handTrackingProvider, IEsimProvider esimProvider); + void registerSystemService(IHttpProvider httpProvider, IWebSocketProvider webSocketProvider, IDnsProvider dnsProvider, ISttProvider sttProvider, ITouchpadProvider touchpadProvider, ILedProvider ledProvider, IHandGestureProvider handGestureProvider, IHandTrackingProvider handTrackingProvider, IEsimProvider esimProvider); void registerSettingsService(ISettingsProvider settingsProvider); void registerShellService(IShellProvider shellProvider); } diff --git a/bridge-shared/aidl/com/penumbraos/bridge/IHandGestureProvider.aidl b/bridge-shared/aidl/com/penumbraos/bridge/IHandGestureProvider.aidl new file mode 100644 index 0000000..d679109 --- /dev/null +++ b/bridge-shared/aidl/com/penumbraos/bridge/IHandGestureProvider.aidl @@ -0,0 +1,8 @@ +package com.penumbraos.bridge; + +import com.penumbraos.bridge.callback.IHandGestureCallback; + +interface IHandGestureProvider { + void registerCallback(IHandGestureCallback callback); + void deregisterCallback(IHandGestureCallback callback); +} \ No newline at end of file diff --git a/bridge-shared/aidl/com/penumbraos/bridge/callback/IHandGestureCallback.aidl b/bridge-shared/aidl/com/penumbraos/bridge/callback/IHandGestureCallback.aidl new file mode 100644 index 0000000..f4ccb69 --- /dev/null +++ b/bridge-shared/aidl/com/penumbraos/bridge/callback/IHandGestureCallback.aidl @@ -0,0 +1,6 @@ +package com.penumbraos.bridge.callback; + +oneway interface IHandGestureCallback { + void onHandClose(); + void onHandPush(); +} \ No newline at end of file diff --git a/bridge-system/src/main/java/com/penumbraos/bridge_system/Entrypoint.kt b/bridge-system/src/main/java/com/penumbraos/bridge_system/Entrypoint.kt index ef45a3c..59914a3 100644 --- a/bridge-system/src/main/java/com/penumbraos/bridge_system/Entrypoint.kt +++ b/bridge-system/src/main/java/com/penumbraos/bridge_system/Entrypoint.kt @@ -11,6 +11,7 @@ import com.penumbraos.bridge_system.esim.LPA_APK_PATH import com.penumbraos.bridge_system.esim.MockFactoryService import com.penumbraos.bridge_system.provider.DnsProvider import com.penumbraos.bridge_system.provider.EsimProvider +import com.penumbraos.bridge_system.provider.HandGestureProvider import com.penumbraos.bridge_system.provider.HandTrackingProvider import com.penumbraos.bridge_system.provider.HttpProvider import com.penumbraos.bridge_system.provider.LedProvider @@ -71,6 +72,7 @@ class Entrypoint { SttProvider(context, looper), TouchpadProvider(looper), LedProvider(context), + HandGestureProvider(looper), HandTrackingProvider(context), EsimProvider(classLoader, context) ) diff --git a/bridge-system/src/main/java/com/penumbraos/bridge_system/provider/HandGestureProvider.kt b/bridge-system/src/main/java/com/penumbraos/bridge_system/provider/HandGestureProvider.kt new file mode 100644 index 0000000..d0a2727 --- /dev/null +++ b/bridge-system/src/main/java/com/penumbraos/bridge_system/provider/HandGestureProvider.kt @@ -0,0 +1,128 @@ +package com.penumbraos.bridge_system.provider + +import android.os.Looper +import android.util.Log +import android.view.InputChannel +import android.view.InputEvent +import android.view.InputEventReceiver +import android.view.KeyEvent +import android.view.MotionEvent +import com.penumbraos.bridge.IHandGestureProvider +import com.penumbraos.bridge.callback.IHandGestureCallback +import com.penumbraos.bridge_system.util.registerTouchpadInputChannel + +private const val TAG = "HandGestureProvider" + +private const val VELOCITY_THRESHOLD = 0.5f +private const val VELOCITY_MIN_MOVING = 0.3f +private const val GESTURE_COOLDOWN_MS = 500L + +class HandGestureProvider(private val looper: Looper) : IHandGestureProvider.Stub() { + private val callbacks = mutableListOf() + private var listener: EventListener? = null + + inner class PushListener { + + private var gestureActive = false + + private var lastDepth = Float.NaN + private var lastTimestamp = 0L + private var gestureEndTime = 0L + + fun processMotionEvent(event: MotionEvent) { + val depth = event.getAxisValue(MotionEvent.AXIS_PRESSURE) + val timestamp = event.eventTime + + if (lastDepth.isNaN()) { + lastDepth = depth + lastTimestamp = timestamp + return + } + + val timeDelta = timestamp - lastTimestamp + if (timeDelta > 0) { + val depthDelta = depth - lastDepth + val velocity = kotlin.math.abs(depthDelta * 1000.0f / timeDelta) + + when { + !gestureActive && velocity >= VELOCITY_THRESHOLD && (timestamp - gestureEndTime) > GESTURE_COOLDOWN_MS -> { + gestureActive = true + onHandPush() + } + + gestureActive && velocity < VELOCITY_MIN_MOVING -> { + gestureActive = false + gestureEndTime = timestamp + } + } + } + + lastDepth = depth + lastTimestamp = timestamp + } + } + + inner class EventListener(inputChannel: InputChannel) : + InputEventReceiver(inputChannel, looper) { + private val pushListener = PushListener() + + override fun onInputEvent(event: InputEvent?) { + if (event != null) { + if (event is MotionEvent) { + pushListener.processMotionEvent(event) + } else if (event is KeyEvent && event.keyCode == KeyEvent.KEYCODE_H) { + onHandClose() + } + } + super.onInputEvent(event) + } + } + + override fun registerCallback(callback: IHandGestureCallback) { + callbacks.add(callback) + registerListenerIfNecessary() + } + + override fun deregisterCallback(callback: IHandGestureCallback) { + callbacks.remove(callback) + if (callbacks.count() < 1) { + Log.w(TAG, "Deregistering hand gesture listener") + listener?.dispose() + listener = null + } + } + + fun onHandPush() { + callCallback { callback -> callback.onHandPush() } + } + + fun onHandClose() { + callCallback { callback -> callback.onHandClose() } + } + + private fun callCallback(withCallback: (IHandGestureCallback) -> Unit) { + val callbacksToRemove = mutableListOf() + callbacks.forEach { callback -> + safeCallback(TAG, { + withCallback(callback) + }, onDeadObject = { + callbacksToRemove.add(callback) + }) + } + callbacksToRemove.forEach { callback -> deregisterCallback(callback) } + } + + private fun registerListenerIfNecessary() { + if (listener != null) { + return + } + + Log.w(TAG, "Registering touchpad listener") + + val inputChannel = registerTouchpadInputChannel(TAG) + + if (inputChannel != null) { + listener = EventListener(inputChannel) + } + } +} diff --git a/bridge-system/src/main/java/com/penumbraos/bridge_system/provider/TouchpadProvider.kt b/bridge-system/src/main/java/com/penumbraos/bridge_system/provider/TouchpadProvider.kt index f19dae2..0778713 100644 --- a/bridge-system/src/main/java/com/penumbraos/bridge_system/provider/TouchpadProvider.kt +++ b/bridge-system/src/main/java/com/penumbraos/bridge_system/provider/TouchpadProvider.kt @@ -1,17 +1,14 @@ package com.penumbraos.bridge_system.provider -import android.hardware.input.IInputManager import android.os.Looper -import android.os.ServiceManager import android.util.Log import android.view.InputChannel import android.view.InputEvent import android.view.InputEventReceiver import com.penumbraos.bridge.ITouchpadProvider import com.penumbraos.bridge.callback.ITouchpadCallback +import com.penumbraos.bridge_system.util.registerTouchpadInputChannel -private const val TOUCHPAD_MONITOR_NAME = "Humane Touchpad Monitor" -private const val TOUCHPAD_DISPLAY_ID = 3344 private const val TOUCHPAD_EVENT_SOURCE = 0x100008 private const val TAG = "TouchpadProvider" @@ -19,11 +16,12 @@ private const val TAG = "TouchpadProvider" class TouchpadProvider(private val looper: Looper) : ITouchpadProvider.Stub() { private val callbacks = mutableListOf() + private var listener: EventListener? = null inner class EventListener(inputChannel: InputChannel) : InputEventReceiver(inputChannel, looper) { override fun onInputEvent(event: InputEvent?) { - if (event != null) { + if (event != null && event.isFromSource(TOUCHPAD_EVENT_SOURCE)) { val callbacksToRemove = mutableListOf() callbacks.forEach { callback -> safeCallback(TAG, { @@ -38,8 +36,6 @@ class TouchpadProvider(private val looper: Looper) : } } - private var listener: EventListener? = null - override fun registerCallback(callback: ITouchpadCallback) { callbacks.add(callback) registerListenerIfNecessary() @@ -61,15 +57,10 @@ class TouchpadProvider(private val looper: Looper) : Log.w(TAG, "Registering touchpad listener") - try { - val inputManagerBinder = ServiceManager.getService("input") - val inputManager = IInputManager.Stub.asInterface(inputManagerBinder) + val inputChannel = registerTouchpadInputChannel(TAG) - val inputMonitor = - inputManager.monitorGestureInput(TOUCHPAD_MONITOR_NAME, TOUCHPAD_DISPLAY_ID) - listener = EventListener(inputMonitor.inputChannel) - } catch (e: Exception) { - Log.e(TAG, "Failed to register touchpad listener", e) + if (inputChannel != null) { + listener = EventListener(inputChannel) } } } \ No newline at end of file diff --git a/bridge-system/src/main/java/com/penumbraos/bridge_system/util/input.kt b/bridge-system/src/main/java/com/penumbraos/bridge_system/util/input.kt new file mode 100644 index 0000000..9d06269 --- /dev/null +++ b/bridge-system/src/main/java/com/penumbraos/bridge_system/util/input.kt @@ -0,0 +1,21 @@ +package com.penumbraos.bridge_system.util + +import android.hardware.input.IInputManager +import android.os.ServiceManager +import android.util.Log +import android.view.InputChannel + +private const val TOUCHPAD_MONITOR_NAME = "Humane Touchpad Monitor" +private const val TOUCHPAD_DISPLAY_ID = 3344 + +fun registerTouchpadInputChannel(tag: String): InputChannel? { + return try { + val inputManagerBinder = ServiceManager.getService("input") + val inputManager = IInputManager.Stub.asInterface(inputManagerBinder) + + inputManager.monitorGestureInput(TOUCHPAD_MONITOR_NAME, TOUCHPAD_DISPLAY_ID).inputChannel + } catch (e: Exception) { + Log.e(tag, "Failed to register touchpad listener", e) + null + } +} \ No newline at end of file diff --git a/sdk/src/main/java/com/penumbraos/sdk/PenumbraClient.kt b/sdk/src/main/java/com/penumbraos/sdk/PenumbraClient.kt index d21e1fc..af74b92 100644 --- a/sdk/src/main/java/com/penumbraos/sdk/PenumbraClient.kt +++ b/sdk/src/main/java/com/penumbraos/sdk/PenumbraClient.kt @@ -9,6 +9,8 @@ import android.os.IBinder import android.util.Log import com.penumbraos.bridge.IBridge import com.penumbraos.bridge.IDnsProvider +import com.penumbraos.bridge.IEsimProvider +import com.penumbraos.bridge.IHandGestureProvider import com.penumbraos.bridge.IHandTrackingProvider import com.penumbraos.bridge.IHttpProvider import com.penumbraos.bridge.ILedProvider @@ -17,10 +19,10 @@ import com.penumbraos.bridge.IShellProvider import com.penumbraos.bridge.ISttProvider import com.penumbraos.bridge.ITouchpadProvider import com.penumbraos.bridge.IWebSocketProvider -import com.penumbraos.bridge.IEsimProvider import com.penumbraos.bridge.external.BRIDGE_SERVICE_READY import com.penumbraos.sdk.api.DnsClient import com.penumbraos.sdk.api.EsimClient +import com.penumbraos.sdk.api.HandGestureClient import com.penumbraos.sdk.api.HandTrackingClient import com.penumbraos.sdk.api.HttpClient import com.penumbraos.sdk.api.LedClient @@ -49,6 +51,7 @@ class PenumbraClient { lateinit var touchpad: TouchpadClient lateinit var led: LedClient + lateinit var handGesture: HandGestureClient lateinit var handTracking: HandTrackingClient lateinit var esim: EsimClient @@ -125,10 +128,12 @@ class PenumbraClient { val touchpadProvider = ITouchpadProvider.Stub.asInterface(service!!.getTouchpadProvider()) val ledProvider = ILedProvider.Stub.asInterface(service!!.getLedProvider()) + val handGestureProvider = + IHandGestureProvider.Stub.asInterface(service!!.getHandGestureProvider()) val handTrackingProvider = IHandTrackingProvider.Stub.asInterface(service!!.getHandTrackingProvider()) - val esimProvider = + val esimProvider = IEsimProvider.Stub.asInterface(service!!.getEsimProvider()) val settingsProvider = @@ -145,6 +150,7 @@ class PenumbraClient { touchpad = TouchpadClient(touchpadProvider) led = LedClient(ledProvider) + handGesture = HandGestureClient(handGestureProvider) handTracking = HandTrackingClient(handTrackingProvider) esim = EsimClient(esimProvider) diff --git a/sdk/src/main/java/com/penumbraos/sdk/api/HandGestureClient.kt b/sdk/src/main/java/com/penumbraos/sdk/api/HandGestureClient.kt new file mode 100644 index 0000000..0f9e947 --- /dev/null +++ b/sdk/src/main/java/com/penumbraos/sdk/api/HandGestureClient.kt @@ -0,0 +1,32 @@ +package com.penumbraos.sdk.api + +import com.penumbraos.bridge.IHandGestureProvider +import com.penumbraos.bridge.callback.IHandGestureCallback +import com.penumbraos.sdk.api.types.HandGestureReceiver +import java.util.concurrent.ConcurrentHashMap + +class HandGestureClient(private val handGestureProvider: IHandGestureProvider) { + private val registeredCallbacks = + ConcurrentHashMap() + + fun register(receiver: HandGestureReceiver) { + val callbackStub = object : IHandGestureCallback.Stub() { + override fun onHandClose() { + receiver.onHandClose() + } + + override fun onHandPush() { + receiver.onHandPush() + } + } + registeredCallbacks[receiver] = callbackStub + handGestureProvider.registerCallback(callbackStub) + } + + fun remove(receiver: HandGestureReceiver) { + val callbackStub = registeredCallbacks.remove(receiver) + if (callbackStub != null) { + handGestureProvider.deregisterCallback(callbackStub) + } + } +} \ No newline at end of file diff --git a/sdk/src/main/java/com/penumbraos/sdk/api/types/HandGestureReceiver.kt b/sdk/src/main/java/com/penumbraos/sdk/api/types/HandGestureReceiver.kt new file mode 100644 index 0000000..2152204 --- /dev/null +++ b/sdk/src/main/java/com/penumbraos/sdk/api/types/HandGestureReceiver.kt @@ -0,0 +1,6 @@ +package com.penumbraos.sdk.api.types + +interface HandGestureReceiver { + fun onHandClose() + fun onHandPush() +} \ No newline at end of file