Make sure LLM client preserves Content-Type

.github/workflows/release_on_push.yml

@@ -89,7 +89,6 @@ jobs:
 
           # Rename plugin APKs
           cp plugins/demo/build/outputs/apk/release/demo-release.apk "release-artifacts/PenumbraOS-Plugin-Demo-${VERSION_NAME}.apk"
-          cp plugins/openai/build/outputs/apk/release/openai-release.apk "release-artifacts/PenumbraOS-Plugin-OpenAI-${VERSION_NAME}.apk"
           cp plugins/aipinsystem/build/outputs/apk/release/aipinsystem-release.apk "release-artifacts/PenumbraOS-Plugin-AiPin-System-${VERSION_NAME}.apk"
           cp plugins/system/build/outputs/apk/release/system-release.apk "release-artifacts/PenumbraOS-Plugin-Generic-System-${VERSION_NAME}.apk"
           cp plugins/googlesearch/build/outputs/apk/release/googlesearch-release.apk "release-artifacts/PenumbraOS-Plugin-GoogleSearch-${VERSION_NAME}.apk"

.idea/runConfigurations/mabl.xml

@@ -58,7 +58,7 @@
       <option name="ADVANCED_PROFILING_ENABLED" value="false" />
       <option name="STARTUP_PROFILING_ENABLED" value="false" />
       <option name="STARTUP_CPU_PROFILING_ENABLED" value="false" />
-      <option name="STARTUP_CPU_PROFILING_CONFIGURATION_NAME" value="Java/Kotlin Method Sample (legacy)" />
+      <option name="STARTUP_CPU_PROFILING_CONFIGURATION_NAME" value="System Trace" />
       <option name="STARTUP_NATIVE_MEMORY_PROFILING_ENABLED" value="false" />
       <option name="NATIVE_MEMORY_SAMPLE_RATE_BYTES" value="2048" />
     </Profilers>
@@ -69,7 +69,6 @@
     <option name="SKIP_ACTIVITY_VALIDATION" value="false" />
     <method v="2">
       <option name="Gradle.BeforeRunTask" enabled="false" tasks="installDemoPlugins" externalProjectPath="$PROJECT_DIR$" vmOptions="" scriptParameters="" />
-      <option name="Gradle.BeforeRunTask" enabled="false" tasks="installOpenAiPlugin" externalProjectPath="$PROJECT_DIR$" vmOptions="" scriptParameters="" />
       <option name="Android.Gradle.BeforeRunTask" enabled="true" />
     </method>
   </configuration>

build.sh

@@ -4,10 +4,9 @@ fi
 if ! [ -f mabl/src/main/assets/minilm-l6-v2-tokenizer.json ]; then
   curl -L -o mabl/src/main/assets/minilm-l6-v2-tokenizer.json https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json?download=true
 fi
-./gradlew :plugins:demo:installDebug :plugins:aipinsystem:installDebug :plugins:system:installDebug :plugins:openai:installDebug :plugins:googlesearch:installDebug :mabl:installAipinDebug
+./gradlew :plugins:demo:installDebug :plugins:aipinsystem:installDebug :plugins:system:installDebug :plugins:googlesearch:installDebug :mabl:installAipinDebug
 adb shell pm grant com.penumbraos.mabl.pin android.permission.CAMERA
 adb shell appops set com.penumbraos.mabl.pin MANAGE_EXTERNAL_STORAGE allow
-adb shell appops set com.penumbraos.plugins.openai MANAGE_EXTERNAL_STORAGE allow
 adb shell pm disable-user --user 0 humane.experience.systemnavigation
 sleep 1
 adb shell cmd package set-home-activity com.penumbraos.mabl.pin/com.penumbraos.mabl.MainActivity

gradle/libs.versions.toml

@@ -16,8 +16,7 @@ penumbraos-sdk = "e1a344b"
 penumbraos-sdk-local = "0.1.0"
 moonlight-ui = "6ae1d5e"
 moonlight-ui-local = "0.1.0"
-openai-client = "dedfa47"
-#openai-client-local = "debugging-SNAPSHOT"
+langchain4j = "67188d8"
 ktor-client = "3.0.0"
 kotlinx-serialization = "1.7.1"
 kotlinx-coroutines = "1.8.1"
@@ -61,8 +60,10 @@ penumbraos-sdk = { group = "com.github.PenumbraOS", name = "sdk", version.ref =
 #penumbraos-sdk = { group = "com.penumbraos", name = "sdk", version.ref = "penumbraos-sdk-local" }
 moonlight-ui = { group = "com.github.agg23", name = "moonlight", version.ref = "moonlight-ui" }
 #moonlight-ui = { group = "com.open.pin", name = "ui", version.ref = "moonlight-ui-local" }
-openai-client = { group = "com.github.agg23.openai-kotlin", name = "openai-client", version.ref = "openai-client" }
-#openai-client = { group = "com.github.agg23.openai-kotlin", name = "openai-client", version.ref = "openai-client-local" }
+langchain4j = { group = "com.github.agg23.langchain4j", name = "langchain4j-bom", version.ref = "langchain4j" }
+langchain4j-gemini = { group = "com.github.agg23.langchain4j", name = "langchain4j-google-ai-gemini", version.ref = "langchain4j" }
+langchain4j-openai = { group = "com.github.agg23.langchain4j", name = "langchain4j-open-ai", version.ref = "langchain4j" }
+langchain4j-kotlin = { group = "com.github.agg23.langchain4j", name = "langchain4j-kotlin", version.ref = "langchain4j" }
 ktor-client-android = { group = "io.ktor", name = "ktor-client-android", version.ref = "ktor-client" }
 ktor-content-negociation = { group = "io.ktor", name = "ktor-client-content-negotiation", version.ref = "ktor-client" }
 ktor-serialization-kotlinx-json = { group = "io.ktor", name = "ktor-serialization-kotlinx-json", version.ref = "ktor-client" }

mabl/build.gradle.kts

@@ -30,11 +30,11 @@ android {
         }
     }
     compileOptions {
-        sourceCompatibility = JavaVersion.VERSION_11
-        targetCompatibility = JavaVersion.VERSION_11
+        sourceCompatibility = JavaVersion.VERSION_17
+        targetCompatibility = JavaVersion.VERSION_17
     }
     kotlinOptions {
-        jvmTarget = "11"
+        jvmTarget = "17"
     }
     buildFeatures {
         compose = true
@@ -81,10 +81,16 @@ dependencies {
     "aipinImplementation"(libs.moonlight.ui)
     "aipinSimulatorImplementation"(libs.moonlight.ui)
 
+    implementation(libs.langchain4j.kotlin)
+    implementation(libs.langchain4j)
+    implementation(libs.langchain4j.openai)
+    implementation(libs.langchain4j.gemini)
+
     implementation(libs.androidx.camera.core)
     implementation(libs.androidx.camera.lifecycle)
     implementation(libs.androidx.camera.camera2)
 
+    implementation(libs.ktor.client.android)
     implementation(libs.kotlinx.serialization.json)
 
     implementation(libs.onnx.runtime.android)

mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/PlatformInputHandler.kt

@@ -65,15 +65,28 @@ open class PlatformInputHandler(
             object : ITouchpadGestureDelegate {
                 override fun onGesture(gesture: TouchpadGesture) {
                     // TODO: Build proper API for Input Handler to perform standardized triggers
-                    if (gesture.kind != TouchpadGestureKind.HOLD_END) {
-                        // Any gesture that isn't a release should halt talking
-                        interactionFlowManager.finishListening()
+                    if (gesture.kind != TouchpadGestureKind.HOLD_END && 
+                        gesture.kind != TouchpadGestureKind.FINGER_DOWN &&
+                        gesture.kind != TouchpadGestureKind.GESTURE_CANCEL) {
+                        // Any gesture that isn't a release (or intermediate finger down/cancel) should halt talking
+                        interactionFlowManager.cancelTalking()
                     }
 
                     when (gesture.kind) {
+                        TouchpadGestureKind.FINGER_DOWN -> {
+                            // Immediately start listening, even if we abort later
+                            interactionFlowManager.startListening()
+                        }
+
+                        TouchpadGestureKind.GESTURE_CANCEL -> {
+                            interactionFlowManager.finishListening(abort = true)
+                        }
+
                         TouchpadGestureKind.DOUBLE_TAP -> {
                             // TODO: Fix double tap with two fingers
 //                            if (gesture.fingerCount == 2) {
+                            // Cancel listening if it is ongoing
+                            interactionFlowManager.finishListening(abort = true)
                             interactionFlowManager.takePicture()
 //                            }
                         }

mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/input/ITouchpadGestureDelegate.kt

@@ -7,6 +7,8 @@ interface ITouchpadGestureDelegate {
 data class TouchpadGesture(val kind: TouchpadGestureKind, val duration: Long, val fingerCount: Int)
 
 enum class TouchpadGestureKind {
+    FINGER_DOWN,
+    GESTURE_CANCEL,
     SINGLE_TAP,
     DOUBLE_TAP,
     HOLD_START,

mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/input/TouchpadGestureManager.kt

@@ -73,6 +73,14 @@ class TouchpadGestureManager(
             MotionEvent.ACTION_DOWN -> {
                 activePointers.add(event.getPointerId(0))
 
+                sendEventIfAllowed(event, updateLastEventTime = false) {
+                    TouchpadGesture(
+                        TouchpadGestureKind.FINGER_DOWN,
+                        0,
+                        activePointers.size
+                    )
+                }
+
                 if (activePointers.size == 1) {
                     holdStartTime = event.eventTime
                     singleFingerHoldHandler = Handler(Looper.getMainLooper())
@@ -91,14 +99,26 @@ class TouchpadGestureManager(
                 activePointers.remove(event.getPointerId(0))
 
                 // Cancel any pending single finger hold
+                val wasPendingHold = singleFingerHoldHandler != null
                 singleFingerHoldHandler?.removeCallbacksAndMessages(null)
                 singleFingerHoldHandler = null
 
+                val duration = event.eventTime - holdStartTime
+
                 // Handle hold end
                 if (isHolding) {
-                    val duration = event.eventTime - holdStartTime
                     delegate.onGesture(TouchpadGesture(TouchpadGestureKind.HOLD_END, duration, 1))
                     isHolding = false
+                } else if (wasPendingHold && activePointers.isEmpty()) {
+                    // Finger was lifted before any gesture started
+                    // Only send if we didn't just send a recognized gesture
+                    sendEventIfAllowed(event, updateLastEventTime = false) {
+                        TouchpadGesture(
+                            TouchpadGestureKind.GESTURE_CANCEL,
+                            duration,
+                            1
+                        )
+                    }
                 }
             }
             // A non-primary touch has changed
@@ -149,12 +169,18 @@ class TouchpadGestureManager(
     /**
      * Send TouchpadGesture if allowed based on time since last event. Specifically to prevent sending gesture start events too close together
      */
-    private fun sendEventIfAllowed(event: MotionEvent, lambda: () -> TouchpadGesture) {
+    private fun sendEventIfAllowed(
+        event: MotionEvent,
+        updateLastEventTime: Boolean = true,
+        lambda: () -> TouchpadGesture,
+    ) {
         if (event.eventTime < lastEventTime + MIN_GESTURE_SEPARATION_MS) {
             return
         }
 
-        lastEventTime = event.eventTime
+        if (updateLastEventTime) {
+            lastEventTime = event.eventTime
+        }
         delegate.onGesture(lambda())
     }
 
@@ -163,8 +189,15 @@ class TouchpadGestureManager(
 
         if (isHolding) {
             delegate.onGesture(TouchpadGesture(TouchpadGestureKind.HOLD_END, duration, 2))
+            isHolding = false
         } else if (duration < 200) {
             delegate.onGesture(TouchpadGesture(TouchpadGestureKind.SINGLE_TAP, duration, 2))
+        } else {
+            // Finger was lifted before any gesture completed
+            // Only send if we didn't just send a recognized gesture
+            sendEventIfAllowed(event, updateLastEventTime = false) {
+                TouchpadGesture(TouchpadGestureKind.GESTURE_CANCEL, duration, 2)
+            }
         }
     }
 }

mabl/src/main/AndroidManifest.xml

@@ -56,6 +56,22 @@
             android:enabled="true"
             android:exported="false"
             android:foregroundServiceType="camera" />
+
+        <service
+            android:name=".plugins.llm.LangchainLlmService"
+            android:exported="true"
+            tools:ignore="ExportedService">
+            <intent-filter>
+                <action android:name="com.penumbraos.mabl.sdk.action.LLM_SERVICE" />
+            </intent-filter>
+
+            <meta-data
+                android:name="com.penumbraos.mabl.sdk.metadata.DISPLAY_NAME"
+                android:value="Langchain LLM Service" />
+            <meta-data
+                android:name="com.penumbraos.mabl.sdk.metadata.DESCRIPTION"
+                android:value="Langchain language model service supporting multiple providers" />
+        </service>
     </application>
 
 </manifest>

mabl/src/main/java/com/penumbraos/mabl/conversation/ConversationManager.kt

@@ -238,6 +238,7 @@ class ConversationManager(
                     })
             }
         } else {
+            Log.d(TAG, "LLM requested 0 tool calls: ${response.text}")
             // No tool calls, this is the final response
             persistAssistantMessage(responseText, emptyArray())
 
@@ -296,19 +297,7 @@ class ConversationManager(
                 }
 
                 override fun onCompleteResponse(response: LlmResponse) {
-                    // This should be the final response after tool execution
-                    val message = BinderConversationMessage().apply {
-                        type = "assistant"
-                        content = response.text ?: ""
-                        toolCalls = emptyArray()
-                        toolCallId = null
-                    }
-                    conversationHistory.add(message)
-
-                    // Persist final assistant response to database
-                    persistMessageSync("assistant", response.text ?: "")
-
-                    callback.onCompleteResponse(response.text ?: "")
+                    handleLlmResponse(response, filteredTools, callback)
                 }
 
                 override fun onError(error: String) {

mabl/src/main/java/com/penumbraos/mabl/interaction/IInteractionFlowManager.kt

@@ -6,7 +6,8 @@ import com.penumbraos.mabl.types.Error
 interface IInteractionFlowManager {
     fun startListening(requestImage: Boolean = false)
     fun startConversationFromInput(userInput: String)
-    fun finishListening()
+    fun finishListening(abort: Boolean = false)
+    fun cancelTalking()
     fun isFlowActive(): Boolean
     fun getCurrentFlowState(): InteractionFlowState
 

mabl/src/main/java/com/penumbraos/mabl/interaction/InteractionFlowManager.kt

@@ -44,6 +44,8 @@ class InteractionFlowManager
     private var stateCallback: InteractionStateCallback? = null
     private var contentCallback: InteractionContentCallback? = null
 
+    private var didAbort: Boolean = false
+
     private var cameraService: CameraService? = null
     private var isCameraServiceBound = false
 
@@ -64,17 +66,30 @@ class InteractionFlowManager
 
     private val sttCallback = object : ISttCallback.Stub() {
         override fun onPartialTranscription(partialText: String) {
+            if (didAbort) {
+                return
+            }
             Log.d(TAG, "STT partial transcription: $partialText")
             contentCallback?.onPartialTranscription(partialText)
         }
 
         override fun onFinalTranscription(finalText: String) {
-            Log.d(TAG, "STT final transcription: $finalText")
-            setState(InteractionFlowState.PROCESSING)
-            contentCallback?.onFinalTranscription(finalText)
+            if (didAbort) {
+                return
+            }
 
-            // Start conversation with the transcribed text
-            startConversationFromInput(finalText)
+            if (finalText.trim().isEmpty()) {
+                Log.d(TAG, "STT transcription was empty, skipping")
+                setState(InteractionFlowState.IDLE)
+                stateCallback?.onError(Error.SttError("Empty transcription"))
+            } else {
+                Log.d(TAG, "STT final transcription: $finalText")
+                setState(InteractionFlowState.PROCESSING)
+                contentCallback?.onFinalTranscription(finalText)
+
+                // Start conversation with the transcribed text
+                startConversationFromInput(finalText)
+            }
         }
 
         override fun onError(errorMessage: String) {
@@ -95,16 +110,22 @@ class InteractionFlowManager
     }
 
     override fun startListening(requestImage: Boolean) {
-        if (currentState != InteractionFlowState.IDLE) {
+        currentModality =
+            if (requestImage) InteractionFlowModality.Vision else InteractionFlowModality.Speech
+
+        if (currentState == InteractionFlowState.LISTENING) {
+            Log.d(TAG, "Already listening. Continuing")
+            return
+        } else if (currentState != InteractionFlowState.IDLE) {
             Log.w(TAG, "Cannot start listening, current state: $currentState")
             return
         }
 
+        didAbort = false
+
         try {
             allControllers.stt.startListening()
             setState(InteractionFlowState.LISTENING)
-            currentModality =
-                if (requestImage) InteractionFlowModality.Vision else InteractionFlowModality.Speech
         } catch (e: Exception) {
             Log.e(TAG, "Failed to start listening: ${e.message}")
             stateCallback?.onError(Error.SttError("Failed to start listening: ${e.message}"))
@@ -150,12 +171,12 @@ class InteractionFlowManager
                     }
 
                     override fun onCompleteResponse(finalResponse: String) {
-                        Log.d(TAG, "LLM complete response: $finalResponse")
                         allControllers.soundEffectManager.stopWaitingEffect()
                         if (currentState == InteractionFlowState.PROCESSING) {
                             // We never sent any partial responses. Make sure we send at least one
                             onPartialResponse(finalResponse)
                         }
+                        Log.d(TAG, "LLM complete response: $finalResponse")
                         contentCallback?.onFinalResponse(finalResponse)
                         setState(InteractionFlowState.IDLE)
                     }
@@ -170,11 +191,12 @@ class InteractionFlowManager
             )
         }
     }
-    
-    override fun finishListening() {
+
+    override fun finishListening(abort: Boolean) {
         Log.d(TAG, "Stopping listening, state: $currentState")
         setState(InteractionFlowState.CANCELLING)
 
+        didAbort = abort
         allControllers.stt.cancelListening()
         allControllers.tts.service?.stopSpeaking()
 
@@ -182,6 +204,10 @@ class InteractionFlowManager
         stateCallback?.onUserFinished()
     }
 
+    override fun cancelTalking() {
+        allControllers.tts.service?.stopSpeaking()
+    }
+
     override fun isFlowActive(): Boolean {
         return currentState != InteractionFlowState.IDLE
     }

mabl/src/main/java/com/penumbraos/mabl/plugins/llm/KtorHttpClient.kt

@@ -0,0 +1,131 @@
+package com.penumbraos.mabl.plugins.llm
+
+import com.penumbraos.sdk.PenumbraClient
+import com.penumbraos.sdk.http.ktor.HttpClientPlugin
+import dev.langchain4j.exception.HttpException
+import dev.langchain4j.http.client.HttpClient
+import dev.langchain4j.http.client.HttpMethod
+import dev.langchain4j.http.client.HttpRequest
+import dev.langchain4j.http.client.SuccessfulHttpResponse
+import dev.langchain4j.http.client.sse.ServerSentEventListener
+import dev.langchain4j.http.client.sse.ServerSentEventParser
+import io.ktor.client.call.body
+import io.ktor.client.plugins.sse.SSE
+import io.ktor.client.request.HttpRequestBuilder
+import io.ktor.client.request.prepareRequest
+import io.ktor.client.request.request
+import io.ktor.client.request.setBody
+import io.ktor.client.request.url
+import io.ktor.client.statement.HttpResponse
+import io.ktor.client.statement.bodyAsChannel
+import io.ktor.content.TextContent
+import io.ktor.http.ContentType
+import io.ktor.http.isSuccess
+import io.ktor.util.toMap
+import io.ktor.utils.io.jvm.javaio.toInputStream
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+
+class KtorHttpClient : HttpClient {
+    private val coroutineScope: CoroutineScope
+    private val ktorClient: io.ktor.client.HttpClient
+
+    constructor(coroutineScope: CoroutineScope, penumbraClient: PenumbraClient) {
+        this.coroutineScope = coroutineScope
+        this.ktorClient = io.ktor.client.HttpClient {
+            // Otherwise ktor strips ContentType
+            useDefaultTransformers = false
+            install(HttpClientPlugin) {
+                this.penumbraClient = penumbraClient
+            }
+            install(SSE)
+        }
+    }
+
+    override fun execute(request: HttpRequest): SuccessfulHttpResponse {
+        return runBlocking {
+            val response = ktorClient.request(request.url()) {
+                buildRequest(this, request)
+            }
+
+            buildResponse(response, true)
+        }
+    }
+
+    override fun execute(
+        request: HttpRequest,
+        parser: ServerSentEventParser,
+        listener: ServerSentEventListener
+    ) {
+        coroutineScope.launch {
+            ktorClient.prepareRequest {
+                buildRequest(this, request)
+            }.execute { response ->
+                if (!response.status.isSuccess()) {
+                    try {
+                        listener.onError(HttpException(response.status.value, response.body()))
+                    } catch (_: Exception) {
+                    }
+
+                    return@execute
+                }
+
+                try {
+                    listener.onOpen(buildResponse(response, false))
+                } catch (_: Exception) {
+                    return@execute
+                }
+
+                try {
+                    val stream = response.bodyAsChannel().toInputStream()
+                    parser.parse(stream, listener)
+                    listener.onClose()
+                } catch (e: Exception) {
+                    listener.onError(e)
+                }
+            }
+        }
+    }
+
+    private fun buildRequest(builder: HttpRequestBuilder, langChainRequest: HttpRequest) {
+        builder.url(langChainRequest.url())
+        builder.method = when (langChainRequest.method()) {
+            HttpMethod.GET -> io.ktor.http.HttpMethod.Get
+            HttpMethod.POST -> io.ktor.http.HttpMethod.Post
+            HttpMethod.DELETE -> io.ktor.http.HttpMethod.Delete
+        }
+        for ((key, values) in langChainRequest.headers()) {
+            builder.headers.appendAll(key, values)
+        }
+        val contentTypeString = langChainRequest.headers()["ContentType"]?.first() ?: ""
+
+        val contentType = try {
+            ContentType.parse(contentTypeString)
+        } catch (_: Exception) {
+            ContentType.Application.Json
+        }
+
+        builder.setBody(
+            TextContent(
+                langChainRequest.body(),
+                contentType
+            )
+        )
+    }
+
+    private suspend fun buildResponse(
+        response: HttpResponse,
+        withBody: Boolean
+    ): SuccessfulHttpResponse {
+        val builder = SuccessfulHttpResponse.builder()
+            .statusCode(response.status.value)
+            .headers(response.headers.toMap())
+
+        if (withBody) {
+            builder.body(response.body())
+        }
+
+        return builder.build()
+    }
+}

mabl/src/main/java/com/penumbraos/mabl/plugins/llm/KtorHttpClientBuilder.kt

@@ -0,0 +1,36 @@
+package com.penumbraos.mabl.plugins.llm
+
+import com.penumbraos.sdk.PenumbraClient
+import dev.langchain4j.http.client.HttpClient
+import dev.langchain4j.http.client.HttpClientBuilder
+import kotlinx.coroutines.CoroutineScope
+import java.time.Duration
+
+class KtorHttpClientBuilder(coroutineScope: CoroutineScope, penumbraClient: PenumbraClient) :
+    HttpClientBuilder {
+    val client = KtorHttpClient(coroutineScope, penumbraClient)
+
+    override fun connectTimeout(): Duration? {
+        // TODO
+        return Duration.ZERO
+    }
+
+    override fun connectTimeout(timeout: Duration?): HttpClientBuilder {
+        // TODO
+        return this
+    }
+
+    override fun readTimeout(): Duration? {
+        // TODO
+        return Duration.ZERO
+    }
+
+    override fun readTimeout(timeout: Duration?): HttpClientBuilder {
+        // TODO
+        return this
+    }
+
+    override fun build(): HttpClient {
+        return client
+    }
+}

mabl/src/main/java/com/penumbraos/mabl/plugins/llm/LangchainLlmService.kt

@@ -0,0 +1,331 @@
+@file:OptIn(ExperimentalEncodingApi::class)
+
+package com.penumbraos.mabl.plugins.llm
+
+import android.annotation.SuppressLint
+import android.content.Intent
+import android.os.IBinder
+import android.system.Os
+import android.system.OsConstants
+import android.util.Log
+import com.penumbraos.mabl.sdk.BinderConversationMessage
+import com.penumbraos.mabl.sdk.ILlmCallback
+import com.penumbraos.mabl.sdk.ILlmService
+import com.penumbraos.mabl.sdk.LlmResponse
+import com.penumbraos.mabl.sdk.MablService
+import com.penumbraos.mabl.sdk.ToolCall
+import com.penumbraos.mabl.sdk.ToolDefinition
+import com.penumbraos.mabl.sdk.ToolParameter
+import com.penumbraos.sdk.PenumbraClient
+import dev.langchain4j.agent.tool.ToolExecutionRequest
+import dev.langchain4j.agent.tool.ToolSpecification
+import dev.langchain4j.data.message.AiMessage.aiMessage
+import dev.langchain4j.data.message.ImageContent
+import dev.langchain4j.data.message.SystemMessage.systemMessage
+import dev.langchain4j.data.message.TextContent
+import dev.langchain4j.data.message.ToolExecutionResultMessage.toolExecutionResultMessage
+import dev.langchain4j.data.message.UserMessage.userMessage
+import dev.langchain4j.kotlin.model.chat.StreamingChatModelReply
+import dev.langchain4j.kotlin.model.chat.chatFlow
+import dev.langchain4j.model.chat.StreamingChatModel
+import dev.langchain4j.model.chat.request.ChatRequestParameters
+import dev.langchain4j.model.chat.request.json.JsonObjectSchema
+import dev.langchain4j.model.chat.response.ChatResponse
+import dev.langchain4j.model.googleai.GoogleAiGeminiStreamingChatModel
+import dev.langchain4j.model.openai.OpenAiStreamingChatModel
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.flow.catch
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.flow.onEach
+import kotlinx.coroutines.launch
+import java.io.ByteArrayOutputStream
+import java.io.FileInputStream
+import kotlin.io.encoding.Base64
+import kotlin.io.encoding.ExperimentalEncodingApi
+
+private const val TAG = "LangchainLlmService"
+
+private const val DEFAULT_PROMPT =
+    """You are the MABL voice assistant. Your response will be spoken aloud to the user, so keep the response short and to the point.
+        |Your core responsibilities:
+        |1. Understand the user's request thoroughly.
+        |2. Identify which of the provided tools can best fulfill the request.
+        |3. Execute the tool(s) and provide a concise, accurate response based on the tool's output.
+        |4. If a tool is necessary to provide up-to-date or factual information (e.g., current news, real-time data), prioritize its use.
+        |5. Do NOT make up information. If a tool is required to get the answer, use it.
+        |6. If a query requires knowledge beyond your training data, especially for current events or news, the `web_search` tool is essential.
+        |7. Do not declare limitations (e.g., "I can only do X") if other relevant tools are available for the user's query. You have access to *all* provided tools.
+        |8. If no adequate tool is available, you are allowed to fall back on your own knowledge, but only when you have a high confidence of the answer."""
+
+class LangchainLlmService : MablService("LangchainLlmService") {
+
+    private val llmScope = CoroutineScope(Dispatchers.IO)
+    private var model: StreamingChatModel? = null
+    private val configManager = LlmConfigManager()
+    private var currentConfig: LlmConfiguration? = null
+
+    @SuppressLint("ForegroundServiceType")
+    override fun onCreate() {
+        super.onCreate()
+
+        llmScope.launch {
+            var client = PenumbraClient(this@LangchainLlmService)
+            client.waitForBridge()
+
+            try {
+                currentConfig = configManager.getAvailableConfigs().first()
+            } catch (e: Exception) {
+                Log.e(TAG, "Failed to load LLM configuration", e)
+            }
+
+            val config = currentConfig
+
+            if (config == null) {
+                Log.e(TAG, "No valid LLM configuration found")
+                return@launch
+            }
+
+            try {
+                Log.d(TAG, "About to create Langchain client")
+                model = when (config) {
+                    is LlmConfiguration.Gemini -> {
+                        GoogleAiGeminiStreamingChatModel.builder()
+                            .allowGoogleSearch(true)
+                            .allowGoogleMaps(true)
+                            .httpClientBuilder(KtorHttpClientBuilder(llmScope, client))
+                            .apiKey(config.apiKey)
+                            .modelName(config.model)
+                            .temperature(config.temperature)
+                            .maxOutputTokens(config.maxTokens).build()
+                    }
+
+                    is LlmConfiguration.OpenAI -> {
+                        OpenAiStreamingChatModel.builder()
+                            .httpClientBuilder(KtorHttpClientBuilder(llmScope, client))
+                            .baseUrl(config.baseUrl)
+                            .apiKey(config.apiKey)
+                            .modelName(config.model)
+                            .temperature(config.temperature)
+                            .maxTokens(config.maxTokens).build()
+                    }
+                }
+
+                Log.w(
+                    TAG,
+                    "${config.type} client initialized successfully with model: ${config.model}"
+                )
+            } catch (e: Exception) {
+                Log.e(TAG, "Failed to initialize Langchain client", e)
+            }
+        }
+    }
+
+    private val binder = object : ILlmService.Stub() {
+        // TODO: Remove
+        override fun setAvailableTools(tools: Array<ToolDefinition>) {
+            Log.d(TAG, "Received ${tools.size} tool definitions")
+        }
+
+        override fun generateResponse(
+            messages: Array<BinderConversationMessage>,
+            tools: Array<ToolDefinition>,
+            callback: ILlmCallback
+        ) {
+            Log.w(
+                TAG,
+                "Submitting ${messages.size} conversation messages with ${tools.size} filtered tools. Last message: \"${messages.last().content}\""
+            )
+
+            if (model == null) {
+                Log.e(TAG, "LLM client not initialized")
+                callback.onError("LLM client not initialized. Check API key configuration.")
+                return
+            }
+
+            llmScope.launch {
+                try {
+                    val responseBuilder = StringBuilder()
+                    val toolCalls = mutableListOf<ToolCall>()
+
+                    val completions = model!!.chatFlow {
+                        this.messages += systemMessage(
+                            currentConfig!!.systemPrompt
+                                ?: DEFAULT_PROMPT.trimMargin()
+                        )
+
+                        this.messages += messages.map { message ->
+                            when (message.type) {
+                                "user" -> {
+                                    if (message.imageFile != null) {
+                                        val fileDescriptor = message.imageFile.fileDescriptor
+                                        // Rewind file descriptor so we can reuse them
+                                        // TODO: This somehow needs to live in MABL core
+                                        Os.lseek(
+                                            fileDescriptor,
+                                            0,
+                                            OsConstants.SEEK_SET
+                                        )
+                                        val imageBytes =
+                                            FileInputStream(fileDescriptor)
+                                        val byteArrayOutputStream = ByteArrayOutputStream()
+                                        val buffer = ByteArray(4096)
+                                        var bytesRead: Int
+                                        while (imageBytes.read(buffer)
+                                                .also { bytesRead = it } != -1
+                                        ) {
+                                            byteArrayOutputStream.write(buffer, 0, bytesRead)
+                                        }
+                                        val imageUrl =
+                                            Base64.Default.encode(byteArrayOutputStream.toByteArray())
+
+                                        userMessage(
+                                            TextContent(message.content),
+                                            ImageContent(
+                                                imageUrl,
+                                                "image/jpeg",
+                                                ImageContent.DetailLevel.HIGH
+                                            )
+                                        )
+                                    } else {
+                                        userMessage(TextContent(message.content))
+                                    }
+                                }
+
+                                "assistant" -> aiMessage(
+                                    message.content,
+                                    message.toolCalls.map { toolCall ->
+                                        ToolExecutionRequest.builder().id(toolCall.id)
+                                            .name(toolCall.name).arguments(toolCall.parameters)
+                                            .build()
+                                    }
+                                )
+
+                                // TODO: This tool name might be wrong/necessary
+                                "tool" -> toolExecutionResultMessage(
+                                    message.toolCallId,
+                                    message.toolCallId,
+                                    message.content
+                                )
+
+                                else -> userMessage(message.content)
+                            }
+                        }
+
+                        this.parameters =
+                            ChatRequestParameters.builder().toolSpecifications(
+                                convertToolDefinitionsToAPI(tools)
+                            ).build()
+                    }
+
+                    var finalResponse: ChatResponse? = null
+
+                    completions
+                        .catch { exception ->
+                            Log.e(TAG, "Error making request", exception)
+                            val content =
+                                "LLM model error: ${exception.message?.removePrefix("Stream error: ")}"
+                            responseBuilder.append(content)
+                            // TODO: This should be onError
+                            callback.onPartialResponse(content)
+                        }
+                        .onEach { chunk ->
+                            when (chunk) {
+                                is StreamingChatModelReply.CompleteResponse -> {
+                                    finalResponse = chunk.response
+                                }
+
+                                is StreamingChatModelReply.PartialResponse -> {
+                                    callback.onPartialResponse(chunk.partialResponse)
+                                }
+
+                                is StreamingChatModelReply.Error -> {
+                                    throw chunk.cause
+                                }
+                            }
+                        }
+                        .collect()
+
+                    if (finalResponse == null) {
+                        // TODO: This should be onError
+                        callback.onCompleteResponse(LlmResponse().apply {
+                            text = "LLM model error: Empty response"
+                        })
+                        return@launch
+                    }
+
+                    // Send final response
+                    val response = LlmResponse().apply {
+                        text = finalResponse.aiMessage().text() ?: ""
+                        this.toolCalls =
+                            finalResponse.aiMessage().toolExecutionRequests().map { request ->
+                                ToolCall().apply {
+                                    id = request.id()
+                                    name = request.name()
+                                    parameters = request.arguments()
+                                    isLLM = true
+                                }
+                            }.toTypedArray()
+                    }
+
+                    val flattenedCalls = toolCalls.joinToString {
+                        "id: ${it.id}, name: ${it.name}, parameters: ${it.parameters}"
+                    }
+                    Log.w(
+                        TAG,
+                        "LLM response received: \"${response.text}\", $flattenedCalls"
+                    )
+                    callback.onCompleteResponse(response)
+                } catch (e: Exception) {
+                    Log.e(TAG, "Error generating response", e)
+                    callback.onError("Error generating response: ${e.message}")
+                }
+            }
+        }
+    }
+
+    private fun convertToolDefinitionsToAPI(toolDefinitions: Array<ToolDefinition>): List<ToolSpecification>? {
+        if (toolDefinitions.isEmpty()) {
+            return null
+        }
+
+        return toolDefinitions.map { toolDef ->
+            ToolSpecification.builder().name(toolDef.name).description(toolDef.description)
+                .parameters(convertParametersToAPI(toolDef.parameters)).build()
+        }
+    }
+
+    private fun convertParametersToAPI(parameters: Array<ToolParameter>): JsonObjectSchema {
+        val builder = JsonObjectSchema.builder()
+        val required = mutableListOf<String>()
+
+        for (parameter in parameters) {
+            if (parameter.required) {
+                required += parameter.name
+            }
+
+            when (parameter.type.lowercase()) {
+                "string" -> builder.addStringProperty(parameter.name, parameter.description)
+                "number", "float", "double", "int" -> builder.addNumberProperty(
+                    parameter.name,
+                    parameter.description
+                )
+
+                "enum" -> builder.addEnumProperty(
+                    parameter.name,
+                    parameter.enumValues.toList(),
+                    parameter.description
+                )
+            }
+        }
+
+        return builder.required(required).build()
+    }
+
+    override fun onBind(intent: Intent?): IBinder = binder
+
+    override fun onDestroy() {
+        super.onDestroy()
+        Log.d(TAG, "Langchain4j LLM service destroyed")
+    }
+}

mabl/src/main/java/com/penumbraos/mabl/plugins/llm/LlmConfig.kt

@@ -0,0 +1,82 @@
+package com.penumbraos.mabl.plugins.llm
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonTransformingSerializer
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.put
+
+private const val DEFAULT_MAX_TOKENS = 1000
+private const val DEFAULT_TEMPERATURE = 0.7
+
+interface LlmConfig {
+    val type: String
+
+    val name: String
+    val apiKey: String
+    val model: String
+    val maxTokens: Int
+    val temperature: Double
+    val systemPrompt: String?
+}
+
+@Serializable
+sealed class LlmConfiguration : LlmConfig {
+
+    @Serializable
+    @SerialName("gemini")
+    data class Gemini(
+        override val type: String = "Gemini",
+
+        override val name: String,
+        override val apiKey: String,
+        override val model: String,
+        override val maxTokens: Int = DEFAULT_MAX_TOKENS,
+        override val temperature: Double = DEFAULT_TEMPERATURE,
+        override val systemPrompt: String? = null
+    ) : LlmConfiguration()
+
+    @Serializable
+    @SerialName("openai")
+    data class OpenAI(
+        override val type: String = "OpenAI",
+
+        override val name: String,
+        override val apiKey: String,
+        override val model: String,
+        val baseUrl: String,
+        override val maxTokens: Int = DEFAULT_MAX_TOKENS,
+        override val temperature: Double = DEFAULT_TEMPERATURE,
+        override val systemPrompt: String? = null
+    ) : LlmConfiguration()
+}
+
+object LlmConfigurationSerializer :
+    JsonTransformingSerializer<LlmConfiguration>(LlmConfiguration.serializer()) {
+    override fun transformDeserialize(element: JsonElement): JsonElement {
+        if (element is JsonObject && "type" !in element) {
+            // If no type field, default to "openai"
+            return buildJsonObject {
+                put("type", "openai")
+                element.forEach { (key, value) ->
+                    put(key, value)
+                }
+            }
+        }
+        return element
+    }
+}
+
+@Serializable
+data class LlmConfigFile(
+    @Serializable(with = LlmConfigurationListSerializer::class)
+    val configs: List<LlmConfiguration>
+)
+
+object LlmConfigurationListSerializer : JsonTransformingSerializer<List<LlmConfiguration>>(
+    kotlinx.serialization.builtins.ListSerializer(LlmConfigurationSerializer)
+) {
+    override fun transformDeserialize(element: JsonElement): JsonElement = element
+}

mabl/src/main/java/com/penumbraos/mabl/plugins/llm/LlmConfigManager.kt

@@ -1,29 +1,12 @@
-package com.penumbraos.plugins.openai
+package com.penumbraos.mabl.plugins.llm
 
 import android.annotation.SuppressLint
 import android.util.Log
-import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.Json
 import java.io.File
 
 private const val TAG = "LlmConfigService"
 
-@Serializable
-data class LlmConfiguration(
-    val name: String,
-    val apiKey: String,
-    val model: String,
-    val maxTokens: Int = 1000,
-    val temperature: Double = 0.7,
-    val systemPrompt: String? = null,
-    val baseUrl: String
-)
-
-@Serializable
-data class LlmConfigFile(
-    val configs: List<LlmConfiguration>
-)
-
 class LlmConfigManager {
 
     private var configs: List<LlmConfiguration>? = null
@@ -50,11 +33,16 @@ class LlmConfigManager {
                 val jsonString = configFile.readText()
                 val configFile = json.decodeFromString<LlmConfigFile>(jsonString)
                 val logMap = configFile.configs.map { config ->
+                    val baseUrlInfo = if (config is LlmConfiguration.OpenAI) {
+                        "Base URL: ${config.baseUrl}\n                        "
+                    } else {
+                        ""
+                    }
                     """
+                        Type: ${config.type}
                         Name: ${config.name}
                         Model: ${config.model}
-                        Base URL: ${config.baseUrl}
-                        Max Tokens: ${config.maxTokens}
+                        ${baseUrlInfo}Max Tokens: ${config.maxTokens}
                         Temperature: ${config.temperature}
                     """.trimIndent()
                 }

mabl/src/main/java/com/penumbraos/mabl/services/AllControllers.kt

@@ -70,7 +70,7 @@ class AllControllers(coroutineScope: CoroutineScope, private val context: Contex
             // In simulator mode, use more resilient connection approach
             // Only connect to services that are known to work
             try {
-                llm.connect(context, "com.penumbraos.plugins.openai")
+                llm.connect(context, "com.penumbraos.mabl.pinsim")
             } catch (e: Exception) {
                 Log.w(TAG, "Failed to connect LLM service in simulator: $e")
             }
@@ -99,7 +99,7 @@ class AllControllers(coroutineScope: CoroutineScope, private val context: Contex
             }
         } else {
             // Normal mode - connect to all external services
-            llm.connect(context, "com.penumbraos.plugins.openai")
+            llm.connect(context, "com.penumbraos.mabl.pin")
             stt.connect(context, "com.penumbraos.plugins.demo")
             tts.connect(context, "com.penumbraos.plugins.demo")
             toolOrchestrator.connectAll()

plugins/demo/src/main/java/com/penumbraos/plugins/demo/DemoSttService.kt

@@ -39,8 +39,14 @@ class DemoSttService : MablService("DemoSttService") {
             client.stt.initialize(object : SttRecognitionListener() {
                 override fun onError(error: Int) {
                     try {
-                        currentCallback?.onError("Recognition error: $error")
-                    } catch (e: RemoteException) {
+                        // RecognitionError.ERROR_NO_MATCH
+                        if (error == 7) {
+                            Log.d("DemoSttService", "No speech recognized")
+                            currentCallback?.onFinalTranscription("")
+                        } else {
+                            currentCallback?.onError("Recognition error: $error")
+                        }
+                    } catch (e: Exception) {
                         Log.e("DemoSttService", "Callback error", e)
                     }
                 }

plugins/openai/.gitignore

@@ -1 +0,0 @@
-/build

plugins/openai/build.gradle.kts

@@ -1,49 +0,0 @@
-plugins {
-    alias(libs.plugins.android.application)
-    alias(libs.plugins.kotlin.android)
-    alias(libs.plugins.kotlin.serialization)
-}
-
-android {
-    namespace = "com.penumbraos.plugins.openai"
-    compileSdk = 35
-
-    defaultConfig {
-        applicationId = "com.penumbraos.plugins.openai"
-        minSdk = 32
-        targetSdk = 35
-        versionCode = (project.findProperty("versionCode") as String?)?.toIntOrNull() ?: 1
-        versionName = project.findProperty("versionName") as String? ?: "1.0"
-    }
-
-    buildTypes {
-        release {
-            isMinifyEnabled = false
-            signingConfig = signingConfigs.getByName("debug")
-        }
-    }
-    compileOptions {
-        sourceCompatibility = JavaVersion.VERSION_11
-        targetCompatibility = JavaVersion.VERSION_11
-    }
-    kotlinOptions {
-        jvmTarget = "11"
-    }
-}
-
-dependencies {
-    implementation(project(":sdk"))
-
-    implementation(libs.penumbraos.sdk)
-
-    implementation(libs.openai.client)
-    implementation(libs.ktor.client.android)
-
-    implementation(libs.kotlinx.serialization.json)
-
-    implementation(libs.kotlinx.coroutines.android)
-
-    implementation(libs.androidx.core.ktx)
-    implementation(libs.androidx.appcompat)
-    implementation(libs.material)
-}

plugins/openai/src/main/AndroidManifest.xml

@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<manifest xmlns:tools="http://schemas.android.com/tools"
-    xmlns:android="http://schemas.android.com/apk/res/android"
-    package="com.penumbraos.plugins.openai">
-
-    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
-    <uses-permission android:name="android.permission.INTERNET" />
-    <uses-permission
-        android:name="android.permission.MANAGE_EXTERNAL_STORAGE"
-        tools:ignore="ScopedStorage" />
-
-    <application tools:ignore="MissingApplicationIcon">
-        <service
-            android:name=".OpenAiLlmService"
-            android:exported="true"
-            tools:ignore="ExportedService">
-            <intent-filter>
-                <action android:name="com.penumbraos.mabl.sdk.action.LLM_SERVICE" />
-            </intent-filter>
-
-            <meta-data
-                android:name="com.penumbraos.mabl.sdk.metadata.DISPLAY_NAME"
-                android:value="OpenAI LLM Service" />
-            <meta-data
-                android:name="com.penumbraos.mabl.sdk.metadata.DESCRIPTION"
-                android:value="OpenAI compatible language model service" />
-        </service>
-    </application>
-
-</manifest>

plugins/openai/src/main/java/com/penumbraos/plugins/openai/OpenAiLlmService.kt

@@ -1,344 +0,0 @@
-@file:OptIn(ExperimentalEncodingApi::class)
-
-package com.penumbraos.plugins.openai
-
-import android.annotation.SuppressLint
-import android.content.Intent
-import android.os.IBinder
-import android.system.Os
-import android.system.OsConstants
-import android.util.Log
-import com.aallam.openai.api.chat.*
-import com.aallam.openai.api.core.Parameters
-import com.aallam.openai.api.model.ModelId
-import com.aallam.openai.client.OpenAI
-import com.aallam.openai.client.OpenAIHost
-import com.aallam.openai.client.RetryStrategy
-import com.penumbraos.mabl.sdk.BinderConversationMessage
-import com.penumbraos.mabl.sdk.DeviceUtils
-import com.penumbraos.mabl.sdk.ILlmCallback
-import com.penumbraos.mabl.sdk.ILlmService
-import com.penumbraos.mabl.sdk.LlmResponse
-import com.penumbraos.mabl.sdk.MablService
-import com.penumbraos.mabl.sdk.ToolCall
-import com.penumbraos.mabl.sdk.ToolDefinition
-import com.penumbraos.mabl.sdk.ToolParameter
-import com.penumbraos.sdk.PenumbraClient
-import com.penumbraos.sdk.http.ktor.HttpClientPlugin
-import io.ktor.client.*
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.flow.catch
-import kotlinx.coroutines.flow.collect
-import kotlinx.coroutines.flow.onEach
-import kotlinx.coroutines.launch
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.json.Json
-import java.io.ByteArrayOutputStream
-import java.io.FileInputStream
-import kotlin.io.encoding.Base64
-import kotlin.io.encoding.ExperimentalEncodingApi
-import kotlin.time.Duration.Companion.milliseconds
-
-private const val TAG = "OpenAiLlmService"
-
-@Serializable
-data class ParameterSchema(
-    val type: String,
-    val properties: Map<String, PropertySchema> = emptyMap(),
-    val required: List<String> = emptyList()
-)
-
-@Serializable
-data class PropertySchema(
-    val type: String,
-    val description: String,
-    val enum: List<String>? = null
-)
-
-private const val DEFAULT_PROMPT =
-    """You are the MABL voice assistant. Your response will be spoken aloud to the user, so keep the response short and to the point.
-        |Your core responsibilities:
-        |1. Understand the user's request thoroughly.
-        |2. Identify which of the provided tools can best fulfill the request.
-        |3. Execute the tool(s) and provide a concise, accurate response based on the tool's output.
-        |4. If a tool is necessary to provide up-to-date or factual information (e.g., current news, real-time data), prioritize its use.
-        |5. Do NOT make up information. If a tool is required to get the answer, use it.
-        |6. If a query requires knowledge beyond your training data, especially for current events or news, the `web_search` tool is essential.
-        |7. Do not declare limitations (e.g., "I can only do X") if other relevant tools are available for the user's query. You have access to *all* provided tools.
-        |8. If no adequate tool is available, you are allowed to fall back on your own knowledge, but only when you have a high confidence of the answer."""
-
-class OpenAiLlmService : MablService("OpenAiLlmService") {
-
-    private val llmScope = CoroutineScope(Dispatchers.IO)
-    private var openAI: OpenAI? = null
-    private val configManager = LlmConfigManager()
-    private var currentConfig: LlmConfiguration? = null
-
-    @SuppressLint("ForegroundServiceType")
-    override fun onCreate() {
-        super.onCreate()
-
-        llmScope.launch {
-            var client: PenumbraClient? = null
-            if (DeviceUtils.isAiPin()) {
-                client = PenumbraClient(this@OpenAiLlmService)
-                client.waitForBridge()
-            }
-
-            try {
-                currentConfig = configManager.getAvailableConfigs().first()
-            } catch (e: Exception) {
-                Log.e(TAG, "Failed to load LLM configuration", e)
-            }
-
-            if (currentConfig == null) {
-                Log.e(TAG, "No valid LLM configuration found")
-                return@launch
-            }
-
-            try {
-                Log.d(TAG, "About to create OpenAI client")
-                val apiKey = currentConfig!!.apiKey
-                val baseUrl = currentConfig!!.baseUrl
-
-                openAI =
-                    OpenAI(
-                        token = apiKey,
-                        host = OpenAIHost(baseUrl),
-                        retry = RetryStrategy(
-                            maxRetries = 1,
-                            base = 0.2,
-                            maxDelay = 200.milliseconds
-                        ),
-                        httpClientConfig = {
-                            if (DeviceUtils.isAiPin()) {
-                                install(HttpClientPlugin) {
-                                    // Should have been initialized at start
-                                    penumbraClient = client!!
-                                }
-                            }
-                        }
-                    )
-                Log.w(
-                    TAG,
-                    "OpenAI client initialized successfully with model: ${currentConfig!!.model}"
-                )
-            } catch (e: Exception) {
-                Log.e(TAG, "Failed to initialize OpenAI client", e)
-            }
-        }
-    }
-
-    private var availableTools: List<Tool>? = null
-
-    private val binder = object : ILlmService.Stub() {
-        override fun setAvailableTools(tools: Array<ToolDefinition>) {
-            Log.d(TAG, "Received ${tools.size} tool definitions")
-            availableTools = convertToolDefinitionsToOpenAI(tools)
-        }
-
-        override fun generateResponse(
-            messages: Array<BinderConversationMessage>,
-            tools: Array<ToolDefinition>,
-            callback: ILlmCallback
-        ) {
-            Log.w(
-                TAG,
-                "Submitting ${messages.size} conversation messages with ${tools.size} filtered tools. Last message: \"${messages.last().content}\""
-            )
-
-            if (openAI == null) {
-                Log.e(TAG, "OpenAI client not initialized")
-                callback.onError("OpenAI client not initialized. Check API key configuration.")
-                return
-            }
-
-            llmScope.launch {
-                try {
-                    val conversationMessages = messages.map { message ->
-                        when (message.type) {
-                            "user" -> {
-                                if (message.imageFile != null) {
-                                    val fileDescriptor = message.imageFile.fileDescriptor
-                                    // Rewind file descriptor so we can reuse them
-                                    // TODO: This somehow needs to live in MABL core
-                                    Os.lseek(
-                                        fileDescriptor,
-                                        0,
-                                        OsConstants.SEEK_SET
-                                    )
-                                    val imageBytes =
-                                        FileInputStream(fileDescriptor)
-                                    val byteArrayOutputStream = ByteArrayOutputStream()
-                                    val buffer = ByteArray(4096)
-                                    var bytesRead: Int
-                                    while (imageBytes.read(buffer).also { bytesRead = it } != -1) {
-                                        byteArrayOutputStream.write(buffer, 0, bytesRead)
-                                    }
-                                    val imageUrl =
-                                        Base64.Default.encode(byteArrayOutputStream.toByteArray())
-
-                                    ChatMessage(
-                                        role = ChatRole.User,
-                                        content = listOf(
-                                            TextPart(message.content),
-                                            ImagePart(url = "data:image/jpeg;base64,$imageUrl")
-                                        )
-                                    )
-                                } else {
-                                    ChatMessage(
-                                        role = ChatRole.User,
-                                        content = message.content
-                                    )
-                                }
-                            }
-
-                            "assistant" -> ChatMessage(
-                                role = ChatRole.Assistant,
-                                content = message.content,
-                                toolCalls = message.toolCalls?.map { toolCall ->
-                                    function {
-                                        id = ToolId(toolCall.id)
-                                        function = FunctionCall(
-                                            toolCall.name,
-                                            toolCall.parameters
-                                        )
-                                    }
-                                }
-                            )
-
-                            "tool" -> ChatMessage(
-                                role = ChatRole.Tool,
-                                content = message.content,
-                                toolCallId = message.toolCallId?.let {
-                                    ToolId(
-                                        it
-                                    )
-                                }
-                            )
-
-                            else -> ChatMessage(
-                                role = ChatRole.User,
-                                content = message.content
-                            )
-                        }
-                    }
-
-                    val chatMessages = listOf(
-                        ChatMessage(
-                            role = ChatRole.System,
-                            content = currentConfig!!.systemPrompt
-                                ?: DEFAULT_PROMPT.trimMargin()
-                        )
-                    ) + conversationMessages
-
-                    val chatCompletionRequest = ChatCompletionRequest(
-                        model = ModelId(currentConfig!!.model),
-                        messages = chatMessages,
-                        maxTokens = currentConfig!!.maxTokens,
-                        temperature = currentConfig!!.temperature,
-                        tools = convertToolDefinitionsToOpenAI(tools)
-                    )
-
-                    val responseBuilder = StringBuilder()
-                    val toolCalls = mutableListOf<ToolCall>()
-
-                    val completions = openAI!!.chatCompletions(chatCompletionRequest)
-                    completions
-                        .catch { exception ->
-                            Log.e(TAG, "Error making request", exception)
-                            val content =
-                                "OpenAI error: ${exception.message?.removePrefix("Stream error: ")}"
-                            responseBuilder.append(content)
-                            // TODO: This should be onError
-                            callback.onPartialResponse(content)
-                        }
-                        .onEach { chunk: ChatCompletionChunk ->
-                            Log.d(TAG, "Received chunk: $chunk")
-                            chunk.choices.forEach { choice ->
-                                choice.delta?.let { delta ->
-                                    delta.content?.let { content ->
-                                        responseBuilder.append(content)
-                                        callback.onPartialResponse(content)
-                                    }
-
-                                    delta.toolCalls?.forEach { toolCall ->
-                                        if (toolCall.function != null) {
-                                            val convertedToolCall = ToolCall().apply {
-                                                id = toolCall.id!!.id
-                                                name = toolCall.function!!.name
-                                                parameters = toolCall.function!!.arguments
-                                                isLLM = true
-                                            }
-                                            toolCalls.add(convertedToolCall)
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                        .collect()
-
-                    // Send final response
-                    val response = LlmResponse().apply {
-                        text = responseBuilder.toString()
-                        this.toolCalls = toolCalls.toTypedArray()
-                    }
-
-                    val flattenedCalls = toolCalls.joinToString {
-                        "id: ${it.id}, name: ${it.name}, parameters: ${it.parameters}"
-                    }
-                    Log.w(
-                        TAG,
-                        "LLM response received: \"${response.text}\", $flattenedCalls"
-                    )
-                    callback.onCompleteResponse(response)
-                } catch (e: Exception) {
-                    Log.e(TAG, "Error generating response", e)
-                    callback.onError("Error generating response: ${e.message}")
-                }
-            }
-        }
-    }
-
-    private fun convertToolDefinitionsToOpenAI(toolDefinitions: Array<ToolDefinition>): List<Tool>? {
-        if (toolDefinitions.isEmpty()) {
-            return null
-        }
-
-        return toolDefinitions.map { toolDef ->
-            Tool.function(
-                name = toolDef.name,
-                description = toolDef.description,
-                parameters = convertParametersToOpenAI(toolDef.parameters)
-            )
-        }
-    }
-
-    private fun convertParametersToOpenAI(parameters: Array<ToolParameter>): Parameters {
-        val properties = parameters.associate { param ->
-            param.name to PropertySchema(
-                type = param.type,
-                description = param.description,
-                enum = if (param.enumValues.isNotEmpty()) param.enumValues.toList() else null
-            )
-        }
-
-        val required = parameters.filter { it.required }.map { it.name }
-
-        val schema = ParameterSchema(
-            type = "object",
-            properties = properties,
-            required = required
-        )
-
-        return Parameters.fromJsonString(Json.encodeToString(ParameterSchema.serializer(), schema))
-    }
-
-    override fun onBind(intent: Intent?): IBinder = binder
-
-    override fun onDestroy() {
-        super.onDestroy()
-        Log.d(TAG, "OpenAI LLM Service destroyed")
-    }
-}